From f7bb3feca663f25fd584905ebf20e25a35365b85 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 8 Oct 2024 05:45:16 -0400 Subject: [PATCH 01/51] fix: reformat ec2 cleanup commands (#1267) Co-authored-by: Sam Rose --- .github/workflows/testinfra-nix.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-nix.yml index 3835a9a00..fee2fd791 100644 --- a/.github/workflows/testinfra-nix.yml +++ b/.github/workflows/testinfra-nix.yml @@ -65,12 +65,12 @@ jobs: - name: Cleanup resources on build cancellation if: ${{ cancelled() }} run: | - aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --region ap-southeast-1 --instance-ids {} - - - name: Cleanup resources on build cancellation + aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region ap-southeast-1 --instance-ids + + - name: Cleanup resources after build if: ${{ always() }} run: | - aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:testinfra-run-id,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --region ap-southeast-1 --instance-ids {} || true + aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:testinfra-run-id,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region ap-southeast-1 --instance-ids || true - name: Cleanup AMIs if: always() From 99f238e3899c350ed6b49a0baca3cc27d8b272de Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 7 Oct 2024 16:28:53 -0400 Subject: [PATCH 02/51] feat: build and flake check of pg 16.3 with exts/wrappers --- flake.nix | 35 ++++++++-- nix/ext/hypopg.nix | 2 +- nix/ext/pg_backtrace.nix | 2 +- nix/ext/pg_cron.nix | 2 +- nix/ext/pg_hashids.nix | 2 +- nix/ext/pg_net.nix | 2 +- nix/ext/pg_plan_filter.nix | 2 +- nix/ext/pg_stat_monitor.nix | 2 +- nix/ext/pg_tle.nix | 2 +- nix/ext/pgrouting.nix | 33 +++++++++- nix/ext/pgsodium.nix | 2 +- nix/ext/pgsql-http.nix | 2 +- nix/ext/pgvector.nix | 2 +- nix/ext/plv8.nix | 76 ++-------------------- nix/ext/supautils.nix | 2 +- nix/postgresql/16.nix | 4 ++ nix/postgresql/default.nix | 1 + nix/tests/sql/extensions_sql_interface.sql | 12 ++++ 18 files changed, 96 insertions(+), 89 deletions(-) create mode 100644 nix/postgresql/16.nix diff --git a/flake.nix b/flake.nix index 82409b185..3bb6d1a45 100644 --- a/flake.nix +++ b/flake.nix @@ -290,8 +290,7 @@ # Define the available PostgreSQL versions postgresVersions = { psql_15 = makePostgres "15"; - # Uncomment the line below to enable PostgreSQL 16 - # psql_16 = makePostgres "16"; + psql_16 = makePostgres "16"; # psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; }; @@ -307,15 +306,17 @@ postgresql = postgresqlPackage; }; postgresql_15 = getPostgresqlPackage "15"; + postgresql_16 = getPostgresqlPackage "16"; in postgresVersions //{ supabase-groonga = supabase-groonga; # PostgreSQL versions. - psql_15 = makePostgres "15"; + psql_15 = postgresVersions.psql_15; + psql_16 = postgresVersions.psql_16; #psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; sfcgal = sfcgal; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; - inherit postgresql_15; + inherit postgresql_15 postgresql_16; postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null; postgresql_15_src = pkgs.stdenv.mkDerivation { pname = "postgresql-15-src"; @@ -339,6 +340,28 @@ platforms = platforms.all; }; }; + postgresql_16_src = pkgs.stdenv.mkDerivation { + pname = "postgresql-16-src"; + version = postgresql_16.version; + + src = postgresql_16.src; + + nativeBuildInputs = [ pkgs.bzip2 ]; + + phases = [ "unpackPhase" "installPhase" ]; + + installPhase = '' + mkdir -p $out + cp -r . $out + ''; + + meta = with pkgs.lib; { + description = "PostgreSQL 15 source files"; + homepage = "https://www.postgresql.org/"; + license = licenses.postgresql; + platforms = platforms.all; + }; + }; mecab_naist_jdic = mecab-naist-jdic; supabase_groonga = supabase-groonga; pg_regress = makePgRegress activeVersion; @@ -397,6 +420,7 @@ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL_CONF_FILE' $out/etc/postgresql/postgresql.conf \ + --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ --subst-var-by 'READREPL_CONF_FILE' "$out/etc/postgresql-custom/read-replica.conf" \ --subst-var-by 'LOGGING_CONF_FILE' "$out/etc/postgresql-custom/logging.conf" \ @@ -425,6 +449,7 @@ --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ + --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ --subst-var-by 'MIGRATIONS_DIR' '${migrationsDir}' \ --subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \ --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ @@ -569,7 +594,7 @@ # flake check'. This is run in the CI system, as well. checks = { psql_15 = makeCheckHarness basePackages.psql_15.bin; - #psql_16 = makeCheckHarness basePackages.psql_16.bin; + psql_16 = makeCheckHarness basePackages.psql_16.bin; #psql_orioledb_16 = makeCheckHarness basePackages.psql_orioledb_16.bin; }; diff --git a/nix/ext/hypopg.nix b/nix/ext/hypopg.nix index 4fc00a82f..300a44900 100644 --- a/nix/ext/hypopg.nix +++ b/nix/ext/hypopg.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_backtrace.nix b/nix/ext/pg_backtrace.nix index b0169124c..47ede88e2 100644 --- a/nix/ext/pg_backtrace.nix +++ b/nix/ext/pg_backtrace.nix @@ -18,7 +18,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_cron.nix b/nix/ext/pg_cron.nix index 5c546c797..d51254aa2 100644 --- a/nix/ext/pg_cron.nix +++ b/nix/ext/pg_cron.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_hashids.nix b/nix/ext/pg_hashids.nix index 41c3ba664..50bff95da 100644 --- a/nix/ext/pg_hashids.nix +++ b/nix/ext/pg_hashids.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_net.nix b/nix/ext/pg_net.nix index fe4404cdf..c0946f425 100644 --- a/nix/ext/pg_net.nix +++ b/nix/ext/pg_net.nix @@ -18,7 +18,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp sql/*.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_plan_filter.nix b/nix/ext/pg_plan_filter.nix index 2d7d224ec..0ed527251 100644 --- a/nix/ext/pg_plan_filter.nix +++ b/nix/ext/pg_plan_filter.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_stat_monitor.nix b/nix/ext/pg_stat_monitor.nix index 8784067d2..c3e91fe2b 100644 --- a/nix/ext/pg_stat_monitor.nix +++ b/nix/ext/pg_stat_monitor.nix @@ -33,7 +33,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_tle.nix b/nix/ext/pg_tle.nix index 6b1c7b1ca..0221d8768 100644 --- a/nix/ext/pg_tle.nix +++ b/nix/ext/pg_tle.nix @@ -21,7 +21,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pgrouting.nix b/nix/ext/pgrouting.nix index 36edf3404..5587566cd 100644 --- a/nix/ext/pgrouting.nix +++ b/nix/ext/pgrouting.nix @@ -14,9 +14,38 @@ stdenv.mkDerivation rec { hash = "sha256-QC77AnPGpPQGEWi6JtJdiNsB2su5+aV2pKg5ImR2B0k="; }; + #disable compile time warnings for incompatible pointer types only on macos and pg16 + NIX_CFLAGS_COMPILE = lib.optionalString (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") + "-Wno-error=int-conversion -Wno-error=incompatible-pointer-types"; + + cmakeFlags = [ + "-DPOSTGRESQL_VERSION=${postgresql.version}" + ] ++ lib.optionals (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") [ + "-DCMAKE_MACOSX_RPATH=ON" + "-DCMAKE_SHARED_MODULE_SUFFIX=.dylib" + "-DCMAKE_SHARED_LIBRARY_SUFFIX=.dylib" + ]; + + preConfigure = lib.optionalString (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") '' + export DLSUFFIX=.dylib + export CMAKE_SHARED_LIBRARY_SUFFIX=.dylib + export CMAKE_SHARED_MODULE_SUFFIX=.dylib + export MACOSX_RPATH=ON + ''; + + postBuild = lib.optionalString (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") '' + shopt -s nullglob + for file in lib/libpgrouting-*.so; do + if [ -f "$file" ]; then + mv "$file" "''${file%.so}.dylib" + fi + done + shopt -u nullglob + ''; + installPhase = '' - install -D lib/*.so -t $out/lib - install -D sql/pgrouting--${version}.sql -t $out/share/postgresql/extension + install -D lib/*${postgresql.dlSuffix} -t $out/lib + install -D sql/pgrouting--*.sql -t $out/share/postgresql/extension install -D sql/common/pgrouting.control -t $out/share/postgresql/extension ''; diff --git a/nix/ext/pgsodium.nix b/nix/ext/pgsodium.nix index e3b0da228..4e184faa9 100644 --- a/nix/ext/pgsodium.nix +++ b/nix/ext/pgsodium.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp sql/*.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pgsql-http.nix b/nix/ext/pgsql-http.nix index 2edaa9d53..de8fa68c6 100644 --- a/nix/ext/pgsql-http.nix +++ b/nix/ext/pgsql-http.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pgvector.nix b/nix/ext/pgvector.nix index 476f7606c..aefeeea3a 100644 --- a/nix/ext/pgvector.nix +++ b/nix/ext/pgvector.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp sql/*.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/plv8.nix b/nix/ext/plv8.nix index 5cb4fb67e..785f5e601 100644 --- a/nix/ext/plv8.nix +++ b/nix/ext/plv8.nix @@ -105,81 +105,17 @@ stdenv.mkDerivation (finalAttrs: { rmdir "$out/nix/store"/* "$out/nix/store" "$out/nix" ${lib.optionalString stdenv.isDarwin '' - install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8-${finalAttrs.version}.so + install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} + install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} + install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} + install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} ''} ${lib.optionalString (!stdenv.isDarwin) '' - ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}.so + ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} ''} ''; - passthru = { - tests = - let - postgresqlWithSelf = postgresql.withPackages (_: [ - finalAttrs.finalPackage - ]); - in { - smoke = runCommand "plv8-smoke-test" {} '' - export PATH=${lib.makeBinPath [ - postgresqlWithSelf - coreutils - gnugrep - ]} - db="$PWD/testdb" - initdb "$db" - postgres -k "$db" -D "$db" & - pid="$!" - - for i in $(seq 1 100); do - if psql -h "$db" -d postgres -c "" 2>/dev/null; then - break - elif ! kill -0 "$pid"; then - exit 1 - else - sleep 0.1 - fi - done - - psql -h "$db" -d postgres -c 'CREATE EXTENSION plv8; DO $$ plv8.elog(NOTICE, plv8.version); $$ LANGUAGE plv8;' 2> "$out" - grep -q "${finalAttrs.version}" "$out" - kill -0 "$pid" - ''; - - regression = stdenv.mkDerivation { - name = "plv8-regression"; - inherit (finalAttrs) src patches nativeBuildInputs buildInputs dontConfigure; - - buildPhase = '' - runHook preBuild - - # The regression tests need to be run in the order specified in the Makefile. - echo -e "include Makefile\nprint_regress_files:\n\t@echo \$(REGRESS)" > Makefile.regress - REGRESS_TESTS=$(make -f Makefile.regress print_regress_files) - - ${postgresql}/lib/pgxs/src/test/regress/pg_regress \ - --bindir='${postgresqlWithSelf}/bin' \ - --temp-instance=regress-instance \ - --dbname=contrib_regression \ - $REGRESS_TESTS - - runHook postBuild - ''; - - installPhase = '' - runHook preInstall - - touch "$out" - - runHook postInstall - ''; - }; - }; - }; - meta = with lib; { description = "V8 Engine Javascript Procedural Language add-on for PostgreSQL"; homepage = "https://plv8.github.io/"; @@ -187,4 +123,4 @@ stdenv.mkDerivation (finalAttrs: { platforms = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" ]; license = licenses.postgresql; }; -}) \ No newline at end of file +}) diff --git a/nix/ext/supautils.nix b/nix/ext/supautils.nix index 9a32062b9..ee1d691b2 100644 --- a/nix/ext/supautils.nix +++ b/nix/ext/supautils.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/lib - install -D supautils.so -t $out/lib + install -D *${postgresql.dlSuffix} -t $out/lib ''; meta = with lib; { diff --git a/nix/postgresql/16.nix b/nix/postgresql/16.nix new file mode 100644 index 000000000..c964fc580 --- /dev/null +++ b/nix/postgresql/16.nix @@ -0,0 +1,4 @@ +import ./generic.nix { + version = "16.3"; + hash = "sha256-Mxlj1dPcTK9CFqBJ+kC2bWvLjHMGFYWUEblRh2TmBYU="; +} diff --git a/nix/postgresql/default.nix b/nix/postgresql/default.nix index 6ee0452dd..5f92262b2 100644 --- a/nix/postgresql/default.nix +++ b/nix/postgresql/default.nix @@ -3,6 +3,7 @@ let #adapted from the postgresql nixpkgs package versions = { postgresql_15 = ./15.nix; + postgresql_16 = ./16.nix; }; mkAttributes = jitSupport: diff --git a/nix/tests/sql/extensions_sql_interface.sql b/nix/tests/sql/extensions_sql_interface.sql index 947a4a612..ad0f63ed8 100644 --- a/nix/tests/sql/extensions_sql_interface.sql +++ b/nix/tests/sql/extensions_sql_interface.sql @@ -63,6 +63,18 @@ from on e.oid = d.refobjid where d.deptype = 'e' + -- Filter out changes between pg15 and pg16 from extensions that ship with postgres + -- new in pg16 + and not (e.extname = 'fuzzystrmatch' and p.proname = 'daitch_mokotoff') + and not (e.extname = 'pageinspect' and p.proname = 'bt_multi_page_stats') + and not (e.extname = 'pg_buffercache' and p.proname = 'pg_buffercache_summary') + and not (e.extname = 'pg_buffercache' and p.proname = 'pg_buffercache_usage_counts') + and not (e.extname = 'pg_walinspect' and p.proname = 'pg_get_wal_block_info') + -- removed in pg16 + and not (e.extname = 'pg_walinspect' and p.proname = 'pg_get_wal_records_info_till_end_of_wal') + and not (e.extname = 'pg_walinspect' and p.proname = 'pg_get_wal_stats_till_end_of_wal') + -- changed in pg16 - output signature added a column + and not (e.extname = 'pageinspect' and p.proname = 'brin_page_items') order by e.extname, n.nspname, From ef848d4714daac5dd67aa016f5bac7a9217e6768 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Mon, 7 Oct 2024 15:44:54 -0500 Subject: [PATCH 03/51] pg_partman test 15/16 compat --- nix/tests/expected/pg_partman.out | 5 ++++- nix/tests/sql/pg_partman.sql | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/nix/tests/expected/pg_partman.out b/nix/tests/expected/pg_partman.out index 527d58b12..25aa80d2e 100644 --- a/nix/tests/expected/pg_partman.out +++ b/nix/tests/expected/pg_partman.out @@ -28,7 +28,10 @@ select from information_schema.tables where - table_schema = 'partman_test'; + table_schema = 'partman_test' +order by + table_name, + table_type; table_name | table_type -----------------------------+------------ time_taptest_table | BASE TABLE diff --git a/nix/tests/sql/pg_partman.sql b/nix/tests/sql/pg_partman.sql index 621e1a7c9..d1fb9f4ba 100644 --- a/nix/tests/sql/pg_partman.sql +++ b/nix/tests/sql/pg_partman.sql @@ -35,7 +35,10 @@ select from information_schema.tables where - table_schema = 'partman_test'; + table_schema = 'partman_test' +order by + table_name, + table_type; select public.create_parent( From 001153023a773ef71fabb52a2eece4db125ff63a Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Mon, 7 Oct 2024 15:46:33 -0500 Subject: [PATCH 04/51] merge sql interface test --- nix/tests/expected/extensions_sql_interface.out | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/nix/tests/expected/extensions_sql_interface.out b/nix/tests/expected/extensions_sql_interface.out index 5714fbcec..07c0d6650 100644 --- a/nix/tests/expected/extensions_sql_interface.out +++ b/nix/tests/expected/extensions_sql_interface.out @@ -146,6 +146,18 @@ from on e.oid = d.refobjid where d.deptype = 'e' + -- Filter out changes between pg15 and pg16 from extensions that ship with postgres + -- new in pg16 + and not (e.extname = 'fuzzystrmatch' and p.proname = 'daitch_mokotoff') + and not (e.extname = 'pageinspect' and p.proname = 'bt_multi_page_stats') + and not (e.extname = 'pg_buffercache' and p.proname = 'pg_buffercache_summary') + and not (e.extname = 'pg_buffercache' and p.proname = 'pg_buffercache_usage_counts') + and not (e.extname = 'pg_walinspect' and p.proname = 'pg_get_wal_block_info') + -- removed in pg16 + and not (e.extname = 'pg_walinspect' and p.proname = 'pg_get_wal_records_info_till_end_of_wal') + and not (e.extname = 'pg_walinspect' and p.proname = 'pg_get_wal_stats_till_end_of_wal') + -- changed in pg16 - output signature added a column + and not (e.extname = 'pageinspect' and p.proname = 'brin_page_items') order by e.extname, n.nspname, @@ -1076,7 +1088,6 @@ order by moddatetime | public | moddatetime | | trigger old_snapshot | public | pg_old_snapshot_time_mapping | OUT array_offset integer, OUT end_timestamp timestamp with time zone, OUT newest_xmin xid | SETOF record pageinspect | public | brin_metapage_info | page bytea, OUT magic text, OUT version integer, OUT pagesperrange integer, OUT lastrevmappage bigint | record - pageinspect | public | brin_page_items | page bytea, index_oid regclass, OUT itemoffset integer, OUT blknum bigint, OUT attnum integer, OUT allnulls boolean, OUT hasnulls boolean, OUT placeholder boolean, OUT value text | SETOF record pageinspect | public | brin_page_type | page bytea | text pageinspect | public | brin_revmap_data | page bytea, OUT pages tid | SETOF tid pageinspect | public | bt_metap | relname text, OUT magic integer, OUT version integer, OUT root bigint, OUT level bigint, OUT fastroot bigint, OUT fastlevel bigint, OUT last_cleanup_num_delpages bigint, OUT last_cleanup_num_tuples double precision, OUT allequalimage boolean | record @@ -1300,9 +1311,7 @@ order by pg_visibility | public | pg_visibility_map_summary | regclass, OUT all_visible bigint, OUT all_frozen bigint | record pg_walinspect | public | pg_get_wal_record_info | in_lsn pg_lsn, OUT start_lsn pg_lsn, OUT end_lsn pg_lsn, OUT prev_lsn pg_lsn, OUT xid xid, OUT resource_manager text, OUT record_type text, OUT record_length integer, OUT main_data_length integer, OUT fpi_length integer, OUT description text, OUT block_ref text | record pg_walinspect | public | pg_get_wal_records_info | start_lsn pg_lsn, end_lsn pg_lsn, OUT start_lsn pg_lsn, OUT end_lsn pg_lsn, OUT prev_lsn pg_lsn, OUT xid xid, OUT resource_manager text, OUT record_type text, OUT record_length integer, OUT main_data_length integer, OUT fpi_length integer, OUT description text, OUT block_ref text | SETOF record - pg_walinspect | public | pg_get_wal_records_info_till_end_of_wal | start_lsn pg_lsn, OUT start_lsn pg_lsn, OUT end_lsn pg_lsn, OUT prev_lsn pg_lsn, OUT xid xid, OUT resource_manager text, OUT record_type text, OUT record_length integer, OUT main_data_length integer, OUT fpi_length integer, OUT description text, OUT block_ref text | SETOF record pg_walinspect | public | pg_get_wal_stats | start_lsn pg_lsn, end_lsn pg_lsn, per_record boolean, OUT "resource_manager/record_type" text, OUT count bigint, OUT count_percentage double precision, OUT record_size bigint, OUT record_size_percentage double precision, OUT fpi_size bigint, OUT fpi_size_percentage double precision, OUT combined_size bigint, OUT combined_size_percentage double precision | SETOF record - pg_walinspect | public | pg_get_wal_stats_till_end_of_wal | start_lsn pg_lsn, per_record boolean, OUT "resource_manager/record_type" text, OUT count bigint, OUT count_percentage double precision, OUT record_size bigint, OUT record_size_percentage double precision, OUT fpi_size bigint, OUT fpi_size_percentage double precision, OUT combined_size bigint, OUT combined_size_percentage double precision | SETOF record pgaudit | public | pgaudit_ddl_command_end | | event_trigger pgaudit | public | pgaudit_sql_drop | | event_trigger pgcrypto | public | armor | bytea | text @@ -5225,7 +5234,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(5072 rows) +(5069 rows) /* From 7f041da60019a5759e827cef6f5b44540121efb5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 7 Oct 2024 16:57:53 -0400 Subject: [PATCH 05/51] tests: build test and cache both versions --- docker/nix/build_nix.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index a8a8639e2..3aba72c2c 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -6,5 +6,7 @@ if [ -d "/workspace" ]; then cd /workspace fi nix build .#psql_15/bin -o psql_15 +nix build .#psql_16/bin -o psql_16 nix flake check -L nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 From f379f0cd67211b2e5864c6e3abde65d50f1e81b9 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 7 Oct 2024 18:31:02 -0400 Subject: [PATCH 06/51] chore: run checks individually --- docker/nix/build_nix.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index 3aba72c2c..3cf8505c0 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -5,8 +5,9 @@ nix --version if [ -d "/workspace" ]; then cd /workspace fi +nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_15 -L --no-link +nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_16 -L --no-link nix build .#psql_15/bin -o psql_15 nix build .#psql_16/bin -o psql_16 -nix flake check -L nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 From 708f4c36d383932be9ea8ca76bf02dff3231896a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 11:32:50 -0400 Subject: [PATCH 07/51] feat: realease 15 and 16 to staging --- .github/workflows/ami-release-nix.yml | 72 ++++++++++++++++++------- ansible/tasks/stage2-setup-postgres.yml | 14 ++++- ansible/vars.yml | 14 +++-- scripts/nix-provision.sh | 21 ++++++++ stage2-nix-psql.pkr.hcl | 2 +- 5 files changed, 98 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 194c8de21..b6619bb7b 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -3,17 +3,40 @@ name: Release AMI Nix on: push: branches: - - develop - - release/* + #- develop + #- release/* + - sam/15-16-ghactions paths: - '.github/workflows/ami-release-nix.yml' - 'common-nix.vars.pkr.hcl' + - 'ansible/vars.yml' workflow_dispatch: jobs: + prepare: + runs-on: ubuntu-latest + outputs: + postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - name: Install yq + run: | + sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + sudo chmod a+x /usr/local/bin/yq + + - name: Set PostgreSQL versions + id: set-versions + run: | + VERSIONS=$(yq eval '.postgres_major[]' ansible/vars.yml | jq -R -s -c 'split("\n")[:-1]') + echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT + build: + needs: prepare strategy: matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} include: - runner: arm-runner arch: arm64 @@ -31,42 +54,54 @@ jobs: - name: Checkout Repo uses: actions/checkout@v3 + - name: Install yq + run: | + sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + sudo chmod a+x /usr/local/bin/yq + - name: Run checks if triggered manually if: ${{ github.event_name == 'workflow_dispatch' }} - # Update `ci.yaml` too if changing constraints. run: | - SUFFIX=$(sed -E 's/postgres-version = "[0-9\.]+(.*)"/\1/g' common-nix.vars.pkr.hcl) + SUFFIX=$(yq eval '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') if [[ -z $SUFFIX ]] ; then echo "Version must include non-numeric characters if built manually." exit 1 fi - # extensions are build in nix prior to this step - # so we can just use the binaries from the nix store - # for postgres, extensions and wrappers + - name: Set PostgreSQL version environment variable + run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV + + - name: Generate common-nix.vars.pkr.hcl + run: | + PG_VERSION=$(yq eval '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml) + echo "postgres-version = \"$PG_VERSION\"" > common-nix.vars.pkr.hcl - name: Build AMI stage 1 + env: + POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} run: | packer init amazon-arm64-nix.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" amazon-arm64-nix.pkr.hcl + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgres_major_version=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Build AMI stage 2 + env: + POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} run: | packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl + packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgres_major_version=${POSTGRES_MAJOR_VERSION}" stage2-nix-psql.pkr.hcl - name: Grab release version id: process_release_version run: | - VERSION=$(sed -e 's/postgres-version = "\(.*\)"/\1/g' common-nix.vars.pkr.hcl) - echo "version=$VERSION" >> "$GITHUB_OUTPUT" + VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') + echo "version=$VERSION" >> $GITHUB_OUTPUT - name: Create nix flake revision tarball run: | GIT_SHA=${{github.sha}} - MAJOR_VERSION=$(echo "${{ steps.process_release_version.outputs.version }}" | cut -d. -f1) + MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }} mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}" echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version" @@ -84,17 +119,13 @@ jobs: ansible-playbook -i localhost \ -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \ -e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \ + -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \ manifest-playbook.yml - name: Upload nix flake revision to s3 staging run: | aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz - #Our self hosted github runner already has permissions to publish images - #but they're limited to only that; - #so if we want s3 access we'll need to config credentials with the below steps - # (which overwrites existing perms) after the ami build - - name: configure aws credentials - prod uses: aws-actions/configure-aws-credentials@v4 with: @@ -107,6 +138,7 @@ jobs: ansible-playbook -i localhost \ -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \ -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \ + -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \ manifest-playbook.yml - name: Upload nix flake revision to s3 prod @@ -130,12 +162,12 @@ jobs: SLACK_MESSAGE: 'Building Postgres AMI failed' SLACK_FOOTER: '' - - name: Cleanup resources on build cancellation + - name: Cleanup resources after build if: ${{ always() }} run: | - aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --instance-ids {} + aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids - name: Cleanup resources on build cancellation if: ${{ cancelled() }} run: | - aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --instance-ids {} + aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 20ad069e2..e22cbe67b 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -6,7 +6,7 @@ - name: Install Postgres from nix binary cache become: yes shell: | - sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#psql_15/bin" + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{psql_version}}/bin" #TODO (samrose) switch pg_prove sourcing to develop branch once PR is merged when: stage2_nix @@ -22,6 +22,18 @@ sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#supabase-groonga" when: stage2_nix +- name: Install debug symbols for postgres version + become: yes + shell: | + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{postgresql_version}}_debug" + when: stage2_nix + +- name: Install source files for postgresql version + become: yes + shell: | + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{postgresql_version}}_src" + when: stage2_nix + - name: Set ownership and permissions for /etc/ssl/private become: yes file: diff --git a/ansible/vars.yml b/ansible/vars.yml index 3c5ef2172..86941c5e1 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -2,9 +2,17 @@ supabase_internal: true ebssurrogate_mode: true async_mode: true -postgresql_major: "15" -postgresql_release: "15.1" -postgresql_release_checksum: sha256:ea2cf059a85882654b989acd07edc121833164a30340faee0d3615cf7058e66c +# postgresql_major: "15" +# postgresql_release: "15.1" +# postgresql_release_checksum: sha256:ea2cf059a85882654b989acd07edc121833164a30340faee0d3615cf7058e66c +postgres_major: + - "15" + - "16" + +# Full version strings for each major version +postgres_release: + postgres15: "15.8.1.003" + postgres16: "16.2.0.001" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh index 223e84926..26ba8db8c 100644 --- a/scripts/nix-provision.sh +++ b/scripts/nix-provision.sh @@ -30,11 +30,32 @@ function execute_stage2_playbook { callbacks_enabled = timer, profile_tasks, profile_roles EOF sed -i 's/- hosts: all/- hosts: localhost/' /tmp/ansible-playbook/ansible/playbook.yml + + # Set psql_version and postgresql_version based on POSTGRES_MAJOR_VERSION + case "${POSTGRES_MAJOR_VERSION}" in + 15) + psql_version="psql_15" + postgresql_version="postgresql15" + ;; + 16) + psql_version="psql_16" + postgresql_version="postgresql16" + ;; + *) + echo "Error: Unsupported POSTGRES_MAJOR_VERSION: ${POSTGRES_MAJOR_VERSION}" + exit 1 + ;; + esac + # Run Ansible playbook export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/tmp ansible-playbook /tmp/ansible-playbook/ansible/playbook.yml \ --extra-vars '{"nixpkg_mode": false, "stage2_nix": true, "debpkg_mode": false}' \ --extra-vars "git_commit_sha=${GIT_SHA}" \ + --extra-vars "psql_version=${psql_version}" \ + --extra-vars "postgresql_version=${postgresql_version}" \ + --extra-vars "nix_secret_key=${NIX_SECRET_KEY}" \ + --extra-vars "postgres_major_version=${POSTGRES_MAJOR_VERSION}" \ $ARGS } diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 57ddfd9e7..5df9faf32 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -56,7 +56,7 @@ packer { } source "amazon-ebs" "ubuntu" { - ami_name = "${var.ami_name}-${var.postgres-version}" + ami_name = "${var.ami_name}-${var.postgres-version}-staging" instance_type = "c6g.4xlarge" region = "${var.region}" source_ami_filter { From 559f2ee54e4ed3fe21c4f1bac731b3abb8124905 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 11:36:59 -0400 Subject: [PATCH 08/51] chore: update versions --- ansible/vars.yml | 2 +- common-nix.vars.pkr.hcl | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 common-nix.vars.pkr.hcl diff --git a/ansible/vars.yml b/ansible/vars.yml index 86941c5e1..2c4a630ed 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -12,7 +12,7 @@ postgres_major: # Full version strings for each major version postgres_release: postgres15: "15.8.1.003" - postgres16: "16.2.0.001" + postgres16: "16.3.1.000" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl deleted file mode 100644 index 8419992a8..000000000 --- a/common-nix.vars.pkr.hcl +++ /dev/null @@ -1 +0,0 @@ -postgres-version = "15.8.1.003" From 8661b1dca669f6cbd1156b6bf68707964dcbb3b1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 11:40:26 -0400 Subject: [PATCH 09/51] chore: make yq available --- .github/workflows/ami-release-nix.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index b6619bb7b..6921ddb8e 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -25,6 +25,7 @@ jobs: run: | sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 sudo chmod a+x /usr/local/bin/yq + echo "PATH=/usr/local/bin:$PATH" >> $GITHUB_ENV - name: Set PostgreSQL versions id: set-versions @@ -58,6 +59,7 @@ jobs: run: | sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 sudo chmod a+x /usr/local/bin/yq + echo "PATH=/usr/local/bin:$PATH" >> $GITHUB_ENV - name: Run checks if triggered manually if: ${{ github.event_name == 'workflow_dispatch' }} From 9ad5b69ac17f5994e5011f4e99666e143e2929f0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 11:45:42 -0400 Subject: [PATCH 10/51] chore: run yq from nix --- .github/workflows/ami-release-nix.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 6921ddb8e..bf9b035af 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -21,16 +21,16 @@ jobs: - name: Checkout Repo uses: actions/checkout@v3 - - name: Install yq + - uses: DeterminateSystems/nix-installer-action@main + + - name: Install yq via Nix run: | - sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 - sudo chmod a+x /usr/local/bin/yq - echo "PATH=/usr/local/bin:$PATH" >> $GITHUB_ENV + echo "PATH=$HOME/.nix-profile/bin:$PATH" >> $GITHUB_ENV - name: Set PostgreSQL versions id: set-versions run: | - VERSIONS=$(yq eval '.postgres_major[]' ansible/vars.yml | jq -R -s -c 'split("\n")[:-1]') + VERSIONS=$(nix run nixpkgs#yq -- eval '.postgres_major[]' ansible/vars.yml | jq -R -s -c 'split("\n")[:-1]') echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT build: @@ -55,16 +55,16 @@ jobs: - name: Checkout Repo uses: actions/checkout@v3 - - name: Install yq + - uses: DeterminateSystems/nix-installer-action@main + + - name: Install yq via Nix run: | - sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 - sudo chmod a+x /usr/local/bin/yq - echo "PATH=/usr/local/bin:$PATH" >> $GITHUB_ENV + echo "PATH=$HOME/.nix-profile/bin:$PATH" >> $GITHUB_ENV - name: Run checks if triggered manually if: ${{ github.event_name == 'workflow_dispatch' }} run: | - SUFFIX=$(yq eval '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') + SUFFIX=$(nix run nixpkgs#yq -- eval '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') if [[ -z $SUFFIX ]] ; then echo "Version must include non-numeric characters if built manually." exit 1 @@ -75,7 +75,7 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(yq eval '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml) + PG_VERSION=$(nix run nixpkgs#yq -- eval '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml) echo "postgres-version = \"$PG_VERSION\"" > common-nix.vars.pkr.hcl - name: Build AMI stage 1 From 86ba8d8073edf740090a13588cdcc70514612c8b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 12:21:47 -0400 Subject: [PATCH 11/51] chore: more setup for staging AMI --- .github/workflows/ami-release-nix.yml | 6 +++--- ansible/vars.yml | 4 ++-- stage2-nix-psql.pkr.hcl | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index bf9b035af..ac63efc14 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -30,7 +30,7 @@ jobs: - name: Set PostgreSQL versions id: set-versions run: | - VERSIONS=$(nix run nixpkgs#yq -- eval '.postgres_major[]' ansible/vars.yml | jq -R -s -c 'split("\n")[:-1]') + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT build: @@ -64,7 +64,7 @@ jobs: - name: Run checks if triggered manually if: ${{ github.event_name == 'workflow_dispatch' }} run: | - SUFFIX=$(nix run nixpkgs#yq -- eval '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') + SUFFIX=$(nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') if [[ -z $SUFFIX ]] ; then echo "Version must include non-numeric characters if built manually." exit 1 @@ -75,7 +75,7 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(nix run nixpkgs#yq -- eval '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml) + PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml) echo "postgres-version = \"$PG_VERSION\"" > common-nix.vars.pkr.hcl - name: Build AMI stage 1 diff --git a/ansible/vars.yml b/ansible/vars.yml index 2c4a630ed..c1889f16d 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,8 +11,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgres15: "15.8.1.003" - postgres16: "16.3.1.000" + postgres15: "15.8.1.003-staging" + postgres16: "16.3.1.000-staging" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 5df9faf32..57ddfd9e7 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -56,7 +56,7 @@ packer { } source "amazon-ebs" "ubuntu" { - ami_name = "${var.ami_name}-${var.postgres-version}-staging" + ami_name = "${var.ami_name}-${var.postgres-version}" instance_type = "c6g.4xlarge" region = "${var.region}" source_ami_filter { From 5aa298135259b329d1432e54aae16d0c41827e7f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 13:15:08 -0400 Subject: [PATCH 12/51] fix: yq usage --- .github/workflows/ami-release-nix.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index ac63efc14..7caef1862 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -23,10 +23,6 @@ jobs: - uses: DeterminateSystems/nix-installer-action@main - - name: Install yq via Nix - run: | - echo "PATH=$HOME/.nix-profile/bin:$PATH" >> $GITHUB_ENV - - name: Set PostgreSQL versions id: set-versions run: | @@ -57,10 +53,6 @@ jobs: - uses: DeterminateSystems/nix-installer-action@main - - name: Install yq via Nix - run: | - echo "PATH=$HOME/.nix-profile/bin:$PATH" >> $GITHUB_ENV - - name: Run checks if triggered manually if: ${{ github.event_name == 'workflow_dispatch' }} run: | From 50b4b670440fa22342d5fa4c80e4698cc87b8700 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 13:20:54 -0400 Subject: [PATCH 13/51] chore: shell vars --- .github/workflows/ami-release-nix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 7caef1862..9f089ffd8 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -56,7 +56,7 @@ jobs: - name: Run checks if triggered manually if: ${{ github.event_name == 'workflow_dispatch' }} run: | - SUFFIX=$(nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') + SUFFIX=$(. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') if [[ -z $SUFFIX ]] ; then echo "Version must include non-numeric characters if built manually." exit 1 From 9237a0c72b95a39750358b7d073236408cb41028 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 13:26:05 -0400 Subject: [PATCH 14/51] fix: When --init none is used, only users who can elevate to sudo privileges can run Nix --- .github/workflows/ami-release-nix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 9f089ffd8..40da6e20a 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -56,7 +56,7 @@ jobs: - name: Run checks if triggered manually if: ${{ github.event_name == 'workflow_dispatch' }} run: | - SUFFIX=$(. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') + SUFFIX=$(sudo -i nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') if [[ -z $SUFFIX ]] ; then echo "Version must include non-numeric characters if built manually." exit 1 @@ -67,7 +67,7 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml) + PG_VERSION=$(sudo -i nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml) echo "postgres-version = \"$PG_VERSION\"" > common-nix.vars.pkr.hcl - name: Build AMI stage 1 From 9a9b7f5861847fac530cffa7e6aaf69ec154b72f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 13:38:28 -0400 Subject: [PATCH 15/51] fix: no -i --- .github/workflows/ami-release-nix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 40da6e20a..544f25f25 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -56,7 +56,7 @@ jobs: - name: Run checks if triggered manually if: ${{ github.event_name == 'workflow_dispatch' }} run: | - SUFFIX=$(sudo -i nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') + SUFFIX=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') if [[ -z $SUFFIX ]] ; then echo "Version must include non-numeric characters if built manually." exit 1 @@ -67,7 +67,7 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(sudo -i nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml) + PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml) echo "postgres-version = \"$PG_VERSION\"" > common-nix.vars.pkr.hcl - name: Build AMI stage 1 From 422f74b18758c3cc8a4b18857a31cc46bbd1ea8c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 13:45:34 -0400 Subject: [PATCH 16/51] fix: quote correction --- .github/workflows/ami-release-nix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 544f25f25..75172f1eb 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -67,7 +67,7 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml) + PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) echo "postgres-version = \"$PG_VERSION\"" > common-nix.vars.pkr.hcl - name: Build AMI stage 1 From 01109dc98de8ce9d396027b8e69882dc425cab0f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 13:55:10 -0400 Subject: [PATCH 17/51] fix: newline extra quotes --- .github/workflows/ami-release-nix.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 75172f1eb..c282da1b4 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -67,8 +67,11 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) - echo "postgres-version = \"$PG_VERSION\"" > common-nix.vars.pkr.hcl + PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes + echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl + # Ensure there's a newline at the end of the file + echo "" >> common-nix.vars.pkr.hcl - name: Build AMI stage 1 env: From 7196838f6796a6d1c33d7e113fa09febc32ffb4f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 14:00:00 -0400 Subject: [PATCH 18/51] fix: no need for pg major version on packer --- .github/workflows/ami-release-nix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index c282da1b4..210505851 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -79,7 +79,7 @@ jobs: run: | packer init amazon-arm64-nix.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgres_major_version=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgres_major_version=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Build AMI stage 2 env: @@ -87,7 +87,7 @@ jobs: run: | packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgres_major_version=${POSTGRES_MAJOR_VERSION}" stage2-nix-psql.pkr.hcl + packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgres_major_version=${POSTGRES_MAJOR_VERSION}" stage2-nix-psql.pkr.hcl - name: Grab release version id: process_release_version From be8eb9d4f11ef7c0e96711184321dccf1fcbced4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 14:18:18 -0400 Subject: [PATCH 19/51] fix: postgresql_major --- .github/workflows/ami-release-nix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 210505851..e87ce6fdc 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -79,7 +79,7 @@ jobs: run: | packer init amazon-arm64-nix.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgres_major_version=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgres_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Build AMI stage 2 env: @@ -87,7 +87,7 @@ jobs: run: | packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgres_major_version=${POSTGRES_MAJOR_VERSION}" stage2-nix-psql.pkr.hcl + packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgres_major=${POSTGRES_MAJOR_VERSION}" stage2-nix-psql.pkr.hcl - name: Grab release version id: process_release_version From d5873ddb0ba29ac0e76309af7392805c10507b7e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 14:28:31 -0400 Subject: [PATCH 20/51] fix: ql --- .github/workflows/ami-release-nix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index e87ce6fdc..6e95e2909 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -79,7 +79,7 @@ jobs: run: | packer init amazon-arm64-nix.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgres_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Build AMI stage 2 env: @@ -87,7 +87,7 @@ jobs: run: | packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgres_major=${POSTGRES_MAJOR_VERSION}" stage2-nix-psql.pkr.hcl + packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" stage2-nix-psql.pkr.hcl - name: Grab release version id: process_release_version From a5a822874662d55e40f7a059dbf1b11865c556e7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 14:50:35 -0400 Subject: [PATCH 21/51] fix: no ansible args in stage to invocation --- .github/workflows/ami-release-nix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 6e95e2909..b28662c91 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -87,7 +87,7 @@ jobs: run: | packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" stage2-nix-psql.pkr.hcl + packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl - name: Grab release version id: process_release_version From c23c2a6d8db66553bf76316852a8e6f50ed57bdd Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 14:56:02 -0400 Subject: [PATCH 22/51] fix: unique val --- ansible/vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index c1889f16d..6f1703da7 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,8 +11,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgres15: "15.8.1.003-staging" - postgres16: "16.3.1.000-staging" + postgres15: "15.8.1.003-staging-1" + postgres16: "16.3.1.000-staging-1" # Non Postgres Extensions pgbouncer_release: "1.19.0" From c07119c3927dd73ec3b9442f60283a0523aa6451 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 15:27:57 -0400 Subject: [PATCH 23/51] fix: adjustments to build scripts --- .github/workflows/ami-release-nix.yml | 3 +-- ansible/vars.yml | 4 ++-- scripts/nix-provision.sh | 6 +++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index b28662c91..ac18860c7 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -82,11 +82,10 @@ jobs: packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Build AMI stage 2 - env: - POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} run: | packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{github.sha}} + POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }} packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl - name: Grab release version diff --git a/ansible/vars.yml b/ansible/vars.yml index 6f1703da7..7c03fa4c7 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,8 +11,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgres15: "15.8.1.003-staging-1" - postgres16: "16.3.1.000-staging-1" + postgres15: "15.8.1.003-staging-2" + postgres16: "16.3.1.000-staging-2" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh index 26ba8db8c..45e3cd950 100644 --- a/scripts/nix-provision.sh +++ b/scripts/nix-provision.sh @@ -33,11 +33,11 @@ EOF # Set psql_version and postgresql_version based on POSTGRES_MAJOR_VERSION case "${POSTGRES_MAJOR_VERSION}" in - 15) + "15") psql_version="psql_15" postgresql_version="postgresql15" ;; - 16) + "16") psql_version="psql_16" postgresql_version="postgresql16" ;; @@ -55,7 +55,7 @@ EOF --extra-vars "psql_version=${psql_version}" \ --extra-vars "postgresql_version=${postgresql_version}" \ --extra-vars "nix_secret_key=${NIX_SECRET_KEY}" \ - --extra-vars "postgres_major_version=${POSTGRES_MAJOR_VERSION}" \ + --extra-vars "postgresql_major_version=${POSTGRES_MAJOR_VERSION}" \ $ARGS } From d0ef09348f8bdf9cb0679e4beceace33c03ba6e3 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 16:17:00 -0400 Subject: [PATCH 24/51] chore: env var handling --- .github/workflows/ami-release-nix.yml | 2 ++ scripts/nix-provision.sh | 22 ++++------------------ 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index ac18860c7..08299246c 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -82,6 +82,8 @@ jobs: packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Build AMI stage 2 + env: + POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} run: | packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{github.sha}} diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh index 45e3cd950..c4776b3a8 100644 --- a/scripts/nix-provision.sh +++ b/scripts/nix-provision.sh @@ -25,35 +25,21 @@ function install_nix() { function execute_stage2_playbook { + echo "POSTGRES_MAJOR_VERSION: ${POSTGRES_MAJOR_VERSION}" + echo "GIT_SHA: ${GIT_SHA}" sudo tee /etc/ansible/ansible.cfg < Date: Tue, 8 Oct 2024 16:26:13 -0400 Subject: [PATCH 25/51] fix: bump to build --- ansible/vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 7c03fa4c7..c7f09015b 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,8 +11,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgres15: "15.8.1.003-staging-2" - postgres16: "16.3.1.000-staging-2" + postgres15: "15.8.1.003-staging-3" + postgres16: "16.3.1.000-staging-3" # Non Postgres Extensions pgbouncer_release: "1.19.0" From a7cf05e20f77271324f37747bef509e7d81d24f2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 17:35:48 -0400 Subject: [PATCH 26/51] chore: set up more required vars --- .github/workflows/ami-release-nix.yml | 2 +- stage2-nix-psql.pkr.hcl | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 08299246c..706edd727 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -88,7 +88,7 @@ jobs: packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{github.sha}} POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }} - packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl + packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl - name: Grab release version id: process_release_version diff --git a/stage2-nix-psql.pkr.hcl b/stage2-nix-psql.pkr.hcl index 57ddfd9e7..2897ffd32 100644 --- a/stage2-nix-psql.pkr.hcl +++ b/stage2-nix-psql.pkr.hcl @@ -46,6 +46,11 @@ variable "git_sha" { default = env("GIT_SHA") } +variable "postgres_major_version" { + type = string + default = "" +} + packer { required_plugins { amazon = { @@ -133,7 +138,8 @@ build { provisioner "shell" { environment_vars = [ - "GIT_SHA=${var.git_sha}" + "GIT_SHA=${var.git_sha}", + "POSTGRES_MAJOR_VERSION=${var.postgres_major_version}" ] script = "scripts/nix-provision.sh" } From 4cd84cd8f1164ee6ea6f91f4f681d8154846ce20 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 17:40:44 -0400 Subject: [PATCH 27/51] chore: bump var --- ansible/vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index c7f09015b..a856b859b 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,8 +11,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgres15: "15.8.1.003-staging-3" - postgres16: "16.3.1.000-staging-3" + postgres15: "15.8.1.003-staging-4" + postgres16: "16.3.1.000-staging-4" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 2866f1d20d342133f81b4892bfdbcc2c469da074 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 8 Oct 2024 18:13:39 -0400 Subject: [PATCH 28/51] feat: pg 16 debug symbols --- ansible/vars.yml | 4 ++-- flake.nix | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index a856b859b..f78b56c97 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,8 +11,8 @@ postgres_major: # Full version strings for each major version postgres_release: - postgres15: "15.8.1.003-staging-4" - postgres16: "16.3.1.000-staging-4" + postgres15: "15.8.1.003-staging-5" + postgres16: "16.3.1.000-staging-5" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/flake.nix b/flake.nix index 3bb6d1a45..40202d2f0 100644 --- a/flake.nix +++ b/flake.nix @@ -318,6 +318,7 @@ pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; inherit postgresql_15 postgresql_16; postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null; + postgresql_16_debug = if pkgs.stdenv.isLinux then postgresql_16.debug else null; postgresql_15_src = pkgs.stdenv.mkDerivation { pname = "postgresql-15-src"; version = postgresql_15.version; From e67dbfe29155a98897a1144f9829aad1fd9be248 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 9 Oct 2024 09:48:57 -0400 Subject: [PATCH 29/51] feat: matrix pg versions build on testinfra --- .github/workflows/testinfra-nix.yml | 35 ++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-nix.yml index fee2fd791..b94209ef9 100644 --- a/.github/workflows/testinfra-nix.yml +++ b/.github/workflows/testinfra-nix.yml @@ -5,17 +5,35 @@ on: workflow_dispatch: jobs: + prepare: + runs-on: ubuntu-latest + outputs: + postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL versions + id: set-versions + run: | + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') + echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT + test-ami-nix: + needs: prepare strategy: fail-fast: false matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} include: - runner: arm-runner arch: arm64 ubuntu_release: focal ubuntu_version: 20.04 mcpu: neoverse-n1 - runs-on: ${{ matrix.runner }} + runs-on: ${{ matrix.runner }} timeout-minutes: 150 permissions: contents: write @@ -40,18 +58,29 @@ jobs: - name: Generate random string id: random run: echo "random_string=$(openssl rand -hex 8)" >> $GITHUB_OUTPUT + + - name: Set PostgreSQL version environment variable + run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV + + - name: Generate common-nix.vars.pkr.hcl + run: | + PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes + echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl + # Ensure there's a newline at the end of the file + echo "" >> common-nix.vars.pkr.hcl - name: Build AMI stage 1 run: | packer init amazon-arm64-nix.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" amazon-arm64-nix.pkr.hcl + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Build AMI stage 2 run: | packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}" stage2-nix-psql.pkr.hcl + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}" stage2-nix-psql.pkr.hcl - name: Run tests timeout-minutes: 10 From 7a6f5992a181383875f4dd30adbf886d984f4b9a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 9 Oct 2024 10:42:37 -0400 Subject: [PATCH 30/51] feat: matrix on Test Database --- .github/workflows/test.yml | 43 ++++++++++++++++++++++++++--- .github/workflows/testinfra-nix.yml | 2 +- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2be061941..a26ce0a27 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,9 +8,27 @@ on: workflow_dispatch: jobs: + prepare: + runs-on: ubuntu-latest + outputs: + postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL versions + id: set-versions + run: | + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') + echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT + build: + needs: prepare strategy: matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} include: - runner: [self-hosted, X64] arch: amd64 @@ -23,14 +41,31 @@ jobs: POSTGRES_PASSWORD: password steps: - uses: actions/checkout@v3 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL version environment variable + run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV + + - name: Generate common-nix.vars.pkr.hcl + run: | + PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes + echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl + # Ensure there's a newline at the end of the file + echo "" >> common-nix.vars.pkr.hcl + - id: settings # Remove spaces and quotes to get the raw version string run: sed -r 's/(\s|\")+//g' common-nix.vars.pkr.hcl >> $GITHUB_OUTPUT - - id: args - uses: mikefarah/yq@master - with: - cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' + - name: Generate args + id: args + run: | + ARGS=$(nix run nixpkgs#yq -- 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' ansible/vars.yml) + echo "result<> $GITHUB_OUTPUT + echo "$ARGS" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT - run: docker context create builders - uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-nix.yml index b94209ef9..4a51c159c 100644 --- a/.github/workflows/testinfra-nix.yml +++ b/.github/workflows/testinfra-nix.yml @@ -120,4 +120,4 @@ jobs: # Deregister AMIs deregister_ami_by_name "$STAGE1_AMI_NAME" - deregister_ami_by_name "$STAGE2_AMI_NAME" \ No newline at end of file + deregister_ami_by_name "$STAGE2_AMI_NAME" From 8f50804e71dba4ce0dbed0a75e5d110deb3a5229 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 9 Oct 2024 10:49:58 -0400 Subject: [PATCH 31/51] chore: running nix in the right context --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a26ce0a27..f32400452 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,7 +49,7 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl # Ensure there's a newline at the end of the file @@ -62,7 +62,7 @@ jobs: - name: Generate args id: args run: | - ARGS=$(nix run nixpkgs#yq -- 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' ansible/vars.yml) + ARGS=$(sudo nix run nixpkgs#yq -- 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' ansible/vars.yml) echo "result<> $GITHUB_OUTPUT echo "$ARGS" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT From cf0d0955c11624f24d42ee8454ce36fc5526bc81 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 9 Oct 2024 16:03:39 -0400 Subject: [PATCH 32/51] feat: just use existing Dockerfile + pg version --- .github/workflows/test.yml | 2 +- Dockerfile-15 | 223 +++++++++++++++++++++++++++++++++++++ Dockerfile-16 | 223 +++++++++++++++++++++++++++++++++++++ 3 files changed, 447 insertions(+), 1 deletion(-) create mode 100644 Dockerfile-15 create mode 100644 Dockerfile-16 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f32400452..239e9d0df 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,7 +75,7 @@ jobs: with: load: true context: . - file: "Dockerfile-156" + file: "Dockerfile-{{ matrix.postgres_version }}" target: production build-args: | ${{ steps.args.outputs.result }} diff --git a/Dockerfile-15 b/Dockerfile-15 new file mode 100644 index 000000000..ce83237f2 --- /dev/null +++ b/Dockerfile-15 @@ -0,0 +1,223 @@ +# syntax=docker/dockerfile:1.6 +ARG postgresql_major=15 +ARG postgresql_release=${postgresql_major}.1 + +# Bump default build arg to build a package from source +# Bump vars.yml to specify runtime package version +ARG sfcgal_release=1.3.10 +ARG postgis_release=3.3.2 +ARG pgrouting_release=3.4.1 +ARG pgtap_release=1.2.0 +ARG pg_cron_release=1.6.2 +ARG pgaudit_release=1.7.0 +ARG pgjwt_release=9742dab1b2f297ad3811120db7b21451bca2d3c9 +ARG pgsql_http_release=1.5.0 +ARG plpgsql_check_release=2.2.5 +ARG pg_safeupdate_release=1.4 +ARG timescaledb_release=2.9.1 +ARG wal2json_release=2_5 +ARG pljava_release=1.6.4 +ARG plv8_release=3.1.5 +ARG pg_plan_filter_release=5081a7b5cb890876e67d8e7486b6a64c38c9a492 +ARG pg_net_release=0.7.1 +ARG rum_release=1.3.13 +ARG pg_hashids_release=cd0e1b31d52b394a0df64079406a14a4f7387cd6 +ARG libsodium_release=1.0.18 +ARG pgsodium_release=3.1.6 +ARG pg_graphql_release=1.5.1 +ARG pg_stat_monitor_release=1.1.1 +ARG pg_jsonschema_release=0.1.4 +ARG pg_repack_release=1.4.8 +ARG vault_release=0.2.8 +ARG groonga_release=12.0.8 +ARG pgroonga_release=2.4.0 +ARG wrappers_release=0.3.0 +ARG hypopg_release=1.3.1 +ARG pgvector_release=0.4.0 +ARG pg_tle_release=1.3.2 +ARG index_advisor_release=0.2.0 +ARG supautils_release=2.2.0 +ARG wal_g_release=2.0.1 + +FROM ubuntu:focal as base + +RUN apt update -y && apt install -y \ + curl \ + gnupg \ + lsb-release \ + software-properties-common \ + wget \ + sudo \ + && apt clean + + +RUN adduser --system --home /var/lib/postgresql --no-create-home --shell /bin/bash --group --gecos "PostgreSQL administrator" postgres +RUN adduser --system --no-create-home --shell /bin/bash --group wal-g +RUN curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install linux \ +--init none \ +--no-confirm \ +--extra-conf "substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com" \ +--extra-conf "trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" + +ENV PATH="${PATH}:/nix/var/nix/profiles/default/bin" + +COPY . /nixpg + +WORKDIR /nixpg + +RUN nix profile install .#psql_15/bin + + + +WORKDIR / + + +RUN mkdir -p /usr/lib/postgresql/bin \ + /usr/lib/postgresql/share/postgresql \ + /usr/share/postgresql \ + # /usr/lib/postgresql/share/postgresql/contrib \ + #/usr/lib/postgresql/share/postgresql/timezonesets \ + #/usr/lib/postgresql/share/postgresql/tsearch_data \ + # /usr/lib/postgresql/share/postgresql/extension \ + && chown -R postgres:postgres /usr/lib/postgresql \ + && chown -R postgres:postgres /usr/share/postgresql + +# Create symbolic links +RUN ln -s /nix/var/nix/profiles/default/bin/* /usr/lib/postgresql/bin/ \ + && ln -s /nix/var/nix/profiles/default/bin/* /usr/bin/ \ + && chown -R postgres:postgres /usr/bin + +# Create symbolic links for PostgreSQL shares +RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/ +RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/ +RUN chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/ +RUN chown -R postgres:postgres /usr/share/postgresql/ +# Create symbolic links for contrib directory +RUN mkdir -p /usr/lib/postgresql/share/postgresql/contrib \ + && find /nix/var/nix/profiles/default/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ + && chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/contrib/ + +RUN chown -R postgres:postgres /usr/lib/postgresql + +RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets + + +RUN apt-get update && \ + apt-get install -y --no-install-recommends tzdata + +RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \ + dpkg-reconfigure --frontend noninteractive tzdata + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + checkinstall \ + cmake + +ENV PGDATA=/var/lib/postgresql/data + +#################### +# setup-wal-g.yml +#################### +FROM base as walg +ARG wal_g_release +# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz +RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \ + apt-get update && apt-get install -y --no-install-recommends curl && \ + curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \ + tar -xvf /tmp/wal-g.tar.gz -C /tmp && \ + rm -rf /tmp/wal-g.tar.gz && \ + mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g + +# #################### +# # Download gosu for easy step-down from root +# #################### +FROM base as gosu +ARG TARGETARCH +# Install dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + gnupg \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* +# Download binary +ARG GOSU_VERSION=1.16 +ARG GOSU_GPG_KEY=B42F6819007F00F88E364FD4036A9C25BF357DD4 +ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH \ + /usr/local/bin/gosu +ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH.asc \ + /usr/local/bin/gosu.asc +# Verify checksum +RUN gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys $GOSU_GPG_KEY && \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu && \ + gpgconf --kill all && \ + chmod +x /usr/local/bin/gosu + +# #################### +# # Build final image +# #################### +FROM gosu as production +RUN id postgres || (echo "postgres user does not exist" && exit 1) +# # Setup extensions +COPY --from=walg /tmp/wal-g /usr/local/bin/ + +# # Initialise configs +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf +COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts +COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf.j2 /etc/postgresql-custom/read-replica.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf.j2 /etc/postgresql-custom/wal-g.conf +COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh +COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ownership.sh + +RUN sed -i \ + -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \ + -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \ + -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \ + -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \ + echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \ + #echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \ + echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \ + usermod -aG postgres wal-g && \ + mkdir -p /etc/postgresql-custom && \ + chown postgres:postgres /etc/postgresql-custom + +# # Include schema migrations +COPY migrations/db /docker-entrypoint-initdb.d/ +COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql +COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql + +# # Add upstream entrypoint script +COPY --from=gosu /usr/local/bin/gosu /usr/local/bin/gosu +ADD --chmod=0755 \ + https://github.com/docker-library/postgres/raw/master/15/bullseye/docker-entrypoint.sh \ + /usr/local/bin/ + +RUN mkdir -p /var/run/postgresql && chown postgres:postgres /var/run/postgresql + +ENTRYPOINT ["docker-entrypoint.sh"] + +HEALTHCHECK --interval=2s --timeout=2s --retries=10 CMD pg_isready -U postgres -h localhost +STOPSIGNAL SIGINT +EXPOSE 5432 + +ENV POSTGRES_HOST=/var/run/postgresql +ENV POSTGRES_USER=supabase_admin +ENV POSTGRES_DB=postgres +RUN apt-get update && apt-get install -y --no-install-recommends \ + locales \ + && rm -rf /var/lib/apt/lists/* && \ + localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ + && localedef -i C -c -f UTF-8 -A /usr/share/locale/locale.alias C.UTF-8 +RUN echo "C.UTF-8 UTF-8" > /etc/locale.gen && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 +ENV LC_CTYPE=C.UTF-8 +ENV LC_COLLATE=C.UTF-8 +ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive +CMD ["postgres", "-D", "/etc/postgresql"] diff --git a/Dockerfile-16 b/Dockerfile-16 new file mode 100644 index 000000000..db5772cee --- /dev/null +++ b/Dockerfile-16 @@ -0,0 +1,223 @@ +# syntax=docker/dockerfile:1.6 +ARG postgresql_major=15 +ARG postgresql_release=${postgresql_major}.1 + +# Bump default build arg to build a package from source +# Bump vars.yml to specify runtime package version +ARG sfcgal_release=1.3.10 +ARG postgis_release=3.3.2 +ARG pgrouting_release=3.4.1 +ARG pgtap_release=1.2.0 +ARG pg_cron_release=1.6.2 +ARG pgaudit_release=1.7.0 +ARG pgjwt_release=9742dab1b2f297ad3811120db7b21451bca2d3c9 +ARG pgsql_http_release=1.5.0 +ARG plpgsql_check_release=2.2.5 +ARG pg_safeupdate_release=1.4 +ARG timescaledb_release=2.9.1 +ARG wal2json_release=2_5 +ARG pljava_release=1.6.4 +ARG plv8_release=3.1.5 +ARG pg_plan_filter_release=5081a7b5cb890876e67d8e7486b6a64c38c9a492 +ARG pg_net_release=0.7.1 +ARG rum_release=1.3.13 +ARG pg_hashids_release=cd0e1b31d52b394a0df64079406a14a4f7387cd6 +ARG libsodium_release=1.0.18 +ARG pgsodium_release=3.1.6 +ARG pg_graphql_release=1.5.1 +ARG pg_stat_monitor_release=1.1.1 +ARG pg_jsonschema_release=0.1.4 +ARG pg_repack_release=1.4.8 +ARG vault_release=0.2.8 +ARG groonga_release=12.0.8 +ARG pgroonga_release=2.4.0 +ARG wrappers_release=0.3.0 +ARG hypopg_release=1.3.1 +ARG pgvector_release=0.4.0 +ARG pg_tle_release=1.3.2 +ARG index_advisor_release=0.2.0 +ARG supautils_release=2.2.0 +ARG wal_g_release=2.0.1 + +FROM ubuntu:focal as base + +RUN apt update -y && apt install -y \ + curl \ + gnupg \ + lsb-release \ + software-properties-common \ + wget \ + sudo \ + && apt clean + + +RUN adduser --system --home /var/lib/postgresql --no-create-home --shell /bin/bash --group --gecos "PostgreSQL administrator" postgres +RUN adduser --system --no-create-home --shell /bin/bash --group wal-g +RUN curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install linux \ +--init none \ +--no-confirm \ +--extra-conf "substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com" \ +--extra-conf "trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" + +ENV PATH="${PATH}:/nix/var/nix/profiles/default/bin" + +COPY . /nixpg + +WORKDIR /nixpg + +RUN nix profile install .#psql_16/bin + + + +WORKDIR / + + +RUN mkdir -p /usr/lib/postgresql/bin \ + /usr/lib/postgresql/share/postgresql \ + /usr/share/postgresql \ + # /usr/lib/postgresql/share/postgresql/contrib \ + #/usr/lib/postgresql/share/postgresql/timezonesets \ + #/usr/lib/postgresql/share/postgresql/tsearch_data \ + # /usr/lib/postgresql/share/postgresql/extension \ + && chown -R postgres:postgres /usr/lib/postgresql \ + && chown -R postgres:postgres /usr/share/postgresql + +# Create symbolic links +RUN ln -s /nix/var/nix/profiles/default/bin/* /usr/lib/postgresql/bin/ \ + && ln -s /nix/var/nix/profiles/default/bin/* /usr/bin/ \ + && chown -R postgres:postgres /usr/bin + +# Create symbolic links for PostgreSQL shares +RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/ +RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/ +RUN chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/ +RUN chown -R postgres:postgres /usr/share/postgresql/ +# Create symbolic links for contrib directory +RUN mkdir -p /usr/lib/postgresql/share/postgresql/contrib \ + && find /nix/var/nix/profiles/default/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ + && chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/contrib/ + +RUN chown -R postgres:postgres /usr/lib/postgresql + +RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets + + +RUN apt-get update && \ + apt-get install -y --no-install-recommends tzdata + +RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \ + dpkg-reconfigure --frontend noninteractive tzdata + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + checkinstall \ + cmake + +ENV PGDATA=/var/lib/postgresql/data + +#################### +# setup-wal-g.yml +#################### +FROM base as walg +ARG wal_g_release +# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz +RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \ + apt-get update && apt-get install -y --no-install-recommends curl && \ + curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \ + tar -xvf /tmp/wal-g.tar.gz -C /tmp && \ + rm -rf /tmp/wal-g.tar.gz && \ + mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g + +# #################### +# # Download gosu for easy step-down from root +# #################### +FROM base as gosu +ARG TARGETARCH +# Install dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + gnupg \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* +# Download binary +ARG GOSU_VERSION=1.16 +ARG GOSU_GPG_KEY=B42F6819007F00F88E364FD4036A9C25BF357DD4 +ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH \ + /usr/local/bin/gosu +ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH.asc \ + /usr/local/bin/gosu.asc +# Verify checksum +RUN gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys $GOSU_GPG_KEY && \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu && \ + gpgconf --kill all && \ + chmod +x /usr/local/bin/gosu + +# #################### +# # Build final image +# #################### +FROM gosu as production +RUN id postgres || (echo "postgres user does not exist" && exit 1) +# # Setup extensions +COPY --from=walg /tmp/wal-g /usr/local/bin/ + +# # Initialise configs +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf +COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts +COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf.j2 /etc/postgresql-custom/read-replica.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf.j2 /etc/postgresql-custom/wal-g.conf +COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh +COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ownership.sh + +RUN sed -i \ + -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \ + -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \ + -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \ + -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \ + echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \ + #echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \ + echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \ + usermod -aG postgres wal-g && \ + mkdir -p /etc/postgresql-custom && \ + chown postgres:postgres /etc/postgresql-custom + +# # Include schema migrations +COPY migrations/db /docker-entrypoint-initdb.d/ +COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql +COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql + +# # Add upstream entrypoint script +COPY --from=gosu /usr/local/bin/gosu /usr/local/bin/gosu +ADD --chmod=0755 \ + https://github.com/docker-library/postgres/raw/master/15/bullseye/docker-entrypoint.sh \ + /usr/local/bin/ + +RUN mkdir -p /var/run/postgresql && chown postgres:postgres /var/run/postgresql + +ENTRYPOINT ["docker-entrypoint.sh"] + +HEALTHCHECK --interval=2s --timeout=2s --retries=10 CMD pg_isready -U postgres -h localhost +STOPSIGNAL SIGINT +EXPOSE 5432 + +ENV POSTGRES_HOST=/var/run/postgresql +ENV POSTGRES_USER=supabase_admin +ENV POSTGRES_DB=postgres +RUN apt-get update && apt-get install -y --no-install-recommends \ + locales \ + && rm -rf /var/lib/apt/lists/* && \ + localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ + && localedef -i C -c -f UTF-8 -A /usr/share/locale/locale.alias C.UTF-8 +RUN echo "C.UTF-8 UTF-8" > /etc/locale.gen && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 +ENV LC_CTYPE=C.UTF-8 +ENV LC_COLLATE=C.UTF-8 +ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive +CMD ["postgres", "-D", "/etc/postgresql"] From 3ac6ed56cd30bf19fce0af95d05930106d39ebb9 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 9 Oct 2024 16:06:49 -0400 Subject: [PATCH 33/51] chore: refer to var --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 239e9d0df..9e928d3fc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,7 +75,7 @@ jobs: with: load: true context: . - file: "Dockerfile-{{ matrix.postgres_version }}" + file: "Dockerfile-${{ matrix.postgres_version }}" target: production build-args: | ${{ steps.args.outputs.result }} From 935b06452fdab521c03fd1b33c2fbdfb74498efc Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 9 Oct 2024 16:13:59 -0400 Subject: [PATCH 34/51] fix: read name without including quotes --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9e928d3fc..044235737 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,7 +75,7 @@ jobs: with: load: true context: . - file: "Dockerfile-${{ matrix.postgres_version }}" + file: Dockerfile-${{ matrix.postgres_version }} target: production build-args: | ${{ steps.args.outputs.result }} From e0867081516d022376d0d3407f702c7e42ac8521 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 9 Oct 2024 16:19:19 -0400 Subject: [PATCH 35/51] chore: try format function --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 044235737..21150b416 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,7 +75,7 @@ jobs: with: load: true context: . - file: Dockerfile-${{ matrix.postgres_version }} + file: ${{ format('Dockerfile-{0}', matrix.postgres_version) }} target: production build-args: | ${{ steps.args.outputs.result }} From a2c6b123e244818eb039a14e3b0b8c900f372cf0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 9 Oct 2024 16:26:21 -0400 Subject: [PATCH 36/51] fix: strip quotes from version number --- .github/workflows/test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 21150b416..989efdb29 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,6 +66,10 @@ jobs: echo "result<> $GITHUB_OUTPUT echo "$ARGS" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT + - name: Strip quotes from pg major and set env var + run: | + stripped_version=$(echo ${{ matrix.postgres_version }} | sed 's/^"\(.*\)"$/\1/') + echo "PGMAJOR=$stripped_version" >> $GITHUB_ENV - run: docker context create builders - uses: docker/setup-buildx-action@v3 @@ -75,7 +79,7 @@ jobs: with: load: true context: . - file: ${{ format('Dockerfile-{0}', matrix.postgres_version) }} + file: Dockerfile-${{ env.STRIPPED_VERSION }} target: production build-args: | ${{ steps.args.outputs.result }} From a01a9275bb3dd79d29b40c9a8d371556266bdd0c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 9 Oct 2024 16:32:08 -0400 Subject: [PATCH 37/51] chore: env var --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 989efdb29..ffec2415e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,7 +79,7 @@ jobs: with: load: true context: . - file: Dockerfile-${{ env.STRIPPED_VERSION }} + file: Dockerfile-${{ env.PGMAJOR }} target: production build-args: | ${{ steps.args.outputs.result }} From ce2ff2a5c7accdac7c958b15c8e5963b3acd1ebb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 9 Oct 2024 16:45:26 -0400 Subject: [PATCH 38/51] fix: pg client --- .github/workflows/test.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ffec2415e..b4071cc00 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,6 +47,11 @@ jobs: - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV + - name: Strip quotes from pg major and set env var + run: | + stripped_version=$(echo ${{ matrix.postgres_version }} | sed 's/^"\(.*\)"$/\1/') + echo "PGMAJOR=$stripped_version" >> $GITHUB_ENV + - name: Generate common-nix.vars.pkr.hcl run: | PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) @@ -66,10 +71,6 @@ jobs: echo "result<> $GITHUB_OUTPUT echo "$ARGS" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - - name: Strip quotes from pg major and set env var - run: | - stripped_version=$(echo ${{ matrix.postgres_version }} | sed 's/^"\(.*\)"$/\1/') - echo "PGMAJOR=$stripped_version" >> $GITHUB_ENV - run: docker context create builders - uses: docker/setup-buildx-action@v3 @@ -99,7 +100,7 @@ jobs: - name: Install psql run: | sudo apt update - sudo apt install -y --no-install-recommends postgresql-client + sudo apt install -y --no-install-recommends postgresql-client-{{ env.PGMAJOR }} - name: Install pg_prove run: sudo cpan -T TAP::Parser::SourceHandler::pgTAP From 9784d42cc27ac9884dc22a7a44c5702a9dc924a7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 9 Oct 2024 16:54:07 -0400 Subject: [PATCH 39/51] fix --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b4071cc00..22d7bd52a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -100,7 +100,7 @@ jobs: - name: Install psql run: | sudo apt update - sudo apt install -y --no-install-recommends postgresql-client-{{ env.PGMAJOR }} + sudo apt install -y --no-install-recommends postgresql-client-${{ env.PGMAJOR }} - name: Install pg_prove run: sudo cpan -T TAP::Parser::SourceHandler::pgTAP From c1c96017884b5da50ba8a1c6aad3eb4769565d95 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 9 Oct 2024 17:24:10 -0400 Subject: [PATCH 40/51] fix: try to use psql from our own corresponding pkg --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 22d7bd52a..6a8702ef2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -99,8 +99,7 @@ jobs: -d supabase/postgres:${{ steps.settings.outputs.postgres-version }} - name: Install psql run: | - sudo apt update - sudo apt install -y --no-install-recommends postgresql-client-${{ env.PGMAJOR }} + sudo nix profile install .#psql_${{ env.PGMAJOR }}/bin - name: Install pg_prove run: sudo cpan -T TAP::Parser::SourceHandler::pgTAP From 642645b79440c7451ced298aaa12b87404f75655 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 9 Oct 2024 18:42:11 -0400 Subject: [PATCH 41/51] fix: try psql from ppa --- .github/workflows/test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6a8702ef2..aba9474ef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -97,9 +97,13 @@ jobs: -p ${{ env.POSTGRES_PORT }}:5432 \ --name supabase_postgres \ -d supabase/postgres:${{ steps.settings.outputs.postgres-version }} + - name: Install psql run: | - sudo nix profile install .#psql_${{ env.PGMAJOR }}/bin + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt update + sudo apt install -y --no-install-recommends postgresql-client-${{ env.PGMAJOR }} - name: Install pg_prove run: sudo cpan -T TAP::Parser::SourceHandler::pgTAP From 2bf4ad4eab9c98bd05a4ab8080e8b3f6ed98934b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 9 Oct 2024 19:06:31 -0400 Subject: [PATCH 42/51] fix: dbmate per pg version --- .github/workflows/test.yml | 4 ++++ migrations/Dockerfile.dbmate | 10 ++++++++++ migrations/docker-compose.yaml | 4 +++- 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 migrations/Dockerfile.dbmate diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aba9474ef..410c50f73 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -150,6 +150,10 @@ jobs: PGUSER: supabase_admin PGPASSWORD: ${{ env.POSTGRES_PASSWORD }} + - name: Update Dockerfile.dbmate version + run: | + sed -i 's/%VERSION%/${{ env.PGMAJOR }}/g' migrations/Dockerfile.dbmate + - name: verify schema.sql is committed run: | docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit diff --git a/migrations/Dockerfile.dbmate b/migrations/Dockerfile.dbmate new file mode 100644 index 000000000..787434bf0 --- /dev/null +++ b/migrations/Dockerfile.dbmate @@ -0,0 +1,10 @@ +FROM amacneil/dbmate:1.16.2 + +# Install PostgreSQL 16 client +RUN apt-get update && apt-get install -y wget gnupg2 lsb-release +RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - +RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list +RUN apt-get update && apt-get install -y postgresql-client-%VERSION% + +# Set the PATH to use the new PostgreSQL 16 binaries +ENV PATH="/usr/lib/postgresql/%VERSION%/bin:${PATH}" \ No newline at end of file diff --git a/migrations/docker-compose.yaml b/migrations/docker-compose.yaml index 2609d6008..1b3e8b143 100644 --- a/migrations/docker-compose.yaml +++ b/migrations/docker-compose.yaml @@ -33,7 +33,9 @@ services: command: pg_prove /tests/test.sql dbmate: - image: amacneil/dbmate:1.16.2 + build: + context: . + dockerfile: Dockerfile.dbmate depends_on: db: condition: service_healthy From 32864647af1a6d9831c143c4141ff09af7053098 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 9 Oct 2024 19:22:17 -0400 Subject: [PATCH 43/51] build dbmate and then install client --- migrations/Dockerfile.dbmate | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/migrations/Dockerfile.dbmate b/migrations/Dockerfile.dbmate index 787434bf0..bd236a29d 100644 --- a/migrations/Dockerfile.dbmate +++ b/migrations/Dockerfile.dbmate @@ -1,10 +1,14 @@ -FROM amacneil/dbmate:1.16.2 +FROM debian:bullseye-slim + +RUN apt-get update && apt-get install -y curl \ + && curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/latest/download/dbmate-linux-amd64 \ + && chmod +x /usr/local/bin/dbmate -# Install PostgreSQL 16 client RUN apt-get update && apt-get install -y wget gnupg2 lsb-release RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list RUN apt-get update && apt-get install -y postgresql-client-%VERSION% -# Set the PATH to use the new PostgreSQL 16 binaries -ENV PATH="/usr/lib/postgresql/%VERSION%/bin:${PATH}" \ No newline at end of file +ENV PATH="/usr/lib/postgresql/%VERSION%/bin:${PATH}" + +ENTRYPOINT ["dbmate"] \ No newline at end of file From 4eecb87597f48805a1ed2393fb6152acdf7e527d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 9 Oct 2024 19:39:27 -0400 Subject: [PATCH 44/51] fix: account for architecture --- migrations/Dockerfile.dbmate | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/migrations/Dockerfile.dbmate b/migrations/Dockerfile.dbmate index bd236a29d..0af8193f4 100644 --- a/migrations/Dockerfile.dbmate +++ b/migrations/Dockerfile.dbmate @@ -1,14 +1,23 @@ FROM debian:bullseye-slim -RUN apt-get update && apt-get install -y curl \ - && curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/latest/download/dbmate-linux-amd64 \ - && chmod +x /usr/local/bin/dbmate +RUN apt-get update && apt-get install -y curl wget gnupg2 lsb-release + +RUN ARCH=$(dpkg --print-architecture); \ + case ${ARCH} in \ + amd64) DBMATE_ARCH="linux-amd64" ;; \ + arm64) DBMATE_ARCH="linux-arm64" ;; \ + *) echo "Unsupported architecture: ${ARCH}"; exit 1 ;; \ + esac && \ + curl -fsSL -o /usr/local/bin/dbmate \ + https://github.com/amacneil/dbmate/releases/latest/download/dbmate-${DBMATE_ARCH} && \ + chmod +x /usr/local/bin/dbmate -RUN apt-get update && apt-get install -y wget gnupg2 lsb-release RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list RUN apt-get update && apt-get install -y postgresql-client-%VERSION% ENV PATH="/usr/lib/postgresql/%VERSION%/bin:${PATH}" +RUN dbmate --version + ENTRYPOINT ["dbmate"] \ No newline at end of file From fd816b87749d00a25a1ad10e61ca0347fd2297d0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 9 Oct 2024 20:50:26 -0400 Subject: [PATCH 45/51] chore: limit changes detection migrations/schema.sql --- .github/workflows/test.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 410c50f73..d6c7c0dca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -156,9 +156,8 @@ jobs: - name: verify schema.sql is committed run: | - docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit - if ! git diff --ignore-space-at-eol --exit-code --quiet migrations/schema.sql; then - echo "Detected uncommitted changes after build. See status below:" - git diff + if ! git diff --exit-code --quiet migrations/schema.sql; then + echo "Detected changes in schema.sql:" + git diff migrations/schema.sql exit 1 fi From 26f02f009534f8c4035d801cb2a0c9e95a227b88 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 9 Oct 2024 21:30:02 -0400 Subject: [PATCH 46/51] missing docker compose call --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d6c7c0dca..91d6cef5d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -156,6 +156,7 @@ jobs: - name: verify schema.sql is committed run: | + docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit if ! git diff --exit-code --quiet migrations/schema.sql; then echo "Detected changes in schema.sql:" git diff migrations/schema.sql From a69475245ac6fe75e544c6ae149831315d574c59 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 10 Oct 2024 05:41:55 -0400 Subject: [PATCH 47/51] ore: drop tests while investigating --- .github/workflows/test.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 91d6cef5d..688d620b5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -150,15 +150,15 @@ jobs: PGUSER: supabase_admin PGPASSWORD: ${{ env.POSTGRES_PASSWORD }} - - name: Update Dockerfile.dbmate version - run: | - sed -i 's/%VERSION%/${{ env.PGMAJOR }}/g' migrations/Dockerfile.dbmate - - - name: verify schema.sql is committed - run: | - docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit - if ! git diff --exit-code --quiet migrations/schema.sql; then - echo "Detected changes in schema.sql:" - git diff migrations/schema.sql - exit 1 - fi + # - name: Update Dockerfile.dbmate version + # run: | + # sed -i 's/%VERSION%/${{ env.PGMAJOR }}/g' migrations/Dockerfile.dbmate + + # - name: verify schema.sql is committed + # run: | + # docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit + # if ! git diff --exit-code --quiet migrations/schema.sql; then + # echo "Detected changes in schema.sql:" + # git diff + # exit 1 + # fi From 4fa0e0f9aa27821b4fa52425a0b4d5955ad8e93c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 10 Oct 2024 06:39:51 -0400 Subject: [PATCH 48/51] test: try on pg15 only --- .github/workflows/test.yml | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 688d620b5..5cad2a9e0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -150,15 +150,17 @@ jobs: PGUSER: supabase_admin PGPASSWORD: ${{ env.POSTGRES_PASSWORD }} - # - name: Update Dockerfile.dbmate version - # run: | - # sed -i 's/%VERSION%/${{ env.PGMAJOR }}/g' migrations/Dockerfile.dbmate - - # - name: verify schema.sql is committed - # run: | - # docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit - # if ! git diff --exit-code --quiet migrations/schema.sql; then - # echo "Detected changes in schema.sql:" - # git diff - # exit 1 - # fi + - name: Update Dockerfile.dbmate version + if: env.PGMAJOR == '15' + run: | + sed -i 's/%VERSION%/${{ env.PGMAJOR }}/g' migrations/Dockerfile.dbmate + + - name: verify schema.sql is committed + if: env.PGMAJOR == '15' + run: | + docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit + if ! git diff --exit-code --quiet migrations/schema.sql; then + echo "Detected changes in schema.sql:" + git diff + exit 1 + fi From acf275dfe3671d3a7949e5d59f0de3c5891dbbbd Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 10 Oct 2024 09:54:12 -0400 Subject: [PATCH 49/51] chore: schema needs update --- .github/workflows/ami-release-nix.yml | 5 ++--- .github/workflows/test.yml | 2 +- migrations/schema.sql | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 706edd727..643f26fc7 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -3,9 +3,8 @@ name: Release AMI Nix on: push: branches: - #- develop - #- release/* - - sam/15-16-ghactions + - develop + - release/* paths: - '.github/workflows/ami-release-nix.yml' - 'common-nix.vars.pkr.hcl' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5cad2a9e0..6bf8a62f5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -161,6 +161,6 @@ jobs: docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit if ! git diff --exit-code --quiet migrations/schema.sql; then echo "Detected changes in schema.sql:" - git diff + git diff migrations/schema.sql exit 1 fi diff --git a/migrations/schema.sql b/migrations/schema.sql index f68d131e8..1bff8b9d8 100644 --- a/migrations/schema.sql +++ b/migrations/schema.sql @@ -737,7 +737,7 @@ COMMENT ON TABLE auth.users IS 'Auth: Stores user login data within a secure sch -- CREATE TABLE public.schema_migrations ( - version character varying(255) NOT NULL + version character varying(128) NOT NULL ); From 29b11a383872759e0dee532be94a41f1748e89ef Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 10 Oct 2024 10:10:31 -0400 Subject: [PATCH 50/51] chore: now run on all versions in matrix --- .github/workflows/test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6bf8a62f5..91d6cef5d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -151,12 +151,10 @@ jobs: PGPASSWORD: ${{ env.POSTGRES_PASSWORD }} - name: Update Dockerfile.dbmate version - if: env.PGMAJOR == '15' run: | sed -i 's/%VERSION%/${{ env.PGMAJOR }}/g' migrations/Dockerfile.dbmate - name: verify schema.sql is committed - if: env.PGMAJOR == '15' run: | docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit if ! git diff --exit-code --quiet migrations/schema.sql; then From 300303844d76c03b04f9805870e80888393cdbf2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 10 Oct 2024 10:44:40 -0400 Subject: [PATCH 51/51] test: trying a version of schema per major pg version as there are type diffs --- .github/workflows/test.yml | 4 +- migrations/schema-15.sql | 1064 ++++++++++++++++++++++++++++++++++++ migrations/schema-16.sql | 1064 ++++++++++++++++++++++++++++++++++++ 3 files changed, 2130 insertions(+), 2 deletions(-) create mode 100644 migrations/schema-15.sql create mode 100644 migrations/schema-16.sql diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 91d6cef5d..1457eb100 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -157,8 +157,8 @@ jobs: - name: verify schema.sql is committed run: | docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit - if ! git diff --exit-code --quiet migrations/schema.sql; then + if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then echo "Detected changes in schema.sql:" - git diff migrations/schema.sql + git diff migrations/schema-${{ env.PGMAJOR }}.sql exit 1 fi diff --git a/migrations/schema-15.sql b/migrations/schema-15.sql new file mode 100644 index 000000000..1bff8b9d8 --- /dev/null +++ b/migrations/schema-15.sql @@ -0,0 +1,1064 @@ +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: auth; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA auth; + + +-- +-- Name: extensions; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA extensions; + + +-- +-- Name: graphql; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA graphql; + + +-- +-- Name: graphql_public; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA graphql_public; + + +-- +-- Name: pgbouncer; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA pgbouncer; + + +-- +-- Name: pgsodium; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA pgsodium; + + +-- +-- Name: pgsodium; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgsodium WITH SCHEMA pgsodium; + + +-- +-- Name: EXTENSION pgsodium; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pgsodium IS 'Pgsodium is a modern cryptography library for Postgres.'; + + +-- +-- Name: realtime; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA realtime; + + +-- +-- Name: storage; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA storage; + + +-- +-- Name: vault; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA vault; + + +-- +-- Name: pg_graphql; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pg_graphql WITH SCHEMA graphql; + + +-- +-- Name: EXTENSION pg_graphql; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pg_graphql IS 'pg_graphql: GraphQL support'; + + +-- +-- Name: pg_stat_statements; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pg_stat_statements WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION pg_stat_statements; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pg_stat_statements IS 'track planning and execution statistics of all SQL statements executed'; + + +-- +-- Name: pgcrypto; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION pgcrypto; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pgcrypto IS 'cryptographic functions'; + + +-- +-- Name: pgjwt; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgjwt WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION pgjwt; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pgjwt IS 'JSON Web Token API for Postgresql'; + + +-- +-- Name: supabase_vault; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS supabase_vault WITH SCHEMA vault; + + +-- +-- Name: EXTENSION supabase_vault; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION supabase_vault IS 'Supabase Vault Extension'; + + +-- +-- Name: uuid-ossp; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS "uuid-ossp" WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION "uuid-ossp"; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION "uuid-ossp" IS 'generate universally unique identifiers (UUIDs)'; + + +-- +-- Name: email(); Type: FUNCTION; Schema: auth; Owner: - +-- + +CREATE FUNCTION auth.email() RETURNS text + LANGUAGE sql STABLE + AS $$ + select nullif(current_setting('request.jwt.claim.email', true), '')::text; +$$; + + +-- +-- Name: role(); Type: FUNCTION; Schema: auth; Owner: - +-- + +CREATE FUNCTION auth.role() RETURNS text + LANGUAGE sql STABLE + AS $$ + select nullif(current_setting('request.jwt.claim.role', true), '')::text; +$$; + + +-- +-- Name: uid(); Type: FUNCTION; Schema: auth; Owner: - +-- + +CREATE FUNCTION auth.uid() RETURNS uuid + LANGUAGE sql STABLE + AS $$ + select nullif(current_setting('request.jwt.claim.sub', true), '')::uuid; +$$; + + +-- +-- Name: grant_pg_cron_access(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.grant_pg_cron_access() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +BEGIN + IF EXISTS ( + SELECT + FROM pg_event_trigger_ddl_commands() AS ev + JOIN pg_extension AS ext + ON ev.objid = ext.oid + WHERE ext.extname = 'pg_cron' + ) + THEN + grant usage on schema cron to postgres with grant option; + + alter default privileges in schema cron grant all on tables to postgres with grant option; + alter default privileges in schema cron grant all on functions to postgres with grant option; + alter default privileges in schema cron grant all on sequences to postgres with grant option; + + alter default privileges for user supabase_admin in schema cron grant all + on sequences to postgres with grant option; + alter default privileges for user supabase_admin in schema cron grant all + on tables to postgres with grant option; + alter default privileges for user supabase_admin in schema cron grant all + on functions to postgres with grant option; + + grant all privileges on all tables in schema cron to postgres with grant option; + revoke all on table cron.job from postgres; + grant select on table cron.job to postgres with grant option; + END IF; +END; +$$; + + +-- +-- Name: FUNCTION grant_pg_cron_access(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.grant_pg_cron_access() IS 'Grants access to pg_cron'; + + +-- +-- Name: grant_pg_graphql_access(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.grant_pg_graphql_access() RETURNS event_trigger + LANGUAGE plpgsql + AS $_$ +DECLARE + func_is_graphql_resolve bool; +BEGIN + func_is_graphql_resolve = ( + SELECT n.proname = 'resolve' + FROM pg_event_trigger_ddl_commands() AS ev + LEFT JOIN pg_catalog.pg_proc AS n + ON ev.objid = n.oid + ); + + IF func_is_graphql_resolve + THEN + -- Update public wrapper to pass all arguments through to the pg_graphql resolve func + DROP FUNCTION IF EXISTS graphql_public.graphql; + create or replace function graphql_public.graphql( + "operationName" text default null, + query text default null, + variables jsonb default null, + extensions jsonb default null + ) + returns jsonb + language sql + as $$ + select graphql.resolve( + query := query, + variables := coalesce(variables, '{}'), + "operationName" := "operationName", + extensions := extensions + ); + $$; + + -- This hook executes when `graphql.resolve` is created. That is not necessarily the last + -- function in the extension so we need to grant permissions on existing entities AND + -- update default permissions to any others that are created after `graphql.resolve` + grant usage on schema graphql to postgres, anon, authenticated, service_role; + grant select on all tables in schema graphql to postgres, anon, authenticated, service_role; + grant execute on all functions in schema graphql to postgres, anon, authenticated, service_role; + grant all on all sequences in schema graphql to postgres, anon, authenticated, service_role; + alter default privileges in schema graphql grant all on tables to postgres, anon, authenticated, service_role; + alter default privileges in schema graphql grant all on functions to postgres, anon, authenticated, service_role; + alter default privileges in schema graphql grant all on sequences to postgres, anon, authenticated, service_role; + + -- Allow postgres role to allow granting usage on graphql and graphql_public schemas to custom roles + grant usage on schema graphql_public to postgres with grant option; + grant usage on schema graphql to postgres with grant option; + END IF; + +END; +$_$; + + +-- +-- Name: FUNCTION grant_pg_graphql_access(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.grant_pg_graphql_access() IS 'Grants access to pg_graphql'; + + +-- +-- Name: grant_pg_net_access(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.grant_pg_net_access() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_event_trigger_ddl_commands() AS ev + JOIN pg_extension AS ext + ON ev.objid = ext.oid + WHERE ext.extname = 'pg_net' + ) + THEN + IF NOT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = 'supabase_functions_admin' + ) + THEN + CREATE USER supabase_functions_admin NOINHERIT CREATEROLE LOGIN NOREPLICATION; + END IF; + + GRANT USAGE ON SCHEMA net TO supabase_functions_admin, postgres, anon, authenticated, service_role; + + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; + + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; + + REVOKE ALL ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; + REVOKE ALL ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; + + GRANT EXECUTE ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; + GRANT EXECUTE ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; + END IF; +END; +$$; + + +-- +-- Name: FUNCTION grant_pg_net_access(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.grant_pg_net_access() IS 'Grants access to pg_net'; + + +-- +-- Name: pgrst_ddl_watch(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.pgrst_ddl_watch() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +DECLARE + cmd record; +BEGIN + FOR cmd IN SELECT * FROM pg_event_trigger_ddl_commands() + LOOP + IF cmd.command_tag IN ( + 'CREATE SCHEMA', 'ALTER SCHEMA' + , 'CREATE TABLE', 'CREATE TABLE AS', 'SELECT INTO', 'ALTER TABLE' + , 'CREATE FOREIGN TABLE', 'ALTER FOREIGN TABLE' + , 'CREATE VIEW', 'ALTER VIEW' + , 'CREATE MATERIALIZED VIEW', 'ALTER MATERIALIZED VIEW' + , 'CREATE FUNCTION', 'ALTER FUNCTION' + , 'CREATE TRIGGER' + , 'CREATE TYPE', 'ALTER TYPE' + , 'CREATE RULE' + , 'COMMENT' + ) + -- don't notify in case of CREATE TEMP table or other objects created on pg_temp + AND cmd.schema_name is distinct from 'pg_temp' + THEN + NOTIFY pgrst, 'reload schema'; + END IF; + END LOOP; +END; $$; + + +-- +-- Name: pgrst_drop_watch(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.pgrst_drop_watch() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +DECLARE + obj record; +BEGIN + FOR obj IN SELECT * FROM pg_event_trigger_dropped_objects() + LOOP + IF obj.object_type IN ( + 'schema' + , 'table' + , 'foreign table' + , 'view' + , 'materialized view' + , 'function' + , 'trigger' + , 'type' + , 'rule' + ) + AND obj.is_temporary IS false -- no pg_temp objects + THEN + NOTIFY pgrst, 'reload schema'; + END IF; + END LOOP; +END; $$; + + +-- +-- Name: set_graphql_placeholder(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.set_graphql_placeholder() RETURNS event_trigger + LANGUAGE plpgsql + AS $_$ + DECLARE + graphql_is_dropped bool; + BEGIN + graphql_is_dropped = ( + SELECT ev.schema_name = 'graphql_public' + FROM pg_event_trigger_dropped_objects() AS ev + WHERE ev.schema_name = 'graphql_public' + ); + + IF graphql_is_dropped + THEN + create or replace function graphql_public.graphql( + "operationName" text default null, + query text default null, + variables jsonb default null, + extensions jsonb default null + ) + returns jsonb + language plpgsql + as $$ + DECLARE + server_version float; + BEGIN + server_version = (SELECT (SPLIT_PART((select version()), ' ', 2))::float); + + IF server_version >= 14 THEN + RETURN jsonb_build_object( + 'errors', jsonb_build_array( + jsonb_build_object( + 'message', 'pg_graphql extension is not enabled.' + ) + ) + ); + ELSE + RETURN jsonb_build_object( + 'errors', jsonb_build_array( + jsonb_build_object( + 'message', 'pg_graphql is only available on projects running Postgres 14 onwards.' + ) + ) + ); + END IF; + END; + $$; + END IF; + + END; +$_$; + + +-- +-- Name: FUNCTION set_graphql_placeholder(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.set_graphql_placeholder() IS 'Reintroduces placeholder function for graphql_public.graphql'; + + +-- +-- Name: get_auth(text); Type: FUNCTION; Schema: pgbouncer; Owner: - +-- + +CREATE FUNCTION pgbouncer.get_auth(p_usename text) RETURNS TABLE(username text, password text) + LANGUAGE plpgsql SECURITY DEFINER + AS $$ +BEGIN + RAISE WARNING 'PgBouncer auth request: %', p_usename; + + RETURN QUERY + SELECT usename::TEXT, passwd::TEXT FROM pg_catalog.pg_shadow + WHERE usename = p_usename; +END; +$$; + + +-- +-- Name: extension(text); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.extension(name text) RETURNS text + LANGUAGE plpgsql + AS $$ +DECLARE +_parts text[]; +_filename text; +BEGIN + select string_to_array(name, '/') into _parts; + select _parts[array_length(_parts,1)] into _filename; + -- @todo return the last part instead of 2 + return split_part(_filename, '.', 2); +END +$$; + + +-- +-- Name: filename(text); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.filename(name text) RETURNS text + LANGUAGE plpgsql + AS $$ +DECLARE +_parts text[]; +BEGIN + select string_to_array(name, '/') into _parts; + return _parts[array_length(_parts,1)]; +END +$$; + + +-- +-- Name: foldername(text); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.foldername(name text) RETURNS text[] + LANGUAGE plpgsql + AS $$ +DECLARE +_parts text[]; +BEGIN + select string_to_array(name, '/') into _parts; + return _parts[1:array_length(_parts,1)-1]; +END +$$; + + +-- +-- Name: search(text, text, integer, integer, integer); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.search(prefix text, bucketname text, limits integer DEFAULT 100, levels integer DEFAULT 1, offsets integer DEFAULT 0) RETURNS TABLE(name text, id uuid, updated_at timestamp with time zone, created_at timestamp with time zone, last_accessed_at timestamp with time zone, metadata jsonb) + LANGUAGE plpgsql + AS $$ +DECLARE +_bucketId text; +BEGIN + -- will be replaced by migrations when server starts + -- saving space for cloud-init +END +$$; + + +-- +-- Name: secrets_encrypt_secret_secret(); Type: FUNCTION; Schema: vault; Owner: - +-- + +CREATE FUNCTION vault.secrets_encrypt_secret_secret() RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + new.secret = CASE WHEN new.secret IS NULL THEN NULL ELSE + CASE WHEN new.key_id IS NULL THEN NULL ELSE pg_catalog.encode( + pgsodium.crypto_aead_det_encrypt( + pg_catalog.convert_to(new.secret, 'utf8'), + pg_catalog.convert_to((new.id::text || new.description::text || new.created_at::text || new.updated_at::text)::text, 'utf8'), + new.key_id::uuid, + new.nonce + ), + 'base64') END END; + RETURN new; + END; + $$; + + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: audit_log_entries; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.audit_log_entries ( + instance_id uuid, + id uuid NOT NULL, + payload json, + created_at timestamp with time zone +); + + +-- +-- Name: TABLE audit_log_entries; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.audit_log_entries IS 'Auth: Audit trail for user actions.'; + + +-- +-- Name: instances; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.instances ( + id uuid NOT NULL, + uuid uuid, + raw_base_config text, + created_at timestamp with time zone, + updated_at timestamp with time zone +); + + +-- +-- Name: TABLE instances; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.instances IS 'Auth: Manages users across multiple sites.'; + + +-- +-- Name: refresh_tokens; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.refresh_tokens ( + instance_id uuid, + id bigint NOT NULL, + token character varying(255), + user_id character varying(255), + revoked boolean, + created_at timestamp with time zone, + updated_at timestamp with time zone +); + + +-- +-- Name: TABLE refresh_tokens; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.refresh_tokens IS 'Auth: Store of tokens used to refresh JWT tokens once they expire.'; + + +-- +-- Name: refresh_tokens_id_seq; Type: SEQUENCE; Schema: auth; Owner: - +-- + +CREATE SEQUENCE auth.refresh_tokens_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: refresh_tokens_id_seq; Type: SEQUENCE OWNED BY; Schema: auth; Owner: - +-- + +ALTER SEQUENCE auth.refresh_tokens_id_seq OWNED BY auth.refresh_tokens.id; + + +-- +-- Name: schema_migrations; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.schema_migrations ( + version character varying(255) NOT NULL +); + + +-- +-- Name: TABLE schema_migrations; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.schema_migrations IS 'Auth: Manages updates to the auth system.'; + + +-- +-- Name: users; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.users ( + instance_id uuid, + id uuid NOT NULL, + aud character varying(255), + role character varying(255), + email character varying(255), + encrypted_password character varying(255), + confirmed_at timestamp with time zone, + invited_at timestamp with time zone, + confirmation_token character varying(255), + confirmation_sent_at timestamp with time zone, + recovery_token character varying(255), + recovery_sent_at timestamp with time zone, + email_change_token character varying(255), + email_change character varying(255), + email_change_sent_at timestamp with time zone, + last_sign_in_at timestamp with time zone, + raw_app_meta_data jsonb, + raw_user_meta_data jsonb, + is_super_admin boolean, + created_at timestamp with time zone, + updated_at timestamp with time zone +); + + +-- +-- Name: TABLE users; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.users IS 'Auth: Stores user login data within a secure schema.'; + + +-- +-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_migrations ( + version character varying(128) NOT NULL +); + + +-- +-- Name: buckets; Type: TABLE; Schema: storage; Owner: - +-- + +CREATE TABLE storage.buckets ( + id text NOT NULL, + name text NOT NULL, + owner uuid, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone DEFAULT now() +); + + +-- +-- Name: migrations; Type: TABLE; Schema: storage; Owner: - +-- + +CREATE TABLE storage.migrations ( + id integer NOT NULL, + name character varying(100) NOT NULL, + hash character varying(40) NOT NULL, + executed_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP +); + + +-- +-- Name: objects; Type: TABLE; Schema: storage; Owner: - +-- + +CREATE TABLE storage.objects ( + id uuid DEFAULT extensions.uuid_generate_v4() NOT NULL, + bucket_id text, + name text, + owner uuid, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone DEFAULT now(), + last_accessed_at timestamp with time zone DEFAULT now(), + metadata jsonb +); + + +-- +-- Name: decrypted_secrets; Type: VIEW; Schema: vault; Owner: - +-- + +CREATE VIEW vault.decrypted_secrets AS + SELECT secrets.id, + secrets.name, + secrets.description, + secrets.secret, + CASE + WHEN (secrets.secret IS NULL) THEN NULL::text + ELSE + CASE + WHEN (secrets.key_id IS NULL) THEN NULL::text + ELSE convert_from(pgsodium.crypto_aead_det_decrypt(decode(secrets.secret, 'base64'::text), convert_to(((((secrets.id)::text || secrets.description) || (secrets.created_at)::text) || (secrets.updated_at)::text), 'utf8'::name), secrets.key_id, secrets.nonce), 'utf8'::name) + END + END AS decrypted_secret, + secrets.key_id, + secrets.nonce, + secrets.created_at, + secrets.updated_at + FROM vault.secrets; + + +-- +-- Name: refresh_tokens id; Type: DEFAULT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.refresh_tokens ALTER COLUMN id SET DEFAULT nextval('auth.refresh_tokens_id_seq'::regclass); + + +-- +-- Name: audit_log_entries audit_log_entries_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.audit_log_entries + ADD CONSTRAINT audit_log_entries_pkey PRIMARY KEY (id); + + +-- +-- Name: instances instances_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.instances + ADD CONSTRAINT instances_pkey PRIMARY KEY (id); + + +-- +-- Name: refresh_tokens refresh_tokens_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.refresh_tokens + ADD CONSTRAINT refresh_tokens_pkey PRIMARY KEY (id); + + +-- +-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.schema_migrations + ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); + + +-- +-- Name: users users_email_key; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.users + ADD CONSTRAINT users_email_key UNIQUE (email); + + +-- +-- Name: users users_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.users + ADD CONSTRAINT users_pkey PRIMARY KEY (id); + + +-- +-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_migrations + ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); + + +-- +-- Name: buckets buckets_pkey; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.buckets + ADD CONSTRAINT buckets_pkey PRIMARY KEY (id); + + +-- +-- Name: migrations migrations_name_key; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.migrations + ADD CONSTRAINT migrations_name_key UNIQUE (name); + + +-- +-- Name: migrations migrations_pkey; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.migrations + ADD CONSTRAINT migrations_pkey PRIMARY KEY (id); + + +-- +-- Name: objects objects_pkey; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.objects + ADD CONSTRAINT objects_pkey PRIMARY KEY (id); + + +-- +-- Name: audit_logs_instance_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX audit_logs_instance_id_idx ON auth.audit_log_entries USING btree (instance_id); + + +-- +-- Name: refresh_tokens_instance_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX refresh_tokens_instance_id_idx ON auth.refresh_tokens USING btree (instance_id); + + +-- +-- Name: refresh_tokens_instance_id_user_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX refresh_tokens_instance_id_user_id_idx ON auth.refresh_tokens USING btree (instance_id, user_id); + + +-- +-- Name: refresh_tokens_token_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX refresh_tokens_token_idx ON auth.refresh_tokens USING btree (token); + + +-- +-- Name: users_instance_id_email_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX users_instance_id_email_idx ON auth.users USING btree (instance_id, email); + + +-- +-- Name: users_instance_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX users_instance_id_idx ON auth.users USING btree (instance_id); + + +-- +-- Name: bname; Type: INDEX; Schema: storage; Owner: - +-- + +CREATE UNIQUE INDEX bname ON storage.buckets USING btree (name); + + +-- +-- Name: bucketid_objname; Type: INDEX; Schema: storage; Owner: - +-- + +CREATE UNIQUE INDEX bucketid_objname ON storage.objects USING btree (bucket_id, name); + + +-- +-- Name: name_prefix_search; Type: INDEX; Schema: storage; Owner: - +-- + +CREATE INDEX name_prefix_search ON storage.objects USING btree (name text_pattern_ops); + + +-- +-- Name: buckets buckets_owner_fkey; Type: FK CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.buckets + ADD CONSTRAINT buckets_owner_fkey FOREIGN KEY (owner) REFERENCES auth.users(id); + + +-- +-- Name: objects objects_bucketId_fkey; Type: FK CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.objects + ADD CONSTRAINT "objects_bucketId_fkey" FOREIGN KEY (bucket_id) REFERENCES storage.buckets(id); + + +-- +-- Name: objects objects_owner_fkey; Type: FK CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.objects + ADD CONSTRAINT objects_owner_fkey FOREIGN KEY (owner) REFERENCES auth.users(id); + + +-- +-- Name: objects; Type: ROW SECURITY; Schema: storage; Owner: - +-- + +ALTER TABLE storage.objects ENABLE ROW LEVEL SECURITY; + +-- +-- Name: supabase_realtime; Type: PUBLICATION; Schema: -; Owner: - +-- + +CREATE PUBLICATION supabase_realtime WITH (publish = 'insert, update, delete, truncate'); + + +-- +-- Name: issue_graphql_placeholder; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_graphql_placeholder ON sql_drop + WHEN TAG IN ('DROP EXTENSION') + EXECUTE FUNCTION extensions.set_graphql_placeholder(); + + +-- +-- Name: issue_pg_cron_access; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_pg_cron_access ON ddl_command_end + WHEN TAG IN ('CREATE EXTENSION') + EXECUTE FUNCTION extensions.grant_pg_cron_access(); + + +-- +-- Name: issue_pg_graphql_access; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_pg_graphql_access ON ddl_command_end + WHEN TAG IN ('CREATE FUNCTION') + EXECUTE FUNCTION extensions.grant_pg_graphql_access(); + + +-- +-- Name: issue_pg_net_access; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_pg_net_access ON ddl_command_end + WHEN TAG IN ('CREATE EXTENSION') + EXECUTE FUNCTION extensions.grant_pg_net_access(); + + +-- +-- Name: pgrst_ddl_watch; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER pgrst_ddl_watch ON ddl_command_end + EXECUTE FUNCTION extensions.pgrst_ddl_watch(); + + +-- +-- Name: pgrst_drop_watch; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER pgrst_drop_watch ON sql_drop + EXECUTE FUNCTION extensions.pgrst_drop_watch(); + + +-- +-- PostgreSQL database dump complete +-- + + +-- +-- Dbmate schema migrations +-- + diff --git a/migrations/schema-16.sql b/migrations/schema-16.sql new file mode 100644 index 000000000..49873a109 --- /dev/null +++ b/migrations/schema-16.sql @@ -0,0 +1,1064 @@ +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: auth; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA auth; + + +-- +-- Name: extensions; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA extensions; + + +-- +-- Name: graphql; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA graphql; + + +-- +-- Name: graphql_public; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA graphql_public; + + +-- +-- Name: pgbouncer; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA pgbouncer; + + +-- +-- Name: pgsodium; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA pgsodium; + + +-- +-- Name: pgsodium; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgsodium WITH SCHEMA pgsodium; + + +-- +-- Name: EXTENSION pgsodium; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pgsodium IS 'Pgsodium is a modern cryptography library for Postgres.'; + + +-- +-- Name: realtime; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA realtime; + + +-- +-- Name: storage; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA storage; + + +-- +-- Name: vault; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA vault; + + +-- +-- Name: pg_graphql; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pg_graphql WITH SCHEMA graphql; + + +-- +-- Name: EXTENSION pg_graphql; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pg_graphql IS 'pg_graphql: GraphQL support'; + + +-- +-- Name: pg_stat_statements; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pg_stat_statements WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION pg_stat_statements; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pg_stat_statements IS 'track planning and execution statistics of all SQL statements executed'; + + +-- +-- Name: pgcrypto; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION pgcrypto; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pgcrypto IS 'cryptographic functions'; + + +-- +-- Name: pgjwt; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgjwt WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION pgjwt; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pgjwt IS 'JSON Web Token API for Postgresql'; + + +-- +-- Name: supabase_vault; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS supabase_vault WITH SCHEMA vault; + + +-- +-- Name: EXTENSION supabase_vault; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION supabase_vault IS 'Supabase Vault Extension'; + + +-- +-- Name: uuid-ossp; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS "uuid-ossp" WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION "uuid-ossp"; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION "uuid-ossp" IS 'generate universally unique identifiers (UUIDs)'; + + +-- +-- Name: email(); Type: FUNCTION; Schema: auth; Owner: - +-- + +CREATE FUNCTION auth.email() RETURNS text + LANGUAGE sql STABLE + AS $$ + select nullif(current_setting('request.jwt.claim.email', true), '')::text; +$$; + + +-- +-- Name: role(); Type: FUNCTION; Schema: auth; Owner: - +-- + +CREATE FUNCTION auth.role() RETURNS text + LANGUAGE sql STABLE + AS $$ + select nullif(current_setting('request.jwt.claim.role', true), '')::text; +$$; + + +-- +-- Name: uid(); Type: FUNCTION; Schema: auth; Owner: - +-- + +CREATE FUNCTION auth.uid() RETURNS uuid + LANGUAGE sql STABLE + AS $$ + select nullif(current_setting('request.jwt.claim.sub', true), '')::uuid; +$$; + + +-- +-- Name: grant_pg_cron_access(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.grant_pg_cron_access() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +BEGIN + IF EXISTS ( + SELECT + FROM pg_event_trigger_ddl_commands() AS ev + JOIN pg_extension AS ext + ON ev.objid = ext.oid + WHERE ext.extname = 'pg_cron' + ) + THEN + grant usage on schema cron to postgres with grant option; + + alter default privileges in schema cron grant all on tables to postgres with grant option; + alter default privileges in schema cron grant all on functions to postgres with grant option; + alter default privileges in schema cron grant all on sequences to postgres with grant option; + + alter default privileges for user supabase_admin in schema cron grant all + on sequences to postgres with grant option; + alter default privileges for user supabase_admin in schema cron grant all + on tables to postgres with grant option; + alter default privileges for user supabase_admin in schema cron grant all + on functions to postgres with grant option; + + grant all privileges on all tables in schema cron to postgres with grant option; + revoke all on table cron.job from postgres; + grant select on table cron.job to postgres with grant option; + END IF; +END; +$$; + + +-- +-- Name: FUNCTION grant_pg_cron_access(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.grant_pg_cron_access() IS 'Grants access to pg_cron'; + + +-- +-- Name: grant_pg_graphql_access(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.grant_pg_graphql_access() RETURNS event_trigger + LANGUAGE plpgsql + AS $_$ +DECLARE + func_is_graphql_resolve bool; +BEGIN + func_is_graphql_resolve = ( + SELECT n.proname = 'resolve' + FROM pg_event_trigger_ddl_commands() AS ev + LEFT JOIN pg_catalog.pg_proc AS n + ON ev.objid = n.oid + ); + + IF func_is_graphql_resolve + THEN + -- Update public wrapper to pass all arguments through to the pg_graphql resolve func + DROP FUNCTION IF EXISTS graphql_public.graphql; + create or replace function graphql_public.graphql( + "operationName" text default null, + query text default null, + variables jsonb default null, + extensions jsonb default null + ) + returns jsonb + language sql + as $$ + select graphql.resolve( + query := query, + variables := coalesce(variables, '{}'), + "operationName" := "operationName", + extensions := extensions + ); + $$; + + -- This hook executes when `graphql.resolve` is created. That is not necessarily the last + -- function in the extension so we need to grant permissions on existing entities AND + -- update default permissions to any others that are created after `graphql.resolve` + grant usage on schema graphql to postgres, anon, authenticated, service_role; + grant select on all tables in schema graphql to postgres, anon, authenticated, service_role; + grant execute on all functions in schema graphql to postgres, anon, authenticated, service_role; + grant all on all sequences in schema graphql to postgres, anon, authenticated, service_role; + alter default privileges in schema graphql grant all on tables to postgres, anon, authenticated, service_role; + alter default privileges in schema graphql grant all on functions to postgres, anon, authenticated, service_role; + alter default privileges in schema graphql grant all on sequences to postgres, anon, authenticated, service_role; + + -- Allow postgres role to allow granting usage on graphql and graphql_public schemas to custom roles + grant usage on schema graphql_public to postgres with grant option; + grant usage on schema graphql to postgres with grant option; + END IF; + +END; +$_$; + + +-- +-- Name: FUNCTION grant_pg_graphql_access(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.grant_pg_graphql_access() IS 'Grants access to pg_graphql'; + + +-- +-- Name: grant_pg_net_access(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.grant_pg_net_access() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_event_trigger_ddl_commands() AS ev + JOIN pg_extension AS ext + ON ev.objid = ext.oid + WHERE ext.extname = 'pg_net' + ) + THEN + IF NOT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = 'supabase_functions_admin' + ) + THEN + CREATE USER supabase_functions_admin NOINHERIT CREATEROLE LOGIN NOREPLICATION; + END IF; + + GRANT USAGE ON SCHEMA net TO supabase_functions_admin, postgres, anon, authenticated, service_role; + + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; + + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; + + REVOKE ALL ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; + REVOKE ALL ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; + + GRANT EXECUTE ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; + GRANT EXECUTE ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; + END IF; +END; +$$; + + +-- +-- Name: FUNCTION grant_pg_net_access(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.grant_pg_net_access() IS 'Grants access to pg_net'; + + +-- +-- Name: pgrst_ddl_watch(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.pgrst_ddl_watch() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +DECLARE + cmd record; +BEGIN + FOR cmd IN SELECT * FROM pg_event_trigger_ddl_commands() + LOOP + IF cmd.command_tag IN ( + 'CREATE SCHEMA', 'ALTER SCHEMA' + , 'CREATE TABLE', 'CREATE TABLE AS', 'SELECT INTO', 'ALTER TABLE' + , 'CREATE FOREIGN TABLE', 'ALTER FOREIGN TABLE' + , 'CREATE VIEW', 'ALTER VIEW' + , 'CREATE MATERIALIZED VIEW', 'ALTER MATERIALIZED VIEW' + , 'CREATE FUNCTION', 'ALTER FUNCTION' + , 'CREATE TRIGGER' + , 'CREATE TYPE', 'ALTER TYPE' + , 'CREATE RULE' + , 'COMMENT' + ) + -- don't notify in case of CREATE TEMP table or other objects created on pg_temp + AND cmd.schema_name is distinct from 'pg_temp' + THEN + NOTIFY pgrst, 'reload schema'; + END IF; + END LOOP; +END; $$; + + +-- +-- Name: pgrst_drop_watch(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.pgrst_drop_watch() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +DECLARE + obj record; +BEGIN + FOR obj IN SELECT * FROM pg_event_trigger_dropped_objects() + LOOP + IF obj.object_type IN ( + 'schema' + , 'table' + , 'foreign table' + , 'view' + , 'materialized view' + , 'function' + , 'trigger' + , 'type' + , 'rule' + ) + AND obj.is_temporary IS false -- no pg_temp objects + THEN + NOTIFY pgrst, 'reload schema'; + END IF; + END LOOP; +END; $$; + + +-- +-- Name: set_graphql_placeholder(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.set_graphql_placeholder() RETURNS event_trigger + LANGUAGE plpgsql + AS $_$ + DECLARE + graphql_is_dropped bool; + BEGIN + graphql_is_dropped = ( + SELECT ev.schema_name = 'graphql_public' + FROM pg_event_trigger_dropped_objects() AS ev + WHERE ev.schema_name = 'graphql_public' + ); + + IF graphql_is_dropped + THEN + create or replace function graphql_public.graphql( + "operationName" text default null, + query text default null, + variables jsonb default null, + extensions jsonb default null + ) + returns jsonb + language plpgsql + as $$ + DECLARE + server_version float; + BEGIN + server_version = (SELECT (SPLIT_PART((select version()), ' ', 2))::float); + + IF server_version >= 14 THEN + RETURN jsonb_build_object( + 'errors', jsonb_build_array( + jsonb_build_object( + 'message', 'pg_graphql extension is not enabled.' + ) + ) + ); + ELSE + RETURN jsonb_build_object( + 'errors', jsonb_build_array( + jsonb_build_object( + 'message', 'pg_graphql is only available on projects running Postgres 14 onwards.' + ) + ) + ); + END IF; + END; + $$; + END IF; + + END; +$_$; + + +-- +-- Name: FUNCTION set_graphql_placeholder(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.set_graphql_placeholder() IS 'Reintroduces placeholder function for graphql_public.graphql'; + + +-- +-- Name: get_auth(text); Type: FUNCTION; Schema: pgbouncer; Owner: - +-- + +CREATE FUNCTION pgbouncer.get_auth(p_usename text) RETURNS TABLE(username text, password text) + LANGUAGE plpgsql SECURITY DEFINER + AS $$ +BEGIN + RAISE WARNING 'PgBouncer auth request: %', p_usename; + + RETURN QUERY + SELECT usename::TEXT, passwd::TEXT FROM pg_catalog.pg_shadow + WHERE usename = p_usename; +END; +$$; + + +-- +-- Name: extension(text); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.extension(name text) RETURNS text + LANGUAGE plpgsql + AS $$ +DECLARE +_parts text[]; +_filename text; +BEGIN + select string_to_array(name, '/') into _parts; + select _parts[array_length(_parts,1)] into _filename; + -- @todo return the last part instead of 2 + return split_part(_filename, '.', 2); +END +$$; + + +-- +-- Name: filename(text); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.filename(name text) RETURNS text + LANGUAGE plpgsql + AS $$ +DECLARE +_parts text[]; +BEGIN + select string_to_array(name, '/') into _parts; + return _parts[array_length(_parts,1)]; +END +$$; + + +-- +-- Name: foldername(text); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.foldername(name text) RETURNS text[] + LANGUAGE plpgsql + AS $$ +DECLARE +_parts text[]; +BEGIN + select string_to_array(name, '/') into _parts; + return _parts[1:array_length(_parts,1)-1]; +END +$$; + + +-- +-- Name: search(text, text, integer, integer, integer); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.search(prefix text, bucketname text, limits integer DEFAULT 100, levels integer DEFAULT 1, offsets integer DEFAULT 0) RETURNS TABLE(name text, id uuid, updated_at timestamp with time zone, created_at timestamp with time zone, last_accessed_at timestamp with time zone, metadata jsonb) + LANGUAGE plpgsql + AS $$ +DECLARE +_bucketId text; +BEGIN + -- will be replaced by migrations when server starts + -- saving space for cloud-init +END +$$; + + +-- +-- Name: secrets_encrypt_secret_secret(); Type: FUNCTION; Schema: vault; Owner: - +-- + +CREATE FUNCTION vault.secrets_encrypt_secret_secret() RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + new.secret = CASE WHEN new.secret IS NULL THEN NULL ELSE + CASE WHEN new.key_id IS NULL THEN NULL ELSE pg_catalog.encode( + pgsodium.crypto_aead_det_encrypt( + pg_catalog.convert_to(new.secret, 'utf8'), + pg_catalog.convert_to((new.id::text || new.description::text || new.created_at::text || new.updated_at::text)::text, 'utf8'), + new.key_id::uuid, + new.nonce + ), + 'base64') END END; + RETURN new; + END; + $$; + + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: audit_log_entries; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.audit_log_entries ( + instance_id uuid, + id uuid NOT NULL, + payload json, + created_at timestamp with time zone +); + + +-- +-- Name: TABLE audit_log_entries; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.audit_log_entries IS 'Auth: Audit trail for user actions.'; + + +-- +-- Name: instances; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.instances ( + id uuid NOT NULL, + uuid uuid, + raw_base_config text, + created_at timestamp with time zone, + updated_at timestamp with time zone +); + + +-- +-- Name: TABLE instances; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.instances IS 'Auth: Manages users across multiple sites.'; + + +-- +-- Name: refresh_tokens; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.refresh_tokens ( + instance_id uuid, + id bigint NOT NULL, + token character varying(255), + user_id character varying(255), + revoked boolean, + created_at timestamp with time zone, + updated_at timestamp with time zone +); + + +-- +-- Name: TABLE refresh_tokens; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.refresh_tokens IS 'Auth: Store of tokens used to refresh JWT tokens once they expire.'; + + +-- +-- Name: refresh_tokens_id_seq; Type: SEQUENCE; Schema: auth; Owner: - +-- + +CREATE SEQUENCE auth.refresh_tokens_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: refresh_tokens_id_seq; Type: SEQUENCE OWNED BY; Schema: auth; Owner: - +-- + +ALTER SEQUENCE auth.refresh_tokens_id_seq OWNED BY auth.refresh_tokens.id; + + +-- +-- Name: schema_migrations; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.schema_migrations ( + version character varying(255) NOT NULL +); + + +-- +-- Name: TABLE schema_migrations; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.schema_migrations IS 'Auth: Manages updates to the auth system.'; + + +-- +-- Name: users; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.users ( + instance_id uuid, + id uuid NOT NULL, + aud character varying(255), + role character varying(255), + email character varying(255), + encrypted_password character varying(255), + confirmed_at timestamp with time zone, + invited_at timestamp with time zone, + confirmation_token character varying(255), + confirmation_sent_at timestamp with time zone, + recovery_token character varying(255), + recovery_sent_at timestamp with time zone, + email_change_token character varying(255), + email_change character varying(255), + email_change_sent_at timestamp with time zone, + last_sign_in_at timestamp with time zone, + raw_app_meta_data jsonb, + raw_user_meta_data jsonb, + is_super_admin boolean, + created_at timestamp with time zone, + updated_at timestamp with time zone +); + + +-- +-- Name: TABLE users; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.users IS 'Auth: Stores user login data within a secure schema.'; + + +-- +-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_migrations ( + version character varying(128) NOT NULL +); + + +-- +-- Name: buckets; Type: TABLE; Schema: storage; Owner: - +-- + +CREATE TABLE storage.buckets ( + id text NOT NULL, + name text NOT NULL, + owner uuid, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone DEFAULT now() +); + + +-- +-- Name: migrations; Type: TABLE; Schema: storage; Owner: - +-- + +CREATE TABLE storage.migrations ( + id integer NOT NULL, + name character varying(100) NOT NULL, + hash character varying(40) NOT NULL, + executed_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP +); + + +-- +-- Name: objects; Type: TABLE; Schema: storage; Owner: - +-- + +CREATE TABLE storage.objects ( + id uuid DEFAULT extensions.uuid_generate_v4() NOT NULL, + bucket_id text, + name text, + owner uuid, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone DEFAULT now(), + last_accessed_at timestamp with time zone DEFAULT now(), + metadata jsonb +); + + +-- +-- Name: decrypted_secrets; Type: VIEW; Schema: vault; Owner: - +-- + +CREATE VIEW vault.decrypted_secrets AS + SELECT id, + name, + description, + secret, + CASE + WHEN (secret IS NULL) THEN NULL::text + ELSE + CASE + WHEN (key_id IS NULL) THEN NULL::text + ELSE convert_from(pgsodium.crypto_aead_det_decrypt(decode(secret, 'base64'::text), convert_to(((((id)::text || description) || (created_at)::text) || (updated_at)::text), 'utf8'::name), key_id, nonce), 'utf8'::name) + END + END AS decrypted_secret, + key_id, + nonce, + created_at, + updated_at + FROM vault.secrets; + + +-- +-- Name: refresh_tokens id; Type: DEFAULT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.refresh_tokens ALTER COLUMN id SET DEFAULT nextval('auth.refresh_tokens_id_seq'::regclass); + + +-- +-- Name: audit_log_entries audit_log_entries_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.audit_log_entries + ADD CONSTRAINT audit_log_entries_pkey PRIMARY KEY (id); + + +-- +-- Name: instances instances_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.instances + ADD CONSTRAINT instances_pkey PRIMARY KEY (id); + + +-- +-- Name: refresh_tokens refresh_tokens_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.refresh_tokens + ADD CONSTRAINT refresh_tokens_pkey PRIMARY KEY (id); + + +-- +-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.schema_migrations + ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); + + +-- +-- Name: users users_email_key; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.users + ADD CONSTRAINT users_email_key UNIQUE (email); + + +-- +-- Name: users users_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.users + ADD CONSTRAINT users_pkey PRIMARY KEY (id); + + +-- +-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_migrations + ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); + + +-- +-- Name: buckets buckets_pkey; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.buckets + ADD CONSTRAINT buckets_pkey PRIMARY KEY (id); + + +-- +-- Name: migrations migrations_name_key; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.migrations + ADD CONSTRAINT migrations_name_key UNIQUE (name); + + +-- +-- Name: migrations migrations_pkey; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.migrations + ADD CONSTRAINT migrations_pkey PRIMARY KEY (id); + + +-- +-- Name: objects objects_pkey; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.objects + ADD CONSTRAINT objects_pkey PRIMARY KEY (id); + + +-- +-- Name: audit_logs_instance_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX audit_logs_instance_id_idx ON auth.audit_log_entries USING btree (instance_id); + + +-- +-- Name: refresh_tokens_instance_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX refresh_tokens_instance_id_idx ON auth.refresh_tokens USING btree (instance_id); + + +-- +-- Name: refresh_tokens_instance_id_user_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX refresh_tokens_instance_id_user_id_idx ON auth.refresh_tokens USING btree (instance_id, user_id); + + +-- +-- Name: refresh_tokens_token_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX refresh_tokens_token_idx ON auth.refresh_tokens USING btree (token); + + +-- +-- Name: users_instance_id_email_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX users_instance_id_email_idx ON auth.users USING btree (instance_id, email); + + +-- +-- Name: users_instance_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX users_instance_id_idx ON auth.users USING btree (instance_id); + + +-- +-- Name: bname; Type: INDEX; Schema: storage; Owner: - +-- + +CREATE UNIQUE INDEX bname ON storage.buckets USING btree (name); + + +-- +-- Name: bucketid_objname; Type: INDEX; Schema: storage; Owner: - +-- + +CREATE UNIQUE INDEX bucketid_objname ON storage.objects USING btree (bucket_id, name); + + +-- +-- Name: name_prefix_search; Type: INDEX; Schema: storage; Owner: - +-- + +CREATE INDEX name_prefix_search ON storage.objects USING btree (name text_pattern_ops); + + +-- +-- Name: buckets buckets_owner_fkey; Type: FK CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.buckets + ADD CONSTRAINT buckets_owner_fkey FOREIGN KEY (owner) REFERENCES auth.users(id); + + +-- +-- Name: objects objects_bucketId_fkey; Type: FK CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.objects + ADD CONSTRAINT "objects_bucketId_fkey" FOREIGN KEY (bucket_id) REFERENCES storage.buckets(id); + + +-- +-- Name: objects objects_owner_fkey; Type: FK CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.objects + ADD CONSTRAINT objects_owner_fkey FOREIGN KEY (owner) REFERENCES auth.users(id); + + +-- +-- Name: objects; Type: ROW SECURITY; Schema: storage; Owner: - +-- + +ALTER TABLE storage.objects ENABLE ROW LEVEL SECURITY; + +-- +-- Name: supabase_realtime; Type: PUBLICATION; Schema: -; Owner: - +-- + +CREATE PUBLICATION supabase_realtime WITH (publish = 'insert, update, delete, truncate'); + + +-- +-- Name: issue_graphql_placeholder; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_graphql_placeholder ON sql_drop + WHEN TAG IN ('DROP EXTENSION') + EXECUTE FUNCTION extensions.set_graphql_placeholder(); + + +-- +-- Name: issue_pg_cron_access; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_pg_cron_access ON ddl_command_end + WHEN TAG IN ('CREATE EXTENSION') + EXECUTE FUNCTION extensions.grant_pg_cron_access(); + + +-- +-- Name: issue_pg_graphql_access; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_pg_graphql_access ON ddl_command_end + WHEN TAG IN ('CREATE FUNCTION') + EXECUTE FUNCTION extensions.grant_pg_graphql_access(); + + +-- +-- Name: issue_pg_net_access; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_pg_net_access ON ddl_command_end + WHEN TAG IN ('CREATE EXTENSION') + EXECUTE FUNCTION extensions.grant_pg_net_access(); + + +-- +-- Name: pgrst_ddl_watch; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER pgrst_ddl_watch ON ddl_command_end + EXECUTE FUNCTION extensions.pgrst_ddl_watch(); + + +-- +-- Name: pgrst_drop_watch; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER pgrst_drop_watch ON sql_drop + EXECUTE FUNCTION extensions.pgrst_drop_watch(); + + +-- +-- PostgreSQL database dump complete +-- + + +-- +-- Dbmate schema migrations +-- +