From c80c24bd139f8fe2d1abee52ab8e49f758d641c7 Mon Sep 17 00:00:00 2001 From: Varjitt Jeeva Date: Tue, 26 Nov 2024 14:00:26 -0500 Subject: [PATCH 1/2] fix: remove dump-dst-indexes, useless --- pgbelt/cmd/schema.py | 13 ------------ pgbelt/util/dump.py | 47 -------------------------------------------- 2 files changed, 60 deletions(-) diff --git a/pgbelt/cmd/schema.py b/pgbelt/cmd/schema.py index fe92d97..538ad72 100644 --- a/pgbelt/cmd/schema.py +++ b/pgbelt/cmd/schema.py @@ -7,7 +7,6 @@ from pgbelt.util.dump import apply_target_schema from pgbelt.util.dump import create_target_indexes from pgbelt.util.dump import dump_source_schema -from pgbelt.util.dump import dump_dst_create_index from pgbelt.util.dump import remove_dst_not_valid_constraints from pgbelt.util.dump import remove_dst_indexes from pgbelt.util.logs import get_logger @@ -69,17 +68,6 @@ async def remove_constraints(config_future: Awaitable[DbupgradeConfig]) -> None: await remove_dst_not_valid_constraints(conf, logger) -@run_with_configs(skip_src=True) -async def dump_indexes(config_future: Awaitable[DbupgradeConfig]) -> None: - """ - Dumps the CREATE INDEX statements from the target database onto disk, in - the schemas directory. - """ - conf = await config_future - logger = get_logger(conf.db, conf.dc, "schema.dst") - await dump_dst_create_index(conf, logger) - - @run_with_configs(skip_src=True) async def remove_indexes(config_future: Awaitable[DbupgradeConfig]) -> None: """ @@ -123,7 +111,6 @@ async def create_indexes(config_future: Awaitable[DbupgradeConfig]) -> None: load_schema, load_constraints, remove_constraints, - dump_indexes, remove_indexes, create_indexes, ] diff --git a/pgbelt/util/dump.py b/pgbelt/util/dump.py index 2858d20..73276d6 100644 --- a/pgbelt/util/dump.py +++ b/pgbelt/util/dump.py @@ -339,53 +339,6 @@ async def apply_target_constraints(config: DbupgradeConfig, logger: Logger) -> N ) -async def dump_dst_create_index(config: DbupgradeConfig, logger: Logger) -> None: - """ - Dump CREATE INDEX statements from the target database. - Used when schema is loaded in outside of pgbelt. - """ - - logger.info("Dumping target CREATE INDEX statements...") - - command = [ - "pg_dump", - "--schema-only", - "--no-owner", - "-n", - config.schema_name, - config.dst.pglogical_dsn, - ] - - out = await _execute_subprocess(command, "Retrieved target schema", logger) - - # No username replacement needs to be done, so replace dst user with the same. - commands_raw = _parse_dump_commands( - out.decode("utf-8"), config.dst.owner_user.name, config.dst.owner_user.name - ) - - commands = [] - for c in commands_raw: - if "CREATE" in command and "INDEX" in command: - regex_matches = search( - r"CREATE [UNIQUE ]*INDEX (?P[a-zA-Z0-9._]+)+.*", - c, - ) - if not regex_matches: - continue - commands.append(c) - - try: - await makedirs(schema_dir(config.db, config.dc)) - except FileExistsError: - pass - - async with aopen(schema_file(config.db, config.dc, ONLY_INDEXES), "w") as out: - for command in commands: - await out.write(command) - - logger.debug("Finished dumping CREATE INDEX statements from the target.") - - async def remove_dst_indexes(config: DbupgradeConfig, logger: Logger) -> None: """ Remove the INDEXes from the schema of the target database. From 137b8555cafc505eeeab522b1cdb6cccfa5eb6a6 Mon Sep 17 00:00:00 2001 From: Varjitt Jeeva Date: Tue, 26 Nov 2024 14:18:26 -0500 Subject: [PATCH 2/2] fix: create index no statement timeout --- pgbelt/util/dump.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pgbelt/util/dump.py b/pgbelt/util/dump.py index 73276d6..28a8aa8 100644 --- a/pgbelt/util/dump.py +++ b/pgbelt/util/dump.py @@ -404,7 +404,10 @@ async def create_target_indexes( index = regex_matches.groupdict()["index"] # Create the index - command = ["psql", config.dst.owner_dsn, "-c", f"{c};"] + # Note that the host DSN must have a statement timeout of 0. + # Example DSN: `host=server-hostname user=user dbname=db_name options='-c statement_timeout=3600000'` + host_dsn = config.dst.owner_dsn + " options='-c statement_timeout=0'" + command = ["psql", host_dsn, "-c", f"{c};"] logger.info(f"Creating index {index} on the target...") try: await _execute_subprocess(