From 7c0cea34aec4abf07afce7bc1caf89c07a6c33af Mon Sep 17 00:00:00 2001 From: Varjitt Jeeva Date: Tue, 19 Nov 2024 14:59:41 -0500 Subject: [PATCH] fix: sync command breakdown update and remove useless commands --- docs/quickstart.md | 24 +++++++++++++++++++----- pgbelt/cmd/sync.py | 35 ----------------------------------- 2 files changed, 19 insertions(+), 40 deletions(-) diff --git a/docs/quickstart.md b/docs/quickstart.md index 572b8dd..bbc5369 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -179,7 +179,7 @@ Therefore the next command will do the following: - Sync sequence values - Dump and load tables without Primary Keys - Add NOT VALID constraints to the target schema (they were removed in Step 1 in the target database) -- Create Indexes (as long as this was run in Step 2, this will be glossed over. If step 2 was missed, indexes will build now amnd this will take longer than expected). +- Create Indexes (as long as this was run in Step 2, this will be glossed over. If step 2 was missed, indexes will build now amd this will take longer than expected). - Validate data (take 100 random rows and 100 last rows of each table, and compare data) - Run ANALYZE to ensure optimal performance @@ -189,13 +189,27 @@ $ belt sync testdatacenter1 database1 If the above command fails, you can diagnose and run the individual steps with the following commands: +### 1. Syncing Sequences: + - `sync-sequences` - reads and sets sequences values from SRC to DST at the time of command execution -- `dump-tables` - dumps only tables without Primary Keys + +### 2. Syncing Tables without Primary Keys: + +- `dump-tables` - dumps only tables without Primary Keys (to ensure only tables without Primary Keys are dumped, DO NOT specify the `--tables` flag for this command) - `load-tables` - load into DST DB the tables from the `dump-tables` command (found on disk) -- `dump-contraints` - dumps NOT VALID constraints from your SRC DB schema onto disk -- `load-constraints` - load NOT VALID constraints from disk to your DST DB schema + +### 3. Syncing NOT VALID Constraints: + +- `dump-schema` - dumps schema from your SRC DB schema onto disk (the files may already be on disk, but run this command just to ensure they exist anyways) +- `load-constraints` - load NOT VALID constraints from disk (obtained by the `dump-schema` command) to your DST DB schema + +### 4. Creating Indexes & Running ANALYZE: + +- `create-indexes` - Create indexes on the target database, and then runs ANALYZE as well. + +### 5. Validating Data: + - `validate-data` - Check random 100 rows and last 100 rows of every table involved in the replication job, and ensure all match exactly. -- `analyze` - Run ANALYZE on the database ## Step 8: Enable write traffic to the destination host diff --git a/pgbelt/cmd/sync.py b/pgbelt/cmd/sync.py index 944f2f0..50c5e36 100644 --- a/pgbelt/cmd/sync.py +++ b/pgbelt/cmd/sync.py @@ -108,40 +108,6 @@ async def load_tables( await load_dumped_tables(conf, tables, logger) -@run_with_configs -async def sync_tables( - config_future: Awaitable[DbupgradeConfig], - tables: list[str] = Option([], help="Specific tables to sync"), -): - """ - Dump and load all tables from the source database to the destination database. - Equivalent to running dump-tables followed by load-tables. Table data will be - saved locally in files. - - You may also provide a list of tables to sync with the - --tables option and only these tables will be synced. - """ - conf = await config_future - src_logger = get_logger(conf.db, conf.dc, "sync.src") - dst_logger = get_logger(conf.db, conf.dc, "sync.dst") - - if tables: - dump_tables = tables.split(",") - else: - async with create_pool(conf.src.pglogical_uri, min_size=1) as src_pool: - _, dump_tables, _ = await analyze_table_pkeys( - src_pool, conf.schema_name, src_logger - ) - - if conf.tables: - dump_tables = [t for t in dump_tables if t in conf.tables] - - await dump_source_tables(conf, dump_tables) - await load_dumped_tables( - conf, [] if not tables and not conf.tables else dump_tables, dst_logger - ) - - @run_with_configs(skip_src=True) async def analyze(config_future: Awaitable[DbupgradeConfig]) -> None: """ @@ -276,7 +242,6 @@ async def sync( sync_sequences, dump_tables, load_tables, - sync_tables, analyze, validate_data, sync,