Skip to content

Commit

Permalink
fix: support for table names with capitals
Browse files Browse the repository at this point in the history
  • Loading branch information
vjeeva committed Mar 18, 2024
1 parent fd95e0d commit d8b9bfb
Show file tree
Hide file tree
Showing 4 changed files with 75 additions and 8 deletions.
27 changes: 25 additions & 2 deletions pgbelt/cmd/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,23 @@ async def _setup_src_node(

pglogical_tables = pkey_tables
if conf.tables:
pglogical_tables = [t for t in pkey_tables if t in conf.tables]
pglogical_tables = [
t
for t in pkey_tables
if t
in list(
map(str.lower, conf.tables)
) # Postgres returns table names in lowercase (in analyze_table_pkeys)
]

# Intentionally throw an error if no tables are found, so that the user can correct their config.
# When reported by a certain user, errors showed when running the status command, but it was ignored,
# then the user ran setup and since that DIDN'T throw an error, they assumed everything was fine.

if not pglogical_tables:
raise ValueError(
f"No tables were targeted to replicate. Please check your config's schema and tables. DB: {conf.db} DC: {conf.dc}, SCHEMA: {conf.schema_name} TABLES: {conf.tables}.\nIf TABLES is [], all tables in the schema should be replicated, but pgbelt still found no tables.\nCheck the schema name or reach out to the pgbelt team for help."
)

await configure_replication_set(
src_root_pool, pglogical_tables, conf.schema_name, src_logger
Expand Down Expand Up @@ -145,7 +161,14 @@ async def setup_back_replication(config_future: Awaitable[DbupgradeConfig]) -> N

pglogical_tables = pkeys
if conf.tables:
pglogical_tables = [t for t in pkeys if t in conf.tables]
pglogical_tables = [
t
for t in pkeys
if t
in list(
map(str.lower, conf.tables)
) # Postgres returns table names in lowercase (in analyze_table_pkeys)
]

await configure_replication_set(
dst_root_pool, pglogical_tables, conf.schema_name, dst_logger
Expand Down
9 changes: 8 additions & 1 deletion pgbelt/cmd/status.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,14 @@ async def status(conf_future: Awaitable[DbupgradeConfig]) -> dict[str, str]:
all_tables = pkey_tables + non_pkey_tables
target_tables = all_tables
if conf.tables:
target_tables = [t for t in all_tables if t in conf.tables]
target_tables = [
t
for t in all_tables
if t
in list(
map(str.lower, conf.tables)
) # Postgres gave us lowercase table names in analyze_table_pkeys
]

if not target_tables:
raise ValueError(
Expand Down
18 changes: 16 additions & 2 deletions pgbelt/cmd/sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,14 @@ async def dump_tables(
_, tables, _ = await analyze_table_pkeys(src_pool, conf.schema_name, logger)

if conf.tables:
tables = [t for t in tables if t in conf.tables]
tables = [
t
for t in tables
if t
in list(
map(str.lower, conf.tables)
) # Postgres returns table names in lowercase (in analyze_table_pkeys)
]

await dump_source_tables(conf, tables, logger)

Expand Down Expand Up @@ -185,7 +192,14 @@ async def _dump_and_load_all_tables(
) -> None:
_, tables, _ = await analyze_table_pkeys(src_pool, conf.schema_name, src_logger)
if conf.tables:
tables = [t for t in tables if t in conf.tables]
tables = [
t
for t in tables
if t
in list(
map(str.lower, conf.tables)
) # Postgres returns table names in lowercase (in analyze_table_pkeys)
]
await dump_source_tables(conf, tables, src_logger)
await load_dumped_tables(conf, tables, dst_logger)

Expand Down
29 changes: 26 additions & 3 deletions pgbelt/util/postgres.py
Original file line number Diff line number Diff line change
Expand Up @@ -372,9 +372,22 @@ async def precheck_info(
AND n.nspname <> 'pglogical'
ORDER BY 1,2;"""
)

# We filter the table list if the user has specified a list of tables to target.
# Note, from issue #420, the above query will return the table names in lowercase,
# so we need to map the target_tables to lowercase.
if target_tables:
result["tables"] = [t for t in result["tables"] if t["Name"] in target_tables]

result["tables"] = [
t
for t in result["tables"]
if t["Name"] in list(map(str.lower, target_tables))
]

# We will not recapitalize the table names in the result["tables"] list,
# to preserve how Postgres sees those tables in its system catalog. Easy
# rabbit hole later if we keep patching the table names to match the user's
# input.

result["sequences"] = await pool.fetch(
"""
Expand All @@ -392,12 +405,22 @@ async def precheck_info(
ORDER BY 1,2;"""
)

# We filter the sequence list if the user has specified a list of sequences to target.
# We filter the table list if the user has specified a list of tables to target.
# Note, from issue #420, the above query will return the table names in lowercase,
# so we need to map the target_tables to lowercase.
if target_sequences:

result["sequences"] = [
s for s in result["sequences"] if s["Name"] in target_sequences
t
for t in result["sequences"]
if t["Name"] in list(map(str.lower, target_sequences))
]

# We will not recapitalize the table names in the result["tables"] list,
# to preserve how Postgres sees those tables in its system catalog. Easy
# rabbit hole later if we keep patching the table names to match the user's
# input.

users = await pool.fetch(
f"""
SELECT r.rolname, r.rolsuper, r.rolinherit,
Expand Down

0 comments on commit d8b9bfb

Please sign in to comment.