diff --git a/.gitignore b/.gitignore index 7c0fd7e..e793a92 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +config.yaml *.log .DS_Store .huskyrc.json diff --git a/Pipfile b/Pipfile index bd10d74..3d197b7 100644 --- a/Pipfile +++ b/Pipfile @@ -7,7 +7,6 @@ name = "pypi" textual = "*" pyyaml = "*" isort = "*" -asyncpg = "*" psycopg2 = "*" boto3 = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 3a492de..ff37b30 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "3b4c5eddb1caeeec3ffdd0982da401df8e0cea0b225e9ac4f6ee4bc947dd08f3" + "sha256": "e015d310d4d62a195dc01704846fceed3c19e6531c9df30318c32e59de1bed1f" }, "pipfile-spec": 6, "requires": { @@ -16,54 +16,6 @@ ] }, "default": { - "asyncpg": { - "hashes": [ - "sha256:0009a300cae37b8c525e5b449233d59cd9868fd35431abc470a3e364d2b85cb9", - "sha256:000c996c53c04770798053e1730d34e30cb645ad95a63265aec82da9093d88e7", - "sha256:012d01df61e009015944ac7543d6ee30c2dc1eb2f6b10b62a3f598beb6531548", - "sha256:039a261af4f38f949095e1e780bae84a25ffe3e370175193174eb08d3cecab23", - "sha256:103aad2b92d1506700cbf51cd8bb5441e7e72e87a7b3a2ca4e32c840f051a6a3", - "sha256:1e186427c88225ef730555f5fdda6c1812daa884064bfe6bc462fd3a71c4b675", - "sha256:2245be8ec5047a605e0b454c894e54bf2ec787ac04b1cb7e0d3c67aa1e32f0fe", - "sha256:37a2ec1b9ff88d8773d3eb6d3784dc7e3fee7756a5317b67f923172a4748a175", - "sha256:48e7c58b516057126b363cec8ca02b804644fd012ef8e6c7e23386b7d5e6ce83", - "sha256:52e8f8f9ff6e21f9b39ca9f8e3e33a5fcdceaf5667a8c5c32bee158e313be385", - "sha256:5340dd515d7e52f4c11ada32171d87c05570479dc01dc66d03ee3e150fb695da", - "sha256:54858bc25b49d1114178d65a88e48ad50cb2b6f3e475caa0f0c092d5f527c106", - "sha256:5b52e46f165585fd6af4863f268566668407c76b2c72d366bb8b522fa66f1870", - "sha256:5bbb7f2cafd8d1fa3e65431833de2642f4b2124be61a449fa064e1a08d27e449", - "sha256:5cad1324dbb33f3ca0cd2074d5114354ed3be2b94d48ddfd88af75ebda7c43cc", - "sha256:6011b0dc29886ab424dc042bf9eeb507670a3b40aece3439944006aafe023178", - "sha256:642a36eb41b6313ffa328e8a5c5c2b5bea6ee138546c9c3cf1bffaad8ee36dd9", - "sha256:6feaf2d8f9138d190e5ec4390c1715c3e87b37715cd69b2c3dfca616134efd2b", - "sha256:72fd0ef9f00aeed37179c62282a3d14262dbbafb74ec0ba16e1b1864d8a12169", - "sha256:746e80d83ad5d5464cfbf94315eb6744222ab00aa4e522b704322fb182b83610", - "sha256:76c3ac6530904838a4b650b2880f8e7af938ee049e769ec2fba7cd66469d7772", - "sha256:797ab8123ebaed304a1fad4d7576d5376c3a006a4100380fb9d517f0b59c1ab2", - "sha256:8d36c7f14a22ec9e928f15f92a48207546ffe68bc412f3be718eedccdf10dc5c", - "sha256:97eb024685b1d7e72b1972863de527c11ff87960837919dac6e34754768098eb", - "sha256:a65c1dcd820d5aea7c7d82a3fdcb70e096f8f70d1a8bf93eb458e49bfad036ac", - "sha256:a921372bbd0aa3a5822dd0409da61b4cd50df89ae85150149f8c119f23e8c408", - "sha256:a9e6823a7012be8b68301342ba33b4740e5a166f6bbda0aee32bc01638491a22", - "sha256:b544ffc66b039d5ec5a7454667f855f7fec08e0dfaf5a5490dfafbb7abbd2cfb", - "sha256:bb1292d9fad43112a85e98ecdc2e051602bce97c199920586be83254d9dafc02", - "sha256:bde17a1861cf10d5afce80a36fca736a86769ab3579532c03e45f83ba8a09c59", - "sha256:cce08a178858b426ae1aa8409b5cc171def45d4293626e7aa6510696d46decd8", - "sha256:cfe73ffae35f518cfd6e4e5f5abb2618ceb5ef02a2365ce64f132601000587d3", - "sha256:d1c49e1f44fffafd9a55e1a9b101590859d881d639ea2922516f5d9c512d354e", - "sha256:d4900ee08e85af01adb207519bb4e14b1cae8fd21e0ccf80fac6aa60b6da37b4", - "sha256:d84156d5fb530b06c493f9e7635aa18f518fa1d1395ef240d211cb563c4e2364", - "sha256:dc600ee8ef3dd38b8d67421359779f8ccec30b463e7aec7ed481c8346decf99f", - "sha256:e0bfe9c4d3429706cf70d3249089de14d6a01192d617e9093a8e941fea8ee775", - "sha256:e17b52c6cf83e170d3d865571ba574577ab8e533e7361a2b8ce6157d02c665d3", - "sha256:f100d23f273555f4b19b74a96840aa27b85e99ba4b1f18d4ebff0734e78dc090", - "sha256:f9ea3f24eb4c49a615573724d88a48bd1b7821c890c2effe04f05382ed9e8810", - "sha256:ff8e8109cd6a46ff852a5e6bab8b0a047d7ea42fcb7ca5ae6eaae97d8eacf397" - ], - "index": "pypi", - "markers": "python_full_version >= '3.8.0'", - "version": "==0.29.0" - }, "boto3": { "hashes": [ "sha256:b633e8fbf7145bdb995ce68a27d096bb89fd393185b0e773418d81cd78db5a03", diff --git a/pg-migration-tool/config.example.yaml b/pg-migration-tool/config.example.yaml index d9a1a86..3d7061e 100644 --- a/pg-migration-tool/config.example.yaml +++ b/pg-migration-tool/config.example.yaml @@ -1,3 +1,5 @@ +common: + kms_key_id: alias/my-key dbs: my-service: source: diff --git a/pg-migration-tool/main.py b/pg-migration-tool/main.py index 6755a47..6f44bfc 100644 --- a/pg-migration-tool/main.py +++ b/pg-migration-tool/main.py @@ -1,19 +1,18 @@ +import asyncio +import base64 import os +import subprocess +import threading +import boto3 +import psycopg2 import yaml -import asyncio -import concurrent.futures -import asyncpg -from asyncpg import PostgresError +from psycopg2 import DatabaseError from textual import on from textual.app import App, ComposeResult -from textual.containers import ScrollableContainer, Horizontal -from textual.widgets import Header, Label, Markdown, Select, Log, Button, Static +from textual.containers import Horizontal from textual.events import Print -import boto3 -import subprocess -import threading -import base64 +from textual.widgets import Button, Header, Label, Log, Markdown, Select root_dir = os.path.dirname(__file__) # <-- absolute dir the script is in config_rel_path = "config.yaml" @@ -33,8 +32,8 @@ def compose(self) -> ComposeResult: yield Header() yield Horizontal(Select(((line, line) for line in LINES), prompt="Select database"), Button.success("Migrate", id="migrate", disabled=True)) yield Markdown(id="db_config_markdown", markdown="") - yield Label("Running source connection test...", id="source_connection_label", classes="invisible") - yield Label("Running target connection test...", id="target_connection_label", classes="invisible") + yield Label("#source Running connection test...", id="source", classes="invisible") + yield Label("#target Running connection test...", id="target", classes="invisible") yield Log(auto_scroll=True) @on(Select.Changed) @@ -59,7 +58,6 @@ def display_db_config(self, db): DB_CONFIG_MARKDOWN = f"""\ # Database Configuration -### Source | key | source | target | | --- | --- | --- | | db_connection_host | {db["source"]["db_connection_host"]} | {db["target"]["db_connection_host"]} | @@ -72,11 +70,8 @@ def display_db_config(self, db): async def check_db_connection(self, event: Select.Changed) -> bool: db = config["dbs"][event.value] - self.query_one("#source_connection_label").set_class(False, 'invisible') - self.query_one("#target_connection_label").set_class(False, 'invisible') - - source_ok = await self.check_connection_for_db(db["source"], "#source_connection_label") - target_ok = await self.check_connection_for_db(db["target"], "#target_connection_label") + source_ok = await self.check_connection_for_db(db["source"], "#source") + target_ok = await self.check_connection_for_db(db["target"], "#target") return source_ok and target_ok @@ -88,22 +83,26 @@ async def check_connection_for_db(self, db, label) -> bool: if not db_password: return False + + self.query_one(label).set_class(False, 'invisible') + self.query_one(label).update(f"{label} Running connection test...") try: - await asyncpg.connect( + await asyncio.to_thread(psycopg2.connect, database=db["db_database_name"], user=db["db_username"], password=db_password, host=db["db_connection_host"], ) - self.query_one(label).update(f"{db["db_connection_host"]} connection successful.") + self.query_one(label).update(f"{label} {db["db_connection_host"]} connection successful.") return True - except PostgresError as e: - self.query_one(label).update(f"{db["db_connection_host"]} connection failed with password {db_password}: {e}") + except DatabaseError as e: + self.query_one(label).update(f"{label} {db["db_connection_host"]} connection failed: {e}") return False async def decrypt_password(self, db, label) -> str: - self.query_one(label).update(f"Decrypting db password...") + self.query_one(label).update(f"{label} Decrypting db password...") + self.query_one(label).set_class(False, 'invisible') try: response = await asyncio.to_thread(client.decrypt, CiphertextBlob=base64.b64decode(db["db_password_encrypted"]), KeyId=config["common"]["kms_key_id"]) @@ -113,7 +112,7 @@ async def decrypt_password(self, db, label) -> str: return decrypted_password except Exception as e: - self.query_one(label).update(f"Failed to decrypt password: {e}") + self.query_one(label).update(f'{label} Failed to decrypt password with kms key \'{config["common"]["kms_key_id"]}\': {e}') return None def generate_pg_dump_and_restore_cmd(self, event: Select.Changed)-> str: