From 678950a5f1842de92863123b79af2c95dc6b2b87 Mon Sep 17 00:00:00 2001 From: hustic <30499756+hustic@users.noreply.github.com> Date: Tue, 9 Jan 2024 14:38:07 +0000 Subject: [PATCH] Move info logs to result logs for Task Tests (#247) --- sayn/logging/log_formatter.py | 15 ++++++++++---- sayn/tasks/copy.py | 23 ++++++++++++--------- sayn/tasks/sql.py | 22 +++++++++++--------- sayn/tasks/task.py | 38 ++++++++++++----------------------- 4 files changed, 49 insertions(+), 49 deletions(-) diff --git a/sayn/logging/log_formatter.py b/sayn/logging/log_formatter.py index 4dd9861d..3c0ae4a1 100644 --- a/sayn/logging/log_formatter.py +++ b/sayn/logging/log_formatter.py @@ -545,10 +545,17 @@ def task_stage_finish(self, stage, task, task_order, total_tasks, details): duration = human(details["duration"]) if details.get("result") is None or details["result"].is_ok: - return { - "level": "info", - "message": self.good(f"Took ({duration})"), - } + if stage == "test": + success_message = details.get("result").value + return { + "level": "info", + "message": self.good(f"Took ({duration}) - {success_message}"), + } + else: + return { + "level": "info", + "message": self.good(f"Took ({duration})"), + } else: return self.error_result(details["duration"], details["result"].error) diff --git a/sayn/tasks/copy.py b/sayn/tasks/copy.py index a2a87c73..20d3da22 100644 --- a/sayn/tasks/copy.py +++ b/sayn/tasks/copy.py @@ -396,26 +396,29 @@ def test(self): failed, self.table, self.schema ) + problematic_report = "\n" + for query in problematic_values_query.split(";"): if query.strip(): header = re.search(r"--.*?--", query).group(0) - self.info("") - self.info(header) - self.info( - "====================================================================" - ) - self.info( + + problematic_report += header + problematic_report += "\n====================================================================\n" + problematic_report += ( re.sub(r"--.*?--", "", query).replace("\n", " ").strip() + ";" ) - self.info( - "====================================================================" - ) - self.info("") + problematic_report += "\n====================================================================\n" self.write_compilation_output( problematic_values_query, "test_problematic_values" ) + + errout.error.details["message"] += ( + problematic_report + + f"\n\tTest Failed. You can find the compiled test query at compile/{self.group}/{self.name}_test.sql. You can find queries to retrieve the problematic values at compile/{self.group}/{self.name}_test_problematic_values.sql" + ) + return errout def execute(self, execute, debug, is_full_load, limit=None): diff --git a/sayn/tasks/sql.py b/sayn/tasks/sql.py index 7a46b328..15813346 100644 --- a/sayn/tasks/sql.py +++ b/sayn/tasks/sql.py @@ -477,25 +477,27 @@ def test(self): failed, self.table, self.schema ) + problematic_report = "\n" + for query in problematic_values_query.split(";"): if query.strip(): header = re.search(r"--.*?--", query).group(0) - self.info("") - self.info(header) - self.info( - "====================================================================" - ) - self.info( + + problematic_report += header + problematic_report += "\n====================================================================\n" + problematic_report += ( re.sub(r"--.*?--", "", query).replace("\n", " ").strip() + ";" ) - self.info( - "====================================================================" - ) - self.info("") + problematic_report += "\n====================================================================\n" self.write_compilation_output( problematic_values_query, "test_problematic_values" ) + errout.error.details["message"] += ( + problematic_report + + f"\n\tTest Failed. You can find the compiled test query at compile/{self.group}/{self.name}_test.sql. You can find queries to retrieve the problematic values at compile/{self.group}/{self.name}_test_problematic_values.sql" + ) + return errout diff --git a/sayn/tasks/task.py b/sayn/tasks/task.py index 75aab334..f9967921 100644 --- a/sayn/tasks/task.py +++ b/sayn/tasks/task.py @@ -139,9 +139,9 @@ def ready(self): """(Deprecated: use `success` instead) Returned on successful execution.""" return Ok() - def success(self): + def success(self, value=None): """Returned on successful execution.""" - return Ok() + return Ok(value=value) def fail(self, msg=None): """Returned on failure in any stage.""" @@ -250,15 +250,12 @@ def test_sucessful(self, breakdown: list) -> Ok: skipped = [brk for brk in breakdown if brk[0] == "SKIPPED"] executed = [brk for brk in breakdown if brk[0] == "EXECUTED"] + message = str() if skipped: - self.info( - f"{Fore.GREEN}{len(skipped)} Column test(s) {Style.BRIGHT}SKIPPED{Style.NORMAL}" - ) - self.info( - f"{Fore.GREEN}{len(executed)} Column test(s) {Style.BRIGHT}EXECUTED{Style.NORMAL}, {len(executed)} succeeded." - ) + message += f"{Fore.GREEN}{len(skipped)} Column test(s) {Style.BRIGHT}SKIPPED{Style.NORMAL}" + message += f"{Fore.GREEN}{len(executed)} Column test(s) {Style.BRIGHT}EXECUTED{Style.NORMAL}, {len(executed)} succeeded." - return self.success() + return self.success(message) def test_failure(self, breakdown: list, result: dict, run_argument: str) -> tuple: """CLI outputs on failed test execution. @@ -282,8 +279,12 @@ def test_failure(self, breakdown: list, result: dict, run_argument: str) -> tupl else: failed.append(brk) if run_argument: + fl_info = f"{Fore.RED}FAILED: " + + if skipped: + fl_info += f"{Fore.RED}{len(skipped)} Column test(s) {Style.BRIGHT}SKIPPED{Style.NORMAL}\n" + fl_info += f"{Fore.RED}{len(executed)+len(failed)} Column test(s) {Style.BRIGHT}EXECUTED{Style.NORMAL}, {len(executed)} succeeded.\n" - fl_info = [f"{Fore.RED}FAILED: "] for info in failed: count = sum( [ @@ -292,22 +293,9 @@ def test_failure(self, breakdown: list, result: dict, run_argument: str) -> tupl if (item["type"] == info[1] and item["col"] == info[2]) ] ) - fl_info.append( - f"{Fore.RED}{Style.BRIGHT}{info[1]} test{Style.NORMAL} on {Style.BRIGHT}{info[2]} FAILED{Style.NORMAL}. {count} offending records." - ) - if skipped: - self.info( - f"{Fore.GREEN}{len(skipped)} Column test(s) {Style.BRIGHT}SKIPPED{Style.NORMAL}" - ) - self.info( - f"{Fore.GREEN}{len(executed)+len(failed)} Column test(s) {Style.BRIGHT}EXECUTED{Style.NORMAL}, {len(executed)} succeeded." - ) - for err in fl_info: - self.info(err) - - errinfo = f"Test Failed. You can find the compiled test query at compile/{self.group}/{self.name}_test.sql. You can find queries to retrieve the problematic values at compile/{self.group}/{self.name}_test_problematic_values.sql" + fl_info += f"\t{Fore.RED}{Style.BRIGHT}{info[1]} test{Style.NORMAL} on {Style.BRIGHT}{info[2]} FAILED{Style.NORMAL}. {count} offending records.\n" - return (self.fail(errinfo), failed) + return (self.fail(fl_info), failed) else: summary = f"{len(executed)+len(failed)} Column tests were ran, {len(executed)} succeeded, " if skipped: