diff --git a/sayn/cli.py b/sayn/cli.py index 187d6bf9..7437cabe 100644 --- a/sayn/cli.py +++ b/sayn/cli.py @@ -24,6 +24,7 @@ def __init__( full_load=False, start_dt=None, end_dt=None, + run_tests=False, fail_fast=False, ): super().__init__() @@ -69,6 +70,9 @@ def __init__( if upstream_prod is not None: self.run_arguments.upstream_prod = upstream_prod + if run_tests is not None: + self.run_arguments.run_tests = run_tests + if fail_fast is not None: self.run_arguments.fail_fast = fail_fast @@ -121,6 +125,13 @@ def parser_process(value, state): "--debug", "-d", is_flag=True, default=False, help="Include debug messages" ) +click_include_tests = click.option( + "--run-tests", + is_flag=True, + default=False, + help="Include Tests in task execution - after task run.", +) + click_fail_fast = click.option( "--fail-fast", is_flag=True, @@ -211,6 +222,7 @@ def compile( full_load, start_dt, end_dt, + run_tests, fail_fast, ): @@ -226,6 +238,7 @@ def compile( full_load, start_dt, end_dt, + run_tests, fail_fast, ) @@ -237,6 +250,7 @@ def compile( @cli.command(help="Run SAYN tasks.") +@click_include_tests @click_run_options def run( debug, @@ -247,6 +261,7 @@ def run( full_load, start_dt, end_dt, + run_tests, fail_fast, ): @@ -262,6 +277,7 @@ def run( full_load, start_dt, end_dt, + run_tests, fail_fast, ) diff --git a/sayn/core/app.py b/sayn/core/app.py index c72529b5..00c46791 100644 --- a/sayn/core/app.py +++ b/sayn/core/app.py @@ -68,6 +68,7 @@ class Folders: command: Command = Command.UNDEFINED upstream_prod: bool = False is_prod: bool = False + run_tests: bool = False fail_fast: bool = False include: Set[str] diff --git a/sayn/tasks/copy.py b/sayn/tasks/copy.py index 0d007b23..570d129f 100644 --- a/sayn/tasks/copy.py +++ b/sayn/tasks/copy.py @@ -272,7 +272,9 @@ def config(self, **config): # noqa: C901 else: return result - if self.run_arguments["command"] == "test" and len(self.columns["columns"]) > 0: + if ( + self.run_arguments["command"] == "test" or self.run_arguments["run_tests"] + ) and len(self.columns["columns"]) > 0: result = self.target_db._construct_tests( self.columns["columns"], self.table, self.schema ) @@ -483,7 +485,6 @@ def execute(self, execute, debug, is_full_load, limit=None): def read_iter(iter): if self.mode == "append": - load_time = datetime.utcnow() for record in iter: yield dict(record, _sayn_load_ts=load_time) diff --git a/sayn/tasks/sql.py b/sayn/tasks/sql.py index 15813346..b9085cc2 100644 --- a/sayn/tasks/sql.py +++ b/sayn/tasks/sql.py @@ -259,7 +259,9 @@ def def_out(obj, level=None): else: self.ddl = result.value - if self.run_arguments["command"] == "test" and len(self.ddl["columns"]) != 0: + if ( + self.run_arguments["command"] == "test" or self.run_arguments["run_tests"] + ) and len(self.ddl["columns"]) != 0: result = self.target_db._construct_tests( self.ddl["columns"], self.table, self.schema ) diff --git a/sayn/tasks/task_wrapper.py b/sayn/tasks/task_wrapper.py index c15544d4..24c8b2cc 100644 --- a/sayn/tasks/task_wrapper.py +++ b/sayn/tasks/task_wrapper.py @@ -76,7 +76,6 @@ def __init__( compiler, db_object_compiler, ): - self.tags = set(tags or set()) self.parent_names = set(parent_names or set()) self.parents = list() @@ -110,6 +109,7 @@ def __init__( self.run_arguments = { "debug": run_arguments.debug, + "run_tests": run_arguments.run_tests, "full_load": run_arguments.full_load, "start_dt": run_arguments.start_dt, "end_dt": run_arguments.end_dt, @@ -382,8 +382,12 @@ def execute_task(self, command): try: if command == "run": result = self.runner.run() + if self.run_arguments["run_tests"] and self.has_tests(): + result = self.runner.test() elif command == "compile": result = self.runner.compile() + if self.run_arguments["run_tests"] and self.has_tests(): + result = self.runner.test() else: result = self.runner.test() diff --git a/tests/__init__.py b/tests/__init__.py index 06be7b03..929cfa25 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -115,6 +115,7 @@ def simulate_task( run_arguments = { "debug": obj_run_arguments.debug, + "run_tests": obj_run_arguments.run_tests, "full_load": obj_run_arguments.full_load, "start_dt": obj_run_arguments.start_dt, "end_dt": obj_run_arguments.end_dt, diff --git a/tests/test_cli.py b/tests/test_cli.py index b9e93294..452c1803 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,9 +15,11 @@ def cli(): @cli.command() +@tcli.click_include_tests @tcli.click_run_options def run( debug, + run_tests, fail_fast, tasks, exclude, @@ -39,6 +41,7 @@ def run( return { "command": "run", "debug": debug, + "run_tests": run_tests, "include": tasks, "exclude": exclude, "profile": profile, @@ -97,6 +100,7 @@ def test_simple_run(): assert output == { "command": "run", "debug": False, + "run_tests": False, "fail_fast": False, "include": [], "exclude": [], @@ -129,6 +133,7 @@ def test_run_one_task(): assert output == { "command": "run", "debug": False, + "run_tests": False, "fail_fast": False, "include": ["something"], "exclude": [], @@ -145,6 +150,7 @@ def test_run_two_tasks_old(): assert output == { "command": "run", "debug": False, + "run_tests": False, "fail_fast": False, "include": ["something", "somethingelse"], "exclude": [], @@ -161,6 +167,7 @@ def test_run_two_tasks_new(): assert output == { "command": "run", "debug": False, + "run_tests": False, "fail_fast": False, "include": ["something", "somethingelse"], "exclude": [], @@ -225,6 +232,7 @@ def test_run_debug(): assert output == { "command": "run", "debug": True, + "run_tests": False, "fail_fast": False, "include": ["something"], "exclude": [], @@ -257,6 +265,7 @@ def test_run_debug_multitasks(): assert output == { "command": "run", "debug": True, + "run_tests": False, "fail_fast": False, "include": ["something", "somethingelse", "somesomeelse"], "exclude": [], @@ -290,6 +299,7 @@ def test_run_fail_fast(): "command": "run", "debug": False, "fail_fast": True, + "run_tests": False, "include": ["something"], "exclude": [], "profile": None, @@ -338,6 +348,7 @@ def test_run_fail_fast_multitasks(): "command": "run", "debug": False, "fail_fast": True, + "run_tests": False, "include": ["something", "somethingelse", "somesomeelse"], "exclude": [], "profile": None, @@ -353,6 +364,7 @@ def test_run_full(): assert output == { "command": "run", "debug": False, + "run_tests": False, "fail_fast": False, "include": ["something"], "exclude": [], @@ -385,6 +397,7 @@ def test_run_exclude(): assert output == { "command": "run", "debug": False, + "run_tests": False, "fail_fast": False, "include": [], "exclude": ["something"], @@ -417,6 +430,7 @@ def test_run_include_exclude(): assert output == { "command": "run", "debug": False, + "run_tests": False, "fail_fast": False, "include": ["somethingelse"], "exclude": ["something"],