diff --git a/sayn/cli.py b/sayn/cli.py index c0351f40..271069d9 100644 --- a/sayn/cli.py +++ b/sayn/cli.py @@ -17,6 +17,7 @@ def __init__( self, command, debug=False, + include_tests=False, include=None, exclude=None, upstream_prod=False, @@ -68,6 +69,9 @@ def __init__( if upstream_prod is not None: self.run_arguments.upstream_prod = upstream_prod + if include_tests is not None: + self.run_arguments.include_tests = include_tests + self.start_app() @@ -117,6 +121,14 @@ def parser_process(value, state): "--debug", "-d", is_flag=True, default=False, help="Include debug messages" ) +click_include_tests = click.option( + "--include-tests", + "-it", + is_flag=True, + default=False, + help="Include Tests in task Run.", +) + def click_filter(func): func = click.option( @@ -168,6 +180,7 @@ def click_incremental(func): def click_run_options(func): func = click_debug(func) + func = click_include_tests(func) func = click.option("--profile", "-p", help="Profile from settings to use")(func) func = click_incremental(func) func = click_filter(func) @@ -190,13 +203,24 @@ def init(sayn_project_name): @cli.command(help="Compile sql tasks.") @click_run_options -def compile(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt, end_dt): +def compile( + debug, + include_tests, + tasks, + exclude, + upstream_prod, + profile, + full_load, + start_dt, + end_dt, +): tasks = [i for t in tasks for i in t.strip().split(" ")] exclude = [i for t in exclude for i in t.strip().split(" ")] app = CliApp( Command.COMPILE, debug, + include_tests, tasks, exclude, upstream_prod, @@ -215,13 +239,24 @@ def compile(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt, @cli.command(help="Run SAYN tasks.") @click_run_options -def run(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt, end_dt): +def run( + debug, + include_tests, + tasks, + exclude, + upstream_prod, + profile, + full_load, + start_dt, + end_dt, +): tasks = [i for t in tasks for i in t.strip().split(" ")] exclude = [i for t in exclude for i in t.strip().split(" ")] app = CliApp( Command.RUN, debug, + include_tests, tasks, exclude, upstream_prod, @@ -240,13 +275,24 @@ def run(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt, end_ @cli.command(help="Test SAYN tasks.") @click_run_options -def test(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt, end_dt): +def test( + debug, + include_tests, + tasks, + exclude, + upstream_prod, + profile, + full_load, + start_dt, + end_dt, +): tasks = [i for t in tasks for i in t.strip().split(" ")] exclude = [i for t in exclude for i in t.strip().split(" ")] app = CliApp( Command.TEST, debug, + include_tests, tasks, exclude, upstream_prod, diff --git a/sayn/core/app.py b/sayn/core/app.py index fb989245..d0204fde 100644 --- a/sayn/core/app.py +++ b/sayn/core/app.py @@ -68,6 +68,7 @@ class Folders: command: Command = Command.UNDEFINED upstream_prod: bool = False is_prod: bool = False + include_tests: bool = False include: Set[str] exclude: Set[str] @@ -622,6 +623,12 @@ def execute_dag(self): if self.run_arguments.command == Command.RUN: result = task.run() + if not result.is_err and self.run_arguments.include_tests: + # Force Task to be Ready + task.status = TaskStatus.READY + test_result = task.test() + if test_result.is_err: + result = test_result elif self.run_arguments.command == Command.COMPILE: result = task.compile() elif self.run_arguments.command == Command.TEST: diff --git a/sayn/tasks/copy.py b/sayn/tasks/copy.py index a2a87c73..cf38eb91 100644 --- a/sayn/tasks/copy.py +++ b/sayn/tasks/copy.py @@ -272,7 +272,7 @@ def config(self, **config): # noqa: C901 else: return result - if self.run_arguments["command"] == "test" and len(self.columns["columns"]) > 0: + if self.run_arguments["include_tests"] and len(self.columns["columns"]) > 0: result = self.target_db._construct_tests( self.columns["columns"], self.table, self.schema ) diff --git a/sayn/tasks/sql.py b/sayn/tasks/sql.py index 7a46b328..667a649d 100644 --- a/sayn/tasks/sql.py +++ b/sayn/tasks/sql.py @@ -259,7 +259,7 @@ def def_out(obj, level=None): else: self.ddl = result.value - if self.run_arguments["command"] == "test" and len(self.ddl["columns"]) != 0: + if self.run_arguments["include_tests"] and len(self.ddl["columns"]) != 0: result = self.target_db._construct_tests( self.ddl["columns"], self.table, self.schema ) diff --git a/sayn/tasks/task_wrapper.py b/sayn/tasks/task_wrapper.py index efa7a0e5..ae122abc 100644 --- a/sayn/tasks/task_wrapper.py +++ b/sayn/tasks/task_wrapper.py @@ -76,7 +76,6 @@ def __init__( compiler, db_object_compiler, ): - self.tags = set(tags or set()) self.parent_names = set(parent_names or set()) self.parents = list() @@ -109,6 +108,9 @@ def __init__( self.run_arguments = { "debug": run_arguments.debug, + "include_tests": ( + run_arguments.command.value == "test" or run_arguments.include_tests + ), "full_load": run_arguments.full_load, "start_dt": run_arguments.start_dt, "end_dt": run_arguments.end_dt, @@ -372,6 +374,7 @@ def execute_task(self, command): self.status = TaskStatus.NOT_IN_QUERY return Err("execution", "task_not_in_query") elif self.status not in (TaskStatus.SETTING_UP, TaskStatus.READY): + print("Are we here?") return Err("execution", "setup_error", status=self.status) else: try: diff --git a/tests/__init__.py b/tests/__init__.py index 06be7b03..ce19c5ec 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -115,6 +115,8 @@ def simulate_task( run_arguments = { "debug": obj_run_arguments.debug, + "include_tests": obj_run_arguments.include_tests + or obj_run_arguments.command.value == "test", "full_load": obj_run_arguments.full_load, "start_dt": obj_run_arguments.start_dt, "end_dt": obj_run_arguments.end_dt, diff --git a/tests/test_cli.py b/tests/test_cli.py index 53c2d195..2f5edc53 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -16,7 +16,17 @@ def cli(): @cli.command() @tcli.click_run_options -def run(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt, end_dt): +def run( + debug, + include_tests, + tasks, + exclude, + upstream_prod, + profile, + full_load, + start_dt, + end_dt, +): tasks = [i for t in tasks for i in t.strip().split(" ")] exclude = [i for t in exclude for i in t.strip().split(" ")] @@ -29,6 +39,7 @@ def run(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt, end_ return { "command": "run", "debug": debug, + "include_tests": include_tests, "include": tasks, "exclude": exclude, "profile": profile, @@ -40,7 +51,17 @@ def run(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt, end_ @cli.command() @tcli.click_run_options -def compile(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt, end_dt): +def compile( + debug, + include_tests, + tasks, + exclude, + upstream_prod, + profile, + full_load, + start_dt, + end_dt, +): tasks = [i for t in tasks for i in t.strip().split(" ")] exclude = [i for t in exclude for i in t.strip().split(" ")] @@ -53,6 +74,7 @@ def compile(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt, return { "command": "compile", "debug": debug, + "include_tests": include_tests, "include": tasks, "exclude": exclude, "profile": profile, @@ -75,6 +97,7 @@ def test_simple_run(): assert output == { "command": "run", "debug": False, + "include_tests": False, "include": [], "exclude": [], "profile": None, @@ -90,6 +113,7 @@ def test_simple_compile(): assert output == { "command": "compile", "debug": False, + "include_tests": False, "include": [], "exclude": [], "profile": None, @@ -105,6 +129,7 @@ def test_run_one_task(): assert output == { "command": "run", "debug": False, + "include_tests": False, "include": ["something"], "exclude": [], "profile": None, @@ -120,6 +145,7 @@ def test_run_two_tasks_old(): assert output == { "command": "run", "debug": False, + "include_tests": False, "include": ["something", "somethingelse"], "exclude": [], "profile": None, @@ -135,6 +161,7 @@ def test_run_two_tasks_new(): assert output == { "command": "run", "debug": False, + "include_tests": False, "include": ["something", "somethingelse"], "exclude": [], "profile": None, @@ -150,6 +177,7 @@ def test_compile_one_task(): assert output == { "command": "compile", "debug": False, + "include_tests": False, "include": ["something"], "exclude": [], "profile": None, @@ -165,6 +193,7 @@ def test_compile_two_tasks_old(): assert output == { "command": "compile", "debug": False, + "include_tests": False, "include": ["something", "somethingelse"], "exclude": [], "profile": None, @@ -180,6 +209,7 @@ def test_compile_two_tasks_new(): assert output == { "command": "compile", "debug": False, + "include_tests": False, "include": ["something", "somethingelse"], "exclude": [], "profile": None, @@ -195,6 +225,7 @@ def test_run_debug(): assert output == { "command": "run", "debug": True, + "include_tests": False, "include": ["something"], "exclude": [], "profile": None, @@ -210,6 +241,7 @@ def test_compile_debug(): assert output == { "command": "compile", "debug": True, + "include_tests": False, "include": ["something"], "exclude": [], "profile": None, @@ -225,6 +257,7 @@ def test_run_debug_multitasks(): assert output == { "command": "run", "debug": True, + "include_tests": False, "include": ["something", "somethingelse", "somesomeelse"], "exclude": [], "profile": None, @@ -240,6 +273,7 @@ def test_compile_debug_multitasks(): assert output == { "command": "compile", "debug": True, + "include_tests": False, "include": ["something", "somethingelse", "somesomeelse"], "exclude": [], "profile": None, @@ -255,6 +289,7 @@ def test_run_full(): assert output == { "command": "run", "debug": False, + "include_tests": False, "include": ["something"], "exclude": [], "profile": None, @@ -270,6 +305,7 @@ def test_compile_full(): assert output == { "command": "compile", "debug": False, + "include_tests": False, "include": ["something"], "exclude": [], "profile": None, @@ -285,6 +321,7 @@ def test_run_exclude(): assert output == { "command": "run", "debug": False, + "include_tests": False, "include": [], "exclude": ["something"], "profile": None, @@ -300,6 +337,7 @@ def test_compile_exclude(): assert output == { "command": "compile", "debug": False, + "include_tests": False, "include": [], "exclude": ["something"], "profile": None, @@ -315,6 +353,7 @@ def test_run_include_exclude(): assert output == { "command": "run", "debug": False, + "include_tests": False, "include": ["somethingelse"], "exclude": ["something"], "profile": None, @@ -330,6 +369,7 @@ def test_compile_include_exclude(): assert output == { "command": "compile", "debug": False, + "include_tests": False, "include": ["somethingelse"], "exclude": ["something"], "profile": None,