Feature: Include-Tests flag (#252)

* Introduced `--run-tests` flag to append Task Test execution after a Task Run or Compile execution.

sayn/cli.py

@@ -24,6 +24,7 @@ def __init__(
         full_load=False,
         start_dt=None,
         end_dt=None,
+        run_tests=False,
         fail_fast=False,
     ):
         super().__init__()
@@ -69,6 +70,9 @@ def __init__(
         if upstream_prod is not None:
             self.run_arguments.upstream_prod = upstream_prod
 
+        if run_tests is not None:
+            self.run_arguments.run_tests = run_tests
+
         if fail_fast is not None:
             self.run_arguments.fail_fast = fail_fast
 
@@ -121,6 +125,13 @@ def parser_process(value, state):
     "--debug", "-d", is_flag=True, default=False, help="Include debug messages"
 )
 
+click_include_tests = click.option(
+    "--run-tests",
+    is_flag=True,
+    default=False,
+    help="Include Tests in task execution - after task run.",
+)
+
 click_fail_fast = click.option(
     "--fail-fast",
     is_flag=True,
@@ -211,6 +222,7 @@ def compile(
     full_load,
     start_dt,
     end_dt,
+    run_tests,
     fail_fast,
 ):
 
@@ -226,6 +238,7 @@ def compile(
         full_load,
         start_dt,
         end_dt,
+        run_tests,
         fail_fast,
     )
 
@@ -237,6 +250,7 @@ def compile(
 
 
 @cli.command(help="Run SAYN tasks.")
+@click_include_tests
 @click_run_options
 def run(
     debug,
@@ -247,6 +261,7 @@ def run(
     full_load,
     start_dt,
     end_dt,
+    run_tests,
     fail_fast,
 ):
 
@@ -262,6 +277,7 @@ def run(
         full_load,
         start_dt,
         end_dt,
+        run_tests,
         fail_fast,
     )
 

sayn/core/app.py

@@ -68,6 +68,7 @@ class Folders:
     command: Command = Command.UNDEFINED
     upstream_prod: bool = False
     is_prod: bool = False
+    run_tests: bool = False
     fail_fast: bool = False
 
     include: Set[str]

sayn/tasks/copy.py

@@ -272,7 +272,9 @@ def config(self, **config):  # noqa: C901
         else:
             return result
 
-        if self.run_arguments["command"] == "test" and len(self.columns["columns"]) > 0:
+        if (
+            self.run_arguments["command"] == "test" or self.run_arguments["run_tests"]
+        ) and len(self.columns["columns"]) > 0:
             result = self.target_db._construct_tests(
                 self.columns["columns"], self.table, self.schema
             )
@@ -483,7 +485,6 @@ def execute(self, execute, debug, is_full_load, limit=None):
 
                 def read_iter(iter):
                     if self.mode == "append":
-
                         load_time = datetime.utcnow()
                         for record in iter:
                             yield dict(record, _sayn_load_ts=load_time)

sayn/tasks/sql.py

@@ -259,7 +259,9 @@ def def_out(obj, level=None):
         else:
             self.ddl = result.value
 
-        if self.run_arguments["command"] == "test" and len(self.ddl["columns"]) != 0:
+        if (
+            self.run_arguments["command"] == "test" or self.run_arguments["run_tests"]
+        ) and len(self.ddl["columns"]) != 0:
             result = self.target_db._construct_tests(
                 self.ddl["columns"], self.table, self.schema
             )

sayn/tasks/task_wrapper.py

@@ -76,7 +76,6 @@ def __init__(
         compiler,
         db_object_compiler,
     ):
-
         self.tags = set(tags or set())
         self.parent_names = set(parent_names or set())
         self.parents = list()
@@ -110,6 +109,7 @@ def __init__(
 
             self.run_arguments = {
                 "debug": run_arguments.debug,
+                "run_tests": run_arguments.run_tests,
                 "full_load": run_arguments.full_load,
                 "start_dt": run_arguments.start_dt,
                 "end_dt": run_arguments.end_dt,
@@ -382,8 +382,12 @@ def execute_task(self, command):
             try:
                 if command == "run":
                     result = self.runner.run()
+                    if self.run_arguments["run_tests"] and self.has_tests():
+                        result = self.runner.test()
                 elif command == "compile":
                     result = self.runner.compile()
+                    if self.run_arguments["run_tests"] and self.has_tests():
+                        result = self.runner.test()
                 else:
                     result = self.runner.test()
 

tests/__init__.py

@@ -115,6 +115,7 @@ def simulate_task(
 
     run_arguments = {
         "debug": obj_run_arguments.debug,
+        "run_tests": obj_run_arguments.run_tests,
         "full_load": obj_run_arguments.full_load,
         "start_dt": obj_run_arguments.start_dt,
         "end_dt": obj_run_arguments.end_dt,

tests/test_cli.py

@@ -15,9 +15,11 @@ def cli():
 
 
 @cli.command()
+@tcli.click_include_tests
 @tcli.click_run_options
 def run(
     debug,
+    run_tests,
     fail_fast,
     tasks,
     exclude,
@@ -39,6 +41,7 @@ def run(
     return {
         "command": "run",
         "debug": debug,
+        "run_tests": run_tests,
         "include": tasks,
         "exclude": exclude,
         "profile": profile,
@@ -97,6 +100,7 @@ def test_simple_run():
     assert output == {
         "command": "run",
         "debug": False,
+        "run_tests": False,
         "fail_fast": False,
         "include": [],
         "exclude": [],
@@ -129,6 +133,7 @@ def test_run_one_task():
     assert output == {
         "command": "run",
         "debug": False,
+        "run_tests": False,
         "fail_fast": False,
         "include": ["something"],
         "exclude": [],
@@ -145,6 +150,7 @@ def test_run_two_tasks_old():
     assert output == {
         "command": "run",
         "debug": False,
+        "run_tests": False,
         "fail_fast": False,
         "include": ["something", "somethingelse"],
         "exclude": [],
@@ -161,6 +167,7 @@ def test_run_two_tasks_new():
     assert output == {
         "command": "run",
         "debug": False,
+        "run_tests": False,
         "fail_fast": False,
         "include": ["something", "somethingelse"],
         "exclude": [],
@@ -225,6 +232,7 @@ def test_run_debug():
     assert output == {
         "command": "run",
         "debug": True,
+        "run_tests": False,
         "fail_fast": False,
         "include": ["something"],
         "exclude": [],
@@ -257,6 +265,7 @@ def test_run_debug_multitasks():
     assert output == {
         "command": "run",
         "debug": True,
+        "run_tests": False,
         "fail_fast": False,
         "include": ["something", "somethingelse", "somesomeelse"],
         "exclude": [],
@@ -290,6 +299,7 @@ def test_run_fail_fast():
         "command": "run",
         "debug": False,
         "fail_fast": True,
+        "run_tests": False,
         "include": ["something"],
         "exclude": [],
         "profile": None,
@@ -338,6 +348,7 @@ def test_run_fail_fast_multitasks():
         "command": "run",
         "debug": False,
         "fail_fast": True,
+        "run_tests": False,
         "include": ["something", "somethingelse", "somesomeelse"],
         "exclude": [],
         "profile": None,
@@ -353,6 +364,7 @@ def test_run_full():
     assert output == {
         "command": "run",
         "debug": False,
+        "run_tests": False,
         "fail_fast": False,
         "include": ["something"],
         "exclude": [],
@@ -385,6 +397,7 @@ def test_run_exclude():
     assert output == {
         "command": "run",
         "debug": False,
+        "run_tests": False,
         "fail_fast": False,
         "include": [],
         "exclude": ["something"],
@@ -417,6 +430,7 @@ def test_run_include_exclude():
     assert output == {
         "command": "run",
         "debug": False,
+        "run_tests": False,
         "fail_fast": False,
         "include": ["somethingelse"],
         "exclude": ["something"],