Introduce Include-Tests flag

sayn/cli.py

@@ -17,6 +17,7 @@ def __init__(
         self,
         command,
         debug=False,
+        include_tests=False,
         include=None,
         exclude=None,
         upstream_prod=False,
@@ -68,6 +69,9 @@ def __init__(
         if upstream_prod is not None:
             self.run_arguments.upstream_prod = upstream_prod
 
+        if include_tests is not None:
+            self.run_arguments.include_tests = include_tests
+
         self.start_app()
 
 
@@ -117,6 +121,14 @@ def parser_process(value, state):
     "--debug", "-d", is_flag=True, default=False, help="Include debug messages"
 )
 
+click_include_tests = click.option(
+    "--include-tests",
+    "-it",
+    is_flag=True,
+    default=False,
+    help="Include Tests in task Run.",
+)
+
 
 def click_filter(func):
     func = click.option(
@@ -168,6 +180,7 @@ def click_incremental(func):
 
 def click_run_options(func):
     func = click_debug(func)
+    func = click_include_tests(func)
     func = click.option("--profile", "-p", help="Profile from settings to use")(func)
     func = click_incremental(func)
     func = click_filter(func)
@@ -190,13 +203,24 @@ def init(sayn_project_name):
 
 @cli.command(help="Compile sql tasks.")
 @click_run_options
-def compile(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt, end_dt):
+def compile(
+    debug,
+    include_tests,
+    tasks,
+    exclude,
+    upstream_prod,
+    profile,
+    full_load,
+    start_dt,
+    end_dt,
+):
 
     tasks = [i for t in tasks for i in t.strip().split(" ")]
     exclude = [i for t in exclude for i in t.strip().split(" ")]
     app = CliApp(
         Command.COMPILE,
         debug,
+        include_tests,
         tasks,
         exclude,
         upstream_prod,
@@ -215,13 +239,24 @@ def compile(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt,
 
 @cli.command(help="Run SAYN tasks.")
 @click_run_options
-def run(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt, end_dt):
+def run(
+    debug,
+    include_tests,
+    tasks,
+    exclude,
+    upstream_prod,
+    profile,
+    full_load,
+    start_dt,
+    end_dt,
+):
 
     tasks = [i for t in tasks for i in t.strip().split(" ")]
     exclude = [i for t in exclude for i in t.strip().split(" ")]
     app = CliApp(
         Command.RUN,
         debug,
+        include_tests,
         tasks,
         exclude,
         upstream_prod,
@@ -240,13 +275,24 @@ def run(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt, end_
 
 @cli.command(help="Test SAYN tasks.")
 @click_run_options
-def test(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt, end_dt):
+def test(
+    debug,
+    include_tests,
+    tasks,
+    exclude,
+    upstream_prod,
+    profile,
+    full_load,
+    start_dt,
+    end_dt,
+):
 
     tasks = [i for t in tasks for i in t.strip().split(" ")]
     exclude = [i for t in exclude for i in t.strip().split(" ")]
     app = CliApp(
         Command.TEST,
         debug,
+        include_tests,
         tasks,
         exclude,
         upstream_prod,

sayn/core/app.py

@@ -68,6 +68,7 @@ class Folders:
     command: Command = Command.UNDEFINED
     upstream_prod: bool = False
     is_prod: bool = False
+    include_tests: bool = False
 
     include: Set[str]
     exclude: Set[str]
@@ -622,6 +623,12 @@ def execute_dag(self):
 
             if self.run_arguments.command == Command.RUN:
                 result = task.run()
+                if not result.is_err and self.run_arguments.include_tests:
+                    # Force Task to be Ready
+                    task.status = TaskStatus.READY
+                    test_result = task.test()
+                    if test_result.is_err:
+                        result = test_result
             elif self.run_arguments.command == Command.COMPILE:
                 result = task.compile()
             elif self.run_arguments.command == Command.TEST:

sayn/tasks/copy.py

@@ -272,7 +272,7 @@ def config(self, **config):  # noqa: C901
         else:
             return result
 
-        if self.run_arguments["command"] == "test" and len(self.columns["columns"]) > 0:
+        if self.run_arguments["include_tests"] and len(self.columns["columns"]) > 0:
             result = self.target_db._construct_tests(
                 self.columns["columns"], self.table, self.schema
             )

sayn/tasks/sql.py

@@ -259,7 +259,7 @@ def def_out(obj, level=None):
         else:
             self.ddl = result.value
 
-        if self.run_arguments["command"] == "test" and len(self.ddl["columns"]) != 0:
+        if self.run_arguments["include_tests"] and len(self.ddl["columns"]) != 0:
             result = self.target_db._construct_tests(
                 self.ddl["columns"], self.table, self.schema
             )

sayn/tasks/task_wrapper.py

@@ -76,7 +76,6 @@ def __init__(
         compiler,
         db_object_compiler,
     ):
-
         self.tags = set(tags or set())
         self.parent_names = set(parent_names or set())
         self.parents = list()
@@ -109,6 +108,9 @@ def __init__(
 
             self.run_arguments = {
                 "debug": run_arguments.debug,
+                "include_tests": (
+                    run_arguments.command.value == "test" or run_arguments.include_tests
+                ),
                 "full_load": run_arguments.full_load,
                 "start_dt": run_arguments.start_dt,
                 "end_dt": run_arguments.end_dt,
@@ -372,6 +374,7 @@ def execute_task(self, command):
             self.status = TaskStatus.NOT_IN_QUERY
             return Err("execution", "task_not_in_query")
         elif self.status not in (TaskStatus.SETTING_UP, TaskStatus.READY):
+            print("Are we here?")
             return Err("execution", "setup_error", status=self.status)
         else:
             try:

tests/__init__.py

@@ -115,6 +115,8 @@ def simulate_task(
 
     run_arguments = {
         "debug": obj_run_arguments.debug,
+        "include_tests": obj_run_arguments.include_tests
+        or obj_run_arguments.command.value == "test",
         "full_load": obj_run_arguments.full_load,
         "start_dt": obj_run_arguments.start_dt,
         "end_dt": obj_run_arguments.end_dt,

tests/test_cli.py

@@ -16,7 +16,17 @@ def cli():
 
 @cli.command()
 @tcli.click_run_options
-def run(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt, end_dt):
+def run(
+    debug,
+    include_tests,
+    tasks,
+    exclude,
+    upstream_prod,
+    profile,
+    full_load,
+    start_dt,
+    end_dt,
+):
 
     tasks = [i for t in tasks for i in t.strip().split(" ")]
     exclude = [i for t in exclude for i in t.strip().split(" ")]
@@ -29,6 +39,7 @@ def run(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt, end_
     return {
         "command": "run",
         "debug": debug,
+        "include_tests": include_tests,
         "include": tasks,
         "exclude": exclude,
         "profile": profile,
@@ -40,7 +51,17 @@ def run(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt, end_
 
 @cli.command()
 @tcli.click_run_options
-def compile(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt, end_dt):
+def compile(
+    debug,
+    include_tests,
+    tasks,
+    exclude,
+    upstream_prod,
+    profile,
+    full_load,
+    start_dt,
+    end_dt,
+):
 
     tasks = [i for t in tasks for i in t.strip().split(" ")]
     exclude = [i for t in exclude for i in t.strip().split(" ")]
@@ -53,6 +74,7 @@ def compile(debug, tasks, exclude, upstream_prod, profile, full_load, start_dt,
     return {
         "command": "compile",
         "debug": debug,
+        "include_tests": include_tests,
         "include": tasks,
         "exclude": exclude,
         "profile": profile,
@@ -75,6 +97,7 @@ def test_simple_run():
     assert output == {
         "command": "run",
         "debug": False,
+        "include_tests": False,
         "include": [],
         "exclude": [],
         "profile": None,
@@ -90,6 +113,7 @@ def test_simple_compile():
     assert output == {
         "command": "compile",
         "debug": False,
+        "include_tests": False,
         "include": [],
         "exclude": [],
         "profile": None,
@@ -105,6 +129,7 @@ def test_run_one_task():
     assert output == {
         "command": "run",
         "debug": False,
+        "include_tests": False,
         "include": ["something"],
         "exclude": [],
         "profile": None,
@@ -120,6 +145,7 @@ def test_run_two_tasks_old():
     assert output == {
         "command": "run",
         "debug": False,
+        "include_tests": False,
         "include": ["something", "somethingelse"],
         "exclude": [],
         "profile": None,
@@ -135,6 +161,7 @@ def test_run_two_tasks_new():
     assert output == {
         "command": "run",
         "debug": False,
+        "include_tests": False,
         "include": ["something", "somethingelse"],
         "exclude": [],
         "profile": None,
@@ -150,6 +177,7 @@ def test_compile_one_task():
     assert output == {
         "command": "compile",
         "debug": False,
+        "include_tests": False,
         "include": ["something"],
         "exclude": [],
         "profile": None,
@@ -165,6 +193,7 @@ def test_compile_two_tasks_old():
     assert output == {
         "command": "compile",
         "debug": False,
+        "include_tests": False,
         "include": ["something", "somethingelse"],
         "exclude": [],
         "profile": None,
@@ -180,6 +209,7 @@ def test_compile_two_tasks_new():
     assert output == {
         "command": "compile",
         "debug": False,
+        "include_tests": False,
         "include": ["something", "somethingelse"],
         "exclude": [],
         "profile": None,
@@ -195,6 +225,7 @@ def test_run_debug():
     assert output == {
         "command": "run",
         "debug": True,
+        "include_tests": False,
         "include": ["something"],
         "exclude": [],
         "profile": None,
@@ -210,6 +241,7 @@ def test_compile_debug():
     assert output == {
         "command": "compile",
         "debug": True,
+        "include_tests": False,
         "include": ["something"],
         "exclude": [],
         "profile": None,
@@ -225,6 +257,7 @@ def test_run_debug_multitasks():
     assert output == {
         "command": "run",
         "debug": True,
+        "include_tests": False,
         "include": ["something", "somethingelse", "somesomeelse"],
         "exclude": [],
         "profile": None,
@@ -240,6 +273,7 @@ def test_compile_debug_multitasks():
     assert output == {
         "command": "compile",
         "debug": True,
+        "include_tests": False,
         "include": ["something", "somethingelse", "somesomeelse"],
         "exclude": [],
         "profile": None,
@@ -255,6 +289,7 @@ def test_run_full():
     assert output == {
         "command": "run",
         "debug": False,
+        "include_tests": False,
         "include": ["something"],
         "exclude": [],
         "profile": None,
@@ -270,6 +305,7 @@ def test_compile_full():
     assert output == {
         "command": "compile",
         "debug": False,
+        "include_tests": False,
         "include": ["something"],
         "exclude": [],
         "profile": None,
@@ -285,6 +321,7 @@ def test_run_exclude():
     assert output == {
         "command": "run",
         "debug": False,
+        "include_tests": False,
         "include": [],
         "exclude": ["something"],
         "profile": None,
@@ -300,6 +337,7 @@ def test_compile_exclude():
     assert output == {
         "command": "compile",
         "debug": False,
+        "include_tests": False,
         "include": [],
         "exclude": ["something"],
         "profile": None,
@@ -315,6 +353,7 @@ def test_run_include_exclude():
     assert output == {
         "command": "run",
         "debug": False,
+        "include_tests": False,
         "include": ["somethingelse"],
         "exclude": ["something"],
         "profile": None,
@@ -330,6 +369,7 @@ def test_compile_include_exclude():
     assert output == {
         "command": "compile",
         "debug": False,
+        "include_tests": False,
         "include": ["somethingelse"],
         "exclude": ["something"],
         "profile": None,