Release/collective update

.pre-commit-config.yaml

@@ -7,12 +7,8 @@ repos:
       - id: no-commit-to-branch
         args: [--branch, staging, --branch, main, --branch, production]
       - id: trailing-whitespace
-  - repo: https://github.com/psf/black
-    rev: 24.3.0
-    hooks:
-      - id: black
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.3.4'
+    rev: 'v0.4.1'
     hooks:
       - id: ruff
         verbose: true

Dockerfile

@@ -31,7 +31,7 @@ ENV PATH="$PATH:/dk/bin:/opt/mssql-tools/bin/"
 RUN TG_METADATA_DB_USER=- TG_METADATA_DB_PASSWORD=- TG_METADATA_DB_HOST=- TG_METADATA_DB_PORT=- testgen ui patch-streamlit
 
 ARG TESTGEN_VERSION
-ENV TESTGEN_VERSION=$TESTGEN_VERSION
+ENV TESTGEN_VERSION=v$TESTGEN_VERSION
 
 WORKDIR /dk
 

invocations/ci.py

@@ -2,13 +2,14 @@
 Release and CI/CD tasks belong here.
 """
 
-__all__ = ["ci_dotenv", "check_valid_release_type"]
+__all__ = ["check_valid_release_type", "ci_dotenv"]
 
 
 import os
 
 import semver
-from invoke import Exit, task
+from invoke.exceptions import Exit
+from invoke.tasks import task
 
 
 @task

invocations/dev.py

@@ -1,10 +1,12 @@
-__all__ = ["install", "lint", "clean", "build_public_image"]
+__all__ = ["build_public_image", "clean", "install", "lint"]
 
 from os.path import exists, join
 from shutil import rmtree, which
 
 import tomli
-from invoke import Exit, task
+from invoke.context import Context
+from invoke.exceptions import Exit
+from invoke.tasks import task
 
 from .toolbox import ensure_tools
 
@@ -13,20 +15,20 @@
 
 
 @task
-def required_tools(ctx):
+def required_tools(ctx: Context) -> None:
     ensure_tools("git", "find", "docker")
 
 
 @task
-def install(ctx, quiet_pip=False):
+def install(ctx: Context, quiet_pip: bool = False) -> None:
     """Installs the package as a developer (editable, all optional dependencies)."""
     if quiet_pip:
         print("testgen package is being re-installed.")
     ctx.run("pip install -e .[dev]", hide=quiet_pip)
 
 
 @task
-def lint(ctx):
+def lint(ctx: Context) -> None:
     """Runs the standard suite of quality/linting tools."""
     ctx.run("isort .")
     ctx.run("black .")
@@ -35,7 +37,7 @@ def lint(ctx):
 
 
 @task
-def precommit(ctx, all_files=False):
+def precommit(ctx: Context, all_files: bool = False) -> None:
     """Runs pre-commit."""
     if which("pre-commit") is None:
         install(ctx)
@@ -47,10 +49,13 @@ def precommit(ctx, all_files=False):
 
 
 @task(pre=(required_tools,))
-def clean(ctx):
+def clean(ctx: Context) -> None:
     """Deletes old python files and build artifacts"""
-    repo_root = ctx.run("git rev-parse --show-toplevel", hide=True).stdout.strip()
+    result = ctx.run("git rev-parse --show-toplevel", hide=True)
+    if not result:
+        raise Exit("Failure running git rev-parse")
 
+    repo_root = result.stdout.strip()
     with open(join(repo_root, "pyproject.toml"), "rb") as f:
         project_name: str = tomli.load(f)["project"]["name"]
 
@@ -62,13 +67,14 @@ def clean(ctx):
 
 
 @task(pre=(required_tools,))
-def build_public_image(ctx, version: str, push=False, local=False):
+def build_public_image(ctx: Context, version: str, push: bool = False, local: bool = False) -> None:
     """Builds and pushes the TestGen image"""
     use_cmd = f"docker buildx use {DOCKER_BUILDER_NAME}"
     if push and local:
         raise Exit("Cannot use --local and --push at the same time.")
 
-    if not ctx.run(use_cmd, hide=True, warn=True).ok:
+
+    if (result := ctx.run(use_cmd, hide=True, warn=True)) and not result.ok:
         ctx.run(f"docker buildx create --name {DOCKER_BUILDER_NAME} --platform {DOCKER_BUILDER_PLATFORMS}")
         ctx.run(use_cmd)
 

invocations/toolbox.py

@@ -1,9 +1,9 @@
 from shutil import which
 
-from invoke import Exit
+from invoke.exceptions import Exit
 
 
-def ensure_tools(*tools):
+def ensure_tools(*tools: str) -> None:
     """
     Check the PATH to see if the required tools exist. e.g.,
 

pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "data-ops-testgen"
-version = "2.0.0"
+version = "2.1.1"
 description = "DataKitchen Inc. Data Quality Engine"
 urls = { "homepage" = "https://datakitchen.io" }
 authors = [
@@ -63,10 +63,8 @@ dependencies = [
 
 [project.optional-dependencies]
 dev = [
-    "black[d]==24.3.0",
     "invoke==2.2.0",
-    "isort==5.13.2",
-    "ruff==0.3.4",
+    "ruff==0.4.1",
     "mypy==1.9.0",
     "pre-commit==3.6.2",
     "types-PyYAML==6.0.12.20240311",
@@ -164,42 +162,33 @@ show_error_context = true
 # Mypy appears smart enough to ignore hidden directories. But it needs to scan __pycache__ for .pyc and pyi files,
 # so it cannot honor gitignore.
 exclude = [
-    '''^(?:.*\/)+[tT]ests?''',
     'conftest.py',
     'venv/',
     'build/',
+    'tests/',
 ]
 
-[tool.isort]
-profile = "black"
-line_length = 120
-
-[tool.black]
-line-length = 120
-target-version = ['py310']
-include = '\.pyi?$'
-exclude = '''
-(
-  /(
-      \.eggs         # exclude a few common directories in the
-    | \.git          # root of the project
-    | \.mypy_cache
-    | \.tox
-    | \.venv
-    | env
-    | venv
-    | _build
-    | build
-    | dist
-  )/
-  | foo.py           # also separately exclude a file named foo.py in
-                     # the root of the project
-)
-'''
-
 [tool.ruff]
 target-version = "py310"
 line-length = 120
+indent-width = 4
+include = [
+    "invocations/**/*.py",
+    "testgen/**/*.py",
+    "tests/**/*.py",
+]
+exclude = [
+    ".eggs",
+    ".git",
+    ".mypy_cache",
+    ".tox",
+    ".venv",
+    "env",
+    "venv",
+    "_build",
+    "build",
+    "dist",
+]
 
 [tool.ruff.lint]
 # see: https://beta.ruff.rs/docs/rules.
@@ -224,8 +213,8 @@ select = ["A", "F", "S", "I", "T10", "B", "UP", "ISC", "T20", "RSE", "Q", "ARG",
 # globally ignore the following error codes
 # * TRY003: Avoid specifying long messages outside the exception class
 # * S608: Hardcoded SQL
-# * I001: Unsorted imports, partially incompatible with isorts suggestions
-ignore = ["TRY003", "S608", "I001"]
+# # F841: Unused local variable (it is instable)
+ignore = ["TRY003", "S608", "S404", "F841"]
 
 # Ignore the following errors in files:
 # F403 - in __init__.py: We use __all__ in our module files so this behavior is acceptable in __init__.py
@@ -235,10 +224,8 @@ ignore = ["TRY003", "S608", "I001"]
 "testgen/__main__.py" = ["ARG001", "S603"]
 "tasks.py" = ["F403"]
 "tests*" = ["S101", "T201"]
-# print and empty arguments are fine here.
 "invocations/**" = ["ARG001", "T201"]
-
-
+"testgen/common/encrypt.py" = ["S413"]
 
 # See: https://coverage.readthedocs.io/en/latest/config.html
 [tool.coverage.run]
@@ -255,9 +242,9 @@ omit = ["tests/*", "templates/*"]
 skip_empty=true
 
 [tool.bumpver]
-current_version = "2.0.0"
+current_version = "2.1.1"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
-commit_message = "Bump version: {old_version} -> {new_version}"
+commit_message = "release: {old_version} -> {new_version}"
 commit = true
 tag = true
 push = false

testgen/__main__.py

@@ -137,10 +137,17 @@ def run_profile(configuration: Configuration, table_group_id: str):
     required=False,
     default=settings.DEFAULT_TEST_SUITE_KEY,
 )
+@click.option(
+    "-gs",
+    "--generation-set",
+    help="A defined subset of tests to generate for your purpose. Use a generation_set defined for your project.",
+    required=False,
+    default=None,
+)
 @pass_configuration
-def run_test_generation(configuration: Configuration, table_group_id: str, test_suite_key: str):
+def run_test_generation(configuration: Configuration, table_group_id: str, test_suite_key: str, generation_set: str):
     LOG.info("CurrentStep: Generate Tests - Main Procedure")
-    message = run_test_gen_queries(table_group_id, test_suite_key)
+    message = run_test_gen_queries(table_group_id, test_suite_key, generation_set)
     LOG.info("Current Step: Generate Tests - Main Procedure Complete")
     display_service.echo("\n" + message)
 

testgen/commands/queries/execute_cat_tests_query.py

@@ -51,7 +51,7 @@ def _ReplaceParms(self, strInputString):
         strInputString = strInputString.replace(
             "{NOW}", date_service.get_now_as_string_with_offset(self.minutes_offset)
         )
-        strInputString = strInputString.replace("{EXCEPTION_MESSAGE}", self.exception_message)
+        strInputString = strInputString.replace("{EXCEPTION_MESSAGE}", self.exception_message.strip())
 
         for parm, value in self.dctTestParms.items():
             strInputString = strInputString.replace("{" + parm.upper() + "}", str(value))

testgen/commands/queries/generate_tests_query.py

@@ -84,3 +84,9 @@ def GetTestQueriesFromGenericFile(self, booClean: bool):
         if booClean:
             strQuery = CleanSQL(strQuery)
         return strQuery
+
+    def GetDeleteOldTestsQuery(self, booClean: bool):
+        strQuery = self.ReplaceParms(read_template_sql_file("gen_delete_old_tests.sql", "generation"))
+        if booClean:
+            strQuery = CleanSQL(strQuery)
+        return strQuery

testgen/commands/run_execute_cat_tests.py

@@ -65,7 +65,9 @@ def FinalizeTestRun(clsCATExecute):
     RunActionQueryList(("DKTG"), lstQueries)
 
 
-def run_cat_test_queries(strTestRunID, strTestTime, strProjectCode, strTestSuite, minutes_offset=0, spinner=None):
+def run_cat_test_queries(
+    strTestRunID, strTestTime, strProjectCode, strTestSuite, error_msg, minutes_offset=0, spinner=None
+):
     # PARAMETERS AND SET-UP
     booErrors = False
     LOG.info("CurrentStep: Retrieving Parameters")
@@ -78,6 +80,7 @@ def run_cat_test_queries(strTestRunID, strTestTime, strProjectCode, strTestSuite
     )
     clsCATExecute.test_run_id = strTestRunID
     clsCATExecute.run_date = strTestTime
+    clsCATExecute.exception_message += error_msg
 
     # Set Project Connection Params in common.db_bridgers from retrieved params
     LOG.info("CurrentStep: Assigning Connection Parms")
@@ -135,17 +138,17 @@ def run_cat_test_queries(strTestRunID, strTestTime, strProjectCode, strTestSuite
                     LOG.info("Test results successfully parsed.")
                 if intErrors > 0:
                     booErrors = True
-                    LOG.warning(
-                        f"Errors were encountered executing aggregate tests. ({intErrors} errors occurred.) Please check log."
-                    )
+                    cat_error_msg = f"Errors were encountered executing aggregate tests. ({intErrors} errors occurred.) Please check log."
+                    LOG.warning(cat_error_msg)
+                    clsCATExecute.exception_message += cat_error_msg
         else:
             LOG.info("No valid tests were available to perform")
 
     except Exception as e:
         booErrors = True
         sqlsplit = e.args[0].split("[SQL", 1)
         errorline = sqlsplit[0].replace("'", "''") if len(sqlsplit) > 0 else "unknown error"
-        clsCATExecute.exception_message = f"{type(e).__name__}: {errorline}"
+        clsCATExecute.exception_message += f"{type(e).__name__}: {errorline}"
         raise
 
     else:

testgen/commands/run_execute_tests.py

@@ -22,6 +22,7 @@
 
 def run_test_queries(strTestRunID, strTestTime, strProjectCode, strTestSuite, minutes_offset=0, spinner=None):
     booErrors = False
+    error_msg = ""
 
     LOG.info("CurrentStep: Retrieving TestExec Parameters")
     dctParms = RetrieveTestExecParms(strProjectCode, strTestSuite)
@@ -76,7 +77,7 @@ def run_test_queries(strTestRunID, strTestTime, strProjectCode, strTestSuite, mi
                     spinner.next()
 
             # Execute list, returning test results
-            LOG.info("CurrentStep: Executing Non-CAT Queries")
+            LOG.info("CurrentStep: Executing Non-CAT Test Queries")
             lstTestResults, colResultNames, intErrors = RunThreadedRetrievalQueryList(
                 "PROJECT", lstTestQueries, dctParms["max_threads"], spinner
             )
@@ -87,9 +88,11 @@ def run_test_queries(strTestRunID, strTestTime, strProjectCode, strTestSuite, mi
                 WriteListToDB("DKTG", lstTestResults, colResultNames, "test_results")
             if intErrors > 0:
                 booErrors = True
-                LOG.warning(
-                    f"Errors were encountered executing query tests. ({intErrors} errors occurred.) Please check log."
+                error_msg = (
+                    f"Errors were encountered executing aggregate tests. ({intErrors} errors occurred.) "
+                    "Please check log. "
                 )
+                LOG.warning(error_msg)
         else:
             LOG.info("No tests found")
 
@@ -103,7 +106,7 @@ def run_test_queries(strTestRunID, strTestTime, strProjectCode, strTestSuite, mi
         raise
 
     else:
-        return booErrors
+        return booErrors, error_msg
 
 
 def run_execution_steps_in_background(strProjectCode, strTestSuite, minutes_offset=0):
@@ -123,6 +126,8 @@ def run_execution_steps_in_background(strProjectCode, strTestSuite, minutes_offs
 def run_execution_steps(strProjectCode, strTestSuite, minutes_offset=0, spinner=None):
     # Initialize required parms for all three steps
     booErrors = False
+    error_msg = ""
+
     strTestRunID = str(uuid.uuid4())
     strTestTime = date_service.get_now_as_string_with_offset(minutes_offset)
 
@@ -133,11 +138,14 @@ def run_execution_steps(strProjectCode, strTestSuite, minutes_offset=0, spinner=
     run_parameter_validation_queries(strTestRunID, strTestTime, strProjectCode, strTestSuite, True)
 
     LOG.info("CurrentStep: Execute Step - Test Execution")
-    if run_test_queries(strTestRunID, strTestTime, strProjectCode, strTestSuite, minutes_offset, spinner):
-        booErrors = True
+    booErrors, error_msg = run_test_queries(
+        strTestRunID, strTestTime, strProjectCode, strTestSuite, minutes_offset, spinner
+    )
 
     LOG.info("CurrentStep: Execute Step - CAT Test Execution")
-    if run_cat_test_queries(strTestRunID, strTestTime, strProjectCode, strTestSuite, minutes_offset, spinner):
+    if run_cat_test_queries(
+        strTestRunID, strTestTime, strProjectCode, strTestSuite, error_msg, minutes_offset, spinner
+    ):
         booErrors = True
 
     if booErrors: