#296 - Refactor testing and workflows

.github/workflows/_build-package.yml

@@ -0,0 +1,22 @@
+name: build-package
+on: 
+  workflow_call:
+jobs:
+  build:
+    name: Build wheel and sdist
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with: 
+          python-version: 3.11
+      - name: Install build dependencies
+        run: pip install --no-cache-dir -U pip .['build']
+      - name: Build package
+        run: ./scripts/cd.py --build
+      - name: Upload built distributions
+        uses: actions/upload-artifact@v3
+        with:
+          name: dist
+          path: dist

.github/workflows/integration-tests.yml → .github/workflows/_integration-tests.yml

@@ -1,9 +1,13 @@
-name: integration-tests [experimental]
-on: [pull_request]
+name: integration-tests
+on: 
+  workflow_call:
+    secrets:
+      DOCKER_TOKEN:
+        required: true
 jobs:
-  integration_tests:
+  integration-tests:
+    name: Run integration tests
     runs-on: ubuntu-latest
-    continue-on-error: true
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python
@@ -15,4 +19,4 @@ jobs:
       - name: Docker login
         run: docker login -u kamforka -p ${{ secrets.DOCKER_TOKEN }}
       - name: Run integration tests
-        run: scripts/ci.py --test
+        run: scripts/ci.py --test

.github/workflows/static-checks.yml → .github/workflows/_static-checks.yml

@@ -1,7 +1,9 @@
 name: static-checks
-on: [pull_request]
+on: 
+  workflow_call:
 jobs:
-  build:
+  static-checks:
+    name: Run static checks
     runs-on: ubuntu-latest
     strategy:
       matrix:

.github/workflows/_upload-package.yml

@@ -0,0 +1,37 @@
+name: upload-package
+on: 
+  workflow_call:
+    secrets:
+      PYPI_TOKEN:
+        required: true
+jobs:
+  upload:
+    name: Upload wheel and sdist
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Compare tag and package version
+        run: |
+          TAG=${GITHUB_REF#refs/*/}
+          VERSION=$(grep -Po '(?<=version = ")[^"]*' pyproject.toml)
+          if [ "$TAG" != "$VERSION" ]; then
+            echo "Tag value and package version are different: ${TAG} != ${VERSION}"
+            exit 1
+          fi
+      - name: Download built distributions
+        uses: actions/download-artifact@v3
+        with:
+          name: dist
+          path: dist
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with: 
+          python-version: 3.11
+      - name: Install build dependencies
+        run: pip install --no-cache-dir -U pip .['build']
+      - name: Upload to PyPI
+        run: ./scripts/cd.py --upload
+        env:
+          TWINE_REPOSITORY_URL: https://upload.pypi.org/legacy/
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}

.github/workflows/main-cicd.yml

@@ -0,0 +1,22 @@
+name: cicd
+on: 
+  push:
+    tags:
+      - "*"
+  pull_request:
+
+jobs:
+  static-checks:
+    uses: ./.github/workflows/_static-checks.yml
+  integration-tests:
+    uses: ./.github/workflows/_integration-tests.yml
+    secrets:
+      DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
+  build-package:
+    uses: ./.github/workflows/_build-package.yml
+  upload-package:
+    uses: ./.github/workflows/_upload-package.yml
+    if: startsWith(github.ref, 'refs/tags/') 
+    needs: [static-checks, integration-tests, build-package]
+    secrets:
+      PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}

scripts/cd.py

@@ -1,103 +1,104 @@
 #!/usr/bin/env python
 import argparse
 import subprocess
+from typing import List
 
 
 def _run_subprocess(
-    args: str,
-    init_message: str,
-    success_message: str,
-    error_message: str,
-    verbose=False,
+    command: str,
+    quiet=False,
 ):
-    print(init_message)
-    proc = subprocess.run(args, shell=True, capture_output=True)
-
-    process_output = proc.stdout.decode() or proc.stderr.decode()
-    indented_process_output = "\n".join(
-        [f"\t{output_line}" for output_line in process_output.splitlines()]
-    )
-
-    if proc.returncode != 0:
-        exit_message = "\n".join([error_message, indented_process_output])
-        exit(exit_message)
-
-    if verbose:
-        print(indented_process_output)
-
-    print(success_message)
+    if not quiet:
+        stdout = stderr = None
+    else:
+        stdout = stderr = subprocess.DEVNULL
+
+    try:
+        subprocess.run(str.split(command), stdout=stdout, stderr=stderr, check=True)
+    except subprocess.CalledProcessError as err:
+        error_output = (
+            f"ERROR: Execution of command '{command}' returned: {err.returncode}\n"
+        )
+        print(error_output)
+        exit(err.returncode)
 
 
-def run_all(verbose=False):
+def run_all(quiet=False):
     print("Run all deployment tasks...")
-    run_build(verbose=verbose)
-    run_publish(verbose=verbose)
+    run_build(quiet=quiet)
+    run_upload(quiet=quiet)
     print("All tasks succeeded!")
 
 
-def run_build(verbose: bool):
+def run_build(quiet: bool):
+    print("Building thehive4py with the build module...")
+    _run_subprocess(
+        command="rm -rf build/ dist/",
+        quiet=quiet,
+    )
     _run_subprocess(
-        args=("rm -rf build/ dist/ && python -m build --sdist --wheel"),
-        init_message="Building the package with the build module...",
-        success_message="Package build succeeded!",
-        error_message="Package build failed due to:",
-        verbose=verbose,
+        command="python -m build --sdist --wheel",
+        quiet=quiet,
     )
+    print("Successfully built thehive4py!")
 
 
-def run_publish(verbose: bool):
+def run_upload(quiet: bool):
+    print("Publishing thehive4py with twine...")
     _run_subprocess(
-        args=("echo 'Publish command is not implemented yet...' && exit 1 "),
-        init_message="Publishing the package with twine...",
-        success_message="Publish succeeded!",
-        error_message="Publish failed due to:",
-        verbose=verbose,
+        command="twine upload dist/*",
+        quiet=quiet,
     )
+    print("Successfully published thehive4py!")
 
 
-def parse_arguments():
-    main_parser = argparse.ArgumentParser(
+def build_run_options() -> List[dict]:
+    return [
+        {"name": "build", "help": "run build step", "func": run_build},
+        {"name": "upload", "help": "run upload step", "func": run_upload},
+    ]
+
+
+def parse_arguments(run_options: List[dict]):
+    parser = argparse.ArgumentParser(
         prog="thehive4py-cd",
-        description="run all cd tasks or use sub commands to run cd tasks individually",
+        description="run all cd steps or use options to run cd steps selectively",
     )
-    main_parser.add_argument(
-        "-v",
-        "--verbose",
+    parser.add_argument(
+        "-q",
+        "--quiet",
         action="store_true",
         default=False,
-        help="generate verbose output",
+        help="silence verbose output",
     )
-    main_parser.set_defaults(func=run_all)
-
-    subparsers = main_parser.add_subparsers(help="commands")
-    subparser_options = [
-        {
-            "name": "build",
-            "help": "task to build the package",
-            "default_func": run_build,
-        },
-        {
-            "name": "publish",
-            "help": "task to publish the package",
-            "default_func": run_publish,
-        },
-    ]
 
-    for subparser_option in subparser_options:
-        _subparser = subparsers.add_parser(
-            name=subparser_option["name"],
-            help=subparser_option["help"],
-            parents=[main_parser],
-            add_help=False,
+    for run_option in run_options:
+        parser.add_argument(
+            f"--{run_option['name']}",
+            help=run_option["help"],
+            action="store_true",
         )
-        _subparser.set_defaults(func=subparser_option["default_func"])
 
-    return main_parser.parse_args()
+    return parser.parse_args()
 
 
 def main():
-    args = parse_arguments()
-    args.func(verbose=args.verbose)
+    run_options = build_run_options()
+    args = parse_arguments(run_options=run_options)
+
+    quiet = args.quiet
+
+    selective_runs = [
+        run_option["func"]
+        for run_option in run_options
+        if getattr(args, run_option["name"])
+    ]
+
+    if selective_runs:
+        for run in selective_runs:
+            run(quiet=quiet)
+    else:
+        run_all(quiet=quiet)
 
 
 if __name__ == "__main__":