Features 0.5 (#130)

* feat: Improving bash functionality, map catalog being specific to iterable

* fix: release process PSR fix to 7.x.x

* fix: release process PSR fix to 7.x.x

* feat: Working container task type

* feat: Adding required accomodations for local container

* feat: accommodating executor

* feat: working container in local with data and parameters synced

* feat: Releasing container task type

* feat: Improving SDK

* feat: Improving SDK

* feat: Improving SDK

* docs: Improving SDK

* fix: WIP working with stricter mypy

* fix: Stricter mypy WIP

* fix: Fixed mypy and test. To resume working on container related stuff

* feat: Final checks to container task type

* feat: Need to check job execution command in a few scenarios

* feat: Better .gitignore

* fix: WUP still in process

* fix: mypy issues

* fix: Adding service_type to BaseProviders

* fix: updating sematic release

* fix: todo for containers

* fix: merging chat.tests

* ci: figuring out the release

* ci: chaning magnus path

* ci: fixing poetry install

* ci: forcing release

* feat: Making integrations simple within magnus

* feat: Moving to Rich and adding examples

* fix: Trying to get the logger working

* fix: logging levels different for client code and magnus

* fix: Adding log level example

* chore: Moving coverage to pyproject.toml, making integrations smoother

* feat: Adding K8's extensions modules into magnus

* docs: Adding a bit more clear examples

* docs: Adding a bit more clear examples

* feat: Getting pyinstaller into play

* fix: remove docker build as it extends scope

* fix: Moving to better statically typed code

* fix: Refactor context

* feat: Proper base services and implemtations moving to extensions

* feat: Moving to base vs implementation model

* perf: Improving the start up time

* fix: Adding the test structure

* style: adding gitlint

* fix: updating to pydantiv v2

* fix: upgrade mypy and bring back standards

* test: more and more tests

* fix: Fixing pydantic nesting

* test: adding more tests

* test: Still working on tests

* test: working with tests and typing

* test: added more tests, renamed as-is to stub

* feat: in the process of adding a generic chunked data store

* fix: generic chunked data store

* feat: Bringing in argo

* fix: W I P working on argo

* fix: W I P working on argo

* feat: Brining in Argo

* fix: mypy fixes

* feat: Adding SDK nodes and testing

* feat: Adding SDK nodes and testing

* feat: workking on parallel sdk

* feat: working on SDK, still wrok inprogress

* feat: Working SDK which looks good

* fix: TODO about the catalog

* feat: improved testing and system to bring in other features

* docs: Add docs

* docs: Adding more docs and making it better

* feat: Improved parameters, still working on it

* feat: Improved parameter passing

* docs: improved documentation

* feat: Shell can now interact with paramters

* feat: improved secrets manager

* docs: more and more docs

* feat: Improving docs and examples

* feat: Improving docs and examples

* test: fixed broken tests

* test: passing tests

* fix: Fixing catalog settings

* test: Adding all examples as tests

* feat: experiment tracking

* feat: incrememntal experiment tracking

* feat: incrememntal experiment tracking

* feat: incrememntal experiment tracking

* feat: mlflow integration

* feat: experiment tracking and more docs

* feat: working local container

* feat: still need to make tests pass

* feat: adding map to sdk, still working

* feat: map in sdk

* feat: map in sdk

* feat: map in sdk

* feat: dynamic configuration file for sdk

* feat: Deeply nested SDK

* feat: brining in local container

* feat: brining in local container

* feat: Brining in step overrides

* feat: Bringing in argo, still working

* feat: Bringing in argo

* docs: more and more docs

* feat: Adding mocked executor

* feat: mocked executors

* docs: trying mermaid

* docs: trying mermaid

* docs: trying mermaid

* docs: trying mermaid

* docs: trying mermaid

* docs: trying mermaid

* docs: trying mermaid

* docs: trying mermaid

* feat: Getting a bit closer to release

* docs: Updated references

* fix: broken test

.github/workflows/pr.yaml

@@ -14,6 +14,22 @@ jobs:
         with:
           python-version: 3.8
       - run: python -m pip install poetry
+      - run: |
+          # Download the binary
+          curl -sLO https://github.com/argoproj/argo-workflows/releases/download/v3.5.4/argo-linux-amd64.gz
+
+          # Unzip
+          gunzip argo-linux-amd64.gz
+
+          # Make binary executable
+          chmod +x argo-linux-amd64
+
+          # Move binary to path
+          mv ./argo-linux-amd64 /usr/local/bin/argo
+
+          # Test installation
+          argo version
+
       - run: |
           python -m poetry install
           poetry run tox

.github/workflows/release.yaml

@@ -4,6 +4,7 @@ on:
       - "magnus/**"
     branches:
       - "main"
+      - "rc"
 
 jobs:
   PRCheck:
@@ -17,6 +18,7 @@ jobs:
       - run: |
           python -m poetry install
           poetry run tox
+
   Release:
     runs-on: ubuntu-latest
     needs: PRCheck
@@ -27,15 +29,28 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: 3.8
-      - run: python -m pip install python-semantic-release==7.34.6
+      - run: python -m pip install python-semantic-release==8.0.7
       - name: Figure version
+        continue-on-error: true
         id: last_tag
         run: |
 
-          VERSION=$(semantic-release print-version --patch)
-          echo $VERSION
+          CURRENT=$(git tag --sort=-committerdate -l | head -n 1)
+          echo "current: $CURRENT"
+
+          VERSION=$(python -m semantic-release --noop --strict version --no-push --no-commit --print)
+          echo "New: $VERSION"
+
+          if [ "$CURRENT" == "$VERSION" ]; then
+            echo "version="" >> $GITHUB_OUTPUT
+            exit 1
+          fi
+
           echo "version=$VERSION" >> $GITHUB_OUTPUT
+          exit 0
+
       - name: Apply new tag
+        if: steps.last_tag.outcome == 'success'
         env:
           VERSION: ${{ steps.last_tag.outputs.version }}
         uses: actions/github-script@v6
@@ -50,6 +65,7 @@ jobs:
               sha: context.sha
             })
       - name: Publish to PyPI
+        if: steps.last_tag.outcome == 'success'
         env:
           PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
           LAST_TAG: ${{ steps.last_tag.outputs.version }}
@@ -60,6 +76,7 @@ jobs:
           poetry publish --build
 
       - name: "Create release"
+        if: steps.last_tag.outcome == 'success'
         env:
           RELEASE_TAG: ${{ steps.last_tag.outputs.version }}
         uses: "actions/github-script@v6"

.gitignore

@@ -141,3 +141,16 @@ cython_debug/
 
 # VSCode specific settings
 .vscode
+
+
+.run_log_store/
+
+.catalog/
+
+cov.xml
+
+.DS_Store
+
+data/
+
+example_bak/

.mypy.ini

@@ -1,6 +1,24 @@
 [mypy]
-implicit_optional = True
-ignore_missing_imports = True
+exclude = _*FF.py
+ignore_missing_imports = True
+plugins = pydantic.mypy
+show_error_codes = True
+
+follow_imports = silent
+warn_redundant_casts = True
+warn_unused_ignores = True
+#disallow_any_generics = True # This complicates it a lot
+check_untyped_defs = True
+implicit_reexport = True
+
+# for strict mypy: (this is the tricky one :-))
+#disallow_untyped_defs = True
+
+[pydantic-mypy]
+init_forbid_extra = True
+init_typed = True
+warn_required_dynamic_aliases = True
+warn_untyped_fields = True
 
 [mypy-ruamel.*]
 ignore_missing_imports = True

.pre-commit-config.yaml

@@ -27,3 +27,16 @@ repos:
         # pre-commit's default_language_version, see
         # https://pre-commit.com/#top_level-default_language_version
         language_version: python3.8
+
+  - repo: https://github.com/jorisroovers/gitlint
+    rev: v0.19.1
+    hooks:
+      - id: gitlint
+        args:
+          [
+            --contrib=CT1,
+            --ignore=title-trailing-punctuation,
+            --ignore=T3,
+            --ignore=B6,
+            --msg-filename,
+          ]