Merge pull request #914 from projectsyn/renovate/kapitan-0.x

Update dependency kapitan to v0.34.0

.github/workflows/test.yml

@@ -72,32 +72,6 @@ jobs:
       run: echo "PYVER=$(echo ${{ matrix.python-version}} |cut -d. -f1,2)" >> $GITHUB_ENV
     - name: Run tests on Python ${{ matrix.python-version }}
       run: make test_py${PYVER}
-  tests_reclass_rs:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
-      with:
-        python-version: '3.11'
-    - uses: Gr1N/setup-poetry@v9
-    - name: Install tox
-      run: |
-        pip install tox
-    - uses: actions/cache@v4
-      with:
-        path: |
-          .tox
-          ~/.cache/pypoetry/virtualenvs
-        key: ${{ runner.os }}-test-py3.11-reclass-rs-${{ hashFiles('poetry.lock', 'tox.ini') }}
-    - name: Install jsonnet-bundler
-      run: |
-        mkdir -p /opt/bin && curl -sLo /opt/bin/jb \
-          https://github.com/projectsyn/jsonnet-bundler/releases/download/v0.6.1/jb_linux_amd64 \
-          && chmod +x /opt/bin/jb
-    - name: Update PATH
-      run: echo "/opt/bin" >> $GITHUB_PATH
-    - name: Run tests on Python 3.11 with reclass-rs
-      run: make test_py3.11_reclass_rs
   benchs:
     runs-on: ubuntu-latest
     strategy:

Dockerfile

@@ -20,10 +20,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY pyproject.toml poetry.lock ./
 
 RUN poetry config virtualenvs.create false \
- && poetry install --no-dev --no-root \
- && curl -L \
- https://raw.githubusercontent.com/projectsyn/reclass-rs/main/hack/kapitan_0.32_reclass_rs.patch \
- | patch -p1 -d /usr/local/lib/python3.11/site-packages/kapitan
+ && poetry install --no-dev --no-root
 
 COPY . ./
 

commodore/cli/__init__.py

@@ -6,6 +6,8 @@
 
 import click
 
+from reclass_rs import Reclass
+
 from dotenv import load_dotenv, find_dotenv
 from commodore import __git_version__, __version__
 from commodore.config import Config
@@ -68,6 +70,7 @@ def commodore(ctx, working_dir, verbose, request_timeout):
 
 def main():
     multiprocessing.set_start_method("spawn")
+    Reclass.set_thread_count(0)
 
     load_dotenv(dotenv_path=find_dotenv(usecwd=True))
     commodore.main(

commodore/component/compile.py

@@ -10,7 +10,6 @@
 
 import click
 import git
-from kapitan.resources import inventory_reclass
 
 from commodore.cluster import generate_target
 from commodore.config import Config
@@ -20,7 +19,7 @@
     create_component_library_aliases,
 )
 from commodore.dependency_mgmt.jsonnet_bundler import fetch_jsonnet_libraries
-from commodore.helpers import kapitan_compile, relsymlink, yaml_dump
+from commodore.helpers import kapitan_inventory, kapitan_compile, relsymlink, yaml_dump
 from commodore.inventory import Inventory
 from commodore.inventory.lint import check_removed_reclass_variables
 from commodore.postprocess import postprocess_components
@@ -82,7 +81,7 @@ def compile_component(
         )
 
         # Verify component alias
-        nodes = inventory_reclass(inv.inventory_dir)["nodes"]
+        nodes = kapitan_inventory(config)
         config.verify_component_aliases(nodes[instance_name]["parameters"])
 
         cluster_params = nodes[instance_name]["parameters"]

commodore/helpers.py

@@ -5,7 +5,9 @@
 import json
 import shutil
 import os
+import sys
 from collections.abc import Callable, Iterable
+from datetime import datetime
 from pathlib import Path as P
 from typing import Optional
 
@@ -24,7 +26,8 @@
 from kapitan.cached import reset_cache as reset_reclass_cache
 from kapitan.refs.base import RefController, PlainRef
 from kapitan.refs.secrets.vaultkv import VaultBackend
-from kapitan.resources import inventory_reclass
+
+from reclass_rs import Reclass
 
 from commodore import __install_dir__
 from commodore.config import Config
@@ -33,8 +36,9 @@
 ArgumentCache = collections.namedtuple(
     "ArgumentCache",
     [
+        "inventory_backend",
         "inventory_path",
-        "yaml_multiline_string_style",
+        "multiline_string_style",
         "yaml_dump_null_as_empty",
     ],
 )
@@ -240,20 +244,20 @@ def kapitan_compile(
     if fake_refs:
         refController.register_backend(FakeVaultBackend())
     click.secho("Compiling catalog...", bold=True)
-    cached.args["compile"] = ArgumentCache(
-        inventory_path=config.inventory.inventory_dir,
-        yaml_multiline_string_style="literal",
-        yaml_dump_null_as_empty=False,
-    )
+    # workaround the non-modifiable Namespace() default value for cached.args
+    cached.args.inventory_backend = "reclass-rs"
+    cached.args.inventory_path = str(config.inventory.inventory_dir)
+    cached.args.multiline_string_style = "literal"
+    cached.args.yaml_dump_null_as_empty = False
+    cached.args.verbose = config.trace
     kapitan_targets.compile_targets(
-        inventory_path=config.inventory.inventory_dir,
+        inventory_path=cached.args.inventory_path,
         search_paths=search_paths,
         output_path=output_dir,
-        targets=targets,
-        parallel=4,
+        desired_targets=targets,
+        parallelism=4,
         labels=None,
         ref_controller=refController,
-        verbose=config.trace,
         prune=False,
         indent=2,
         reveal=reveal,
@@ -272,14 +276,23 @@ def kapitan_inventory(
     config: Config, key: str = "nodes", ignore_class_notfound: bool = False
 ) -> dict:
     """
-    Reset reclass cache and render inventory.
     Returns the top-level key according to the kwarg.
     """
-    reset_reclass_cache()
-    inv = inventory_reclass(
-        config.inventory.inventory_dir, ignore_class_notfound=ignore_class_notfound
+    r = Reclass(
+        nodes_path=str(config.inventory.targets_dir),
+        classes_path=str(config.inventory.classes_dir),
+        ignore_class_notfound=ignore_class_notfound,
     )
-    return inv[key]
+    print("running reclass_rs", file=sys.stderr)
+    start = datetime.now()
+    try:
+        inv = r.inventory()
+    except ValueError as e:
+        raise click.ClickException(f"While rendering inventory: {e}")
+    elapsed = datetime.now() - start
+    print(f"Inventory (reclass_rs) took {elapsed}", file=sys.stderr)
+
+    return inv.as_dict()[key]
 
 
 def rm_tree_contents(basedir):

commodore/postprocess/jsonnet.py

@@ -32,7 +32,7 @@ def _try_path(basedir: P, rel: str):
     if not full_path.is_file():
         return full_path.name, None
     with open(full_path, encoding="utf-8") as f:
-        return full_path.name, f.read()
+        return full_path.name, f.read().encode("utf-8")
 
 
 def _import_callback_with_searchpath(search: Iterable[P], basedir: P, rel: str):

docs/modules/ROOT/pages/explanation/running-commodore.adoc

@@ -89,30 +89,6 @@ brew install libmagic
 commodore --version
 ----
 
-. Finally, patch Kapitan to use `reclass-rs` over the bundled Python implementation.
-This significantly improves compilation times of Commodore cluster catalogs.
-+
-[source,bash]
-----
-KAPITAN_PATH=$(python -c 'import kapitan; print(kapitan.__path__[0])')
-curl -L \
-  https://raw.githubusercontent.com/projectsyn/reclass-rs/main/hack/kapitan_0.32_reclass_rs.patch \
-  | patch -p1 -d "${KAPITAN_PATH}"
-----
-+
-[NOTE]
-====
-The `reclass-rs` Python package is installed automatically as a dependency of Commodore.
-For officially supported platforms (Linux, macOS), prebuilt wheels are available on PyPI.
-====
-+
-[TIP]
-====
-This step is optional, but we highly recommend using `reclass-rs` over Kapitan's bundled Reclass implementation.
-
-We're actively working with Kapitan to upstream support for `reclass-rs`, so this step may disappear again in future versions of this page.
-====
-
 === Usage
 
 If you've installed Commodore into a virtualenv, you either need to activate the virtualenv whenever you want to run Commodore, or you can create a symlink to the `commodore` entrypoint script in a directory which is part of your `$PATH`.