diff --git a/Makefile b/Makefile index 7434d33ea..ded989a72 100644 --- a/Makefile +++ b/Makefile @@ -57,3 +57,10 @@ local_serve_documentation: mkdocs_gh_deploy: # to run locally assuming git ssh access docker build -f Dockerfile.docs --no-cache -t kapitan-docs . docker run --rm -it -v $(PWD):/src -v ~/.ssh:/root/.ssh -w /src kapitan-docs gh-deploy -f ./mkdocs.yml + +pull_oc: + rm -rf omegaconf + git clone --branch 1080-add-list-deep-merging https://github.com/nexenio/omegaconf.git oc + pip install -r oc/requirements/dev.txt -e oc/ + mv oc/omegaconf . + rm -rf oc diff --git a/kapitan/cli.py b/kapitan/cli.py index a4fca3225..63e748518 100644 --- a/kapitan/cli.py +++ b/kapitan/cli.py @@ -306,6 +306,20 @@ def build_parser(): help="dumps all none-type entries as empty, default is dumping as 'null'", ) + compile_parser.add_argument( + "--omegaconf", + help="use omegaconf as inventory backend", + action="store_true", + default=from_dot_kapitan("compile", "omegaconf", False), + ) + + compile_parser.add_argument( + "--migrate", + help="migrate inventory to omegaconf", + action="store_true", + default=from_dot_kapitan("compile", "migrate", False), + ) + compile_selector_parser = compile_parser.add_mutually_exclusive_group() compile_selector_parser.add_argument( "--targets", @@ -379,6 +393,12 @@ def build_parser(): default=from_dot_kapitan("inventory", "multiline-string-style", "double-quotes"), help="set multiline string style to STYLE, default is 'double-quotes'", ) + inventory_parser.add_argument( + "--omegaconf", + help="use omegaconf as inventory backend", + action="store_true", + default=from_dot_kapitan("inventory", "omegaconf", False), + ) searchvar_parser = subparser.add_parser( "searchvar", aliases=["sv"], help="show all inventory files where var is declared" @@ -505,6 +525,12 @@ def build_parser(): action="store_true", default=from_dot_kapitan("refs", "verbose", False), ) + refs_parser.add_argument( + "--omegaconf", + help="use omegaconf as inventory backend", + action="store_true", + default=from_dot_kapitan("inventory", "omegaconf", False), + ) lint_parser = subparser.add_parser("lint", aliases=["l"], help="linter for inventory and refs") lint_parser.set_defaults(func=start_lint, name="lint") @@ -548,6 +574,12 @@ def build_parser(): default=from_dot_kapitan("lint", "inventory-path", "./inventory"), help='set inventory path, default is "./inventory"', ) + lint_parser.add_argument( + "--omegaconf", + help="use omegaconf as inventory backend", + action="store_true", + default=from_dot_kapitan("inventory", "omegaconf", False), + ) init_parser = subparser.add_parser( "init", help="initialize a directory with the recommended kapitan project skeleton." diff --git a/kapitan/omegaconf_inv.py b/kapitan/omegaconf_inv.py new file mode 100644 index 000000000..0a2eb7dbc --- /dev/null +++ b/kapitan/omegaconf_inv.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 + +# Copyright 2023 nexenio +import logging +import os +import time + +import regex + +from kapitan.errors import InventoryError +from kapitan.resolvers import register_resolvers +from omegaconf import ListMergeMode, Node, OmegaConf, errors + +logger = logging.getLogger(__name__) + + +def inventory_omegaconf( + inventory_path: str, + ignore_class_notfound: bool = False, + targets: list = [], + compose_node_name: bool = False, +) -> dict: + """ + generates inventory from yaml files using OmegaConf + """ + + # add config option to specify paths + targets_searchpath = os.path.join(inventory_path, "targets") + classes_searchpath = os.path.join(inventory_path, "classes") + + register_resolvers(inventory_path) + + selected_targets = [] + + # loop through targets searchpath and load all targets + for root, dirs, files in os.walk(targets_searchpath): + for target_name in files: + target_path = os.path.join(root, target_name) + + # split file extension and check if yml/yaml + target_name, ext = os.path.splitext(target_name) + if ext not in (".yml", ".yaml"): + logger.debug(f"{target_name}: targets have to be .yml or .yaml files.") + # RAISE ERROR + continue + + # skip targets if they are not specified with -t flag + if targets and target_name not in targets: + continue + + # compose node name + if compose_node_name: + target_name = str(os.path.splitext(target_path)[0]).replace(targets_searchpath + os.sep, "") + target_name = target_name.replace("/", ".") + + selected_targets.append({"name": target_name, "path": target_path}) + + # using nodes for reclass legacy code + inv = {"nodes": {}} + + # prepare logging + logger.info(f"Found {len(selected_targets)} targets") + + # load targets + for target in selected_targets: + try: + # start = time.time() + name, config = load_target(target, classes_searchpath, ignore_class_notfound) + inv["nodes"][name] = config + # print(time.time() - start) + except Exception as e: + raise InventoryError(f"{target['name']}: {e}") + + return inv + + +def load_target(target: dict, classes_searchpath: str, ignore_class_notfound: bool = False): + """ + load only one target with all its classes + """ + + target_name = target["name"] + target_path = target["path"] + + target_config = OmegaConf.load(target_path) + target_config_classes = target_config.get("classes", []) + target_config_parameters = OmegaConf.create(target_config.get("parameters", {})) + target_config = {} + + classes_redundancy_check = set() + + # load classes for targets + for class_name in target_config_classes: + # resolve class path + class_path = os.path.join(classes_searchpath, *class_name.split(".")) + + if class_path in classes_redundancy_check: + continue + + classes_redundancy_check.add(class_path) + + if os.path.isfile(class_path + ".yml"): + class_path += ".yml" + elif os.path.isdir(class_path): + # search for init file + init_path = os.path.join(classes_searchpath, *class_name.split("."), "init") + ".yml" + if os.path.isfile(init_path): + class_path = init_path + elif ignore_class_notfound: + logger.debug(f"Could not find {class_path}") + continue + else: + raise InventoryError(f"Class {class_name} not found.") + + # load classes recursively + class_config = OmegaConf.load(class_path) + + # resolve relative class names + new_classes = class_config.pop("classes", []) + for new in new_classes: + if new.startswith("."): + new = ".".join(class_name.split(".")[0:-1]) + new + + target_config_classes.append(new) + + class_config_parameters = OmegaConf.create(class_config.get("parameters", {})) + + # merge target with loaded classes + if target_config_parameters: + target_config_parameters = OmegaConf.unsafe_merge( + class_config_parameters, target_config_parameters, list_merge_mode=ListMergeMode.EXTEND + ) + else: + target_config_parameters = class_config_parameters + + if not target_config_parameters: + raise InventoryError("empty target") + + # append meta data (legacy: _reclass_) + target_config_parameters["_reclass_"] = { + "name": { + "full": target_name, + "parts": target_name.split("."), + "path": target_name.replace(".", "/"), + "short": target_name.split(".")[-1], + } + } + + # resolve references / interpolate values + OmegaConf.resolve(target_config_parameters) + target_config["parameters"] = OmegaConf.to_object(target_config_parameters) + + # obtain target name to insert in inv dict + try: + target_name = target_config["parameters"]["kapitan"]["vars"]["target"] + except KeyError: + logger.warning(f"Could not resolve target name on target {target_name}") + + return target_name, target_config + + +def migrate(inventory_path: str) -> None: + """migrates all .yml/.yaml files in the given path to omegaconfs syntax""" + + for root, subdirs, files in os.walk(inventory_path): + for file in files: + file = os.path.join(root, file) + name, ext = os.path.splitext(file) + + if ext not in (".yml", ".yaml"): + continue + + try: + with open(file, "r+") as file: + content = file.read() + file.seek(0) + + # replace colons in tags and replace _reclass_ with _meta_ + updated_content = regex.sub( + r"(? None: + """register pre-defined and user-defined resolvers""" + + # yaml key utility functions + OmegaConf.register_new_resolver("key", key) + OmegaConf.register_new_resolver("parentkey", parentkey) + OmegaConf.register_new_resolver("fullkey", fullkey) + OmegaConf.register_new_resolver("relpath", relpath) + + # yaml object utility functions + OmegaConf.register_new_resolver("tag", escape_tag) + OmegaConf.register_new_resolver("merge", merge) + OmegaConf.register_new_resolver("dict", to_dict) + OmegaConf.register_new_resolver("list", to_list) + + # kapitan helpers / templates + OmegaConf.register_new_resolver("helm_dep", helm_dep) + OmegaConf.register_new_resolver("helm_input", helm_input) + + # user defined resolvers + user_resolver_file = os.path.join(inventory_path, "resolvers.py") + if os.path.exists(user_resolver_file): + try: + register_user_resolvers(inventory_path) + except: + logger.debug(f"Couldn't import {os.join(inventory_path, 'resolvers.py')}") + + +def register_user_resolvers(inventory_path: str) -> None: + """import user resolvers specified in inventory/resolvers.py""" + try: + import_path = os.path.join(os.getcwd(), inventory_path) + sys.path.append(import_path) + from resolvers import pass_resolvers + + funcs = pass_resolvers() + except: + logger.warning("resolvers.py must contain function 'pass_resolvers()'") + return + + if not isinstance(funcs, dict): + logger.warning("pass_resolvers() should return a dict") + return + + import resolvers + + for name, func in funcs.items(): + try: + OmegaConf.register_new_resolver(name, func, replace=True) + except: + logger.warning(f"Could not load resolver {name}") diff --git a/kapitan/resources.py b/kapitan/resources.py index e77f4c885..b573ba715 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -14,19 +14,21 @@ import logging import os import sys +import time from functools import partial import jsonschema -import kapitan.cached as cached +import reclass +import reclass.core import yaml +from reclass.errors import NotFoundError, ReclassException + +import kapitan.cached as cached from kapitan import __file__ as kapitan_install_path from kapitan.errors import CompileError, InventoryError, KapitanError +from kapitan.omegaconf_inv import inventory_omegaconf, migrate from kapitan.utils import PrettyDumper, deep_get, flatten_dict, render_jinja2_file, sha256_string -import reclass -import reclass.core -from reclass.errors import NotFoundError, ReclassException - logger = logging.getLogger(__name__) try: @@ -280,14 +282,14 @@ def inventory(search_paths, target, inventory_path=None): raise InventoryError(f"Inventory not found in search paths: {search_paths}") if target is None: - return inventory_reclass(full_inv_path)["nodes"] + return get_inventory(full_inv_path)["nodes"] - return inventory_reclass(full_inv_path)["nodes"][target] + return get_inventory(full_inv_path)["nodes"][target] def generate_inventory(args): try: - inv = inventory_reclass(args.inventory_path) + inv = get_inventory(args.inventory_path) if args.target_name != "": inv = inv["nodes"][args.target_name] if args.pattern != "": @@ -304,6 +306,52 @@ def generate_inventory(args): sys.exit(1) +def get_inventory(inventory_path, ignore_class_notfound=False, targets=[]): + """ + generic inventory function that makes inventory backend pluggable + default backend is reclass + """ + + # if inventory is already cached theres nothing to do + if cached.inv: + return cached.inv + + # get parsed args from cached.py + args = list(cached.args.values())[0] + use_omegaconf = args.omegaconf + try: + migrate_omegaconf = args.migrate + except: + migrate_omegaconf = False + + if use_omegaconf: + # show warning + logger.debug("Using omegaconf as inventory backend") + logger.warning("\033[0;33mNOTE: OmegaConf inventory is currently in experimental mode.\033[0m") + + # migrate a reclass inventory to omegaConf + if migrate_omegaconf: + migrate_start = time.time() + migrate(inventory_path) + logger.info("Migrated inventory to OmegaConf (%.2fs)", time.time() - migrate_start) + try: + # inv_start = time.time() + inv = inventory_omegaconf(inventory_path, ignore_class_notfound, targets) + # logger.info("REAL_TIME (%.2fs)", time.time() - inv_start) + except Exception as e: + if not migrate_omegaconf: + logger.warning("Make sure to migrate your inventory using --migrate") + raise InventoryError(e) + else: + logger.debug("Using reclass as inventory backend") + # inv_start = time.time() + inv = inventory_reclass(inventory_path, ignore_class_notfound) + # logger.info("REAL_TIME (%.2fs)", time.time() - inv_start) + + cached.inv = inv + return inv + + def inventory_reclass(inventory_path, ignore_class_notfound=False): """ Runs a reclass inventory in inventory_path @@ -314,12 +362,8 @@ def inventory_reclass(inventory_path, ignore_class_notfound=False): Does not throw errors if a class is not found while --fetch flag is enabled """ - # if inventory is already cached theres nothing to do - if cached.inv: - return cached.inv - # set default values initially - reclass_config = reclass_config_defaults = { + reclass_config = { "storage_type": "yaml_fs", "inventory_base_uri": inventory_path, "nodes_uri": "targets", @@ -358,12 +402,10 @@ def inventory_reclass(inventory_path, ignore_class_notfound=False): class_mappings = reclass_config.get("class_mappings") # this defaults to None (disabled) _reclass = reclass.core.Core(storage, class_mappings, reclass.settings.Settings(reclass_config)) - cached.inv = _reclass.inventory() + return _reclass.inventory() except ReclassException as e: if isinstance(e, NotFoundError): logger.error("Inventory reclass error: inventory not found") else: logger.error("Inventory reclass error: %s", e.message) raise InventoryError(e.message) - - return cached.inv diff --git a/kapitan/targets.py b/kapitan/targets.py index c5e18e36f..48b6532da 100644 --- a/kapitan/targets.py +++ b/kapitan/targets.py @@ -32,7 +32,7 @@ from kapitan.inputs.kadet import Kadet from kapitan.inputs.remove import Remove from kapitan.remoteinventory.fetch import fetch_inventories, list_sources -from kapitan.resources import inventory_reclass +from kapitan.resources import get_inventory from kapitan.utils import dictionary_hash, directory_hash, hashable_lru_cache from kapitan.validator.kubernetes_validator import KubernetesManifestValidator @@ -241,7 +241,7 @@ def generate_inv_cache_hashes(inventory_path, targets, cache_paths): ... } """ - inv = inventory_reclass(inventory_path) + inv = get_inventory(inventory_path) cached.inv_cache = {} cached.inv_cache["inventory"] = {} cached.inv_cache["folder"] = {} @@ -295,7 +295,7 @@ def generate_inv_cache_hashes(inventory_path, targets, cache_paths): def changed_targets(inventory_path, output_path): """returns a list of targets that have changed since last compilation""" targets = [] - inv = inventory_reclass(inventory_path) + inv = get_inventory(inventory_path) saved_inv_cache = None saved_inv_cache_path = os.path.join(output_path, "compiled/.kapitan_cache") @@ -388,7 +388,7 @@ def save_inv_cache(compile_path, targets): def load_target_inventory(inventory_path, targets, ignore_class_notfound=False): """returns a list of target objects from the inventory""" target_objs = [] - inv = inventory_reclass(inventory_path, ignore_class_notfound) + inv = get_inventory(inventory_path, ignore_class_notfound, targets) # if '-t' is set on compile, only loop through selected targets if targets: @@ -431,7 +431,7 @@ def search_targets(inventory_path, targets, labels): ) targets_found = [] - inv = inventory_reclass(inventory_path) + inv = get_inventory(inventory_path) for target_name in inv["nodes"]: matched_all_labels = False diff --git a/poetry.lock b/poetry.lock index d33ff334f..12cd99a0e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -12,6 +12,17 @@ files = [ {file = "addict-2.4.0.tar.gz", hash = "sha256:b3b2210e0e067a281f5646c8c5db92e99b7231ea8b0eb5f74dbdf9e259d4e494"}, ] +[[package]] +name = "antlr4-python3-runtime" +version = "4.9.3" +description = "ANTLR 4.9.3 runtime for Python 3.7" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b"}, +] + [[package]] name = "attrs" version = "22.2.0" @@ -813,6 +824,26 @@ portalocker = [ {version = ">=1.6,<3", markers = "python_version >= \"3.5\" and platform_system == \"Windows\""}, ] +[[package]] +name = "omegaconf" +version = "2.4.0.dev0" +description = "A flexible configuration library" +category = "main" +optional = false +python-versions = ">=3.6" +files = [] +develop = false + +[package.dependencies] +antlr4-python3-runtime = ">=4.9.0,<4.10.0" +PyYAML = ">=5.1.0" + +[package.source] +type = "git" +url = "https://github.com/omry/omegaconf.git" +reference = "HEAD" +resolved_reference = "7dae67e4a3869b584fd6d6408b2f916c176755db" + [[package]] name = "packaging" version = "23.0" @@ -1200,6 +1231,104 @@ files = [ {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +[[package]] +name = "regex" +version = "2023.5.5" +description = "Alternative regular expression module, to replace re." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "regex-2023.5.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:48c9ec56579d4ba1c88f42302194b8ae2350265cb60c64b7b9a88dcb7fbde309"}, + {file = "regex-2023.5.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02f4541550459c08fdd6f97aa4e24c6f1932eec780d58a2faa2068253df7d6ff"}, + {file = "regex-2023.5.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:53e22e4460f0245b468ee645156a4f84d0fc35a12d9ba79bd7d79bdcd2f9629d"}, + {file = "regex-2023.5.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b870b6f632fc74941cadc2a0f3064ed8409e6f8ee226cdfd2a85ae50473aa94"}, + {file = "regex-2023.5.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:171c52e320fe29260da550d81c6b99f6f8402450dc7777ef5ced2e848f3b6f8f"}, + {file = "regex-2023.5.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad5524c2aedaf9aa14ef1bc9327f8abd915699dea457d339bebbe2f0d218f86"}, + {file = "regex-2023.5.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a0f874ee8c0bc820e649c900243c6d1e6dc435b81da1492046716f14f1a2a96"}, + {file = "regex-2023.5.5-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e645c757183ee0e13f0bbe56508598e2d9cd42b8abc6c0599d53b0d0b8dd1479"}, + {file = "regex-2023.5.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a4c5da39bca4f7979eefcbb36efea04471cd68db2d38fcbb4ee2c6d440699833"}, + {file = "regex-2023.5.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5e3f4468b8c6fd2fd33c218bbd0a1559e6a6fcf185af8bb0cc43f3b5bfb7d636"}, + {file = "regex-2023.5.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:59e4b729eae1a0919f9e4c0fc635fbcc9db59c74ad98d684f4877be3d2607dd6"}, + {file = "regex-2023.5.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ba73a14e9c8f9ac409863543cde3290dba39098fc261f717dc337ea72d3ebad2"}, + {file = "regex-2023.5.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0bbd5dcb19603ab8d2781fac60114fb89aee8494f4505ae7ad141a3314abb1f9"}, + {file = "regex-2023.5.5-cp310-cp310-win32.whl", hash = "sha256:40005cbd383438aecf715a7b47fe1e3dcbc889a36461ed416bdec07e0ef1db66"}, + {file = "regex-2023.5.5-cp310-cp310-win_amd64.whl", hash = "sha256:59597cd6315d3439ed4b074febe84a439c33928dd34396941b4d377692eca810"}, + {file = "regex-2023.5.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f08276466fedb9e36e5193a96cb944928301152879ec20c2d723d1031cd4ddd"}, + {file = "regex-2023.5.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cd46f30e758629c3ee91713529cfbe107ac50d27110fdcc326a42ce2acf4dafc"}, + {file = "regex-2023.5.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2910502f718828cecc8beff004917dcf577fc5f8f5dd40ffb1ea7612124547b"}, + {file = "regex-2023.5.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:445d6f4fc3bd9fc2bf0416164454f90acab8858cd5a041403d7a11e3356980e8"}, + {file = "regex-2023.5.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18196c16a584619c7c1d843497c069955d7629ad4a3fdee240eb347f4a2c9dbe"}, + {file = "regex-2023.5.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33d430a23b661629661f1fe8395be2004006bc792bb9fc7c53911d661b69dd7e"}, + {file = "regex-2023.5.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:72a28979cc667e5f82ef433db009184e7ac277844eea0f7f4d254b789517941d"}, + {file = "regex-2023.5.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f764e4dfafa288e2eba21231f455d209f4709436baeebb05bdecfb5d8ddc3d35"}, + {file = "regex-2023.5.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:23d86ad2121b3c4fc78c58f95e19173790e22ac05996df69b84e12da5816cb17"}, + {file = "regex-2023.5.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:690a17db524ee6ac4a27efc5406530dd90e7a7a69d8360235323d0e5dafb8f5b"}, + {file = "regex-2023.5.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:1ecf3dcff71f0c0fe3e555201cbe749fa66aae8d18f80d2cc4de8e66df37390a"}, + {file = "regex-2023.5.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:811040d7f3dd9c55eb0d8b00b5dcb7fd9ae1761c454f444fd9f37fe5ec57143a"}, + {file = "regex-2023.5.5-cp311-cp311-win32.whl", hash = "sha256:c8c143a65ce3ca42e54d8e6fcaf465b6b672ed1c6c90022794a802fb93105d22"}, + {file = "regex-2023.5.5-cp311-cp311-win_amd64.whl", hash = "sha256:586a011f77f8a2da4b888774174cd266e69e917a67ba072c7fc0e91878178a80"}, + {file = "regex-2023.5.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b6365703e8cf1644b82104cdd05270d1a9f043119a168d66c55684b1b557d008"}, + {file = "regex-2023.5.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a56c18f21ac98209da9c54ae3ebb3b6f6e772038681d6cb43b8d53da3b09ee81"}, + {file = "regex-2023.5.5-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8b942d8b3ce765dbc3b1dad0a944712a89b5de290ce8f72681e22b3c55f3cc8"}, + {file = "regex-2023.5.5-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:844671c9c1150fcdac46d43198364034b961bd520f2c4fdaabfc7c7d7138a2dd"}, + {file = "regex-2023.5.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2ce65bdeaf0a386bb3b533a28de3994e8e13b464ac15e1e67e4603dd88787fa"}, + {file = "regex-2023.5.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fee0016cc35a8a91e8cc9312ab26a6fe638d484131a7afa79e1ce6165328a135"}, + {file = "regex-2023.5.5-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:18f05d14f14a812fe9723f13afafefe6b74ca042d99f8884e62dbd34dcccf3e2"}, + {file = "regex-2023.5.5-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:941b3f1b2392f0bcd6abf1bc7a322787d6db4e7457be6d1ffd3a693426a755f2"}, + {file = "regex-2023.5.5-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:921473a93bcea4d00295799ab929522fc650e85c6b9f27ae1e6bb32a790ea7d3"}, + {file = "regex-2023.5.5-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:e2205a81f815b5bb17e46e74cc946c575b484e5f0acfcb805fb252d67e22938d"}, + {file = "regex-2023.5.5-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:385992d5ecf1a93cb85adff2f73e0402dd9ac29b71b7006d342cc920816e6f32"}, + {file = "regex-2023.5.5-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:890a09cb0a62198bff92eda98b2b507305dd3abf974778bae3287f98b48907d3"}, + {file = "regex-2023.5.5-cp36-cp36m-win32.whl", hash = "sha256:821a88b878b6589c5068f4cc2cfeb2c64e343a196bc9d7ac68ea8c2a776acd46"}, + {file = "regex-2023.5.5-cp36-cp36m-win_amd64.whl", hash = "sha256:7918a1b83dd70dc04ab5ed24c78ae833ae8ea228cef84e08597c408286edc926"}, + {file = "regex-2023.5.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:338994d3d4ca4cf12f09822e025731a5bdd3a37aaa571fa52659e85ca793fb67"}, + {file = "regex-2023.5.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a69cf0c00c4d4a929c6c7717fd918414cab0d6132a49a6d8fc3ded1988ed2ea"}, + {file = "regex-2023.5.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f5e06df94fff8c4c85f98c6487f6636848e1dc85ce17ab7d1931df4a081f657"}, + {file = "regex-2023.5.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8906669b03c63266b6a7693d1f487b02647beb12adea20f8840c1a087e2dfb5"}, + {file = "regex-2023.5.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fda3e50abad8d0f48df621cf75adc73c63f7243cbe0e3b2171392b445401550"}, + {file = "regex-2023.5.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ac2b7d341dc1bd102be849d6dd33b09701223a851105b2754339e390be0627a"}, + {file = "regex-2023.5.5-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:fb2b495dd94b02de8215625948132cc2ea360ae84fe6634cd19b6567709c8ae2"}, + {file = "regex-2023.5.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:aa7d032c1d84726aa9edeb6accf079b4caa87151ca9fabacef31fa028186c66d"}, + {file = "regex-2023.5.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3d45864693351c15531f7e76f545ec35000d50848daa833cead96edae1665559"}, + {file = "regex-2023.5.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21e90a288e6ba4bf44c25c6a946cb9b0f00b73044d74308b5e0afd190338297c"}, + {file = "regex-2023.5.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:10250a093741ec7bf74bcd2039e697f519b028518f605ff2aa7ac1e9c9f97423"}, + {file = "regex-2023.5.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6b8d0c153f07a953636b9cdb3011b733cadd4178123ef728ccc4d5969e67f3c2"}, + {file = "regex-2023.5.5-cp37-cp37m-win32.whl", hash = "sha256:10374c84ee58c44575b667310d5bbfa89fb2e64e52349720a0182c0017512f6c"}, + {file = "regex-2023.5.5-cp37-cp37m-win_amd64.whl", hash = "sha256:9b320677521aabf666cdd6e99baee4fb5ac3996349c3b7f8e7c4eee1c00dfe3a"}, + {file = "regex-2023.5.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:afb1c70ec1e594a547f38ad6bf5e3d60304ce7539e677c1429eebab115bce56e"}, + {file = "regex-2023.5.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cf123225945aa58b3057d0fba67e8061c62d14cc8a4202630f8057df70189051"}, + {file = "regex-2023.5.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a99757ad7fe5c8a2bb44829fc57ced11253e10f462233c1255fe03888e06bc19"}, + {file = "regex-2023.5.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a623564d810e7a953ff1357f7799c14bc9beeab699aacc8b7ab7822da1e952b8"}, + {file = "regex-2023.5.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ced02e3bd55e16e89c08bbc8128cff0884d96e7f7a5633d3dc366b6d95fcd1d6"}, + {file = "regex-2023.5.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1cbe6b5be3b9b698d8cc4ee4dee7e017ad655e83361cd0ea8e653d65e469468"}, + {file = "regex-2023.5.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a6e4b0e0531223f53bad07ddf733af490ba2b8367f62342b92b39b29f72735a"}, + {file = "regex-2023.5.5-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2e9c4f778514a560a9c9aa8e5538bee759b55f6c1dcd35613ad72523fd9175b8"}, + {file = "regex-2023.5.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:256f7f4c6ba145f62f7a441a003c94b8b1af78cee2cccacfc1e835f93bc09426"}, + {file = "regex-2023.5.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bd7b68fd2e79d59d86dcbc1ccd6e2ca09c505343445daaa4e07f43c8a9cc34da"}, + {file = "regex-2023.5.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4a5059bd585e9e9504ef9c07e4bc15b0a621ba20504388875d66b8b30a5c4d18"}, + {file = "regex-2023.5.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:6893544e06bae009916a5658ce7207e26ed17385149f35a3125f5259951f1bbe"}, + {file = "regex-2023.5.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:c64d5abe91a3dfe5ff250c6bb267ef00dbc01501518225b45a5f9def458f31fb"}, + {file = "regex-2023.5.5-cp38-cp38-win32.whl", hash = "sha256:7923470d6056a9590247ff729c05e8e0f06bbd4efa6569c916943cb2d9b68b91"}, + {file = "regex-2023.5.5-cp38-cp38-win_amd64.whl", hash = "sha256:4035d6945cb961c90c3e1c1ca2feb526175bcfed44dfb1cc77db4fdced060d3e"}, + {file = "regex-2023.5.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:50fd2d9b36938d4dcecbd684777dd12a407add4f9f934f235c66372e630772b0"}, + {file = "regex-2023.5.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d19e57f888b00cd04fc38f5e18d0efbd91ccba2d45039453ab2236e6eec48d4d"}, + {file = "regex-2023.5.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd966475e963122ee0a7118ec9024388c602d12ac72860f6eea119a3928be053"}, + {file = "regex-2023.5.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db09e6c18977a33fea26fe67b7a842f706c67cf8bda1450974d0ae0dd63570df"}, + {file = "regex-2023.5.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6164d4e2a82f9ebd7752a06bd6c504791bedc6418c0196cd0a23afb7f3e12b2d"}, + {file = "regex-2023.5.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84397d3f750d153ebd7f958efaa92b45fea170200e2df5e0e1fd4d85b7e3f58a"}, + {file = "regex-2023.5.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9c3efee9bb53cbe7b285760c81f28ac80dc15fa48b5fe7e58b52752e642553f1"}, + {file = "regex-2023.5.5-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:144b5b017646b5a9392a5554a1e5db0000ae637be4971c9747566775fc96e1b2"}, + {file = "regex-2023.5.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1189fbbb21e2c117fda5303653b61905aeeeea23de4a94d400b0487eb16d2d60"}, + {file = "regex-2023.5.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f83fe9e10f9d0b6cf580564d4d23845b9d692e4c91bd8be57733958e4c602956"}, + {file = "regex-2023.5.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:72aa4746993a28c841e05889f3f1b1e5d14df8d3daa157d6001a34c98102b393"}, + {file = "regex-2023.5.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:de2f780c3242ea114dd01f84848655356af4dd561501896c751d7b885ea6d3a1"}, + {file = "regex-2023.5.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:290fd35219486dfbc00b0de72f455ecdd63e59b528991a6aec9fdfc0ce85672e"}, + {file = "regex-2023.5.5-cp39-cp39-win32.whl", hash = "sha256:732176f5427e72fa2325b05c58ad0b45af341c459910d766f814b0584ac1f9ac"}, + {file = "regex-2023.5.5-cp39-cp39-win_amd64.whl", hash = "sha256:1307aa4daa1cbb23823d8238e1f61292fd07e4e5d8d38a6efff00b67a7cdb764"}, + {file = "regex-2023.5.5.tar.gz", hash = "sha256:7d76a8a1fc9da08296462a18f16620ba73bcbf5909e42383b253ef34d9d5141e"}, +] + [[package]] name = "requests" version = "2.31.0" @@ -1426,4 +1555,4 @@ test = ["docker"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "705734ada27559c4ecbd058884e639fff92deaae1154b6b9ac900d2487fc24cb" +content-hash = "e8c1a9fdd874ef3e43ed6e11e70445e23c2a03c2b5a2245bf9419077ee81daaa" \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 07e37a725..febd7a25d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ jsonschema = "^4.17.3" kadet = "^0.2.2" python-gnupg = "^0.4.7" pyyaml = "^6.0" +omegaconf = { git = "https://github.com/neXenio/omegaconf.git", branch = "dev" } requests = "^2.28.2" six = "^1.16.0" toml = "^0.10.2" @@ -63,6 +64,7 @@ packaging = "^23.0" typing-extensions = "^4.0.0" gojsonnet = { version = "^0.17.0", optional = true } docker = { version = "^5.0.0", optional = true } +regex = "^2023.5.5" [tool.poetry.extras] gojsonnet = ["gojsonnet"] diff --git a/tests/test_compile.py b/tests/test_compile.py index 5196ae99b..f65e209dd 100644 --- a/tests/test_compile.py +++ b/tests/test_compile.py @@ -20,7 +20,7 @@ from kapitan.utils import directory_hash from kapitan.cached import reset_cache from kapitan.targets import validate_matching_target_name -from kapitan.resources import inventory_reclass +from kapitan.resources import get_inventory from kapitan.errors import InventoryError @@ -175,7 +175,7 @@ def test_compile_not_matching_targets(self): def test_compile_vars_target_missing(self): inventory_path = "inventory" target_filename = "minikube-es" - target_obj = inventory_reclass(inventory_path)["nodes"][target_filename]["parameters"]["kapitan"] + target_obj = get_inventory(inventory_path)["nodes"][target_filename]["parameters"]["kapitan"] # delete vars.target del target_obj["vars"]["target"]