From 2d39fb767dfbff381ab599ff944b8800b43dad49 Mon Sep 17 00:00:00 2001 From: boekhorstb1 <91957243+boekhorstb1@users.noreply.github.com> Date: Thu, 14 Mar 2024 12:50:49 +0100 Subject: [PATCH 1/4] feat: adding makefile to install first pre-commit and then venv (#29) * feat: makefile to install first pre-commit and then venv, documentation in README.md Signed-off-by: R.A. te Boekhorst * fix: adaptation of requirements.txt to include correct cryptography package Signed-off-by: R.A. te Boekhorst * fix: Makefile installs with requirements that has no rados 2.0.0, but it checks if rados is installed locally in the correct way Signed-off-by: R.A. te Boekhorst * fix: PHONY to Makefile and reset of previous cryptography version in requirements Signed-off-by: R.A. te Boekhorst --------- Signed-off-by: R.A. te Boekhorst --- Makefile | 52 ++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 11 +++++++++- requirements.txt | 1 - 3 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a99da0d --- /dev/null +++ b/Makefile @@ -0,0 +1,52 @@ +COLOUR_GREEN=\033[0;32m +COLOUR_RED=\033[0;31m +COLOUR_BLUE=\033[0;34m +COLOUR_END=\033[0m + +.DEFAULT_GOAL:=help + +# Get needed paths and information from locally installed librados +RADOSLIB_VERSION := 2.0.0 +GENERAL_LIB_LOCATION := $(shell pip show rados | grep -oP "(?<=Location: ).*") +RADOSLIB_INSTALLED_VERSION := $(shell pip show rados | grep Version | awk '{print $$2}') + +.PHONY: help +help: ## Display this help message + @echo -e '${COLOUR_RED}Usage: make ${COLOUR_END}' + @cat $(MAKEFILE_LIST) | grep '^[a-zA-Z]' | \ + awk -F ':.*?## ' 'NF==2 {printf " %-26s%s\n\n", $$1, "${COLOUR_GREEN}"$$2"${COLOUR_END}"}' + +.PHONY: setup +setup: setup-pre-commit check-radoslib setup-venv ## Setup the pre-commit environment and then the venv environment + +.PHONY: setup-pre-commit +setup-pre-commit: + pip install --user pre-commit && pre-commit install + +.PHONY: setup-venv +setup-venv: + python -m venv --system-site-packages ./.venv && \ + source ./.venv/bin/activate && \ + pip install --ignore-installed -r requirements.txt + +.PHONY: update-requirements +update-requirements: ## Update the requirements.txt with newer versions of pip packages + source ./.venv/bin/activate && \ + pip freeze -l > requirements.txt + +.PHONY: check-radoslib +check-radoslib: ## Checks if radoslib is installed and if it contains the right version + @if [ -z "$(GENERAL_LIB_LOCATION)" ]; then \ + echo -e "${COLOUR_RED}ERROR: 'rados' library not found. Please make sure it's installed.${COLOUR_END}"; \ + exit 1; \ + fi + @if [ "$(RADOSLIB_INSTALLED_VERSION)" != "$(RADOSLIB_VERSION)" ]; then \ + echo -e "${COLOUR_RED}ERROR: Incorrect version of 'rados' library found. Expected version $(RADOSLIB_VERSION), found $$RADOSLIB_INSTALLED_VERSION.${COLOUR_END}"; \ + exit 1; \ + fi + +.PHONY: run-local-rookify +run-local-rookify: ## Runs rookify in the local development environment (requires setup-venv) + $(eval PYTHONPATH="${PYTHONPATH}:$(pwd)/src") + source ./.venv/bin/activate && \ + cd src && python3 -m rookify diff --git a/README.md b/README.md index ea09fbe..7c04972 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Rookify is designed to facilitate a smooth and efficient transition for existing - Access to a Kubernetes cluster with sufficient resources to host the migrated Ceph cluster. - Kubernetes nodes should be rolled out at least on the OSD nodes - Rook operator version 1.13 or higher installed in the Kubernetes cluster. +- _local development enivornment_ requires radoslib version 2.0.0 installed ## Installation 1. Clone the repository: @@ -28,11 +29,19 @@ git clone https://github.com/SovereignCloudStack/rookify cd rookify ``` -3. __TODO:__ Install script +3. To install the local development environment +(_Note: This will install pre-commit in your local user context_): +```bash +make setup +``` + +4. __TODO:__ Install script for container-based setup ## Usage __TODO__ +Type `make` to get a list of available development specific commands. + ## Support For issues, questions, or contributions, please open an issue or pull request in the GitHub repository. We welcome community feedback and contributions to enhance rookify. diff --git a/requirements.txt b/requirements.txt index e23bdf7..9de726f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,6 @@ pycparser==2.21 PyNaCl==1.5.0 python-dateutil==2.8.2 PyYAML==6.0.1 -rados==2.0.0 requests==2.31.0 requests-oauthlib==1.3.1 rsa==4.9 From cd8eccefad677b028bd561d0b3303e840dc11131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Marten=20Br=C3=BCggemann?= Date: Thu, 14 Mar 2024 13:04:53 +0100 Subject: [PATCH 2/4] Prs/add mypy to pre commit checks (#35) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add mypy to pre-commit hooks Signed-off-by: Jan-Marten Brüggemann * fix typing Signed-off-by: Jan-Marten Brüggemann * Fixed missing/wrong type hints Signed-off-by: Gondermann --------- Signed-off-by: Jan-Marten Brüggemann Signed-off-by: Gondermann Co-authored-by: Gondermann --- .pre-commit-config.yaml | 7 ++++ setup.py | 9 ++--- src/rookify/__main__.py | 2 +- src/rookify/modules/__init__.py | 29 +++++++++++----- src/rookify/modules/analyze_ceph/__init__.py | 1 + src/rookify/modules/analyze_ceph/main.py | 7 ++-- src/rookify/modules/example/__init__.py | 1 + src/rookify/modules/example/main.py | 8 +++-- .../modules/migrate_monitors/__init__.py | 1 + src/rookify/modules/migrate_osds/__init__.py | 1 + src/rookify/modules/migrate_osds/main.py | 9 +++-- src/rookify/modules/module.py | 33 +++++++++++-------- src/rookify/yaml.py | 9 +++-- 13 files changed, 77 insertions(+), 40 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f715b3c..898981e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,3 +15,10 @@ repos: - id: mixed-line-ending - id: end-of-file-fixer - id: trailing-whitespace + - repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v1.9.0' + hooks: + - id: mypy + args: [--strict, --ignore-missing-imports, --check-untyped-defs] + additional_dependencies: + - types-PyYAML diff --git a/setup.py b/setup.py index abf4a76..af65979 100644 --- a/setup.py +++ b/setup.py @@ -16,14 +16,15 @@ limitations under the License. """ +from typing import Dict, Any + try: from setuptools import find_packages, setup except ImportError: - from distutils import find_packages, setup -# + from distutils import find_packages, setup # type: ignore[attr-defined, unused-ignore] -def get_version(): +def get_version() -> str: """ Returns the version currently in development. @@ -36,7 +37,7 @@ def get_version(): # -_setup = { +_setup: Dict[str, Any] = { "version": get_version()[1:], "data_files": [("docs", ["LICENSE", "README.md"])], "test_suite": "tests", diff --git a/src/rookify/__main__.py b/src/rookify/__main__.py index 44c71fe..bada9ba 100644 --- a/src/rookify/__main__.py +++ b/src/rookify/__main__.py @@ -6,7 +6,7 @@ from .yaml import load_yaml, save_yaml -def main(): +def main() -> None: try: config = load_yaml("config.yaml") except FileNotFoundError as err: diff --git a/src/rookify/modules/__init__.py b/src/rookify/modules/__init__.py index aaf0140..3167e96 100644 --- a/src/rookify/modules/__init__.py +++ b/src/rookify/modules/__init__.py @@ -2,6 +2,7 @@ import importlib import types +from typing import List from collections import OrderedDict from .module import ModuleHandler @@ -23,7 +24,9 @@ def __init__(self, module_name: str, message: str): self.message = message -def load_modules(module_names: list) -> tuple[list, list]: +def load_modules( + module_names: List[str], +) -> tuple[List[types.ModuleType], List[types.ModuleType]]: """ Dynamically loads modules from the 'modules' package. @@ -32,7 +35,7 @@ def load_modules(module_names: list) -> tuple[list, list]: """ # Sanity checks for modules - def check_module_sanity(module_name: str, module: types.ModuleType): + def check_module_sanity(module_name: str, module: types.ModuleType) -> None: for attr_type, attr_name in ( (ModuleHandler, "HANDLER_CLASS"), (str, "MODULE_NAME"), @@ -52,9 +55,11 @@ def check_module_sanity(module_name: str, module: types.ModuleType): ) # Load the modules in the given list and recursivley load required modules - required_modules = OrderedDict() + required_modules: OrderedDict[str, types.ModuleType] = OrderedDict() - def load_required_modules(modules_out: OrderedDict, module_names: list) -> None: + def load_required_modules( + modules_out: OrderedDict[str, types.ModuleType], module_names: List[str] + ) -> None: for module_name in module_names: if module_name in modules_out: continue @@ -70,10 +75,12 @@ def load_required_modules(modules_out: OrderedDict, module_names: list) -> None: load_required_modules(required_modules, module_names) # Recursively load the modules in the PREFLIGHT_REQUIRES attribute of the given modules - preflight_modules = OrderedDict() + preflight_modules: OrderedDict[str, types.ModuleType] = OrderedDict() def load_preflight_modules( - modules_in: OrderedDict, modules_out: OrderedDict, module_names: list + modules_in: OrderedDict[str, types.ModuleType], + modules_out: OrderedDict[str, types.ModuleType], + module_names: List[str], ) -> None: for module_name in module_names: if module_name in modules_out: @@ -94,13 +101,17 @@ def load_preflight_modules( if module_name not in modules_in: modules_out[module_name] = module - load_preflight_modules(required_modules, preflight_modules, required_modules.keys()) + load_preflight_modules( + required_modules, preflight_modules, list(required_modules.keys()) + ) # Sort the modules by the AFTER keyword - modules = OrderedDict() + modules: OrderedDict[str, types.ModuleType] = OrderedDict() def sort_modules( - modules_in: OrderedDict, modules_out: OrderedDict, module_names: list + modules_in: OrderedDict[str, types.ModuleType], + modules_out: OrderedDict[str, types.ModuleType], + module_names: List[str], ) -> None: for module_name in module_names: if module_name not in modules_in: diff --git a/src/rookify/modules/analyze_ceph/__init__.py b/src/rookify/modules/analyze_ceph/__init__.py index c3726e4..d821800 100644 --- a/src/rookify/modules/analyze_ceph/__init__.py +++ b/src/rookify/modules/analyze_ceph/__init__.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# type: ignore from .main import AnalyzeCephHandler diff --git a/src/rookify/modules/analyze_ceph/main.py b/src/rookify/modules/analyze_ceph/main.py index 4cb570c..cc90105 100644 --- a/src/rookify/modules/analyze_ceph/main.py +++ b/src/rookify/modules/analyze_ceph/main.py @@ -1,14 +1,15 @@ # -*- coding: utf-8 -*- - from ..module import ModuleHandler +from typing import Any, Dict + class AnalyzeCephHandler(ModuleHandler): - def run(self) -> dict: + def run(self) -> Dict[str, Any]: commands = ["mon dump", "osd dump", "device ls", "fs dump", "node ls"] - results = dict() + results: Dict[str, Any] = dict() for command in commands: parts = command.split(" ") leaf = results diff --git a/src/rookify/modules/example/__init__.py b/src/rookify/modules/example/__init__.py index 8496365..ff0dcf5 100644 --- a/src/rookify/modules/example/__init__.py +++ b/src/rookify/modules/example/__init__.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# type: ignore from .main import ExampleHandler diff --git a/src/rookify/modules/example/main.py b/src/rookify/modules/example/main.py index 12f337f..2d0391e 100644 --- a/src/rookify/modules/example/main.py +++ b/src/rookify/modules/example/main.py @@ -2,12 +2,14 @@ from ..module import ModuleHandler, ModuleException +from typing import Any, Dict + class ExampleHandler(ModuleHandler): - def preflight_check(self): + def preflight_check(self) -> None: # Do something for checking if all needed preconditions are met else throw ModuleException raise ModuleException("Example module was loaded, so aborting!") - def run(self) -> dict: + def run(self) -> Dict[str, Any]: # Run the migration tasks - pass + return {} diff --git a/src/rookify/modules/migrate_monitors/__init__.py b/src/rookify/modules/migrate_monitors/__init__.py index 168ce94..ff3c337 100644 --- a/src/rookify/modules/migrate_monitors/__init__.py +++ b/src/rookify/modules/migrate_monitors/__init__.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# type: ignore from .main import MigrateMonitorsHandler diff --git a/src/rookify/modules/migrate_osds/__init__.py b/src/rookify/modules/migrate_osds/__init__.py index f3b00b6..86041ae 100644 --- a/src/rookify/modules/migrate_osds/__init__.py +++ b/src/rookify/modules/migrate_osds/__init__.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# type: ignore from .main import MigrateOSDsHandler diff --git a/src/rookify/modules/migrate_osds/main.py b/src/rookify/modules/migrate_osds/main.py index 3b47496..d6b4a34 100644 --- a/src/rookify/modules/migrate_osds/main.py +++ b/src/rookify/modules/migrate_osds/main.py @@ -2,15 +2,17 @@ from ..module import ModuleHandler +from typing import Any, Dict + class MigrateOSDsHandler(ModuleHandler): - def preflight_check(self): + def preflight_check(self) -> None: pass # result = self.ceph.mon_command("osd dump") # raise ModuleException('test error') - def run(self) -> dict: - osd_config = dict() + def run(self) -> Dict[str, Any]: + osd_config: Dict[str, Any] = dict() for node, osds in self._data["analyze_ceph"]["node"]["ls"]["osd"].items(): osd_config[node] = {"osds": {}} for osd in osds: @@ -33,3 +35,4 @@ def run(self) -> dict: break print(osd_config) + return {} diff --git a/src/rookify/modules/module.py b/src/rookify/modules/module.py index d2882b9..84208e5 100644 --- a/src/rookify/modules/module.py +++ b/src/rookify/modules/module.py @@ -5,6 +5,7 @@ import rados import kubernetes import fabric +from typing import Any, Dict, List, Optional class ModuleException(Exception): @@ -17,7 +18,7 @@ class ModuleHandler: """ class __Ceph: - def __init__(self, config: dict): + def __init__(self, config: Dict[str, Any]): try: self.__ceph = rados.Rados( conffile=config["conf_file"], conf={"keyring": config["keyring"]} @@ -26,16 +27,20 @@ def __init__(self, config: dict): except rados.ObjectNotFound as err: raise ModuleException(f"Could not connect to ceph: {err}") - def mon_command(self, command: str, **kwargs) -> dict: + def mon_command( + self, command: str, **kwargs: str + ) -> Dict[str, Any] | List[Any]: cmd = {"prefix": command, "format": "json"} - cmd.update(kwargs) + cmd.update(**kwargs) result = self.__ceph.mon_command(json.dumps(cmd), b"") if result[0] != 0: raise ModuleException(f"Ceph did return an error: {result}") - return json.loads(result[1]) + data = json.loads(result[1]) + assert isinstance(data, dict) or isinstance(data, list) + return data class __K8s: - def __init__(self, config: dict): + def __init__(self, config: Dict[str, Any]): k8s_config = kubernetes.client.Configuration() k8s_config.api_key = config["api_key"] k8s_config.host = config["host"] @@ -54,7 +59,7 @@ def NodeV1Api(self) -> kubernetes.client.NodeV1Api: return kubernetes.client.NodeV1Api(self.__client) class __SSH: - def __init__(self, config: dict): + def __init__(self, config: Dict[str, Any]): self.__config = config def command(self, host: str, command: str) -> fabric.runners.Result: @@ -77,7 +82,7 @@ def command(self, host: str, command: str) -> fabric.runners.Result: ).run(command, hide=True) return result - def __init__(self, config: dict, data: dict): + def __init__(self, config: Dict[str, Any], data: Dict[str, Any]): """ Construct a new 'ModuleHandler' object. @@ -86,9 +91,9 @@ def __init__(self, config: dict, data: dict): """ self._config = config self._data = data - self.__ceph = None - self.__k8s = None - self.__ssh = None + self.__ceph: Optional[ModuleHandler.__Ceph] = None + self.__k8s: Optional[ModuleHandler.__K8s] = None + self.__ssh: Optional[ModuleHandler.__SSH] = None @abc.abstractmethod def preflight_check(self) -> None: @@ -98,7 +103,7 @@ def preflight_check(self) -> None: pass @abc.abstractmethod - def run(self) -> dict: + def run(self) -> Dict[str, Any]: """ Run the modules tasks @@ -109,17 +114,17 @@ def run(self) -> dict: @property def ceph(self) -> __Ceph: if self.__ceph is None: - self.__ceph = self.__Ceph(self._config["ceph"]) + self.__ceph = ModuleHandler.__Ceph(self._config["ceph"]) return self.__ceph @property def k8s(self) -> __K8s: if self.__k8s is None: - self.__k8s = self.__K8s(self._config["kubernetes"]) + self.__k8s = ModuleHandler.__K8s(self._config["kubernetes"]) return self.__k8s @property def ssh(self) -> __SSH: if self.__ssh is None: - self.__ssh = self.__SSH(self._config["ssh"]) + self.__ssh = ModuleHandler.__SSH(self._config["ssh"]) return self.__ssh diff --git a/src/rookify/yaml.py b/src/rookify/yaml.py index 9cf6ad7..856716f 100644 --- a/src/rookify/yaml.py +++ b/src/rookify/yaml.py @@ -1,13 +1,16 @@ # -*- coding: utf-8 -*- import yaml +from typing import Any, Dict -def load_yaml(path: str) -> dict: +def load_yaml(path: str) -> Dict[str, Any]: with open(path, "r") as file: - return yaml.safe_load(file) + data = yaml.safe_load(file) + assert isinstance(data, dict) + return data -def save_yaml(path: str, data: dict) -> None: +def save_yaml(path: str, data: Dict[str, Any]) -> None: with open(path, "w") as file: yaml.safe_dump(data, file) From 75eef77b10263a697a9565fde648205e15501277 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Tue, 19 Mar 2024 19:57:37 +0100 Subject: [PATCH 3/4] Move "setup.cfg" metadata to `pyproject.toml` Signed-off-by: Tobias Wolf --- pyproject.toml | 20 ++++++++++++++++++++ setup.cfg | 16 ---------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 63a7d09..60b8972 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,4 +3,24 @@ requires = [ "setuptools", "wheel" ] build-backend = "setuptools.build_meta" [project] +dynamic = ["version"] requires-python = ">=3.9" + +name = "Rookify" +description = "Enable Ceph-Cluster migrations to Rook" +readme = "README.md" +authors = [{name = "Sovereign Cloud Stack Developers", email = "scs@osb-alliance.com"}] +license = {file = "LICENSE"} +classifiers = [ + "Intended Audience :: System Administrators", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Environment :: Console", + "Topic :: System :: Clustering", + "Topic :: System :: Systems Administration", + "Topic :: Utilities" +] + +[project.urls] +Homepage = "https://scs.community" diff --git a/setup.cfg b/setup.cfg index 4183ecc..3921288 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,18 +1,2 @@ [metadata] -name = Rookify -description = Enable Ceph-Cluster migrations to Rook -long_description = Rookify is designed to facilitate a smooth and efficient transition for existing Ceph clusters to a Rook-managed Ceph cluster environment. platforms = any -author = Sovereign Cloud Stack Developers -author_email = scs@osb-alliance.com -license = Apache -url = https://scs.community -classifiers = - Intended Audience :: System Administrators - License :: OSI Approved :: Apache License - Operating System :: OS Independent - Programming Language :: Python - Environment :: Console - Topic :: System :: Clustering - Topic :: System :: Systems Administration - Topic :: Utilities From cdb0c2e53aa465f2345dca2e3b32672216f198c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Marten=20Br=C3=BCggemann?= Date: Mon, 18 Mar 2024 14:57:47 +0100 Subject: [PATCH 4/4] module enhancement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rename prefligt_check to preflight implement jinja2 templating for modules Signed-off-by: Jan-Marten Brüggemann --- .gitignore | 1 + requirements.txt | 2 + src/config.example.yaml | 10 ++- src/rookify/__main__.py | 22 +++++- src/rookify/modules/analyze_ceph/main.py | 2 +- src/rookify/modules/example/main.py | 6 +- src/rookify/modules/migrate_osds/main.py | 4 +- src/rookify/modules/module.py | 94 +++++++++++++++++++++--- 8 files changed, 119 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index d8dd9a3..8174be4 100644 --- a/.gitignore +++ b/.gitignore @@ -89,3 +89,4 @@ cython_debug/ data.yaml config.yaml .ceph +.k8s diff --git a/requirements.txt b/requirements.txt index 9de726f..0d819fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,9 @@ fabric==3.2.2 google-auth==2.28.1 idna==3.6 invoke==2.2.0 +Jinja2==3.1.3 kubernetes==29.0.0 +MarkupSafe==2.1.5 oauthlib==3.2.2 paramiko==3.4.0 pyasn1==0.5.1 diff --git a/src/config.example.yaml b/src/config.example.yaml index 7e60f64..4ed0fdd 100644 --- a/src/config.example.yaml +++ b/src/config.example.yaml @@ -19,8 +19,14 @@ ssh: user: dragon kubernetes: - host: 192.168.22.10 - api_key: abc + config: ../.k8s/config + +rook: + cluster: + name: osism-ceph + namespace: rook-ceph + ceph: + image: quay.io/ceph/ceph:v18.2.1 migration_modules: - migrate_osds diff --git a/src/rookify/__main__.py b/src/rookify/__main__.py index bada9ba..cac2ed7 100644 --- a/src/rookify/__main__.py +++ b/src/rookify/__main__.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import os import rookify.modules from types import MappingProxyType @@ -21,21 +22,34 @@ def main() -> None: except FileNotFoundError: pass + # Get absolute path of the rookify instance + rookify_path = os.path.dirname(__file__) + # Run preflight requirement modules for preflight_module in preflight_modules: + module_path = os.path.join( + rookify_path, "modules", preflight_module.MODULE_NAME + ) handler = preflight_module.HANDLER_CLASS( - config=MappingProxyType(config), data=MappingProxyType(module_data) + config=MappingProxyType(config), + data=MappingProxyType(module_data), + module_path=module_path, ) result = handler.run() module_data[preflight_module.MODULE_NAME] = result - # Run preflight checks and append handlers to list + # Run preflight and append handlers to list handlers = list() for migration_module in migration_modules: + module_path = os.path.join( + rookify_path, "modules", migration_module.MODULE_NAME + ) handler = migration_module.HANDLER_CLASS( - config=MappingProxyType(config), data=MappingProxyType(module_data) + config=MappingProxyType(config), + data=MappingProxyType(module_data), + module_path=module_path, ) - handler.preflight_check() + handler.preflight() handlers.append((migration_module, handler)) # Run migration modules diff --git a/src/rookify/modules/analyze_ceph/main.py b/src/rookify/modules/analyze_ceph/main.py index cc90105..6cb023f 100644 --- a/src/rookify/modules/analyze_ceph/main.py +++ b/src/rookify/modules/analyze_ceph/main.py @@ -6,7 +6,7 @@ class AnalyzeCephHandler(ModuleHandler): - def run(self) -> Dict[str, Any]: + def run(self) -> Any: commands = ["mon dump", "osd dump", "device ls", "fs dump", "node ls"] results: Dict[str, Any] = dict() diff --git a/src/rookify/modules/example/main.py b/src/rookify/modules/example/main.py index 2d0391e..e62394c 100644 --- a/src/rookify/modules/example/main.py +++ b/src/rookify/modules/example/main.py @@ -2,14 +2,14 @@ from ..module import ModuleHandler, ModuleException -from typing import Any, Dict +from typing import Any class ExampleHandler(ModuleHandler): - def preflight_check(self) -> None: + def preflight(self) -> None: # Do something for checking if all needed preconditions are met else throw ModuleException raise ModuleException("Example module was loaded, so aborting!") - def run(self) -> Dict[str, Any]: + def run(self) -> Any: # Run the migration tasks return {} diff --git a/src/rookify/modules/migrate_osds/main.py b/src/rookify/modules/migrate_osds/main.py index d6b4a34..5377671 100644 --- a/src/rookify/modules/migrate_osds/main.py +++ b/src/rookify/modules/migrate_osds/main.py @@ -6,12 +6,12 @@ class MigrateOSDsHandler(ModuleHandler): - def preflight_check(self) -> None: + def preflight(self) -> None: pass # result = self.ceph.mon_command("osd dump") # raise ModuleException('test error') - def run(self) -> Dict[str, Any]: + def run(self) -> Any: osd_config: Dict[str, Any] = dict() for node, osds in self._data["analyze_ceph"]["node"]["ls"]["osd"].items(): osd_config[node] = {"osds": {}} diff --git a/src/rookify/modules/module.py b/src/rookify/modules/module.py index 84208e5..8d1a75a 100644 --- a/src/rookify/modules/module.py +++ b/src/rookify/modules/module.py @@ -1,10 +1,13 @@ # -*- coding: utf-8 -*- +import os +import yaml import json import abc import rados import kubernetes import fabric +import jinja2 from typing import Any, Dict, List, Optional @@ -21,7 +24,7 @@ class __Ceph: def __init__(self, config: Dict[str, Any]): try: self.__ceph = rados.Rados( - conffile=config["conf_file"], conf={"keyring": config["keyring"]} + conffile=config["config"], conf={"keyring": config["keyring"]} ) self.__ceph.connect() except rados.ObjectNotFound as err: @@ -41,23 +44,61 @@ def mon_command( class __K8s: def __init__(self, config: Dict[str, Any]): - k8s_config = kubernetes.client.Configuration() - k8s_config.api_key = config["api_key"] - k8s_config.host = config["host"] + k8s_config = kubernetes.config.load_kube_config( + config_file=config["config"] + ) self.__client = kubernetes.client.ApiClient(k8s_config) + self.__dynamic_client: Optional[kubernetes.dynamic.DynamicClient] = None @property - def CoreV1Api(self) -> kubernetes.client.CoreV1Api: + def core_v1_api(self) -> kubernetes.client.CoreV1Api: return kubernetes.client.CoreV1Api(self.__client) @property - def AppsV1Api(self) -> kubernetes.client.AppsV1Api: + def apps_v1_api(self) -> kubernetes.client.AppsV1Api: return kubernetes.client.AppsV1Api(self.__client) @property - def NodeV1Api(self) -> kubernetes.client.NodeV1Api: + def node_v1_api(self) -> kubernetes.client.NodeV1Api: return kubernetes.client.NodeV1Api(self.__client) + @property + def custom_objects_api(self) -> kubernetes.client.CustomObjectsApi: + return kubernetes.client.CustomObjectsApi(self.__client) + + @property + def dynamic_client(self) -> kubernetes.dynamic.DynamicClient: + if not self.__dynamic_client: + self.__dynamic_client = kubernetes.dynamic.DynamicClient(self.__client) + return self.__dynamic_client + + def crd_api( + self, api_version: str, kind: str + ) -> kubernetes.dynamic.resource.Resource: + return self.dynamic_client.resources.get(api_version=api_version, kind=kind) + + def crd_api_apply( + self, manifest: Dict[Any, Any] + ) -> kubernetes.dynamic.resource.ResourceInstance: + """ + This applies a manifest for custom CRDs + See https://github.com/kubernetes-client/python/issues/1792 for more information + :param manifest: Dict of the kubernetes manifest + """ + api_version = manifest["apiVersion"] + kind = manifest["kind"] + resource_name = manifest["metadata"]["name"] + namespace = manifest["metadata"]["namespace"] + crd_api = self.crd_api(api_version=api_version, kind=kind) + + try: + crd_api.get(namespace=namespace, name=resource_name) + return crd_api.patch( + body=manifest, content_type="application/merge-patch+json" + ) + except kubernetes.dynamic.exceptions.NotFoundError: + return crd_api.create(body=manifest, namespace=namespace) + class __SSH: def __init__(self, config: Dict[str, Any]): self.__config = config @@ -82,21 +123,48 @@ def command(self, host: str, command: str) -> fabric.runners.Result: ).run(command, hide=True) return result - def __init__(self, config: Dict[str, Any], data: Dict[str, Any]): + class __Template: + def __init__(self, template_path: str): + self.__result_raw: Optional[str] = None + self.__result_yaml: Optional[Any] = None + self.__template_path: str = template_path + with open(template_path) as file: + self.__template = jinja2.Template(file.read()) + + def render(self, **variables: Any) -> None: + self.__result_raw = self.__template.render(**variables) + self.__result_yaml = None + + @property + def raw(self) -> str: + if not self.__result_raw: + raise ModuleException("Template was not rendered") + return self.__result_raw + + @property + def yaml(self) -> Any: + if not self.__result_yaml: + self.__result_yaml = yaml.safe_load(self.raw) + return self.__result_yaml + + def __init__(self, config: Dict[str, Any], data: Dict[str, Any], module_path: str): """ Construct a new 'ModuleHandler' object. - :param module_data: The config and results from modules + :param config: The global config file + :param data: The output of modules required by this module + :param module_path: The filesystem path of this module :return: returns nothing """ self._config = config self._data = data + self.__module_path = module_path self.__ceph: Optional[ModuleHandler.__Ceph] = None self.__k8s: Optional[ModuleHandler.__K8s] = None self.__ssh: Optional[ModuleHandler.__SSH] = None @abc.abstractmethod - def preflight_check(self) -> None: + def preflight(self) -> None: """ Run the modules preflight check """ @@ -128,3 +196,9 @@ def ssh(self) -> __SSH: if self.__ssh is None: self.__ssh = ModuleHandler.__SSH(self._config["ssh"]) return self.__ssh + + def load_template(self, filename: str, **variables: Any) -> __Template: + template_path = os.path.join(self.__module_path, "templates", filename) + template = ModuleHandler.__Template(template_path) + template.render(**variables) + return template