From 102c2a6b79e91d24a51b85753f5f65c1a25792cb Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Wed, 6 Mar 2024 15:17:05 +0100 Subject: [PATCH 1/4] Add Python build specific files (#23) Signed-off-by: Tobias Wolf --- MANIFEST.in | 3 +++ pyproject.toml | 3 +++ setup.cfg | 18 ++++++++++++++++ setup.py | 46 +++++++++++++++++++++++++++++++++++++++++ src/rookify/__init__.py | 0 tests/__init__.py | 0 6 files changed, 70 insertions(+) create mode 100644 MANIFEST.in create mode 100644 pyproject.toml create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 src/rookify/__init__.py create mode 100644 tests/__init__.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..1cd3ffd --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +include pyproject.toml README.md requirements.txt + +recursive-include tests * diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b768650 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = [ "setuptools", "wheel" ] +build-backend = "setuptools.build_meta" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..4183ecc --- /dev/null +++ b/setup.cfg @@ -0,0 +1,18 @@ +[metadata] +name = Rookify +description = Enable Ceph-Cluster migrations to Rook +long_description = Rookify is designed to facilitate a smooth and efficient transition for existing Ceph clusters to a Rook-managed Ceph cluster environment. +platforms = any +author = Sovereign Cloud Stack Developers +author_email = scs@osb-alliance.com +license = Apache +url = https://scs.community +classifiers = + Intended Audience :: System Administrators + License :: OSI Approved :: Apache License + Operating System :: OS Independent + Programming Language :: Python + Environment :: Console + Topic :: System :: Clustering + Topic :: System :: Systems Administration + Topic :: Utilities diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..b7ae01f --- /dev/null +++ b/setup.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +""" +Copyright (c) Sovereign Cloud Stack Developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. + +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from os import makedirs, path + +try: + from setuptools import find_packages, setup +except ImportError: + from distutils import find_packages, setup +# + +def get_version(): + """ +Returns the version currently in development. + +:return: (str) Version string +:since: v0.0.1 + """ + + return "v0.0.1" +# + +_setup = { "version": get_version()[1:], + "data_files": [ ( "docs", [ "LICENSE", "README.md" ]) ], + "test_suite": "tests" + } + +_setup['package_dir'] = { "": "src" } +_setup['packages'] = find_packages("src") + +setup(**_setup) diff --git a/src/rookify/__init__.py b/src/rookify/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 From cdd12120110c444c38fc62c60ecd3e03ce9b75fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Marten=20Br=C3=BCggemann?= Date: Wed, 6 Mar 2024 16:01:04 +0100 Subject: [PATCH 2/4] fix module naming behavior (#26) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan-Marten Brüggemann --- src/rookify/__main__.py | 4 ++-- src/rookify/modules/__init__.py | 1 + src/rookify/modules/analyze_ceph/__init__.py | 1 + src/rookify/modules/example/__init__.py | 1 + src/rookify/modules/migrate_monitors/__init__.py | 1 + src/rookify/modules/migrate_osds/__init__.py | 1 + src/rookify/modules/migrate_osds/main.py | 6 +++--- 7 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/rookify/__main__.py b/src/rookify/__main__.py index 1ea9c40..674d2a0 100644 --- a/src/rookify/__main__.py +++ b/src/rookify/__main__.py @@ -25,7 +25,7 @@ def main(): if module.RUN_IN_PREFLIGHT: handler.preflight_check() result = handler.run() - module_data[module.__name__] = result + module_data[module.MODULE_NAME] = result else: handlers.append((module, handler)) @@ -36,7 +36,7 @@ def main(): # Run handlers for module, handler in handlers: result = handler.run() - module_data[module.__name__] = result + module_data[module.MODULE_NAME] = result save_yaml(config['general']['module_data_file'], module_data) diff --git a/src/rookify/modules/__init__.py b/src/rookify/modules/__init__.py index b048498..898668e 100644 --- a/src/rookify/modules/__init__.py +++ b/src/rookify/modules/__init__.py @@ -39,6 +39,7 @@ def load_required_modules(module_names: list, modules: OrderedDict) -> None: for attr_type, attr_name in ( (ModuleHandler, 'HANDLER_CLASS'), + (str, 'MODULE_NAME'), (list, 'REQUIRES'), (list, 'AFTER'), (bool, 'RUN_IN_PREFLIGHT') diff --git a/src/rookify/modules/analyze_ceph/__init__.py b/src/rookify/modules/analyze_ceph/__init__.py index 373b776..9c18032 100644 --- a/src/rookify/modules/analyze_ceph/__init__.py +++ b/src/rookify/modules/analyze_ceph/__init__.py @@ -2,6 +2,7 @@ from .main import AnalyzeCephHandler +MODULE_NAME = 'analyze_ceph' HANDLER_CLASS = AnalyzeCephHandler RUN_IN_PREFLIGHT = True REQUIRES = [] diff --git a/src/rookify/modules/example/__init__.py b/src/rookify/modules/example/__init__.py index fb9b8cb..76da30a 100644 --- a/src/rookify/modules/example/__init__.py +++ b/src/rookify/modules/example/__init__.py @@ -2,6 +2,7 @@ from .main import ExampleHandler +MODULE_NAME = 'example' # Name of the module HANDLER_CLASS = ExampleHandler # Define the handler class for this module RUN_IN_PREFLIGHT = False # This executes the run method during preflight checks. This is neccessary for analyze modules. REQUIRES = ['analyze_ceph'] # A list of modules that are required to run before this module. Modules in this list will be imported, even if they are not configured diff --git a/src/rookify/modules/migrate_monitors/__init__.py b/src/rookify/modules/migrate_monitors/__init__.py index 2bd03a8..a0f7553 100644 --- a/src/rookify/modules/migrate_monitors/__init__.py +++ b/src/rookify/modules/migrate_monitors/__init__.py @@ -2,6 +2,7 @@ from .main import MigrateMonitorsHandler +MODULE_NAME = 'migrate_monitors' HANDLER_CLASS = MigrateMonitorsHandler RUN_IN_PREFLIGHT = False REQUIRES = ['analyze_ceph'] diff --git a/src/rookify/modules/migrate_osds/__init__.py b/src/rookify/modules/migrate_osds/__init__.py index 11f571f..f272dc5 100644 --- a/src/rookify/modules/migrate_osds/__init__.py +++ b/src/rookify/modules/migrate_osds/__init__.py @@ -2,6 +2,7 @@ from .main import MigrateOSDsHandler +MODULE_NAME = 'migrate_osds' HANDLER_CLASS = MigrateOSDsHandler RUN_IN_PREFLIGHT = False REQUIRES = ['analyze_ceph'] diff --git a/src/rookify/modules/migrate_osds/main.py b/src/rookify/modules/migrate_osds/main.py index 141a23a..d0fe192 100644 --- a/src/rookify/modules/migrate_osds/main.py +++ b/src/rookify/modules/migrate_osds/main.py @@ -10,13 +10,13 @@ def preflight_check(self): def run(self) -> dict: osd_config = dict() - for node, osds in self._data['modules.analyze_ceph']['node']['ls']['osd'].items(): + for node, osds in self._data['analyze_ceph']['node']['ls']['osd'].items(): osd_config[node] = {'osds': {}} for osd in osds: osd_config[node]['osds'][osd] = dict() - for osd in self._data['modules.analyze_ceph']['osd']['dump']['osds']: + for osd in self._data['analyze_ceph']['osd']['dump']['osds']: number = osd['osd'] uuid = osd['uuid'] for host in osd_config.values(): @@ -25,7 +25,7 @@ def run(self) -> dict: break for node, values in osd_config.items(): - devices = self._data['modules.analyze_ceph']['ssh']['osd'][node]['devices'] + devices = self._data['analyze_ceph']['ssh']['osd'][node]['devices'] for osd in values['osds'].values(): for device in devices: if osd['uuid'] in device: From b2f286ea5222b670c9a4e053c6b268152bead033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Marten=20Br=C3=BCggemann?= Date: Wed, 6 Mar 2024 16:25:52 +0100 Subject: [PATCH 3/4] Implement PREFLIGHT_REQUIRES attribute for modules (#27) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan-Marten Brüggemann --- src/rookify/__main__.py | 32 +++++---- src/rookify/modules/__init__.py | 66 +++++++++++++------ src/rookify/modules/analyze_ceph/__init__.py | 2 +- src/rookify/modules/example/__init__.py | 4 +- .../modules/migrate_monitors/__init__.py | 4 +- src/rookify/modules/migrate_osds/__init__.py | 4 +- 6 files changed, 67 insertions(+), 45 deletions(-) diff --git a/src/rookify/__main__.py b/src/rookify/__main__.py index 674d2a0..9d9b08c 100644 --- a/src/rookify/__main__.py +++ b/src/rookify/__main__.py @@ -10,7 +10,7 @@ def main(): config = load_yaml("config.yaml") except FileNotFoundError as err: raise SystemExit(f'Could not load config: {err}') - migration_modules = rookify.modules.load_modules(config['migration_modules']) + preflight_modules, migration_modules = rookify.modules.load_modules(config['migration_modules']) module_data = dict() try: @@ -18,25 +18,23 @@ def main(): except FileNotFoundError: pass - # Get a list of handlers and run handlers if they should be run in preflight + # Run preflight requirement modules + for preflight_module in preflight_modules: + handler = preflight_module.HANDLER_CLASS(config=MappingProxyType(config), data=MappingProxyType(module_data)) + result = handler.run() + module_data[preflight_module.MODULE_NAME] = result + + # Run preflight checks and append handlers to list handlers = list() - for module in migration_modules: - handler = module.HANDLER_CLASS(config=MappingProxyType(config), data=MappingProxyType(module_data)) - if module.RUN_IN_PREFLIGHT: - handler.preflight_check() - result = handler.run() - module_data[module.MODULE_NAME] = result - else: - handlers.append((module, handler)) - - # Do preflight check of all other handlers - for module, handler in handlers: + for migration_module in migration_modules: + handler = migration_module.HANDLER_CLASS(config=MappingProxyType(config), data=MappingProxyType(module_data)) handler.preflight_check() - - # Run handlers - for module, handler in handlers: + handlers.append((migration_module, handler)) + + # Run migration modules + for migration_module, handler in handlers: result = handler.run() - module_data[module.MODULE_NAME] = result + module_data[migration_module.MODULE_NAME] = result save_yaml(config['general']['module_data_file'], module_data) diff --git a/src/rookify/modules/__init__.py b/src/rookify/modules/__init__.py index 898668e..3d9cb5a 100644 --- a/src/rookify/modules/__init__.py +++ b/src/rookify/modules/__init__.py @@ -21,41 +21,65 @@ def __init__(self, module_name: str, message: str): self.module_name = module_name self.message = message -def load_modules(module_names: list) -> list: +def load_modules(module_names: list) -> tuple[list, list]: """ Dynamically loads modules from the 'modules' package. :param module_names: The module names to load + :return: returns tuple of preflight_modules, modules """ + # Sanity checks for modules + def check_module_sanity(module_name: str, module: module): + for attr_type, attr_name in ( + (ModuleHandler, 'HANDLER_CLASS'), + (str, 'MODULE_NAME'), + (list, 'REQUIRES'), + (list, 'AFTER'), + (list, 'PREFLIGHT_REQUIRES') + ): + if not hasattr(module, attr_name): + raise ModuleLoadException(module_name, f'Module has no attribute {attr_name}') + + attr = getattr(module, attr_name) + if not isinstance(attr, attr_type) and not issubclass(attr, attr_type): + raise ModuleLoadException(module_name, f'Attribute {attr_name} is not type {attr_type}') + # Load the modules in the given list and recursivley load required modules required_modules = OrderedDict() - def load_required_modules(module_names: list, modules: OrderedDict) -> None: + def load_required_modules(modules_out: OrderedDict, module_names: list) -> None: for module_name in module_names: - if module_name in modules: + if module_name in modules_out: continue - module = importlib.import_module(f"rookify.modules.{module_name}") + module = importlib.import_module(f".{module_name}", 'rookify.modules') + check_module_sanity(module_name, module) - for attr_type, attr_name in ( - (ModuleHandler, 'HANDLER_CLASS'), - (str, 'MODULE_NAME'), - (list, 'REQUIRES'), - (list, 'AFTER'), - (bool, 'RUN_IN_PREFLIGHT') - ): - if not hasattr(module, attr_name): - raise ModuleLoadException(module_name, f'Module has no attribute {attr_name}') + load_required_modules(modules_out, module.REQUIRES) + module.AFTER.extend(module.REQUIRES) - attr = getattr(module, attr_name) - if not isinstance(attr, attr_type) and not issubclass(attr, attr_type): - raise ModuleLoadException(module_name, f'Attribute {attr_name} is not type {attr_type}') + modules_out[module_name] = module + load_required_modules(required_modules, module_names) - load_required_modules(module.REQUIRES, modules) - module.AFTER.extend(module.REQUIRES) + # Recursively load the modules in the PREFLIGHT_REQUIRES attribute of the given modules + preflight_modules = OrderedDict() + def load_preflight_modules(modules_in: OrderedDict, modules_out: OrderedDict, module_names: list) -> None: + for module_name in module_names: + if module_name in modules_out: + continue + + module = importlib.import_module(f".{module_name}", 'rookify.modules') + check_module_sanity(module_name, module) - modules[module_name] = module - load_required_modules(module_names, required_modules) + # We have to check, if the preflight_requires list is already loaded as migration requirement + for preflight_requirement in module.PREFLIGHT_REQUIRES: + if preflight_requirement in modules_in: + raise ModuleLoadException(module_name, f'Module {preflight_requirement} is already loaded as migration requirement') + + load_preflight_modules(modules_in, modules_out, module.PREFLIGHT_REQUIRES) + if module_name not in modules_in: + modules_out[module_name] = module + load_preflight_modules(required_modules, preflight_modules, required_modules.keys()) # Sort the modules by the AFTER keyword modules = OrderedDict() @@ -73,4 +97,4 @@ def sort_modules(modules_in: OrderedDict, modules_out: OrderedDict, module_names modules_out[module_name] = modules_in[module_name] sort_modules(required_modules, modules, list(required_modules.keys())) - return list(modules.values()) + return list(preflight_modules.values()), list(modules.values()) diff --git a/src/rookify/modules/analyze_ceph/__init__.py b/src/rookify/modules/analyze_ceph/__init__.py index 9c18032..343c33c 100644 --- a/src/rookify/modules/analyze_ceph/__init__.py +++ b/src/rookify/modules/analyze_ceph/__init__.py @@ -4,6 +4,6 @@ MODULE_NAME = 'analyze_ceph' HANDLER_CLASS = AnalyzeCephHandler -RUN_IN_PREFLIGHT = True REQUIRES = [] AFTER = [] +PREFLIGHT_REQUIRES = [] \ No newline at end of file diff --git a/src/rookify/modules/example/__init__.py b/src/rookify/modules/example/__init__.py index 76da30a..bcd4da6 100644 --- a/src/rookify/modules/example/__init__.py +++ b/src/rookify/modules/example/__init__.py @@ -4,6 +4,6 @@ MODULE_NAME = 'example' # Name of the module HANDLER_CLASS = ExampleHandler # Define the handler class for this module -RUN_IN_PREFLIGHT = False # This executes the run method during preflight checks. This is neccessary for analyze modules. -REQUIRES = ['analyze_ceph'] # A list of modules that are required to run before this module. Modules in this list will be imported, even if they are not configured +REQUIRES = [] # A list of modules that are required to run before this module. Modules in this list will be imported, even if they are not configured AFTER = ['migrate_monitors'] # A list of modules that should be run before this module, if they are defined in config +PREFLIGHT_REQUIRES = ['analyze_ceph'] # A list of modules that are required to run the preflight_check of this module. Modules in this list will be imported and run in preflight stage. \ No newline at end of file diff --git a/src/rookify/modules/migrate_monitors/__init__.py b/src/rookify/modules/migrate_monitors/__init__.py index a0f7553..a4a8601 100644 --- a/src/rookify/modules/migrate_monitors/__init__.py +++ b/src/rookify/modules/migrate_monitors/__init__.py @@ -4,6 +4,6 @@ MODULE_NAME = 'migrate_monitors' HANDLER_CLASS = MigrateMonitorsHandler -RUN_IN_PREFLIGHT = False -REQUIRES = ['analyze_ceph'] +REQUIRES = [] AFTER = [] +PREFLIGHT_REQUIRES = ['analyze_ceph'] \ No newline at end of file diff --git a/src/rookify/modules/migrate_osds/__init__.py b/src/rookify/modules/migrate_osds/__init__.py index f272dc5..ecfcfb1 100644 --- a/src/rookify/modules/migrate_osds/__init__.py +++ b/src/rookify/modules/migrate_osds/__init__.py @@ -4,6 +4,6 @@ MODULE_NAME = 'migrate_osds' HANDLER_CLASS = MigrateOSDsHandler -RUN_IN_PREFLIGHT = False -REQUIRES = ['analyze_ceph'] +REQUIRES = [] AFTER = ['migrate_monitors'] +PREFLIGHT_REQUIRES = [ 'analyze_ceph' ] \ No newline at end of file From 14541a660c0d76dd6ad78c3023c15aee05ce660d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Marten=20Br=C3=BCggemann?= Date: Thu, 7 Mar 2024 13:19:06 +0100 Subject: [PATCH 4/4] Add pre-commit to project (#28) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add pre-commit to project Signed-off-by: Jan-Marten Brüggemann * add github workflow for pre-commit Signed-off-by: Jan-Marten Brüggemann * run pre-commit hook on all files Signed-off-by: Jan-Marten Brüggemann --------- Signed-off-by: Jan-Marten Brüggemann --- .github/workflows/pre-commit.yml | 14 +++++ .gitignore | 2 +- .pre-commit-config.yaml | 17 ++++++ pyproject.toml | 3 + setup.py | 24 ++++---- src/rookify/__main__.py | 22 +++++--- src/rookify/modules/__init__.py | 49 ++++++++++++----- src/rookify/modules/analyze_ceph/__init__.py | 4 +- src/rookify/modules/analyze_ceph/main.py | 26 +++------ src/rookify/modules/example/__init__.py | 14 +++-- src/rookify/modules/example/main.py | 4 +- .../modules/migrate_monitors/__init__.py | 4 +- src/rookify/modules/migrate_monitors/main.py | 1 + src/rookify/modules/migrate_osds/__init__.py | 6 +- src/rookify/modules/migrate_osds/main.py | 34 ++++++------ src/rookify/modules/module.py | 55 +++++++++++-------- src/rookify/yaml.py | 6 +- 17 files changed, 177 insertions(+), 108 deletions(-) create mode 100644 .github/workflows/pre-commit.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..2b11178 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,14 @@ +name: pre-commit + +on: + pull_request: + push: + branches: [main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - uses: pre-commit/action@v3.0.1 diff --git a/.gitignore b/.gitignore index e4bf442..d8dd9a3 100644 --- a/.gitignore +++ b/.gitignore @@ -88,4 +88,4 @@ cython_debug/ # Project specific data.yaml config.yaml -.ceph \ No newline at end of file +.ceph diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..f715b3c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,17 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.3.1 + hooks: + - id: ruff + args: [ --fix ] + - id: ruff-format + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-yaml + - id: check-json + - id: check-toml + - id: check-merge-conflict + - id: mixed-line-ending + - id: end-of-file-fixer + - id: trailing-whitespace diff --git a/pyproject.toml b/pyproject.toml index b768650..63a7d09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,6 @@ [build-system] requires = [ "setuptools", "wheel" ] build-backend = "setuptools.build_meta" + +[project] +requires-python = ">=3.9" diff --git a/setup.py b/setup.py index b7ae01f..abf4a76 100644 --- a/setup.py +++ b/setup.py @@ -16,31 +16,33 @@ limitations under the License. """ -from os import makedirs, path - try: from setuptools import find_packages, setup except ImportError: from distutils import find_packages, setup # + def get_version(): """ -Returns the version currently in development. + Returns the version currently in development. -:return: (str) Version string -:since: v0.0.1 + :return: (str) Version string + :since: v0.0.1 """ return "v0.0.1" + + # -_setup = { "version": get_version()[1:], - "data_files": [ ( "docs", [ "LICENSE", "README.md" ]) ], - "test_suite": "tests" - } +_setup = { + "version": get_version()[1:], + "data_files": [("docs", ["LICENSE", "README.md"])], + "test_suite": "tests", +} -_setup['package_dir'] = { "": "src" } -_setup['packages'] = find_packages("src") +_setup["package_dir"] = {"": "src"} +_setup["packages"] = find_packages("src") setup(**_setup) diff --git a/src/rookify/__main__.py b/src/rookify/__main__.py index 9d9b08c..44c71fe 100644 --- a/src/rookify/__main__.py +++ b/src/rookify/__main__.py @@ -5,38 +5,46 @@ from types import MappingProxyType from .yaml import load_yaml, save_yaml + def main(): try: config = load_yaml("config.yaml") except FileNotFoundError as err: - raise SystemExit(f'Could not load config: {err}') - preflight_modules, migration_modules = rookify.modules.load_modules(config['migration_modules']) + raise SystemExit(f"Could not load config: {err}") + preflight_modules, migration_modules = rookify.modules.load_modules( + config["migration_modules"] + ) module_data = dict() try: - module_data.update(load_yaml(config['general']['module_data_file'])) + module_data.update(load_yaml(config["general"]["module_data_file"])) except FileNotFoundError: pass # Run preflight requirement modules for preflight_module in preflight_modules: - handler = preflight_module.HANDLER_CLASS(config=MappingProxyType(config), data=MappingProxyType(module_data)) + handler = preflight_module.HANDLER_CLASS( + config=MappingProxyType(config), data=MappingProxyType(module_data) + ) result = handler.run() module_data[preflight_module.MODULE_NAME] = result # Run preflight checks and append handlers to list handlers = list() for migration_module in migration_modules: - handler = migration_module.HANDLER_CLASS(config=MappingProxyType(config), data=MappingProxyType(module_data)) + handler = migration_module.HANDLER_CLASS( + config=MappingProxyType(config), data=MappingProxyType(module_data) + ) handler.preflight_check() handlers.append((migration_module, handler)) - + # Run migration modules for migration_module, handler in handlers: result = handler.run() module_data[migration_module.MODULE_NAME] = result - save_yaml(config['general']['module_data_file'], module_data) + save_yaml(config["general"]["module_data_file"], module_data) + if __name__ == "__main__": main() diff --git a/src/rookify/modules/__init__.py b/src/rookify/modules/__init__.py index 3d9cb5a..aaf0140 100644 --- a/src/rookify/modules/__init__.py +++ b/src/rookify/modules/__init__.py @@ -1,16 +1,17 @@ # -*- coding: utf-8 -*- -import functools import importlib +import types -from typing import Optional from collections import OrderedDict from .module import ModuleHandler + class ModuleLoadException(Exception): """ ModuleLoadException is an exception class that can be raised during the dynamic load process for modules. """ + def __init__(self, module_name: str, message: str): """ Construct a new 'ModuleLoadException' object. @@ -21,6 +22,7 @@ def __init__(self, module_name: str, message: str): self.module_name = module_name self.message = message + def load_modules(module_names: list) -> tuple[list, list]: """ Dynamically loads modules from the 'modules' package. @@ -30,60 +32,76 @@ def load_modules(module_names: list) -> tuple[list, list]: """ # Sanity checks for modules - def check_module_sanity(module_name: str, module: module): + def check_module_sanity(module_name: str, module: types.ModuleType): for attr_type, attr_name in ( - (ModuleHandler, 'HANDLER_CLASS'), - (str, 'MODULE_NAME'), - (list, 'REQUIRES'), - (list, 'AFTER'), - (list, 'PREFLIGHT_REQUIRES') + (ModuleHandler, "HANDLER_CLASS"), + (str, "MODULE_NAME"), + (list, "REQUIRES"), + (list, "AFTER"), + (list, "PREFLIGHT_REQUIRES"), ): if not hasattr(module, attr_name): - raise ModuleLoadException(module_name, f'Module has no attribute {attr_name}') + raise ModuleLoadException( + module_name, f"Module has no attribute {attr_name}" + ) attr = getattr(module, attr_name) if not isinstance(attr, attr_type) and not issubclass(attr, attr_type): - raise ModuleLoadException(module_name, f'Attribute {attr_name} is not type {attr_type}') + raise ModuleLoadException( + module_name, f"Attribute {attr_name} is not type {attr_type}" + ) # Load the modules in the given list and recursivley load required modules required_modules = OrderedDict() + def load_required_modules(modules_out: OrderedDict, module_names: list) -> None: for module_name in module_names: if module_name in modules_out: continue - module = importlib.import_module(f".{module_name}", 'rookify.modules') + module = importlib.import_module(f".{module_name}", "rookify.modules") check_module_sanity(module_name, module) load_required_modules(modules_out, module.REQUIRES) module.AFTER.extend(module.REQUIRES) modules_out[module_name] = module + load_required_modules(required_modules, module_names) # Recursively load the modules in the PREFLIGHT_REQUIRES attribute of the given modules preflight_modules = OrderedDict() - def load_preflight_modules(modules_in: OrderedDict, modules_out: OrderedDict, module_names: list) -> None: + + def load_preflight_modules( + modules_in: OrderedDict, modules_out: OrderedDict, module_names: list + ) -> None: for module_name in module_names: if module_name in modules_out: continue - module = importlib.import_module(f".{module_name}", 'rookify.modules') + module = importlib.import_module(f".{module_name}", "rookify.modules") check_module_sanity(module_name, module) # We have to check, if the preflight_requires list is already loaded as migration requirement for preflight_requirement in module.PREFLIGHT_REQUIRES: if preflight_requirement in modules_in: - raise ModuleLoadException(module_name, f'Module {preflight_requirement} is already loaded as migration requirement') + raise ModuleLoadException( + module_name, + f"Module {preflight_requirement} is already loaded as migration requirement", + ) load_preflight_modules(modules_in, modules_out, module.PREFLIGHT_REQUIRES) if module_name not in modules_in: modules_out[module_name] = module + load_preflight_modules(required_modules, preflight_modules, required_modules.keys()) # Sort the modules by the AFTER keyword modules = OrderedDict() - def sort_modules(modules_in: OrderedDict, modules_out: OrderedDict, module_names: list) -> None: + + def sort_modules( + modules_in: OrderedDict, modules_out: OrderedDict, module_names: list + ) -> None: for module_name in module_names: if module_name not in modules_in: continue @@ -95,6 +113,7 @@ def sort_modules(modules_in: OrderedDict, modules_out: OrderedDict, module_names sort_modules(modules_in, modules_out, after_modules_name) modules_out[module_name] = modules_in[module_name] + sort_modules(required_modules, modules, list(required_modules.keys())) return list(preflight_modules.values()), list(modules.values()) diff --git a/src/rookify/modules/analyze_ceph/__init__.py b/src/rookify/modules/analyze_ceph/__init__.py index 343c33c..c3726e4 100644 --- a/src/rookify/modules/analyze_ceph/__init__.py +++ b/src/rookify/modules/analyze_ceph/__init__.py @@ -2,8 +2,8 @@ from .main import AnalyzeCephHandler -MODULE_NAME = 'analyze_ceph' +MODULE_NAME = "analyze_ceph" HANDLER_CLASS = AnalyzeCephHandler REQUIRES = [] AFTER = [] -PREFLIGHT_REQUIRES = [] \ No newline at end of file +PREFLIGHT_REQUIRES = [] diff --git a/src/rookify/modules/analyze_ceph/main.py b/src/rookify/modules/analyze_ceph/main.py index f9042f5..4cb570c 100644 --- a/src/rookify/modules/analyze_ceph/main.py +++ b/src/rookify/modules/analyze_ceph/main.py @@ -1,24 +1,16 @@ # -*- coding: utf-8 -*- -import json from ..module import ModuleHandler -class AnalyzeCephHandler(ModuleHandler): +class AnalyzeCephHandler(ModuleHandler): def run(self) -> dict: - - commands = [ - 'mon dump', - 'osd dump', - 'device ls', - 'fs dump', - 'node ls' - ] + commands = ["mon dump", "osd dump", "device ls", "fs dump", "node ls"] results = dict() for command in commands: - parts = command.split(' ') + parts = command.split(" ") leaf = results for idx, part in enumerate(parts): if idx < len(parts) - 1: @@ -27,12 +19,10 @@ def run(self) -> dict: leaf[part] = self.ceph.mon_command(command) leaf = leaf[part] - results['ssh'] = dict() - results['ssh']['osd'] = dict() - for node, values in results['node']['ls']['osd'].items(): - devices = self.ssh.command(node, 'find /dev/ceph-*/*').stdout.splitlines() - results['ssh']['osd'][node] = { - 'devices': devices - } + results["ssh"] = dict() + results["ssh"]["osd"] = dict() + for node, values in results["node"]["ls"]["osd"].items(): + devices = self.ssh.command(node, "find /dev/ceph-*/*").stdout.splitlines() + results["ssh"]["osd"][node] = {"devices": devices} return results diff --git a/src/rookify/modules/example/__init__.py b/src/rookify/modules/example/__init__.py index bcd4da6..8496365 100644 --- a/src/rookify/modules/example/__init__.py +++ b/src/rookify/modules/example/__init__.py @@ -2,8 +2,12 @@ from .main import ExampleHandler -MODULE_NAME = 'example' # Name of the module -HANDLER_CLASS = ExampleHandler # Define the handler class for this module -REQUIRES = [] # A list of modules that are required to run before this module. Modules in this list will be imported, even if they are not configured -AFTER = ['migrate_monitors'] # A list of modules that should be run before this module, if they are defined in config -PREFLIGHT_REQUIRES = ['analyze_ceph'] # A list of modules that are required to run the preflight_check of this module. Modules in this list will be imported and run in preflight stage. \ No newline at end of file +MODULE_NAME = "example" # Name of the module +HANDLER_CLASS = ExampleHandler # Define the handler class for this module +REQUIRES = [] # A list of modules that are required to run before this module. Modules in this list will be imported, even if they are not configured +AFTER = [ + "migrate_monitors" +] # A list of modules that should be run before this module, if they are defined in config +PREFLIGHT_REQUIRES = [ + "analyze_ceph" +] # A list of modules that are required to run the preflight_check of this module. Modules in this list will be imported and run in preflight stage. diff --git a/src/rookify/modules/example/main.py b/src/rookify/modules/example/main.py index e3fa465..12f337f 100644 --- a/src/rookify/modules/example/main.py +++ b/src/rookify/modules/example/main.py @@ -2,11 +2,11 @@ from ..module import ModuleHandler, ModuleException -class ExampleHandler(ModuleHandler): +class ExampleHandler(ModuleHandler): def preflight_check(self): # Do something for checking if all needed preconditions are met else throw ModuleException - raise ModuleException('Example module was loaded, so aborting!') + raise ModuleException("Example module was loaded, so aborting!") def run(self) -> dict: # Run the migration tasks diff --git a/src/rookify/modules/migrate_monitors/__init__.py b/src/rookify/modules/migrate_monitors/__init__.py index a4a8601..168ce94 100644 --- a/src/rookify/modules/migrate_monitors/__init__.py +++ b/src/rookify/modules/migrate_monitors/__init__.py @@ -2,8 +2,8 @@ from .main import MigrateMonitorsHandler -MODULE_NAME = 'migrate_monitors' +MODULE_NAME = "migrate_monitors" HANDLER_CLASS = MigrateMonitorsHandler REQUIRES = [] AFTER = [] -PREFLIGHT_REQUIRES = ['analyze_ceph'] \ No newline at end of file +PREFLIGHT_REQUIRES = ["analyze_ceph"] diff --git a/src/rookify/modules/migrate_monitors/main.py b/src/rookify/modules/migrate_monitors/main.py index 359c34c..367a7ca 100644 --- a/src/rookify/modules/migrate_monitors/main.py +++ b/src/rookify/modules/migrate_monitors/main.py @@ -2,5 +2,6 @@ from ..module import ModuleHandler + class MigrateMonitorsHandler(ModuleHandler): pass diff --git a/src/rookify/modules/migrate_osds/__init__.py b/src/rookify/modules/migrate_osds/__init__.py index ecfcfb1..f3b00b6 100644 --- a/src/rookify/modules/migrate_osds/__init__.py +++ b/src/rookify/modules/migrate_osds/__init__.py @@ -2,8 +2,8 @@ from .main import MigrateOSDsHandler -MODULE_NAME = 'migrate_osds' +MODULE_NAME = "migrate_osds" HANDLER_CLASS = MigrateOSDsHandler REQUIRES = [] -AFTER = ['migrate_monitors'] -PREFLIGHT_REQUIRES = [ 'analyze_ceph' ] \ No newline at end of file +AFTER = ["migrate_monitors"] +PREFLIGHT_REQUIRES = ["analyze_ceph"] diff --git a/src/rookify/modules/migrate_osds/main.py b/src/rookify/modules/migrate_osds/main.py index d0fe192..3b47496 100644 --- a/src/rookify/modules/migrate_osds/main.py +++ b/src/rookify/modules/migrate_osds/main.py @@ -1,35 +1,35 @@ # -*- coding: utf-8 -*- -from ..module import ModuleHandler, ModuleException +from ..module import ModuleHandler -class MigrateOSDsHandler(ModuleHandler): +class MigrateOSDsHandler(ModuleHandler): def preflight_check(self): - result = self.ceph.mon_command('osd dump') - #raise ModuleException('test error') + pass + # result = self.ceph.mon_command("osd dump") + # raise ModuleException('test error') def run(self) -> dict: osd_config = dict() - for node, osds in self._data['analyze_ceph']['node']['ls']['osd'].items(): - osd_config[node] = {'osds': {}} + for node, osds in self._data["analyze_ceph"]["node"]["ls"]["osd"].items(): + osd_config[node] = {"osds": {}} for osd in osds: - osd_config[node]['osds'][osd] = dict() - + osd_config[node]["osds"][osd] = dict() - for osd in self._data['analyze_ceph']['osd']['dump']['osds']: - number = osd['osd'] - uuid = osd['uuid'] + for osd in self._data["analyze_ceph"]["osd"]["dump"]["osds"]: + number = osd["osd"] + uuid = osd["uuid"] for host in osd_config.values(): - if number in host['osds']: - host['osds'][number]['uuid'] = uuid + if number in host["osds"]: + host["osds"][number]["uuid"] = uuid break for node, values in osd_config.items(): - devices = self._data['analyze_ceph']['ssh']['osd'][node]['devices'] - for osd in values['osds'].values(): + devices = self._data["analyze_ceph"]["ssh"]["osd"][node]["devices"] + for osd in values["osds"].values(): for device in devices: - if osd['uuid'] in device: - osd['device'] = device + if osd["uuid"] in device: + osd["device"] = device break print(osd_config) diff --git a/src/rookify/modules/module.py b/src/rookify/modules/module.py index 7eef9a7..d2882b9 100644 --- a/src/rookify/modules/module.py +++ b/src/rookify/modules/module.py @@ -6,9 +6,11 @@ import kubernetes import fabric + class ModuleException(Exception): pass + class ModuleHandler: """ ModuleHandler is an abstract class that modules have to extend. @@ -17,27 +19,26 @@ class ModuleHandler: class __Ceph: def __init__(self, config: dict): try: - self.__ceph = rados.Rados(conffile=config['conf_file'], conf={'keyring': config['keyring']}) + self.__ceph = rados.Rados( + conffile=config["conf_file"], conf={"keyring": config["keyring"]} + ) self.__ceph.connect() except rados.ObjectNotFound as err: - raise ModuleException(f'Could not connect to ceph: {err}') + raise ModuleException(f"Could not connect to ceph: {err}") def mon_command(self, command: str, **kwargs) -> dict: - cmd = { - 'prefix': command, - 'format': 'json' - } + cmd = {"prefix": command, "format": "json"} cmd.update(kwargs) - result = self.__ceph.mon_command(json.dumps(cmd), b'') + result = self.__ceph.mon_command(json.dumps(cmd), b"") if result[0] != 0: - raise ModuleException(f'Ceph did return an error: {result}') + raise ModuleException(f"Ceph did return an error: {result}") return json.loads(result[1]) class __K8s: def __init__(self, config: dict): k8s_config = kubernetes.client.Configuration() - k8s_config.api_key = config['api_key'] - k8s_config.host = config['host'] + k8s_config.api_key = config["api_key"] + k8s_config.host = config["host"] self.__client = kubernetes.client.ApiClient(k8s_config) @property @@ -58,14 +59,22 @@ def __init__(self, config: dict): def command(self, host: str, command: str) -> fabric.runners.Result: try: - address = self.__config['hosts'][host]['address'] - user = self.__config['hosts'][host]['user'] - port = self.__config['hosts'][host]['port'] if 'port' in self.__config['hosts'][host] else 22 - private_key = self.__config['private_key'] + address = self.__config["hosts"][host]["address"] + user = self.__config["hosts"][host]["user"] + port = ( + self.__config["hosts"][host]["port"] + if "port" in self.__config["hosts"][host] + else 22 + ) + private_key = self.__config["private_key"] except KeyError as err: - raise ModuleException(f'Could not find settings for {host} in config: {err}') - connect_kwargs = {'key_filename': private_key} - result = fabric.Connection(address, user=user, port=port, connect_kwargs=connect_kwargs).run(command, hide=True) + raise ModuleException( + f"Could not find settings for {host} in config: {err}" + ) + connect_kwargs = {"key_filename": private_key} + result = fabric.Connection( + address, user=user, port=port, connect_kwargs=connect_kwargs + ).run(command, hide=True) return result def __init__(self, config: dict, data: dict): @@ -99,18 +108,18 @@ def run(self) -> dict: @property def ceph(self) -> __Ceph: - if self.__ceph == None: - self.__ceph = self.__Ceph(self._config['ceph']) + if self.__ceph is None: + self.__ceph = self.__Ceph(self._config["ceph"]) return self.__ceph @property def k8s(self) -> __K8s: - if self.__k8s == None: - self.__k8s = self.__K8s(self._config['kubernetes']) + if self.__k8s is None: + self.__k8s = self.__K8s(self._config["kubernetes"]) return self.__k8s @property def ssh(self) -> __SSH: - if self.__ssh == None: - self.__ssh = self.__SSH(self._config['ssh']) + if self.__ssh is None: + self.__ssh = self.__SSH(self._config["ssh"]) return self.__ssh diff --git a/src/rookify/yaml.py b/src/rookify/yaml.py index 44d6e49..9cf6ad7 100644 --- a/src/rookify/yaml.py +++ b/src/rookify/yaml.py @@ -2,10 +2,12 @@ import yaml + def load_yaml(path: str) -> dict: - with open(path, 'r') as file: + with open(path, "r") as file: return yaml.safe_load(file) + def save_yaml(path: str, data: dict) -> None: - with open(path, 'w') as file: + with open(path, "w") as file: yaml.safe_dump(data, file)