Merge branch 'main' into prs/python-build-scripts

.gitignore

@@ -89,3 +89,4 @@ cython_debug/
 data.yaml
 config.yaml
 .ceph
+.k8s

.pre-commit-config.yaml

@@ -15,3 +15,10 @@ repos:
       - id: mixed-line-ending
       - id: end-of-file-fixer
       - id: trailing-whitespace
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: 'v1.9.0'
+    hooks:
+    -   id: mypy
+        args: [--strict, --ignore-missing-imports, --check-untyped-defs]
+        additional_dependencies:
+          - types-PyYAML

Makefile

@@ -0,0 +1,52 @@
+COLOUR_GREEN=\033[0;32m
+COLOUR_RED=\033[0;31m
+COLOUR_BLUE=\033[0;34m
+COLOUR_END=\033[0m
+
+.DEFAULT_GOAL:=help
+
+# Get needed paths and information from locally installed librados
+RADOSLIB_VERSION := 2.0.0
+GENERAL_LIB_LOCATION := $(shell pip show rados | grep -oP "(?<=Location: ).*")
+RADOSLIB_INSTALLED_VERSION := $(shell pip show rados | grep Version | awk '{print $$2}')
+
+.PHONY: help
+help: ## Display this help message
+	@echo -e '${COLOUR_RED}Usage: make <command>${COLOUR_END}'
+	@cat $(MAKEFILE_LIST) | grep '^[a-zA-Z]'  | \
+	    awk -F ':.*?## ' 'NF==2 {printf "  %-26s%s\n\n", $$1, "${COLOUR_GREEN}"$$2"${COLOUR_END}"}'
+
+.PHONY: setup
+setup: setup-pre-commit check-radoslib setup-venv ## Setup the pre-commit environment and then the venv environment
+
+.PHONY: setup-pre-commit
+setup-pre-commit:
+	pip install --user pre-commit && pre-commit install
+
+.PHONY: setup-venv
+setup-venv:
+	python -m venv --system-site-packages ./.venv && \
+	source ./.venv/bin/activate && \
+	pip install --ignore-installed -r requirements.txt
+
+.PHONY: update-requirements
+update-requirements: ## Update the requirements.txt with newer versions of pip packages
+	source ./.venv/bin/activate && \
+	pip freeze -l > requirements.txt
+
+.PHONY: check-radoslib
+check-radoslib: ## Checks if radoslib is installed and if it contains the right version
+	@if [ -z "$(GENERAL_LIB_LOCATION)" ]; then \
+		echo -e "${COLOUR_RED}ERROR: 'rados' library not found. Please make sure it's installed.${COLOUR_END}"; \
+		exit 1; \
+	fi
+	@if [ "$(RADOSLIB_INSTALLED_VERSION)" != "$(RADOSLIB_VERSION)" ]; then \
+		echo -e "${COLOUR_RED}ERROR: Incorrect version of 'rados' library found. Expected version $(RADOSLIB_VERSION), found $$RADOSLIB_INSTALLED_VERSION.${COLOUR_END}"; \
+		exit 1; \
+	fi
+
+.PHONY: run-local-rookify
+run-local-rookify: ## Runs rookify in the local development environment (requires setup-venv)
+	$(eval PYTHONPATH="${PYTHONPATH}:$(pwd)/src")
+	source ./.venv/bin/activate && \
+	cd src && python3 -m rookify

README.md

@@ -16,6 +16,7 @@ Rookify is designed to facilitate a smooth and efficient transition for existing
 - Access to a Kubernetes cluster with sufficient resources to host the migrated Ceph cluster.
   - Kubernetes nodes should be rolled out at least on the OSD nodes
 - Rook operator version 1.13 or higher installed in the Kubernetes cluster.
+- _local development enivornment_ requires radoslib version 2.0.0 installed
 
 ## Installation
 1. Clone the repository:
@@ -28,11 +29,19 @@ git clone https://github.com/SovereignCloudStack/rookify
 cd rookify
 ```
 
-3. __TODO:__ Install script
+3. To install the local development environment
+(_Note: This will install pre-commit in your local user context_):
+```bash
+make setup
+```
+
+4. __TODO:__ Install script for container-based setup
 
 ## Usage
 __TODO__
 
+Type `make` to get a list of available development specific commands.
+
 ## Support
 For issues, questions, or contributions, please open an issue or pull request in the GitHub repository. We welcome community feedback and contributions to enhance rookify.
 

pyproject.toml

@@ -3,4 +3,24 @@ requires = [ "setuptools", "wheel" ]
 build-backend = "setuptools.build_meta"
 
 [project]
+dynamic = ["version"]
 requires-python = ">=3.9"
+
+name = "Rookify"
+description = "Enable Ceph-Cluster migrations to Rook"
+readme = "README.md"
+authors = [{name = "Sovereign Cloud Stack Developers", email = "[email protected]"}]
+license = {file = "LICENSE"}
+classifiers = [
+    "Intended Audience :: System Administrators",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Environment :: Console",
+    "Topic :: System :: Clustering",
+    "Topic :: System :: Systems Administration",
+    "Topic :: Utilities"
+]
+
+[project.urls]
+Homepage = "https://scs.community"

requirements.txt

@@ -10,7 +10,9 @@ fabric==3.2.2
 google-auth==2.28.1
 idna==3.6
 invoke==2.2.0
+Jinja2==3.1.3
 kubernetes==29.0.0
+MarkupSafe==2.1.5
 oauthlib==3.2.2
 paramiko==3.4.0
 pyasn1==0.5.1
@@ -19,7 +21,6 @@ pycparser==2.21
 PyNaCl==1.5.0
 python-dateutil==2.8.2
 PyYAML==6.0.1
-rados==2.0.0
 requests==2.31.0
 requests-oauthlib==1.3.1
 rsa==4.9

setup.cfg

@@ -1,21 +1,5 @@
 [metadata]
-name = Rookify
-description = Enable Ceph-Cluster migrations to Rook
-long_description = Rookify is designed to facilitate a smooth and efficient transition for existing Ceph clusters to a Rook-managed Ceph cluster environment.
 platforms = any
-author = Sovereign Cloud Stack Developers
-author_email = [email protected]
-license = Apache
-url = https://scs.community
-classifiers =
-    Intended Audience :: System Administrators
-    License :: OSI Approved :: Apache License
-    Operating System :: OS Independent
-    Programming Language :: Python
-    Environment :: Console
-    Topic :: System :: Clustering
-    Topic :: System :: Systems Administration
-    Topic :: Utilities
 
 [options]
 install_requires=file:requirements.txt

setup.py

@@ -17,14 +17,15 @@
 """
 
 import os
+from typing import Dict, Any
 
 try:
     from setuptools import find_packages, setup
 except ImportError:
-    from distutils import find_packages, setup
+    from distutils import find_packages, setup  # type: ignore[attr-defined, unused-ignore]
 
 
-def get_version():
+def get_version() -> str:
     """
     Returns the version currently in development.
 
@@ -35,7 +36,7 @@ def get_version():
     return os.environ.get("ROOKIFY_VERSION", "0.0.0-dev")
 
 
-_setup = {
+_setup: Dict[str, Any] = {
     "version": get_version(),
     "data_files": [("docs", ["LICENSE", "README.md"])],
     "entry_points": {"console_scripts": ["rookify = rookify.__main__:main"]},

src/config.example.yaml

@@ -19,8 +19,14 @@ ssh:
       user: dragon
 
 kubernetes:
-  host: 192.168.22.10
-  api_key: abc
+  config: ../.k8s/config
+
+rook:
+  cluster:
+    name: osism-ceph
+    namespace: rook-ceph
+  ceph:
+    image: quay.io/ceph/ceph:v18.2.1
 
 migration_modules:
 - migrate_osds

src/rookify/__main__.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
 
+import os
 import rookify.modules
 
 from types import MappingProxyType
 from .yaml import load_yaml, save_yaml
 
 
-def main():
+def main() -> None:
     try:
         config = load_yaml("config.yaml")
     except FileNotFoundError as err:
@@ -21,21 +22,34 @@ def main():
     except FileNotFoundError:
         pass
 
+    # Get absolute path of the rookify instance
+    rookify_path = os.path.dirname(__file__)
+
     # Run preflight requirement modules
     for preflight_module in preflight_modules:
+        module_path = os.path.join(
+            rookify_path, "modules", preflight_module.MODULE_NAME
+        )
         handler = preflight_module.HANDLER_CLASS(
-            config=MappingProxyType(config), data=MappingProxyType(module_data)
+            config=MappingProxyType(config),
+            data=MappingProxyType(module_data),
+            module_path=module_path,
         )
         result = handler.run()
         module_data[preflight_module.MODULE_NAME] = result
 
-    # Run preflight checks and append handlers to list
+    # Run preflight and append handlers to list
     handlers = list()
     for migration_module in migration_modules:
+        module_path = os.path.join(
+            rookify_path, "modules", migration_module.MODULE_NAME
+        )
         handler = migration_module.HANDLER_CLASS(
-            config=MappingProxyType(config), data=MappingProxyType(module_data)
+            config=MappingProxyType(config),
+            data=MappingProxyType(module_data),
+            module_path=module_path,
         )
-        handler.preflight_check()
+        handler.preflight()
         handlers.append((migration_module, handler))
 
     # Run migration modules

src/rookify/modules/__init__.py

@@ -2,6 +2,7 @@
 
 import importlib
 import types
+from typing import List
 
 from collections import OrderedDict
 from .module import ModuleHandler
@@ -23,7 +24,9 @@ def __init__(self, module_name: str, message: str):
         self.message = message
 
 
-def load_modules(module_names: list) -> tuple[list, list]:
+def load_modules(
+    module_names: List[str],
+) -> tuple[List[types.ModuleType], List[types.ModuleType]]:
     """
     Dynamically loads modules from the 'modules' package.
 
@@ -32,7 +35,7 @@ def load_modules(module_names: list) -> tuple[list, list]:
     """
 
     # Sanity checks for modules
-    def check_module_sanity(module_name: str, module: types.ModuleType):
+    def check_module_sanity(module_name: str, module: types.ModuleType) -> None:
         for attr_type, attr_name in (
             (ModuleHandler, "HANDLER_CLASS"),
             (str, "MODULE_NAME"),
@@ -52,9 +55,11 @@ def check_module_sanity(module_name: str, module: types.ModuleType):
                 )
 
     # Load the modules in the given list and recursivley load required modules
-    required_modules = OrderedDict()
+    required_modules: OrderedDict[str, types.ModuleType] = OrderedDict()
 
-    def load_required_modules(modules_out: OrderedDict, module_names: list) -> None:
+    def load_required_modules(
+        modules_out: OrderedDict[str, types.ModuleType], module_names: List[str]
+    ) -> None:
         for module_name in module_names:
             if module_name in modules_out:
                 continue
@@ -70,10 +75,12 @@ def load_required_modules(modules_out: OrderedDict, module_names: list) -> None:
     load_required_modules(required_modules, module_names)
 
     # Recursively load the modules in the PREFLIGHT_REQUIRES attribute of the given modules
-    preflight_modules = OrderedDict()
+    preflight_modules: OrderedDict[str, types.ModuleType] = OrderedDict()
 
     def load_preflight_modules(
-        modules_in: OrderedDict, modules_out: OrderedDict, module_names: list
+        modules_in: OrderedDict[str, types.ModuleType],
+        modules_out: OrderedDict[str, types.ModuleType],
+        module_names: List[str],
     ) -> None:
         for module_name in module_names:
             if module_name in modules_out:
@@ -94,13 +101,17 @@ def load_preflight_modules(
             if module_name not in modules_in:
                 modules_out[module_name] = module
 
-    load_preflight_modules(required_modules, preflight_modules, required_modules.keys())
+    load_preflight_modules(
+        required_modules, preflight_modules, list(required_modules.keys())
+    )
 
     # Sort the modules by the AFTER keyword
-    modules = OrderedDict()
+    modules: OrderedDict[str, types.ModuleType] = OrderedDict()
 
     def sort_modules(
-        modules_in: OrderedDict, modules_out: OrderedDict, module_names: list
+        modules_in: OrderedDict[str, types.ModuleType],
+        modules_out: OrderedDict[str, types.ModuleType],
+        module_names: List[str],
     ) -> None:
         for module_name in module_names:
             if module_name not in modules_in:

src/rookify/modules/analyze_ceph/__init__.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# type: ignore
 
 from .main import AnalyzeCephHandler
 

src/rookify/modules/analyze_ceph/main.py

@@ -1,14 +1,15 @@
 # -*- coding: utf-8 -*-
 
-
 from ..module import ModuleHandler
 
+from typing import Any, Dict
+
 
 class AnalyzeCephHandler(ModuleHandler):
-    def run(self) -> dict:
+    def run(self) -> Any:
         commands = ["mon dump", "osd dump", "device ls", "fs dump", "node ls"]
 
-        results = dict()
+        results: Dict[str, Any] = dict()
         for command in commands:
             parts = command.split(" ")
             leaf = results

src/rookify/modules/example/__init__.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# type: ignore
 
 from .main import ExampleHandler
 

src/rookify/modules/example/main.py

@@ -2,12 +2,14 @@
 
 from ..module import ModuleHandler, ModuleException
 
+from typing import Any
+
 
 class ExampleHandler(ModuleHandler):
-    def preflight_check(self):
+    def preflight(self) -> None:
         # Do something for checking if all needed preconditions are met else throw ModuleException
         raise ModuleException("Example module was loaded, so aborting!")
 
-    def run(self) -> dict:
+    def run(self) -> Any:
         # Run the migration tasks
-        pass
+        return {}

src/rookify/modules/migrate_monitors/__init__.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# type: ignore
 
 from .main import MigrateMonitorsHandler
 

src/rookify/modules/migrate_osds/__init__.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# type: ignore
 
 from .main import MigrateOSDsHandler