From fbef4972f8578f8844be50fa4eb84922ef443325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Marten=20Br=C3=BCggemann?= Date: Wed, 13 Mar 2024 17:26:15 +0100 Subject: [PATCH 1/3] Add mypy to pre-commit hooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan-Marten Brüggemann --- .pre-commit-config.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f715b3c..898981e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,3 +15,10 @@ repos: - id: mixed-line-ending - id: end-of-file-fixer - id: trailing-whitespace + - repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v1.9.0' + hooks: + - id: mypy + args: [--strict, --ignore-missing-imports, --check-untyped-defs] + additional_dependencies: + - types-PyYAML From f19d0276c5de04ac4cda90b49bfe9c9816756db7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Marten=20Br=C3=BCggemann?= Date: Wed, 13 Mar 2024 17:26:47 +0100 Subject: [PATCH 2/3] fix typing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan-Marten Brüggemann --- setup.py | 9 +++++---- src/rookify/modules/__init__.py | 10 ++++++---- src/rookify/modules/analyze_ceph/__init__.py | 1 + src/rookify/modules/analyze_ceph/main.py | 4 ++-- src/rookify/modules/example/__init__.py | 1 + src/rookify/modules/example/main.py | 2 +- src/rookify/modules/migrate_monitors/__init__.py | 1 + src/rookify/modules/migrate_osds/__init__.py | 1 + src/rookify/modules/migrate_osds/main.py | 5 ++++- src/rookify/modules/module.py | 13 +++++++------ 10 files changed, 29 insertions(+), 18 deletions(-) diff --git a/setup.py b/setup.py index abf4a76..ef6b75a 100644 --- a/setup.py +++ b/setup.py @@ -16,14 +16,15 @@ limitations under the License. """ +from typing import Dict, Any + try: from setuptools import find_packages, setup except ImportError: - from distutils import find_packages, setup -# + from distutils import find_packages, setup # type: ignore -def get_version(): +def get_version() -> str: """ Returns the version currently in development. @@ -36,7 +37,7 @@ def get_version(): # -_setup = { +_setup: Dict[str, Any] = { "version": get_version()[1:], "data_files": [("docs", ["LICENSE", "README.md"])], "test_suite": "tests", diff --git a/src/rookify/modules/__init__.py b/src/rookify/modules/__init__.py index aaf0140..6e670b9 100644 --- a/src/rookify/modules/__init__.py +++ b/src/rookify/modules/__init__.py @@ -52,7 +52,7 @@ def check_module_sanity(module_name: str, module: types.ModuleType): ) # Load the modules in the given list and recursivley load required modules - required_modules = OrderedDict() + required_modules: OrderedDict[str, types.ModuleType] = OrderedDict() def load_required_modules(modules_out: OrderedDict, module_names: list) -> None: for module_name in module_names: @@ -70,7 +70,7 @@ def load_required_modules(modules_out: OrderedDict, module_names: list) -> None: load_required_modules(required_modules, module_names) # Recursively load the modules in the PREFLIGHT_REQUIRES attribute of the given modules - preflight_modules = OrderedDict() + preflight_modules: OrderedDict[str, types.ModuleType] = OrderedDict() def load_preflight_modules( modules_in: OrderedDict, modules_out: OrderedDict, module_names: list @@ -94,10 +94,12 @@ def load_preflight_modules( if module_name not in modules_in: modules_out[module_name] = module - load_preflight_modules(required_modules, preflight_modules, required_modules.keys()) + load_preflight_modules( + required_modules, preflight_modules, list(required_modules.keys()) + ) # Sort the modules by the AFTER keyword - modules = OrderedDict() + modules: OrderedDict[str, types.ModuleType] = OrderedDict() def sort_modules( modules_in: OrderedDict, modules_out: OrderedDict, module_names: list diff --git a/src/rookify/modules/analyze_ceph/__init__.py b/src/rookify/modules/analyze_ceph/__init__.py index c3726e4..d821800 100644 --- a/src/rookify/modules/analyze_ceph/__init__.py +++ b/src/rookify/modules/analyze_ceph/__init__.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# type: ignore from .main import AnalyzeCephHandler diff --git a/src/rookify/modules/analyze_ceph/main.py b/src/rookify/modules/analyze_ceph/main.py index 4cb570c..266e92e 100644 --- a/src/rookify/modules/analyze_ceph/main.py +++ b/src/rookify/modules/analyze_ceph/main.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- - +from typing import Dict from ..module import ModuleHandler @@ -8,7 +8,7 @@ class AnalyzeCephHandler(ModuleHandler): def run(self) -> dict: commands = ["mon dump", "osd dump", "device ls", "fs dump", "node ls"] - results = dict() + results: Dict[str, dict] = dict() for command in commands: parts = command.split(" ") leaf = results diff --git a/src/rookify/modules/example/__init__.py b/src/rookify/modules/example/__init__.py index 8496365..ff0dcf5 100644 --- a/src/rookify/modules/example/__init__.py +++ b/src/rookify/modules/example/__init__.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# type: ignore from .main import ExampleHandler diff --git a/src/rookify/modules/example/main.py b/src/rookify/modules/example/main.py index 12f337f..6642a22 100644 --- a/src/rookify/modules/example/main.py +++ b/src/rookify/modules/example/main.py @@ -10,4 +10,4 @@ def preflight_check(self): def run(self) -> dict: # Run the migration tasks - pass + return {} diff --git a/src/rookify/modules/migrate_monitors/__init__.py b/src/rookify/modules/migrate_monitors/__init__.py index 168ce94..ff3c337 100644 --- a/src/rookify/modules/migrate_monitors/__init__.py +++ b/src/rookify/modules/migrate_monitors/__init__.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# type: ignore from .main import MigrateMonitorsHandler diff --git a/src/rookify/modules/migrate_osds/__init__.py b/src/rookify/modules/migrate_osds/__init__.py index f3b00b6..86041ae 100644 --- a/src/rookify/modules/migrate_osds/__init__.py +++ b/src/rookify/modules/migrate_osds/__init__.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# type: ignore from .main import MigrateOSDsHandler diff --git a/src/rookify/modules/migrate_osds/main.py b/src/rookify/modules/migrate_osds/main.py index 3b47496..97ada77 100644 --- a/src/rookify/modules/migrate_osds/main.py +++ b/src/rookify/modules/migrate_osds/main.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from typing import Dict + from ..module import ModuleHandler @@ -10,7 +12,7 @@ def preflight_check(self): # raise ModuleException('test error') def run(self) -> dict: - osd_config = dict() + osd_config: Dict[str, dict] = dict() for node, osds in self._data["analyze_ceph"]["node"]["ls"]["osd"].items(): osd_config[node] = {"osds": {}} for osd in osds: @@ -33,3 +35,4 @@ def run(self) -> dict: break print(osd_config) + return {} diff --git a/src/rookify/modules/module.py b/src/rookify/modules/module.py index d2882b9..055448b 100644 --- a/src/rookify/modules/module.py +++ b/src/rookify/modules/module.py @@ -5,6 +5,7 @@ import rados import kubernetes import fabric +from typing import Any class ModuleException(Exception): @@ -17,7 +18,7 @@ class ModuleHandler: """ class __Ceph: - def __init__(self, config: dict): + def __init__(self, config: dict[str, Any]): try: self.__ceph = rados.Rados( conffile=config["conf_file"], conf={"keyring": config["keyring"]} @@ -26,9 +27,9 @@ def __init__(self, config: dict): except rados.ObjectNotFound as err: raise ModuleException(f"Could not connect to ceph: {err}") - def mon_command(self, command: str, **kwargs) -> dict: + def mon_command(self, command: str, **kwargs: dict[str, str]) -> dict[str, Any]: cmd = {"prefix": command, "format": "json"} - cmd.update(kwargs) + cmd.update(**kwargs) result = self.__ceph.mon_command(json.dumps(cmd), b"") if result[0] != 0: raise ModuleException(f"Ceph did return an error: {result}") @@ -86,9 +87,9 @@ def __init__(self, config: dict, data: dict): """ self._config = config self._data = data - self.__ceph = None - self.__k8s = None - self.__ssh = None + self.__ceph: self.__Ceph = None # type: ignore + self.__k8s: self.__K8s = None # type: ignore + self.__ssh: self.__SSH = None # type: ignore @abc.abstractmethod def preflight_check(self) -> None: From b35bb934fa3dd116f19cb292c7eb61fbc33a68b7 Mon Sep 17 00:00:00 2001 From: Gondermann Date: Thu, 14 Mar 2024 12:14:34 +0100 Subject: [PATCH 3/3] Fixed missing/wrong type hints Signed-off-by: Gondermann --- setup.py | 2 +- src/rookify/__main__.py | 2 +- src/rookify/modules/__init__.py | 19 ++++++++++---- src/rookify/modules/analyze_ceph/main.py | 7 +++--- src/rookify/modules/example/main.py | 6 +++-- src/rookify/modules/migrate_osds/main.py | 10 ++++---- src/rookify/modules/module.py | 32 +++++++++++++----------- src/rookify/yaml.py | 9 ++++--- 8 files changed, 53 insertions(+), 34 deletions(-) diff --git a/setup.py b/setup.py index ef6b75a..af65979 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ try: from setuptools import find_packages, setup except ImportError: - from distutils import find_packages, setup # type: ignore + from distutils import find_packages, setup # type: ignore[attr-defined, unused-ignore] def get_version() -> str: diff --git a/src/rookify/__main__.py b/src/rookify/__main__.py index 44c71fe..bada9ba 100644 --- a/src/rookify/__main__.py +++ b/src/rookify/__main__.py @@ -6,7 +6,7 @@ from .yaml import load_yaml, save_yaml -def main(): +def main() -> None: try: config = load_yaml("config.yaml") except FileNotFoundError as err: diff --git a/src/rookify/modules/__init__.py b/src/rookify/modules/__init__.py index 6e670b9..3167e96 100644 --- a/src/rookify/modules/__init__.py +++ b/src/rookify/modules/__init__.py @@ -2,6 +2,7 @@ import importlib import types +from typing import List from collections import OrderedDict from .module import ModuleHandler @@ -23,7 +24,9 @@ def __init__(self, module_name: str, message: str): self.message = message -def load_modules(module_names: list) -> tuple[list, list]: +def load_modules( + module_names: List[str], +) -> tuple[List[types.ModuleType], List[types.ModuleType]]: """ Dynamically loads modules from the 'modules' package. @@ -32,7 +35,7 @@ def load_modules(module_names: list) -> tuple[list, list]: """ # Sanity checks for modules - def check_module_sanity(module_name: str, module: types.ModuleType): + def check_module_sanity(module_name: str, module: types.ModuleType) -> None: for attr_type, attr_name in ( (ModuleHandler, "HANDLER_CLASS"), (str, "MODULE_NAME"), @@ -54,7 +57,9 @@ def check_module_sanity(module_name: str, module: types.ModuleType): # Load the modules in the given list and recursivley load required modules required_modules: OrderedDict[str, types.ModuleType] = OrderedDict() - def load_required_modules(modules_out: OrderedDict, module_names: list) -> None: + def load_required_modules( + modules_out: OrderedDict[str, types.ModuleType], module_names: List[str] + ) -> None: for module_name in module_names: if module_name in modules_out: continue @@ -73,7 +78,9 @@ def load_required_modules(modules_out: OrderedDict, module_names: list) -> None: preflight_modules: OrderedDict[str, types.ModuleType] = OrderedDict() def load_preflight_modules( - modules_in: OrderedDict, modules_out: OrderedDict, module_names: list + modules_in: OrderedDict[str, types.ModuleType], + modules_out: OrderedDict[str, types.ModuleType], + module_names: List[str], ) -> None: for module_name in module_names: if module_name in modules_out: @@ -102,7 +109,9 @@ def load_preflight_modules( modules: OrderedDict[str, types.ModuleType] = OrderedDict() def sort_modules( - modules_in: OrderedDict, modules_out: OrderedDict, module_names: list + modules_in: OrderedDict[str, types.ModuleType], + modules_out: OrderedDict[str, types.ModuleType], + module_names: List[str], ) -> None: for module_name in module_names: if module_name not in modules_in: diff --git a/src/rookify/modules/analyze_ceph/main.py b/src/rookify/modules/analyze_ceph/main.py index 266e92e..cc90105 100644 --- a/src/rookify/modules/analyze_ceph/main.py +++ b/src/rookify/modules/analyze_ceph/main.py @@ -1,14 +1,15 @@ # -*- coding: utf-8 -*- -from typing import Dict from ..module import ModuleHandler +from typing import Any, Dict + class AnalyzeCephHandler(ModuleHandler): - def run(self) -> dict: + def run(self) -> Dict[str, Any]: commands = ["mon dump", "osd dump", "device ls", "fs dump", "node ls"] - results: Dict[str, dict] = dict() + results: Dict[str, Any] = dict() for command in commands: parts = command.split(" ") leaf = results diff --git a/src/rookify/modules/example/main.py b/src/rookify/modules/example/main.py index 6642a22..2d0391e 100644 --- a/src/rookify/modules/example/main.py +++ b/src/rookify/modules/example/main.py @@ -2,12 +2,14 @@ from ..module import ModuleHandler, ModuleException +from typing import Any, Dict + class ExampleHandler(ModuleHandler): - def preflight_check(self): + def preflight_check(self) -> None: # Do something for checking if all needed preconditions are met else throw ModuleException raise ModuleException("Example module was loaded, so aborting!") - def run(self) -> dict: + def run(self) -> Dict[str, Any]: # Run the migration tasks return {} diff --git a/src/rookify/modules/migrate_osds/main.py b/src/rookify/modules/migrate_osds/main.py index 97ada77..d6b4a34 100644 --- a/src/rookify/modules/migrate_osds/main.py +++ b/src/rookify/modules/migrate_osds/main.py @@ -1,18 +1,18 @@ # -*- coding: utf-8 -*- -from typing import Dict - from ..module import ModuleHandler +from typing import Any, Dict + class MigrateOSDsHandler(ModuleHandler): - def preflight_check(self): + def preflight_check(self) -> None: pass # result = self.ceph.mon_command("osd dump") # raise ModuleException('test error') - def run(self) -> dict: - osd_config: Dict[str, dict] = dict() + def run(self) -> Dict[str, Any]: + osd_config: Dict[str, Any] = dict() for node, osds in self._data["analyze_ceph"]["node"]["ls"]["osd"].items(): osd_config[node] = {"osds": {}} for osd in osds: diff --git a/src/rookify/modules/module.py b/src/rookify/modules/module.py index 055448b..84208e5 100644 --- a/src/rookify/modules/module.py +++ b/src/rookify/modules/module.py @@ -5,7 +5,7 @@ import rados import kubernetes import fabric -from typing import Any +from typing import Any, Dict, List, Optional class ModuleException(Exception): @@ -18,7 +18,7 @@ class ModuleHandler: """ class __Ceph: - def __init__(self, config: dict[str, Any]): + def __init__(self, config: Dict[str, Any]): try: self.__ceph = rados.Rados( conffile=config["conf_file"], conf={"keyring": config["keyring"]} @@ -27,16 +27,20 @@ def __init__(self, config: dict[str, Any]): except rados.ObjectNotFound as err: raise ModuleException(f"Could not connect to ceph: {err}") - def mon_command(self, command: str, **kwargs: dict[str, str]) -> dict[str, Any]: + def mon_command( + self, command: str, **kwargs: str + ) -> Dict[str, Any] | List[Any]: cmd = {"prefix": command, "format": "json"} cmd.update(**kwargs) result = self.__ceph.mon_command(json.dumps(cmd), b"") if result[0] != 0: raise ModuleException(f"Ceph did return an error: {result}") - return json.loads(result[1]) + data = json.loads(result[1]) + assert isinstance(data, dict) or isinstance(data, list) + return data class __K8s: - def __init__(self, config: dict): + def __init__(self, config: Dict[str, Any]): k8s_config = kubernetes.client.Configuration() k8s_config.api_key = config["api_key"] k8s_config.host = config["host"] @@ -55,7 +59,7 @@ def NodeV1Api(self) -> kubernetes.client.NodeV1Api: return kubernetes.client.NodeV1Api(self.__client) class __SSH: - def __init__(self, config: dict): + def __init__(self, config: Dict[str, Any]): self.__config = config def command(self, host: str, command: str) -> fabric.runners.Result: @@ -78,7 +82,7 @@ def command(self, host: str, command: str) -> fabric.runners.Result: ).run(command, hide=True) return result - def __init__(self, config: dict, data: dict): + def __init__(self, config: Dict[str, Any], data: Dict[str, Any]): """ Construct a new 'ModuleHandler' object. @@ -87,9 +91,9 @@ def __init__(self, config: dict, data: dict): """ self._config = config self._data = data - self.__ceph: self.__Ceph = None # type: ignore - self.__k8s: self.__K8s = None # type: ignore - self.__ssh: self.__SSH = None # type: ignore + self.__ceph: Optional[ModuleHandler.__Ceph] = None + self.__k8s: Optional[ModuleHandler.__K8s] = None + self.__ssh: Optional[ModuleHandler.__SSH] = None @abc.abstractmethod def preflight_check(self) -> None: @@ -99,7 +103,7 @@ def preflight_check(self) -> None: pass @abc.abstractmethod - def run(self) -> dict: + def run(self) -> Dict[str, Any]: """ Run the modules tasks @@ -110,17 +114,17 @@ def run(self) -> dict: @property def ceph(self) -> __Ceph: if self.__ceph is None: - self.__ceph = self.__Ceph(self._config["ceph"]) + self.__ceph = ModuleHandler.__Ceph(self._config["ceph"]) return self.__ceph @property def k8s(self) -> __K8s: if self.__k8s is None: - self.__k8s = self.__K8s(self._config["kubernetes"]) + self.__k8s = ModuleHandler.__K8s(self._config["kubernetes"]) return self.__k8s @property def ssh(self) -> __SSH: if self.__ssh is None: - self.__ssh = self.__SSH(self._config["ssh"]) + self.__ssh = ModuleHandler.__SSH(self._config["ssh"]) return self.__ssh diff --git a/src/rookify/yaml.py b/src/rookify/yaml.py index 9cf6ad7..856716f 100644 --- a/src/rookify/yaml.py +++ b/src/rookify/yaml.py @@ -1,13 +1,16 @@ # -*- coding: utf-8 -*- import yaml +from typing import Any, Dict -def load_yaml(path: str) -> dict: +def load_yaml(path: str) -> Dict[str, Any]: with open(path, "r") as file: - return yaml.safe_load(file) + data = yaml.safe_load(file) + assert isinstance(data, dict) + return data -def save_yaml(path: str, data: dict) -> None: +def save_yaml(path: str, data: Dict[str, Any]) -> None: with open(path, "w") as file: yaml.safe_dump(data, file)