From f9091255e59e036d9f516a9ec1bb3a268627d7d7 Mon Sep 17 00:00:00 2001 From: lvoloshyn-sekoia Date: Mon, 5 Aug 2024 14:10:25 +0300 Subject: [PATCH] Fixes and improvements --- pyproject.toml | 4 +-- sekoia_automation/cli.py | 2 +- sekoia_automation/scripts/check_compliance.py | 18 ++++++------- .../scripts/compliance/validators/__init__.py | 7 +++-- .../compliance/validators/actions_json.py | 27 +++++++++---------- .../scripts/compliance/validators/base.py | 1 - .../compliance/validators/changelog.py | 7 +++-- .../compliance/validators/connectors_json.py | 23 ++++++++-------- .../scripts/compliance/validators/deps.py | 2 -- .../compliance/validators/dockerfile.py | 2 -- .../scripts/compliance/validators/logo.py | 3 +-- .../scripts/compliance/validators/main.py | 1 - .../scripts/compliance/validators/manifest.py | 5 ++-- .../scripts/compliance/validators/module.py | 1 - .../scripts/compliance/validators/tests.py | 1 - .../compliance/validators/triggers_json.py | 27 +++++++++---------- 16 files changed, 59 insertions(+), 72 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 448b867..59f0fef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,14 +125,14 @@ testpaths = [ ] [tool.ruff] -select = ["A", "ARG", "E", "F", "I", "N", "RUF", "UP", "W"] +lint.select = ["A", "ARG", "E", "F", "I", "N", "RUF", "UP", "W"] exclude = [ "tests/expectations/sample_module/main.py", "tests/aio/", "sekoia_automation/scripts/new_module/template/" ] -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "tests/*" = ["ARG"] # Ignore unusued args because of pytest fixtures [tool.mypy] diff --git a/sekoia_automation/cli.py b/sekoia_automation/cli.py index 45383a6..0f8d841 100644 --- a/sekoia_automation/cli.py +++ b/sekoia_automation/cli.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from typing import Optional, Annotated +from typing import Optional import typer from cookiecutter.main import cookiecutter diff --git a/sekoia_automation/scripts/check_compliance.py b/sekoia_automation/scripts/check_compliance.py index b82cdbf..6ef2280 100644 --- a/sekoia_automation/scripts/check_compliance.py +++ b/sekoia_automation/scripts/check_compliance.py @@ -101,7 +101,7 @@ def format_errors(self, mod_val: ModuleValidator, ignored_paths: set[Path]) -> s errors = mod_val.result.errors module_name = mod_val.path.name return "\n".join( - "%s:%s:%s" % (module_name, error.filepath.name, error.error) + f"{module_name}:{error.filepath.name}:{error.error}" for error in errors if error.filepath not in ignored_paths ) @@ -121,12 +121,12 @@ def find_modules(self, root_path: Path) -> list[Path]: return result def fix_set_uuid(self, file_path: Path, uuid: str) -> None: - with open(file_path, "rt") as file: + with open(file_path) as file: manifest = json.load(file) manifest["uuid"] = uuid - with open(file_path, "wt") as file: + with open(file_path, "w") as file: json.dump(manifest, file, indent=2) def check_uniqueness(self, items, error_msg: str): @@ -175,13 +175,12 @@ def check_docker_params(self, validators: list[ModuleValidator]): continue if data["connector"] != data["trigger"]: - filename_to_fix = "connector_%s" % suffix + filename_to_fix = f"connector_{suffix}" filepath = module_path / filename_to_fix validator.result.errors.append( CheckError( filepath=filepath, - error=f"`docker_parameters` is not consistent with trigger_%s" - % suffix, + error=f"`docker_parameters` is not consistent with trigger_{suffix}", ) ) # We don't want to check these further @@ -236,14 +235,13 @@ def check_uuids_and_slugs(self, validators: list[ModuleValidator]): continue if data["connector"] != data["trigger"]: - filename_to_fix = "connector_%s" % suffix + filename_to_fix = f"connector_{suffix}" filepath = module_path / filename_to_fix validator.result.errors.append( CheckError( filepath=filepath, - error=f"UUID is not consistent with trigger_%s" % suffix, - fix_label="Set the same UUID for trigger_%s and connector_%s" - % (suffix, suffix), + error=f"UUID is not consistent with trigger_{suffix}", + fix_label=f"Set the same UUID for trigger_{suffix} and connector_{suffix}", fix=partial( self.fix_set_uuid, file_path=filepath, diff --git a/sekoia_automation/scripts/compliance/validators/__init__.py b/sekoia_automation/scripts/compliance/validators/__init__.py index fdce3fe..b9e6c52 100644 --- a/sekoia_automation/scripts/compliance/validators/__init__.py +++ b/sekoia_automation/scripts/compliance/validators/__init__.py @@ -8,9 +8,12 @@ from .logo import LogoValidator from .main import MainPYValidator from .manifest import ManifestValidator +from .module import ModuleValidator from .tests import TestsValidator from .triggers_json import TriggersJSONValidator -from .module import ModuleValidator - MODULES_PATH = Path(__file__).parent.parent.parent.parent + +__all__ = ( +ActionsJSONValidator, ChangelogValidator, ConnectorsJSONValidator, DependenciesValidator, DockerfileValidator, +LogoValidator, MainPYValidator, ManifestValidator, ModuleValidator, TestsValidator, TriggersJSONValidator) diff --git a/sekoia_automation/scripts/compliance/validators/actions_json.py b/sekoia_automation/scripts/compliance/validators/actions_json.py index c9ef5c1..ede5b8b 100644 --- a/sekoia_automation/scripts/compliance/validators/actions_json.py +++ b/sekoia_automation/scripts/compliance/validators/actions_json.py @@ -1,4 +1,3 @@ -import argparse import json import uuid from functools import partial @@ -27,23 +26,23 @@ def validate(cls, result: CheckResult) -> None: @classmethod def validate_action_json(cls, path: Path, result: CheckResult): try: - with open(path, "rt") as file: + with open(path) as file: raw = json.load(file) except ValueError: - result.errors.append(CheckError(filepath=path, error=f"can't load JSON")) + result.errors.append(CheckError(filepath=path, error="can't load JSON")) return if not isinstance(raw.get("name"), str): result.errors.append( - CheckError(filepath=path, error=f"`name` is not present") + CheckError(filepath=path, error="`name` is not present") ) if not isinstance(raw.get("uuid"), str): result.errors.append( CheckError( filepath=path, - error=f"`uuid` is not present", + error="`uuid` is not present", fix_label="Generate random UUID", fix=partial(cls.set_random_uuid, path=path), ) @@ -57,13 +56,13 @@ def validate_action_json(cls, path: Path, result: CheckResult): if not isinstance(raw.get("description"), str): result.errors.append( - CheckError(filepath=path, error=f"`description` is not present") + CheckError(filepath=path, error="`description` is not present") ) # @todo track this to main.py ? if not isinstance(raw.get("docker_parameters"), str): result.errors.append( - CheckError(filepath=path, error=f"`docker_parameters` is not present") + CheckError(filepath=path, error="`docker_parameters` is not present") ) else: @@ -78,22 +77,22 @@ def validate_action_json(cls, path: Path, result: CheckResult): if not isinstance(raw.get("arguments"), dict): result.errors.append( - CheckError(filepath=path, error=f"`arguments` is not present") + CheckError(filepath=path, error="`arguments` is not present") ) elif not cls.is_valid_json_schema(raw["arguments"]): result.errors.append( - CheckError(filepath=path, error=f"`arguments` is not valid JSON schema") + CheckError(filepath=path, error="`arguments` is not valid JSON schema") ) if not isinstance(raw.get("results"), dict): result.errors.append( - CheckError(filepath=path, error=f"`results` is not present") + CheckError(filepath=path, error="`results` is not present") ) elif not cls.is_valid_json_schema(raw["results"]): result.errors.append( - CheckError(filepath=path, error=f"`results` is not valid JSON schema") + CheckError(filepath=path, error="`results` is not valid JSON schema") ) @staticmethod @@ -101,15 +100,15 @@ def is_valid_json_schema(schema: dict) -> bool: try: Draft7Validator.check_schema(schema) return True - except SchemaError as e: + except SchemaError: return False @staticmethod def set_random_uuid(path: Path): - with open(path, "rt") as file: + with open(path) as file: manifest = json.load(file) manifest["uuid"] = str(uuid.uuid4()) - with open(path, "wt") as file: + with open(path, "w") as file: json.dump(manifest, file, indent=2) diff --git a/sekoia_automation/scripts/compliance/validators/base.py b/sekoia_automation/scripts/compliance/validators/base.py index dc73994..b750194 100644 --- a/sekoia_automation/scripts/compliance/validators/base.py +++ b/sekoia_automation/scripts/compliance/validators/base.py @@ -1,4 +1,3 @@ -import argparse from abc import ABC from .models import CheckResult diff --git a/sekoia_automation/scripts/compliance/validators/changelog.py b/sekoia_automation/scripts/compliance/validators/changelog.py index c1f4b04..b3a6748 100644 --- a/sekoia_automation/scripts/compliance/validators/changelog.py +++ b/sekoia_automation/scripts/compliance/validators/changelog.py @@ -1,4 +1,3 @@ -import argparse import re from pathlib import Path @@ -25,7 +24,7 @@ def validate(cls, result: CheckResult) -> None: result.options["changelog_path"] = changelog_path - with open(changelog_path, "rt") as file: + with open(changelog_path) as file: text = file.read() cls.validate_changelog_content(text=text, path=changelog_path, result=result) @@ -163,7 +162,7 @@ def validate(self, path: Path, result: CheckResult): versions = self.versions() if len(versions) == 0: result.errors.append( - CheckError(filepath=path, error=f"No entries in changelog") + CheckError(filepath=path, error="No entries in changelog") ) for v in versions: @@ -256,7 +255,7 @@ def validate_version_sections( if body and not body.startswith("###"): result.errors.append( CheckError( - filepath=path, error=f"Version has content outside of a section." + filepath=path, error="Version has content outside of a section." ) ) diff --git a/sekoia_automation/scripts/compliance/validators/connectors_json.py b/sekoia_automation/scripts/compliance/validators/connectors_json.py index 3013ac5..0b6fdc2 100644 --- a/sekoia_automation/scripts/compliance/validators/connectors_json.py +++ b/sekoia_automation/scripts/compliance/validators/connectors_json.py @@ -1,4 +1,3 @@ -import argparse import json import uuid from functools import partial @@ -27,23 +26,23 @@ def validate(cls, result: CheckResult) -> None: @classmethod def validate_connector_json(cls, path: Path, result: CheckResult): try: - with open(path, "rt") as file: + with open(path) as file: raw = json.load(file) except ValueError: - result.errors.append(CheckError(filepath=path, error=f"can't load JSON")) + result.errors.append(CheckError(filepath=path, error="can't load JSON")) return if not isinstance(raw.get("name"), str): result.errors.append( - CheckError(filepath=path, error=f"`name` is not present") + CheckError(filepath=path, error="`name` is not present") ) if not isinstance(raw.get("uuid"), str): result.errors.append( CheckError( filepath=path, - error=f"`uuid` is not present", + error="`uuid` is not present", fix_label="Generate random UUID", fix=partial(cls.set_random_uuid, path=path), ) @@ -57,12 +56,12 @@ def validate_connector_json(cls, path: Path, result: CheckResult): if not isinstance(raw.get("description"), str): result.errors.append( - CheckError(filepath=path, error=f"`description` is not present") + CheckError(filepath=path, error="`description` is not present") ) if not isinstance(raw.get("docker_parameters"), str): result.errors.append( - CheckError(filepath=path, error=f"`docker_parameters` is not present") + CheckError(filepath=path, error="`docker_parameters` is not present") ) else: if "docker_parameters" not in result.options: @@ -76,12 +75,12 @@ def validate_connector_json(cls, path: Path, result: CheckResult): if not isinstance(raw.get("arguments"), dict): result.errors.append( - CheckError(filepath=path, error=f"`arguments` is not present") + CheckError(filepath=path, error="`arguments` is not present") ) elif not cls.is_valid_json_schema(raw["arguments"]): result.errors.append( - CheckError(filepath=path, error=f"`arguments` is not valid JSON schema") + CheckError(filepath=path, error="`arguments` is not valid JSON schema") ) # @todo perhaps check if results present? (they shouldn't be) @@ -91,15 +90,15 @@ def is_valid_json_schema(schema: dict) -> bool: try: Draft7Validator.check_schema(schema) return True - except SchemaError as e: + except SchemaError: return False @staticmethod def set_random_uuid(path: Path): - with open(path, "rt") as file: + with open(path) as file: manifest = json.load(file) manifest["uuid"] = str(uuid.uuid4()) - with open(path, "wt") as file: + with open(path, "w") as file: json.dump(manifest, file, indent=2) diff --git a/sekoia_automation/scripts/compliance/validators/deps.py b/sekoia_automation/scripts/compliance/validators/deps.py index 0e0a09f..e1f7e81 100644 --- a/sekoia_automation/scripts/compliance/validators/deps.py +++ b/sekoia_automation/scripts/compliance/validators/deps.py @@ -1,5 +1,3 @@ -import argparse -import os from pathlib import Path from .base import Validator diff --git a/sekoia_automation/scripts/compliance/validators/dockerfile.py b/sekoia_automation/scripts/compliance/validators/dockerfile.py index efd7170..101c8ab 100644 --- a/sekoia_automation/scripts/compliance/validators/dockerfile.py +++ b/sekoia_automation/scripts/compliance/validators/dockerfile.py @@ -1,5 +1,3 @@ -import argparse -import os from pathlib import Path from .base import Validator diff --git a/sekoia_automation/scripts/compliance/validators/logo.py b/sekoia_automation/scripts/compliance/validators/logo.py index 040e7dc..5ca26af 100644 --- a/sekoia_automation/scripts/compliance/validators/logo.py +++ b/sekoia_automation/scripts/compliance/validators/logo.py @@ -1,4 +1,3 @@ -import argparse import os from functools import partial from pathlib import Path @@ -30,7 +29,7 @@ def check_logo_image(result: CheckResult) -> None: image_path = module_dir / "logo.png" if not image_path.is_file(): - result.errors.append(CheckError(filepath=image_path, error=f"Logo is missing")) + result.errors.append(CheckError(filepath=image_path, error="Logo is missing")) return image = Image.open(image_path) diff --git a/sekoia_automation/scripts/compliance/validators/main.py b/sekoia_automation/scripts/compliance/validators/main.py index 32868aa..a8d622b 100644 --- a/sekoia_automation/scripts/compliance/validators/main.py +++ b/sekoia_automation/scripts/compliance/validators/main.py @@ -1,4 +1,3 @@ -import argparse from pathlib import Path from pyastgrep.search import FileFinished, search_python_files diff --git a/sekoia_automation/scripts/compliance/validators/manifest.py b/sekoia_automation/scripts/compliance/validators/manifest.py index 1f72b9d..081021c 100644 --- a/sekoia_automation/scripts/compliance/validators/manifest.py +++ b/sekoia_automation/scripts/compliance/validators/manifest.py @@ -1,4 +1,3 @@ -import argparse import json import re from pathlib import Path @@ -29,7 +28,7 @@ def validate(cls, result: CheckResult) -> None: result.options["manifest_path"] = manifest_path try: - with open(manifest_path, "rt") as f: + with open(manifest_path) as f: manifest = json.load(f) except ValueError: @@ -123,5 +122,5 @@ def is_valid_json_schema(schema: dict) -> bool: try: Draft7Validator.check_schema(schema) return True - except SchemaError as e: + except SchemaError: return False diff --git a/sekoia_automation/scripts/compliance/validators/module.py b/sekoia_automation/scripts/compliance/validators/module.py index de1887e..6dac690 100644 --- a/sekoia_automation/scripts/compliance/validators/module.py +++ b/sekoia_automation/scripts/compliance/validators/module.py @@ -1,4 +1,3 @@ -import argparse from pathlib import Path from . import ( diff --git a/sekoia_automation/scripts/compliance/validators/tests.py b/sekoia_automation/scripts/compliance/validators/tests.py index caab3bc..3101d9d 100644 --- a/sekoia_automation/scripts/compliance/validators/tests.py +++ b/sekoia_automation/scripts/compliance/validators/tests.py @@ -1,4 +1,3 @@ -import argparse from pathlib import Path from .base import Validator diff --git a/sekoia_automation/scripts/compliance/validators/triggers_json.py b/sekoia_automation/scripts/compliance/validators/triggers_json.py index 586f6d0..af93114 100644 --- a/sekoia_automation/scripts/compliance/validators/triggers_json.py +++ b/sekoia_automation/scripts/compliance/validators/triggers_json.py @@ -1,4 +1,3 @@ -import argparse import json import uuid from functools import partial @@ -27,23 +26,23 @@ def validate(cls, result: CheckResult) -> None: @classmethod def validate_trigger_json(cls, path: Path, result: CheckResult): try: - with open(path, "rt") as file: + with open(path) as file: raw = json.load(file) except ValueError: - result.errors.append(CheckError(filepath=path, error=f"can't load JSON")) + result.errors.append(CheckError(filepath=path, error="can't load JSON")) return if not isinstance(raw.get("name"), str): result.errors.append( - CheckError(filepath=path, error=f"`name` is not present") + CheckError(filepath=path, error="`name` is not present") ) if not isinstance(raw.get("uuid"), str): result.errors.append( CheckError( filepath=path, - error=f"`uuid` is not present", + error="`uuid` is not present", fix_label="Generate random UUID", fix=partial(cls.set_random_uuid, path=path), ) @@ -57,12 +56,12 @@ def validate_trigger_json(cls, path: Path, result: CheckResult): if not isinstance(raw.get("description"), str): result.errors.append( - CheckError(filepath=path, error=f"`description` is not present") + CheckError(filepath=path, error="`description` is not present") ) if not isinstance(raw.get("docker_parameters"), str): result.errors.append( - CheckError(filepath=path, error=f"`docker_parameters` is not present") + CheckError(filepath=path, error="`docker_parameters` is not present") ) else: @@ -77,22 +76,22 @@ def validate_trigger_json(cls, path: Path, result: CheckResult): if not isinstance(raw.get("arguments"), dict): result.errors.append( - CheckError(filepath=path, error=f"`arguments` is not present") + CheckError(filepath=path, error="`arguments` is not present") ) elif not cls.is_valid_json_schema(raw["arguments"]): result.errors.append( - CheckError(filepath=path, error=f"`arguments` is not valid JSON schema") + CheckError(filepath=path, error="`arguments` is not valid JSON schema") ) if not isinstance(raw.get("results"), dict): result.errors.append( - CheckError(filepath=path, error=f"`results` is not present") + CheckError(filepath=path, error="`results` is not present") ) elif not cls.is_valid_json_schema(raw["results"]): result.errors.append( - CheckError(filepath=path, error=f"`results` is not valid JSON schema") + CheckError(filepath=path, error="`results` is not valid JSON schema") ) @staticmethod @@ -100,15 +99,15 @@ def is_valid_json_schema(schema: dict) -> bool: try: Draft7Validator.check_schema(schema) return True - except SchemaError as e: + except SchemaError: return False @staticmethod def set_random_uuid(path: Path): - with open(path, "rt") as file: + with open(path) as file: manifest = json.load(file) manifest["uuid"] = str(uuid.uuid4()) - with open(path, "wt") as file: + with open(path, "w") as file: json.dump(manifest, file, indent=2)