Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move compliance checks to SDK #128

Open
wants to merge 25 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
3081d7d
Move compliance checks to SDK
lvoloshyn-sekoia Aug 5, 2024
fd90341
Fix default path
lvoloshyn-sekoia Aug 5, 2024
f909125
Fixes and improvements
lvoloshyn-sekoia Aug 5, 2024
83dd0cf
Fixes and improvements
lvoloshyn-sekoia Aug 5, 2024
1da1f31
Fix linting
lvoloshyn-sekoia Aug 5, 2024
6f4e736
Fixes for mypy
lvoloshyn-sekoia Aug 5, 2024
31f6019
Fixes for mypy
lvoloshyn-sekoia Aug 5, 2024
f8e2e4e
Fix tests
lvoloshyn-sekoia Aug 5, 2024
a94ea93
Enhance test fixtures
lvoloshyn-sekoia Aug 5, 2024
f36a820
Merge branch 'main' into lv/move_compliance_check_to_sdk
lvoloshyn-sekoia Aug 5, 2024
3a7e674
Improve linting
lvoloshyn-sekoia Aug 5, 2024
dcff650
Merge remote-tracking branch 'origin/lv/move_compliance_check_to_sdk'…
lvoloshyn-sekoia Aug 5, 2024
3686420
Fix test
lvoloshyn-sekoia Aug 5, 2024
794317d
Fix test
lvoloshyn-sekoia Aug 5, 2024
bb3dd89
Omit cov for action tester
lvoloshyn-sekoia Aug 6, 2024
428167f
Move compliance deps in a separate group
lvoloshyn-sekoia Sep 10, 2024
0052d25
Merge branch 'main' into lv/move_compliance_check_to_sdk
lvoloshyn-sekoia Sep 10, 2024
2ea3823
Update deps
lvoloshyn-sekoia Sep 10, 2024
9675651
Exclude compliance checks from coverage
lvoloshyn-sekoia Sep 10, 2024
2e84881
Merge branch 'main' into lv/move_compliance_check_to_sdk
lvoloshyn-sekoia Sep 10, 2024
7f06dbc
Update poetry.lock
lvoloshyn-sekoia Sep 10, 2024
8e1fef2
Merge branch 'main' into lv/move_compliance_check_to_sdk
lvoloshyn-sekoia Sep 20, 2024
46c87b9
Update poetry.lock
lvoloshyn-sekoia Sep 20, 2024
4b9a75d
Merge branch 'main' into lv/move_compliance_check_to_sdk
lvoloshyn-sekoia Oct 7, 2024
d5ef2e8
Update deps
lvoloshyn-sekoia Oct 7, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fixes and improvements
lvoloshyn-sekoia committed Aug 5, 2024
commit 83dd0cf17045bcfd20e4f2358a1398fcb60af64c
30 changes: 21 additions & 9 deletions sekoia_automation/scripts/check_compliance.py
Original file line number Diff line number Diff line change
@@ -41,7 +41,8 @@ def run(self, fix: bool = False) -> None:
r = self.check_module(module)
all_validators.append(r)

# We have to check all the modules, but show results only for the selected ones
# We have to check all the modules, but
# show results only for the selected ones
if module in self.modules:
selected_validators.append(r)

@@ -67,7 +68,10 @@ def run(self, fix: bool = False) -> None:
print("🛠 Available automatic fixes (run with `fix` command):")
for error in errors_to_fix:
print(
f"FIX {error.filepath.relative_to(self.modules_path)}:{error.fix_label}"
f"FIX "
f"{error.filepath.relative_to(self.modules_path)}"
f":"
f"{error.fix_label}"
)

else:
@@ -77,7 +81,10 @@ def run(self, fix: bool = False) -> None:
print("Fixing...")
for error in errors_to_fix:
print(
f"FIX {error.filepath.relative_to(self.modules_path)}:{error.fix_label}"
f"FIX "
f"{error.filepath.relative_to(self.modules_path)}"
f":"
f"{error.fix_label}"
)
error.fix()

@@ -135,8 +142,9 @@ def check_uniqueness(self, items, error_msg: str):
for file_name, val in v:
path = val.result.options["path"] / file_name

# We don't add fix call (e.g. generating new UUID) here, because it would create
# a lot of error-prone corner cases
# We don't add fix call (e.g. generating new UUID)
# here, because it would create a lot of
# error-prone corner cases
val.result.errors.append(
CheckError(
filepath=path,
@@ -170,7 +178,8 @@ def check_docker_params(self, validators: list[ModuleValidator]):
] = docker

for suffix, data in suffix_to_docker.items():
# ignore cases where we have only either `trigger_` or `connector_` files
# ignore cases where we have only
# either `trigger_` or `connector_` files
if "connector" not in data or "trigger" not in data:
continue

@@ -180,7 +189,8 @@ def check_docker_params(self, validators: list[ModuleValidator]):
validator.result.errors.append(
CheckError(
filepath=filepath,
error=f"`docker_parameters` is not consistent with trigger_{suffix}",
error=f"`docker_parameters` is not "
f"consistent with trigger_{suffix}",
)
)
# We don't want to check these further
@@ -230,7 +240,8 @@ def check_uuids_and_slugs(self, validators: list[ModuleValidator]):
suffix_to_uuid[filename.lstrip("connector_")]["connector"] = uuid

for suffix, data in suffix_to_uuid.items():
# ignore cases where we have only either `trigger_` or `connector_` files
# ignore cases where we have only either
# `trigger_` or `connector_` files
if "connector" not in data or "trigger" not in data:
continue

@@ -241,7 +252,8 @@ def check_uuids_and_slugs(self, validators: list[ModuleValidator]):
CheckError(
filepath=filepath,
error=f"UUID is not consistent with trigger_{suffix}",
fix_label=f"Set the same UUID for trigger_{suffix} and connector_{suffix}",
fix_label=f"Set the same UUID for "
f"trigger_{suffix} and connector_{suffix}",
fix=partial(
self.fix_set_uuid,
file_path=filepath,
6 changes: 4 additions & 2 deletions sekoia_automation/scripts/compliance/validators/__init__.py
Original file line number Diff line number Diff line change
@@ -15,5 +15,7 @@
MODULES_PATH = Path(__file__).parent.parent.parent.parent

__all__ = (
ActionsJSONValidator, ChangelogValidator, ConnectorsJSONValidator, DependenciesValidator, DockerfileValidator,
LogoValidator, MainPYValidator, ManifestValidator, ModuleValidator, TestsValidator, TriggersJSONValidator)
ActionsJSONValidator, ChangelogValidator, ConnectorsJSONValidator,
DependenciesValidator,DockerfileValidator,LogoValidator, MainPYValidator,
ManifestValidator, ModuleValidator, TestsValidator, TriggersJSONValidator
)
3 changes: 2 additions & 1 deletion sekoia_automation/scripts/compliance/validators/base.py
Original file line number Diff line number Diff line change
@@ -5,7 +5,8 @@

class Validator(ABC):
"""
Every validator class changes mutable `result` data class either by adding data for the downstream validators
Every validator class changes mutable `result` data class
either by adding data for the downstream validators
or enhancing list of detected errors
"""

15 changes: 10 additions & 5 deletions sekoia_automation/scripts/compliance/validators/changelog.py
Original file line number Diff line number Diff line change
@@ -113,8 +113,10 @@ def validate(self, path: Path, result: CheckResult):
allowed_header_titles = {"Changelog", "Change log"}
default_content = [
"All notable changes to this project will be documented in this file.",
"The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), "
"and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).",
"The format is based on "
"[Keep a Changelog](https://keepachangelog.com/en/1.0.0/), "
"and this project adheres to "
"[Semantic Versioning](https://semver.org/spec/v2.0.0.html).",
]

# 1. assert only one header and starts on first line
@@ -145,7 +147,8 @@ def validate(self, path: Path, result: CheckResult):
result.errors.append(
CheckError(
filepath=path,
error=f"Header title not valid. Options are [{','.join(allowed_header_titles)}]",
error=f"Header title not valid. Options "
f"are [{','.join(allowed_header_titles)}]",
)
)

@@ -224,7 +227,8 @@ def validate_version_date(
result.errors.append(
CheckError(
filepath=path,
error="Versions need to be decorated with a release date in the following format 'YYYY-MM-DD'",
error="Versions need to be decorated with a release "
"date in the following format 'YYYY-MM-DD'",
)
)
return
@@ -276,7 +280,8 @@ def validate_version_sections(
result.errors.append(
CheckError(
filepath=path,
error=f'"{title}" is not a valid section for a version. Options are [{",".join(allowed_version_sections)}]',
error=f'"{title}" is not a valid section for a version. '
f'Options are [{",".join(allowed_version_sections)}]',
)
)

6 changes: 4 additions & 2 deletions sekoia_automation/scripts/compliance/validators/main.py
Original file line number Diff line number Diff line change
@@ -59,7 +59,8 @@ def validate(cls, result: CheckResult) -> None:
result.errors.append(
CheckError(
filepath=main_path,
error=f"Docker parameter `{item}` used multiple times in main.py",
error=f"Docker parameter `{item}`"
f" used multiple times in main.py",
)
)

@@ -79,6 +80,7 @@ def validate(cls, result: CheckResult) -> None:
result.errors.append(
CheckError(
filepath=main_path,
error=f"`Docker parameter {item}` is registered in main.py, but absent in any JSON",
error=f"`Docker parameter {item}` is registered in main.py,"
f" but absent in any JSON",
)
)
3 changes: 2 additions & 1 deletion sekoia_automation/scripts/compliance/validators/manifest.py
Original file line number Diff line number Diff line change
@@ -96,7 +96,8 @@ def validate(cls, result: CheckResult) -> None:
result.errors.append(
CheckError(
filepath=manifest_path,
error=f"'{module_version}' is not valid semantic version. Read more: https://semver.org/",
error=f"'{module_version}' is not valid semantic version. "
f"Read more: https://semver.org/",
)
)