Merge branch 'main' into fix/tests_for_session

# Conflicts:
#	CHANGELOG.md
#	pyproject.toml

CHANGELOG.md

@@ -7,12 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-
 ### Fixed
 
+## 1.16.1 - 2024-10-16
+
 - Fix tests for async version of connector.
 
-## [1.15.1] - 2024-10-04
+## 1.16.0 - 2024-10-16
+
+### Changed
+
+- Update documentation generation command to follow new structure
+
+## 1.15.1 - 2024-10-04
 
 ### Changed
 

pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api"
 [tool.poetry]
 name = "sekoia-automation-sdk"
 
-version = "1.15.2"
+version = "1.16.1"
 description = "SDK to create Sekoia.io playbook modules"
 license = "MIT"
 readme = "README.md"

sekoia_automation/scripts/documentation/generate.py

@@ -1,6 +1,7 @@
 import copy
 import json
 import os
+from collections import defaultdict
 from dataclasses import dataclass
 from pathlib import Path
 from shutil import copyfile
@@ -16,11 +17,10 @@
 class DocumentationModule:
     name: str
     file_name: str
+    categories: list[str]
 
 
 class DocumentationGenerator:
-    MKDOCS_SUB_NAV = "Features/Automate/Actions Library"
-
     @property
     def modules_paths(self):
         """
@@ -52,6 +52,12 @@ def generate_module_doc(self, module_path: Path) -> DocumentationModule | None:
         with (module_path / "manifest.json").open("rb") as fd:
             module_manifest = json.load(fd)
 
+        # Load connectors uuids
+        connector_uuids = set()
+        for connector_file in sorted(module_path.glob("connector_*.json")):
+            with connector_file.open("rb") as fd:
+                connector_uuids.add(json.load(fd)["uuid"])
+
         # Load actions
         module_actions = []
         for action_file in sorted(module_path.glob("action_*.json")):
@@ -62,7 +68,20 @@ def generate_module_doc(self, module_path: Path) -> DocumentationModule | None:
         module_triggers = []
         for trigger_file in sorted(module_path.glob("trigger_*.json")):
             with trigger_file.open("rb") as fd:
-                module_triggers.append(json.load(fd))
+                loaded = json.load(fd)
+                if loaded["uuid"] in connector_uuids:
+                    print(
+                        f"[white][!] {module_path.name}: Trigger {loaded['name']}"
+                        " ignored because it is a connector[/white]"
+                    )
+                else:
+                    module_triggers.append(loaded)
+
+        if not module_actions and not module_triggers:
+            print(
+                f"[orange3][!] {module_path.name}: No actions/triggers found[/orange3]"
+            )
+            return None
 
         # Copy the logo
         module_logo_filename = self._copy_logo(module_path, module_manifest)
@@ -92,7 +111,9 @@ def generate_module_doc(self, module_path: Path) -> DocumentationModule | None:
             fd.write(output.encode("utf-8"))
 
         return DocumentationModule(
-            name=module_manifest["name"], file_name=module_doc_filename
+            name=module_manifest["name"],
+            file_name=module_doc_filename,
+            categories=module_manifest.get("categories", []),
         )
 
     def _copy_logo(self, module_path: Path, module_manifest: dict) -> str | None:
@@ -121,8 +142,22 @@ def _update_submenu(
                     if append_only:
                         item.setdefault(root_target, [])
                         # Sort by the name of the key in the dict
+                        for category_dict in value:
+                            category = next(iter(category_dict))
+                            for root_categories_dict in item[root_target]:
+                                root_category = next(iter(root_categories_dict))
+                                if root_category == category:
+                                    root_categories_dict[root_category] = sorted(
+                                        root_categories_dict[root_category]
+                                        + category_dict[category],
+                                        key=lambda x: next(iter(x)),
+                                    )
+                                    break
+                            else:
+                                item[root_target].append(category_dict)
+
                         item[root_target] = sorted(
-                            item[root_target] + value, key=lambda x: next(iter(x))
+                            item[root_target], key=lambda x: next(iter(x))
                         )
                     else:
                         item[root_target] = value
@@ -164,15 +199,31 @@ def _update_documentation_menu(self, modules: list[DocumentationModule]):
             mkdocs_conf = yaml.safe_load(fd)
 
             root_menu_items = {
-                f"Sekoia.io XDR/{self.MKDOCS_SUB_NAV}": "xdr/features/automate/library",
-                f"Sekoia.io TIP/{self.MKDOCS_SUB_NAV}": "tip/features/automate/library",
+                "Sekoia.io TIP/Features/"
+                "Automate/Actions Library": "tip/features/automate/library",
+                "Integrations/List of Playbooks Actions": "integration/action_library",
             }
             append_only = True if self.module else False
             for root, directory in root_menu_items.items():
-                sub_content = [
-                    {module.name: f"{directory}/{module.file_name}"}
-                    for module in modules
-                ]
+                categories_mapping = defaultdict(list)
+                for module in modules:
+                    for category in module.categories:
+                        categories_mapping[category].append(
+                            {module.name: f"{directory}/{module.file_name}"}
+                        )
+
+                sub_content: list[dict[str, str | list]] = []
+                for category, pages in sorted(
+                    categories_mapping.items(), key=lambda item: item[0]
+                ):
+                    sub_content.append(
+                        {category: sorted(pages, key=lambda x: next(iter(x)))}
+                    )
+                if not append_only:
+                    sub_content = [
+                        {"Overview": f"{directory}/overview.md"},
+                        *sub_content,
+                    ]
 
                 self._update_submenu(
                     mkdocs_conf["nav"],

sekoia_automation/scripts/documentation/templates/module.md

@@ -1,4 +1,9 @@
+
 {#- -*- mode: jinja2 -*- -#}
+uuid: {{manifest.uuid}}
+name: {{manifest.name}}
+type: playbook
+
 # {{manifest.name}}
 
 {%- if logo_filename %}

tests/data/documentation/mkdocs.yml

@@ -1,12 +1,12 @@
 nav:
-- Sekoia.io XDR:
-  - Features:
-    - Automate:
-      - Actions Library:
-        - Sekoia.io: xdr/features/automate/library/sekoia-io.md
+- Integrations:
+  - List of Playbooks Actions:
+    - Endpoint:
+      - Sekoia.io: integration/action_library/sekoia-io.md
 - Sekoia.io TIP:
   - Features:
     - Automate:
       - Actions Library:
-        - Sekoia.io: tip/features/automate/library/sekoia-io.md
+        - Endpoint:
+          - Sekoia.io: tip/features/automate/library/sekoia-io.md
 site_name: Test

tests/data/sample_module/manifest.json

@@ -4,5 +4,9 @@
     "uuid": "7cf9ccf6-e20a-4e53-bd77-3c19cdc94154",
     "docker": "sekoia-automation-module-sample",
     "configuration": {},
-    "version": "0.1"
+    "version": "0.1",
+    "categories": [
+        "Endpoint",
+        "Network"
+    ]
 }

tests/data/sample_module/trigger_test.json

@@ -0,0 +1,26 @@
+{
+    "arguments": {
+        "$schema": "https://json-schema.org/draft-07/schema#",
+        "properties": {
+            "rule_filter": {
+                "type": "string",
+                "description": "Some description"
+            }
+        },
+        "type": "object",
+        "title": "Connector configuration"
+    },
+    "description": "Duplicate of connector",
+    "docker_parameters": "connector",
+    "name": "Connector",
+    "results": {
+        "$schema": "https://json-schema.org/draft-07/schema#",
+        "properties": {
+        },
+        "required": [
+        ],
+        "title": "Results",
+        "type": "object"
+    },
+    "uuid": "667f7e89-d907-4086-a578-a2324c9a277a"
+}

tests/expectations/sample_module/manifest.json

@@ -14,5 +14,6 @@
         "title": "ModuleConfiguration",
         "type": "object"
     },
-    "version": "0.1"
+    "version": "0.1",
+    "categories": ["Endpoint", "Network"]
 }

tests/scripts/test_sync_library.py

@@ -1,6 +1,7 @@
 import json
 import re
 from pathlib import Path
+from shutil import copytree
 from tempfile import mkdtemp
 from unittest.mock import patch
 
@@ -44,13 +45,20 @@ def connector():
         return connector
 
 
+@pytest.fixture
+def tmp_module(tmp_path):
+    copytree(Path("tests/data"), str(tmp_path), dirs_exist_ok=True)
+    tmp_path.joinpath("sample_module").joinpath("trigger_test.json").unlink()
+    yield tmp_path
+
+
 @requests_mock.Mocker(kw="m")
-def test_no_module_success(module, action, trigger, connector, **kwargs):
+def test_no_module_success(tmp_module, module, action, trigger, connector, **kwargs):
     kwargs["m"].register_uri(
         "GET", re.compile(f"{SYMPOHNY_URL}.*"), status_code=200, json={}
     )
     kwargs["m"].register_uri("PATCH", re.compile(f"{SYMPOHNY_URL}.*"))
-    sync_lib = SyncLibrary(SYMPOHNY_URL, API_KEY, Path("tests/data"))
+    sync_lib = SyncLibrary(SYMPOHNY_URL, API_KEY, tmp_module)
     sync_lib.execute()
 
     history = kwargs["m"].request_history
@@ -82,12 +90,12 @@ def test_no_module_success(module, action, trigger, connector, **kwargs):
 
 
 @requests_mock.Mocker(kw="m")
-def test_no_module_404(module, action, trigger, connector, **kwargs):
+def test_no_module_404(tmp_module, module, action, trigger, connector, **kwargs):
     kwargs["m"].register_uri(
         "GET", re.compile(f"{SYMPOHNY_URL}.*"), status_code=404, json={}
     )
     kwargs["m"].register_uri("POST", re.compile(f"{SYMPOHNY_URL}.*"))
-    sync_lib = SyncLibrary(SYMPOHNY_URL, API_KEY, Path("tests/data"))
+    sync_lib = SyncLibrary(SYMPOHNY_URL, API_KEY, tmp_module)
     sync_lib.execute()
 
     history = kwargs["m"].request_history
@@ -119,12 +127,12 @@ def test_no_module_404(module, action, trigger, connector, **kwargs):
 
 
 @requests_mock.Mocker(kw="m")
-def test_no_module_other_code(module, action, trigger, connector, **kwargs):
+def test_no_module_other_code(tmp_module, module, action, trigger, connector, **kwargs):
     kwargs["m"].register_uri(
         "GET", re.compile(f"{SYMPOHNY_URL}.*"), status_code=418, json={}
     )
     kwargs["m"].register_uri("POST", re.compile(f"{SYMPOHNY_URL}.*"))
-    sync_lib = SyncLibrary(SYMPOHNY_URL, API_KEY, Path("tests/data"))
+    sync_lib = SyncLibrary(SYMPOHNY_URL, API_KEY, tmp_module)
     sync_lib.execute()
 
     history = kwargs["m"].request_history
@@ -144,14 +152,12 @@ def test_no_module_other_code(module, action, trigger, connector, **kwargs):
 
 
 @requests_mock.Mocker(kw="m")
-def test_with_module(module, action, trigger, connector, **kwargs):
+def test_with_module(tmp_module, module, action, trigger, connector, **kwargs):
     kwargs["m"].register_uri(
         "GET", re.compile(f"{SYMPOHNY_URL}.*"), status_code=200, json={}
     )
     kwargs["m"].register_uri("PATCH", re.compile(f"{SYMPOHNY_URL}.*"))
-    sync_lib = SyncLibrary(
-        SYMPOHNY_URL, API_KEY, Path("tests/data"), module="sample_module"
-    )
+    sync_lib = SyncLibrary(SYMPOHNY_URL, API_KEY, tmp_module, module="sample_module")
     sync_lib.execute()
 
     history = kwargs["m"].request_history

tests/test_cli.py

@@ -95,16 +95,19 @@ def test_generate_documentation(tmp_path):
     with documentation_path.joinpath("mkdocs.yml").open("r") as fp:
         manifest = yaml.load(fp, Loader=Loader)
 
+    actions = manifest["nav"][0]["Integrations"][0]["List of Playbooks Actions"]
+    assert actions[0]["Overview"] == "integration/action_library/overview.md"
     assert (
-        manifest["nav"][0]["Sekoia.io XDR"][0]["Features"][0]["Automate"][0][
-            "Actions Library"
-        ][0]["Test Module"]
-        == "xdr/features/automate/library/test-module.md"
+        actions[1]["Endpoint"][0]["Test Module"]
+        == "integration/action_library/test-module.md"
     )
+
+    actions = manifest["nav"][1]["Sekoia.io TIP"][0]["Features"][0]["Automate"][0][
+        "Actions Library"
+    ]
+    assert actions[0]["Overview"] == "tip/features/automate/library/overview.md"
     assert (
-        manifest["nav"][1]["Sekoia.io TIP"][0]["Features"][0]["Automate"][0][
-            "Actions Library"
-        ][0]["Test Module"]
+        actions[1]["Endpoint"][0]["Test Module"]
         == "tip/features/automate/library/test-module.md"
     )
 
@@ -138,17 +141,23 @@ def test_generate_documentation_specific_module(tmp_path):
 
     # Make sure we didn't remove other modules
     assert (
-        manifest["nav"][0]["Sekoia.io XDR"][0]["Features"][0]["Automate"][0][
-            "Actions Library"
+        manifest["nav"][0]["Integrations"][0]["List of Playbooks Actions"][0][
+            "Endpoint"
         ][0]["Sekoia.io"]
-        == "xdr/features/automate/library/sekoia-io.md"
+        == "integration/action_library/sekoia-io.md"
     )
     # And our module has been added
     assert (
-        manifest["nav"][0]["Sekoia.io XDR"][0]["Features"][0]["Automate"][0][
-            "Actions Library"
+        manifest["nav"][0]["Integrations"][0]["List of Playbooks Actions"][0][
+            "Endpoint"
         ][1]["Test Module"]
-        == "xdr/features/automate/library/test-module.md"
+        == "integration/action_library/test-module.md"
+    )
+    assert (
+        manifest["nav"][0]["Integrations"][0]["List of Playbooks Actions"][1][
+            "Network"
+        ][0]["Test Module"]
+        == "integration/action_library/test-module.md"
     )