From 05e116eae4409160c44d2ded8f48e2a244602048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexandre=20L=C3=A9onardi?= Date: Mon, 9 Oct 2023 15:08:49 +0000 Subject: [PATCH] Add support for Connector objects in SDK Connectors are new objects that are close to Triggers. They are meant to replace the usecase where Triggers were used to collect and send data. The modifications to the SDK are as follows: * Modules have an optionnal connector_configuration_uuid * Modules have dedicated properties for logs_ and secrets_url on top of callback_url * Connectors use a dedicated configuration JSON file * Scripts file_generator and library_sync have been updated for connectors * Tests have been updated --- CHANGELOG.md | 6 ++ poetry.lock | 107 ++++++++++++++++--- sekoia_automation/connector/__init__.py | 54 +++++++++- sekoia_automation/module.py | 38 ++++++- sekoia_automation/scripts/files_generator.py | 29 +++++ sekoia_automation/scripts/sync_library.py | 30 ++++++ sekoia_automation/trigger.py | 9 +- tests/conftest.py | 14 ++- tests/connectors/test_connector.py | 30 +++++- tests/data/sample_module/connector_test.json | 26 +++++ tests/scripts/test_sync_library.py | 44 ++++++-- tests/test_trigger.py | 18 +++- 12 files changed, 365 insertions(+), 40 deletions(-) create mode 100644 tests/data/sample_module/connector_test.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 2582363..f1e686b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.5.3] - 2023-10-13 + +### Changed + +- Added support for Connectors + ## [1.5.2] - 2023-10-04 ### Fixed diff --git a/poetry.lock b/poetry.lock index 0ed6188..24f66e8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,9 +1,10 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "aiobotocore" version = "2.6.0" description = "Async client for aws services using botocore and aiohttp" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -25,6 +26,7 @@ boto3 = ["boto3 (>=1.28.17,<1.28.18)"] name = "aiocsv" version = "1.2.4" description = "Asynchronous CSV reading/writing" +category = "main" optional = true python-versions = ">=3.6, <4" files = [ @@ -59,6 +61,7 @@ files = [ name = "aiofiles" version = "23.2.1" description = "File support for asyncio." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -70,6 +73,7 @@ files = [ name = "aiohttp" version = "3.8.5" description = "Async http client/server framework (asyncio)" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -178,6 +182,7 @@ speedups = ["Brotli", "aiodns", "cchardet"] name = "aioitertools" version = "0.11.0" description = "itertools and builtins for AsyncIO and mixed iterables" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -189,6 +194,7 @@ files = [ name = "aiolimiter" version = "1.1.0" description = "asyncio rate limiter, a leaky bucket implementation" +category = "main" optional = true python-versions = ">=3.7,<4.0" files = [ @@ -200,6 +206,7 @@ files = [ name = "aioresponses" version = "0.7.4" description = "Mock out requests made by ClientSession from aiohttp package" +category = "dev" optional = false python-versions = "*" files = [ @@ -214,6 +221,7 @@ aiohttp = ">=2.0.0,<4.0.0" name = "aiosignal" version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -228,6 +236,7 @@ frozenlist = ">=1.1.0" name = "arrow" version = "1.3.0" description = "Better dates & times for Python" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -241,12 +250,13 @@ types-python-dateutil = ">=2.8.10" [package.extras] doc = ["doc8", "sphinx (>=7.0.0)", "sphinx-autobuild", "sphinx-autodoc-typehints", "sphinx_rtd_theme (>=1.3.0)"] -test = ["dateparser (==1.*)", "pre-commit", "pytest", "pytest-cov", "pytest-mock", "pytz (==2021.1)", "simplejson (==3.*)"] +test = ["dateparser (>=1.0.0,<2.0.0)", "pre-commit", "pytest", "pytest-cov", "pytest-mock", "pytz (==2021.1)", "simplejson (>=3.0.0,<4.0.0)"] [[package]] name = "astroid" version = "3.0.0" description = "An abstract syntax tree for Python with inference support." +category = "dev" optional = false python-versions = ">=3.8.0" files = [ @@ -261,6 +271,7 @@ typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} name = "async-timeout" version = "4.0.3" description = "Timeout context manager for asyncio programs" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -272,6 +283,7 @@ files = [ name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -290,6 +302,7 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "binaryornot" version = "0.4.4" description = "Ultra-lightweight pure Python package to check if a file is binary or text." +category = "main" optional = false python-versions = "*" files = [ @@ -304,6 +317,7 @@ chardet = ">=3.0.2" name = "black" version = "23.9.1" description = "The uncompromising code formatter." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -350,6 +364,7 @@ uvloop = ["uvloop (>=0.15.2)"] name = "boto3" version = "1.28.17" description = "The AWS SDK for Python" +category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -369,6 +384,7 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] name = "botocore" version = "1.31.17" description = "Low-level, data-driven core of boto 3." +category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -388,6 +404,7 @@ crt = ["awscrt (==0.16.26)"] name = "certifi" version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -399,6 +416,7 @@ files = [ name = "cfgv" version = "3.4.0" description = "Validate configuration and produce human readable error messages." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -410,6 +428,7 @@ files = [ name = "chardet" version = "5.2.0" description = "Universal encoding detector for Python 3" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -421,6 +440,7 @@ files = [ name = "charset-normalizer" version = "3.3.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -520,6 +540,7 @@ files = [ name = "click" version = "8.1.7" description = "Composable command line interface toolkit" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -534,6 +555,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -545,6 +567,7 @@ files = [ name = "commonmark" version = "0.9.1" description = "Python parser for the CommonMark Markdown spec" +category = "main" optional = false python-versions = "*" files = [ @@ -559,6 +582,7 @@ test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] name = "cookiecutter" version = "2.4.0" description = "A command-line utility that creates projects from project templates, e.g. creating a Python package project from a Python package project template." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -580,6 +604,7 @@ rich = "*" name = "coverage" version = "7.3.2" description = "Code coverage measurement for Python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -647,6 +672,7 @@ toml = ["tomli"] name = "dill" version = "0.3.7" description = "serialize all of Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -661,6 +687,7 @@ graph = ["objgraph (>=1.7.2)"] name = "distlib" version = "0.3.7" description = "Distribution utilities" +category = "dev" optional = false python-versions = "*" files = [ @@ -672,6 +699,7 @@ files = [ name = "exceptiongroup" version = "1.1.3" description = "Backport of PEP 654 (exception groups)" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -686,6 +714,7 @@ test = ["pytest (>=6)"] name = "execnet" version = "2.0.2" description = "execnet: rapid multi-Python deployment" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -700,6 +729,7 @@ testing = ["hatch", "pre-commit", "pytest", "tox"] name = "faker" version = "19.6.2" description = "Faker is a Python package that generates fake data for you." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -714,6 +744,7 @@ python-dateutil = ">=2.4" name = "filelock" version = "3.12.4" description = "A platform independent file lock." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -730,6 +761,7 @@ typing = ["typing-extensions (>=4.7.1)"] name = "frozenlist" version = "1.4.0" description = "A list-like structure which implements collections.abc.MutableSequence" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -800,6 +832,7 @@ files = [ name = "identify" version = "2.5.30" description = "File identification library for Python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -814,6 +847,7 @@ license = ["ukkonen"] name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -825,6 +859,7 @@ files = [ name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -836,6 +871,7 @@ files = [ name = "isort" version = "5.12.0" description = "A Python utility / library to sort Python imports." +category = "dev" optional = false python-versions = ">=3.8.0" files = [ @@ -853,6 +889,7 @@ requirements-deprecated-finder = ["pip-api", "pipreqs"] name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -870,6 +907,7 @@ i18n = ["Babel (>=2.7)"] name = "jmespath" version = "1.0.1" description = "JSON Matching Expressions" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -881,6 +919,7 @@ files = [ name = "loguru" version = "0.7.2" description = "Python logging made (stupidly) simple" +category = "main" optional = true python-versions = ">=3.5" files = [ @@ -899,6 +938,7 @@ dev = ["Sphinx (==7.2.5)", "colorama (==0.4.5)", "colorama (==0.4.6)", "exceptio name = "lxml" version = "4.9.3" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" files = [ @@ -1006,6 +1046,7 @@ source = ["Cython (>=0.29.35)"] name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1065,6 +1106,7 @@ files = [ name = "mccabe" version = "0.7.0" description = "McCabe checker, plugin for flake8" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1076,6 +1118,7 @@ files = [ name = "multidict" version = "6.0.4" description = "multidict implementation" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1159,6 +1202,7 @@ files = [ name = "mypy" version = "1.5.1" description = "Optional static typing for Python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1205,6 +1249,7 @@ reports = ["lxml"] name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -1216,6 +1261,7 @@ files = [ name = "nodeenv" version = "1.8.0" description = "Node.js virtual environment builder" +category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" files = [ @@ -1230,6 +1276,7 @@ setuptools = "*" name = "orjson" version = "3.9.7" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1299,6 +1346,7 @@ files = [ name = "packaging" version = "23.2" description = "Core utilities for Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1310,6 +1358,7 @@ files = [ name = "pathspec" version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1321,6 +1370,7 @@ files = [ name = "platformdirs" version = "3.11.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1336,6 +1386,7 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co name = "pluggy" version = "1.3.0" description = "plugin and hook calling mechanisms for python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1351,6 +1402,7 @@ testing = ["pytest", "pytest-benchmark"] name = "pre-commit" version = "3.4.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1369,6 +1421,7 @@ virtualenv = ">=20.10.0" name = "prometheus-client" version = "0.16.0" description = "Python client for the Prometheus monitoring system." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1383,6 +1436,7 @@ twisted = ["twisted"] name = "pydantic" version = "1.10.13" description = "Data validation and settings management using python type hints" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1435,6 +1489,7 @@ email = ["email-validator (>=1.0.3)"] name = "pygments" version = "2.16.1" description = "Pygments is a syntax highlighting package written in Python." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1449,6 +1504,7 @@ plugins = ["importlib-metadata"] name = "pylint" version = "3.0.0" description = "python code static checker" +category = "dev" optional = false python-versions = ">=3.8.0" files = [ @@ -1477,6 +1533,7 @@ testutils = ["gitpython (>3)"] name = "pytest" version = "7.4.2" description = "pytest: simple powerful testing with Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1499,6 +1556,7 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-asyncio" version = "0.21.1" description = "Pytest support for asyncio" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1517,6 +1575,7 @@ testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy name = "pytest-cov" version = "4.1.0" description = "Pytest plugin for measuring coverage." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1535,6 +1594,7 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale name = "pytest-env" version = "1.0.1" description = "py.test plugin that allows you to add environment variables." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1552,6 +1612,7 @@ test = ["coverage (>=7.2.7)", "pytest-mock (>=3.10)"] name = "pytest-xdist" version = "3.3.1" description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1572,6 +1633,7 @@ testing = ["filelock"] name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -1586,6 +1648,7 @@ six = ">=1.5" name = "python-slugify" version = "5.0.2" description = "A Python Slugify application that handles Unicode" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1603,6 +1666,7 @@ unidecode = ["Unidecode (>=1.1.1)"] name = "pyyaml" version = "6.0.1" description = "YAML parser and emitter for Python" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1611,7 +1675,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1619,15 +1682,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1644,7 +1700,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1652,7 +1707,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1662,6 +1716,7 @@ files = [ name = "requests" version = "2.31.0" description = "Python HTTP for Humans." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1683,6 +1738,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "requests-mock" version = "1.11.0" description = "Mock out responses from the requests package" +category = "dev" optional = false python-versions = "*" files = [ @@ -1702,6 +1758,7 @@ test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "tes name = "rich" version = "12.6.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "main" optional = false python-versions = ">=3.6.3,<4.0.0" files = [ @@ -1720,6 +1777,7 @@ jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] name = "ruff" version = "0.0.292" description = "An extremely fast Python linter, written in Rust." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1746,6 +1804,7 @@ files = [ name = "s3path" version = "0.5.0" description = "" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1761,6 +1820,7 @@ smart-open = "*" name = "s3transfer" version = "0.6.2" description = "An Amazon S3 Transfer Manager" +category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -1778,6 +1838,7 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] name = "sentry-sdk" version = "1.31.0" description = "Python client for Sentry (https://sentry.io)" +category = "main" optional = false python-versions = "*" files = [ @@ -1823,6 +1884,7 @@ tornado = ["tornado (>=5)"] name = "setuptools" version = "68.2.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1839,6 +1901,7 @@ testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jar name = "shellingham" version = "1.5.3" description = "Tool to Detect Surrounding Shell" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1850,6 +1913,7 @@ files = [ name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1861,6 +1925,7 @@ files = [ name = "smart-open" version = "6.4.0" description = "Utils for streaming large files (S3, HDFS, GCS, Azure Blob Storage, gzip, bz2...)" +category = "main" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -1882,6 +1947,7 @@ webhdfs = ["requests"] name = "tenacity" version = "8.2.3" description = "Retry code until it succeeds" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1896,6 +1962,7 @@ doc = ["reno", "sphinx", "tornado (>=4.5)"] name = "text-unidecode" version = "1.3" description = "The most basic Text::Unidecode port" +category = "main" optional = false python-versions = "*" files = [ @@ -1907,6 +1974,7 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1918,6 +1986,7 @@ files = [ name = "tomlkit" version = "0.12.1" description = "Style preserving TOML library" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1929,6 +1998,7 @@ files = [ name = "typer" version = "0.7.0" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1952,6 +2022,7 @@ test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6. name = "types-aiofiles" version = "23.2.0.0" description = "Typing stubs for aiofiles" +category = "dev" optional = false python-versions = "*" files = [ @@ -1963,6 +2034,7 @@ files = [ name = "types-python-dateutil" version = "2.8.19.14" description = "Typing stubs for python-dateutil" +category = "main" optional = false python-versions = "*" files = [ @@ -1974,6 +2046,7 @@ files = [ name = "types-python-slugify" version = "8.0.0.3" description = "Typing stubs for python-slugify" +category = "dev" optional = false python-versions = "*" files = [ @@ -1985,6 +2058,7 @@ files = [ name = "types-pyyaml" version = "6.0.12.12" description = "Typing stubs for PyYAML" +category = "dev" optional = false python-versions = "*" files = [ @@ -1996,6 +2070,7 @@ files = [ name = "types-requests" version = "2.31.0.6" description = "Typing stubs for requests" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2010,6 +2085,7 @@ types-urllib3 = "*" name = "types-urllib3" version = "1.26.25.14" description = "Typing stubs for urllib3" +category = "dev" optional = false python-versions = "*" files = [ @@ -2021,6 +2097,7 @@ files = [ name = "typing-extensions" version = "4.8.0" description = "Backported and Experimental Type Hints for Python 3.8+" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2032,6 +2109,7 @@ files = [ name = "unittest-xml-reporting" version = "3.2.0" description = "unittest-based test runner with Ant/JUnit like XML reporting." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2046,6 +2124,7 @@ lxml = "*" name = "urllib3" version = "1.26.17" description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -2062,6 +2141,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "virtualenv" version = "20.24.5" description = "Virtual Python Environment builder" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2082,6 +2162,7 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess name = "win32-setctime" version = "1.1.0" description = "A small Python utility to set file creation time on Windows" +category = "main" optional = true python-versions = ">=3.5" files = [ @@ -2096,6 +2177,7 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] name = "wrapt" version = "1.15.0" description = "Module for decorators, wrappers and monkey patching." +category = "main" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -2180,6 +2262,7 @@ files = [ name = "yarl" version = "1.9.2" description = "Yet another URL library" +category = "main" optional = false python-versions = ">=3.7" files = [ diff --git a/sekoia_automation/connector/__init__.py b/sekoia_automation/connector/__init__.py index 1f3cfb2..ac5a3be 100644 --- a/sekoia_automation/connector/__init__.py +++ b/sekoia_automation/connector/__init__.py @@ -11,12 +11,24 @@ import orjson import requests +import sentry_sdk from pydantic import BaseModel from requests import Response -from tenacity import Retrying, stop_after_delay, wait_exponential +from tenacity import ( + Retrying, + stop_after_delay, + wait_exponential, +) from sekoia_automation.constants import CHUNK_BYTES_MAX_SIZE, EVENT_BYTES_MAX_SIZE +from sekoia_automation.exceptions import ( + TriggerConfigurationError, +) from sekoia_automation.trigger import Trigger +from sekoia_automation.utils import ( + get_annotation_for, + get_as_model, +) # Connector are a kind of trigger that fetch events from remote sources. # We should add the content of push_events_to_intakes @@ -29,9 +41,42 @@ class DefaultConnectorConfiguration(BaseModel): class Connector(Trigger, ABC): + CONNECTOR_CONFIGURATION_FILE_NAME = "connector-configuration" + seconds_without_events = 3600 * 6 + + # Required for Pydantic to correctly type the configuration object configuration: DefaultConnectorConfiguration - seconds_without_events = 3600 * 6 + @property # type: ignore[override, no-redef] + def configuration(self) -> DefaultConnectorConfiguration: + if self._configuration is None: + try: + self.configuration = self.module.load_config( + self.CONNECTOR_CONFIGURATION_FILE_NAME, "json" + ) + except FileNotFoundError: + return super().configuration # type: ignore[return-value] + return self._configuration # type: ignore[return-value] + + @configuration.setter + def configuration(self, configuration: dict) -> None: + """ + Set the connector configuration. + + Args: + configuration: dict + """ + try: + self._configuration = get_as_model( + get_annotation_for(self.__class__, "configuration"), configuration + ) + except Exception as e: + raise TriggerConfigurationError(str(e)) + + if isinstance(self._configuration, BaseModel): + sentry_sdk.set_context( + "connector_configuration", self._configuration.dict() + ) def __init__(self, *args, **kwargs): executor_max_worker = kwargs.pop("executor_max_worker", 4) @@ -64,7 +109,10 @@ def _send_chunk( collect_ids: dict[int, list[str]], ): try: - request_body = {"intake_key": self.configuration.intake_key, "jsons": chunk} + request_body = { + "intake_key": self.configuration.intake_key, + "jsons": chunk, + } for attempt in self._retry(): with attempt: diff --git a/sekoia_automation/module.py b/sekoia_automation/module.py index ece91b6..3c3eeda 100644 --- a/sekoia_automation/module.py +++ b/sekoia_automation/module.py @@ -3,6 +3,7 @@ import sys import time from abc import ABC, abstractmethod +from functools import cached_property from pathlib import Path from typing import Any, cast @@ -32,6 +33,7 @@ class Module: PLAYBOOK_RUN_UUID_FILE_NAME = "playbook_run_uuid" NODE_RUN_UUID_FILE_NAME = "node_run_uuid" TRIGGER_CONFIGURATION_UUID_FILE_NAME = "trigger_configuration_uuid" + CONNECTOR_CONFIGURATION_UUID_FILE_NAME = "connector_configuration_uuid" SENTRY_FILE_NAME = "sentry_dsn" ENVIRONMENT_FILE_NAME = "environment" @@ -46,6 +48,7 @@ def __init__(self): self._playbook_run_uuid: str | None = None self._node_run_uuid: str | None = None self._trigger_configuration_uuid: str | None = None + self._connector_configuration_uuid: str | None = None self._name = None self.init_sentry() @@ -247,6 +250,15 @@ def trigger_configuration_uuid(self) -> str | None: return self._trigger_configuration_uuid + @property + def connector_configuration_uuid(self) -> str | None: + if self._connector_configuration_uuid is None: + self._connector_configuration_uuid = self.load_config( + self.CONNECTOR_CONFIGURATION_UUID_FILE_NAME, non_exist_ok=True + ) + + return self._connector_configuration_uuid + def load_config(self, file_name: str, type_: str = "str", non_exist_ok=False): return load_config(file_name, type_, non_exist_ok=non_exist_ok) @@ -287,6 +299,10 @@ def init_sentry(self): sentry_sdk.set_tag( "trigger_configuration_uuid", self.trigger_configuration_uuid ) + if self.connector_configuration_uuid: + sentry_sdk.set_tag( + "connector_configuration_uuid", self.connector_configuration_uuid + ) def _load_sentry_dsn(self) -> str | None: try: @@ -304,6 +320,8 @@ def _load_environment(self) -> str | None: class ModuleItem(ABC): TOKEN_FILE_NAME = "token" CALLBACK_URL_FILE_NAME = "url_callback" + SECRETS_URL_FILE_NAME = "url_secrets" + LOGS_URL_FILE_NAME = "url_logs" name: str | None = None description: str | None = None @@ -315,7 +333,6 @@ def __init__(self, module: Module | None = None, data_path: Path | None = None): self.module: Module = module or Module() self._token: str | None = None - self._callback_url: str | None = None # Name may be set by the action/trigger class or the module during the register # Worse case we use the class name @@ -384,12 +401,23 @@ def log_exception(self, exception: Exception, **kwargs): scope.set_extra(key, value) sentry_sdk.capture_exception(exception) - @property + @cached_property def callback_url(self) -> str: - if self._callback_url is None: - self._callback_url = self.module.load_config(self.CALLBACK_URL_FILE_NAME) + return self.module.load_config(self.CALLBACK_URL_FILE_NAME) + + @cached_property + def logs_url(self) -> str: + try: + return self.module.load_config(self.LOGS_URL_FILE_NAME) + except FileNotFoundError: + return self.callback_url.replace("/callback", "/logs") - return self._callback_url + @cached_property + def secrets_url(self) -> str: + try: + return self.module.load_config(self.SECRETS_URL_FILE_NAME) + except FileNotFoundError: + return self.callback_url.replace("/callback", "/secrets") @property def _headers(self) -> dict[str, str]: diff --git a/sekoia_automation/scripts/files_generator.py b/sekoia_automation/scripts/files_generator.py index 858cc28..3bb926b 100644 --- a/sekoia_automation/scripts/files_generator.py +++ b/sekoia_automation/scripts/files_generator.py @@ -12,6 +12,7 @@ from rich import print from sekoia_automation.action import Action +from sekoia_automation.connector import Connector from sekoia_automation.module import Module from sekoia_automation.trigger import Trigger from sekoia_automation.utils import get_annotation_for @@ -70,6 +71,7 @@ def execute(self): self.generate_main(module, actions, triggers) self.generate_action_manifests(actions) self.generate_trigger_manifests(triggers) + self.generate_connector_manifests(triggers) self.update_module_manifest(module) sys.path = _old_path @@ -164,6 +166,33 @@ def generate_trigger_manifests(self, triggers: set[type[Trigger]]): print(f"[green][+][/green] Generated {filepath}") + def generate_connector_manifests(self, connectors: set[type[Connector]]): + for connector in connectors: + name = connector.name or connector.__name__ + filepath = ( + self.base_path / f"connector_{name.lower().replace(' ', '_')}.json" + ) + + manifest: dict[str, str | dict | None] = { + "name": name, + "description": connector.description, + "uuid": str(uuid5(self.module_uuid, name)), + "docker_parameters": connector.__name__, + "arguments": {}, + "results": {}, + } + + if connector.results_model: + manifest["results"] = connector.results_model.schema() + + if configuration_model := get_annotation_for(connector, "configuration"): + manifest["arguments"] = configuration_model.schema() + + with filepath.open("w") as out: + out.write(json.dumps(manifest, indent=2)) + + print(f"[green][+][/green] Generated {filepath}") + def update_module_manifest(self, module: type[Module]): configuration_model = get_annotation_for(module, "configuration") diff --git a/sekoia_automation/scripts/sync_library.py b/sekoia_automation/scripts/sync_library.py index 799202b..c192dc6 100755 --- a/sekoia_automation/scripts/sync_library.py +++ b/sekoia_automation/scripts/sync_library.py @@ -227,6 +227,27 @@ def load_triggers(self, module_path: Path) -> list: return triggers + def load_connectors(self, module_path: Path) -> list: + """Load JSON files representing the connectors linked to a module + + Args: + module_path (Path): Path of the parent module + + Returns: + list: List of connectors related to the parent module + """ + connectors = [] + + for filename in module_path.iterdir(): + if filename.name.endswith(".json") and filename.name.startswith( + "connector_" + ): + connector_path = module_path / filename + with connector_path.open() as fd: + connectors.append(json.load(fd)) + + return connectors + def set_docker(self, manifests: list, module: dict) -> list: """Loops over the Docker name of objets linked to a module and adds the Docker version if missing @@ -334,6 +355,7 @@ def load_module(self, module_path: Path): raise typer.Exit(code=1) triggers = self.set_docker(self.load_triggers(module_path), module_info) + connectors = self.set_docker(self.load_connectors(module_path), module_info) actions = self.set_docker(self.load_actions(module_path), module_info) module_uuid: str = module_info["uuid"] @@ -360,6 +382,14 @@ def load_module(self, module_path: Path): name="action", ) print() + if connectors: + self.sync_list( + module_name=module_name, + module_uuid=module_uuid, + list_objects=connectors, + name="connector", + ) + print() def load(self, library_path: Path): """Lods all modules that can be found in a given library diff --git a/sekoia_automation/trigger.py b/sekoia_automation/trigger.py index e768170..659a77f 100644 --- a/sekoia_automation/trigger.py +++ b/sekoia_automation/trigger.py @@ -4,7 +4,6 @@ from collections.abc import Generator from contextlib import contextmanager from datetime import datetime, timedelta -from functools import cached_property from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path from threading import Event, Thread @@ -90,7 +89,7 @@ def _get_secrets_from_server(self) -> dict[str, Any]: if self.module.has_secrets(): try: response = requests.get( - self.callback_url.replace("/callback", "/secrets"), + self.secrets_url, headers=self._headers, timeout=30, ) @@ -287,10 +286,6 @@ def send_event( remove_directory, ) - @cached_property - def _log_url(self): - return self.callback_url.replace("/callback", "/logs") - # Try to send the log record to the API # If it can't be done, give up after 10 attempts and capture the logging error @@ -332,7 +327,7 @@ def _send_logs_to_api(self): return data = {"logs": self._logs} response = requests.request( - "POST", self._log_url, json=data, headers=self._headers, timeout=30 + "POST", self.logs_url, json=data, headers=self._headers, timeout=30 ) response.raise_for_status() self._logs = [] diff --git a/tests/conftest.py b/tests/conftest.py index 26b95da..2f548fd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -77,11 +77,23 @@ def tls_storage(): @pytest.fixture def mocked_trigger_logs(): with patch.object( + Trigger, + "logs_url", + new_callable=PropertyMock, + return_value="http://sekoia-playbooks/logs", + ), patch.object( + Trigger, + "secrets_url", + new_callable=PropertyMock, + return_value="http://sekoia-playbooks/secrets", + ), patch.object( Trigger, "callback_url", new_callable=PropertyMock, return_value="http://sekoia-playbooks/callback", - ), patch.object(Trigger, "token", return_value="secure_token"): + ), patch.object( + Trigger, "token", return_value="secure_token" + ): with requests_mock.Mocker() as mock: mock.post("http://sekoia-playbooks/logs") diff --git a/tests/connectors/test_connector.py b/tests/connectors/test_connector.py index 0e04c6a..d027aa4 100644 --- a/tests/connectors/test_connector.py +++ b/tests/connectors/test_connector.py @@ -1,11 +1,13 @@ -from unittest.mock import Mock, patch +from unittest.mock import Mock, PropertyMock, patch import pytest from tenacity import Retrying, stop_after_attempt, wait_none -from sekoia_automation.connector import Connector +from sekoia_automation.connector import Connector, DefaultConnectorConfiguration from sekoia_automation.constants import CHUNK_BYTES_MAX_SIZE, EVENT_BYTES_MAX_SIZE from sekoia_automation.exceptions import TriggerConfigurationError +from sekoia_automation.module import Module +from sekoia_automation.trigger import Trigger from tests.utils import match_events EVENTS = ["foo", "bar"] @@ -194,3 +196,27 @@ def test_query_exception_api(test_connector, requests_mock): ) test_connector._retry = lambda: Retrying(reraise=True) assert test_connector.push_events_to_intakes(EVENTS) == ["001", "002"] + + +def test_connector_configuration(test_connector): + test_connector._configuration = None + config = DefaultConnectorConfiguration(intake_key="foo") + with patch.object( + Module, "load_config", return_value=config + ) as mock_load_config, patch("sentry_sdk.set_context") as mock_set_sentry_context: + assert test_connector.configuration == config + mock_set_sentry_context.assert_called_with( + "connector_configuration", config.dict() + ) + mock_load_config.assert_called_with( + test_connector.CONNECTOR_CONFIGURATION_FILE_NAME, "json" + ) + + +def test_connector_configuration_file_not_found(test_connector): + test_connector._configuration = None + config = "foo" + with patch.object( + Trigger, "configuration", new_callable=PropertyMock, return_value=config + ): + assert test_connector.configuration == config diff --git a/tests/data/sample_module/connector_test.json b/tests/data/sample_module/connector_test.json new file mode 100644 index 0000000..fe78793 --- /dev/null +++ b/tests/data/sample_module/connector_test.json @@ -0,0 +1,26 @@ +{ + "arguments": { + "$schema": "https://json-schema.org/draft-07/schema#", + "properties": { + "rule_filter": { + "type": "string", + "description": "Some description" + } + }, + "type": "object", + "title": "Connector configuration" + }, + "description": "Sekoia: connecting... things", + "docker_parameters": "connector", + "name": "Connector", + "results": { + "$schema": "https://json-schema.org/draft-07/schema#", + "properties": { + }, + "required": [ + ], + "title": "Results", + "type": "object" + }, + "uuid": "667f7e89-d907-4086-a578-a2324c9a277a" +} diff --git a/tests/scripts/test_sync_library.py b/tests/scripts/test_sync_library.py index 0de0350..1b69b2b 100644 --- a/tests/scripts/test_sync_library.py +++ b/tests/scripts/test_sync_library.py @@ -37,8 +37,15 @@ def trigger(): return trigger +@pytest.fixture +def connector(): + with open("tests/data/sample_module/connector_test.json") as f: + connector = json.load(f) + return connector + + @requests_mock.Mocker(kw="m") -def test_no_module_success(module, action, trigger, **kwargs): +def test_no_module_success(module, action, trigger, connector, **kwargs): kwargs["m"].register_uri( "GET", re.compile(f"{SYMPOHNY_URL}.*"), status_code=200, json={} ) @@ -47,7 +54,7 @@ def test_no_module_success(module, action, trigger, **kwargs): sync_lib.execute() history = kwargs["m"].request_history - assert len(history) == 6 + assert len(history) == 8 assert history[0].method == "GET" assert history[0].url == f"{SYMPOHNY_URL}/modules/{module['uuid']}" assert history[0].headers["Authorization"] == f"Bearer {API_KEY}" @@ -66,10 +73,16 @@ def test_no_module_success(module, action, trigger, **kwargs): assert history[5].method == "PATCH" assert history[5].url == f"{SYMPOHNY_URL}/actions/{action['uuid']}" assert history[5].headers["Authorization"] == f"Bearer {API_KEY}" + assert history[6].method == "GET" + assert history[6].url == f"{SYMPOHNY_URL}/connectors/{connector['uuid']}" + assert history[6].headers["Authorization"] == f"Bearer {API_KEY}" + assert history[7].method == "PATCH" + assert history[7].url == f"{SYMPOHNY_URL}/connectors/{connector['uuid']}" + assert history[7].headers["Authorization"] == f"Bearer {API_KEY}" @requests_mock.Mocker(kw="m") -def test_no_module_404(module, action, trigger, **kwargs): +def test_no_module_404(module, action, trigger, connector, **kwargs): kwargs["m"].register_uri( "GET", re.compile(f"{SYMPOHNY_URL}.*"), status_code=404, json={} ) @@ -78,7 +91,7 @@ def test_no_module_404(module, action, trigger, **kwargs): sync_lib.execute() history = kwargs["m"].request_history - assert len(history) == 6 + assert len(history) == 8 assert history[0].method == "GET" assert history[0].url == f"{SYMPOHNY_URL}/modules/{module['uuid']}" assert history[0].headers["Authorization"] == f"Bearer {API_KEY}" @@ -97,10 +110,16 @@ def test_no_module_404(module, action, trigger, **kwargs): assert history[5].method == "POST" assert history[5].url == f"{SYMPOHNY_URL}/actions" assert history[5].headers["Authorization"] == f"Bearer {API_KEY}" + assert history[6].method == "GET" + assert history[6].url == f"{SYMPOHNY_URL}/connectors/{connector['uuid']}" + assert history[6].headers["Authorization"] == f"Bearer {API_KEY}" + assert history[7].method == "POST" + assert history[7].url == f"{SYMPOHNY_URL}/connectors" + assert history[7].headers["Authorization"] == f"Bearer {API_KEY}" @requests_mock.Mocker(kw="m") -def test_no_module_other_code(module, action, trigger, **kwargs): +def test_no_module_other_code(module, action, trigger, connector, **kwargs): kwargs["m"].register_uri( "GET", re.compile(f"{SYMPOHNY_URL}.*"), status_code=418, json={} ) @@ -109,7 +128,7 @@ def test_no_module_other_code(module, action, trigger, **kwargs): sync_lib.execute() history = kwargs["m"].request_history - assert len(history) == 3 + assert len(history) == 4 assert history[0].method == "GET" assert history[0].url == f"{SYMPOHNY_URL}/modules/{module['uuid']}" assert history[0].headers["Authorization"] == f"Bearer {API_KEY}" @@ -119,10 +138,13 @@ def test_no_module_other_code(module, action, trigger, **kwargs): assert history[2].method == "GET" assert history[2].url == f"{SYMPOHNY_URL}/actions/{action['uuid']}" assert history[2].headers["Authorization"] == f"Bearer {API_KEY}" + assert history[3].method == "GET" + assert history[3].url == f"{SYMPOHNY_URL}/connectors/{connector['uuid']}" + assert history[3].headers["Authorization"] == f"Bearer {API_KEY}" @requests_mock.Mocker(kw="m") -def test_with_module(module, action, trigger, **kwargs): +def test_with_module(module, action, trigger, connector, **kwargs): kwargs["m"].register_uri( "GET", re.compile(f"{SYMPOHNY_URL}.*"), status_code=200, json={} ) @@ -133,7 +155,7 @@ def test_with_module(module, action, trigger, **kwargs): sync_lib.execute() history = kwargs["m"].request_history - assert len(history) == 6 + assert len(history) == 8 assert history[0].method == "GET" assert history[0].url == f"{SYMPOHNY_URL}/modules/{module['uuid']}" assert history[0].headers["Authorization"] == f"Bearer {API_KEY}" @@ -152,6 +174,12 @@ def test_with_module(module, action, trigger, **kwargs): assert history[5].method == "PATCH" assert history[5].url == f"{SYMPOHNY_URL}/actions/{action['uuid']}" assert history[5].headers["Authorization"] == f"Bearer {API_KEY}" + assert history[6].method == "GET" + assert history[6].url == f"{SYMPOHNY_URL}/connectors/{connector['uuid']}" + assert history[6].headers["Authorization"] == f"Bearer {API_KEY}" + assert history[7].method == "PATCH" + assert history[7].url == f"{SYMPOHNY_URL}/connectors/{connector['uuid']}" + assert history[7].headers["Authorization"] == f"Bearer {API_KEY}" def test_with_module_invalid_name(): diff --git a/tests/test_trigger.py b/tests/test_trigger.py index 8276dc4..8a5b51f 100644 --- a/tests/test_trigger.py +++ b/tests/test_trigger.py @@ -54,6 +54,20 @@ def test_callback_url(): mock.assert_called_with(trigger.CALLBACK_URL_FILE_NAME) +def test_secrets_url(): + trigger = DummyTrigger() + with patch.object(Module, "load_config", return_value="secrets") as mock: + assert trigger.secrets_url == "secrets" + mock.assert_called_with(trigger.SECRETS_URL_FILE_NAME) + + +def test_logs_url(): + trigger = DummyTrigger() + with patch.object(Module, "load_config", return_value="logs") as mock: + assert trigger.logs_url == "logs" + mock.assert_called_with(trigger.LOGS_URL_FILE_NAME) + + def test_trigger_configuration(): trigger = DummyTrigger() with patch.object( @@ -401,9 +415,9 @@ def test_trigger_log_critical_only_once(mocked_trigger_logs): @patch.object(Module, "has_secrets", return_value=True) @patch.object( Trigger, - "callback_url", + "secrets_url", new_callable=PropertyMock, - return_value="http://sekoia-playbooks/callback", + return_value="http://sekoia-playbooks/secrets", ) @patch.object(Trigger, "token", return_value="secure_token") def test_get_secrets(_, __, ___):