diff --git a/CHANGELOG.md b/CHANGELOG.md index 2582363..f1e686b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.5.3] - 2023-10-13 + +### Changed + +- Added support for Connectors + ## [1.5.2] - 2023-10-04 ### Fixed diff --git a/poetry.lock b/poetry.lock index 0ed6188..24f66e8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,9 +1,10 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "aiobotocore" version = "2.6.0" description = "Async client for aws services using botocore and aiohttp" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -25,6 +26,7 @@ boto3 = ["boto3 (>=1.28.17,<1.28.18)"] name = "aiocsv" version = "1.2.4" description = "Asynchronous CSV reading/writing" +category = "main" optional = true python-versions = ">=3.6, <4" files = [ @@ -59,6 +61,7 @@ files = [ name = "aiofiles" version = "23.2.1" description = "File support for asyncio." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -70,6 +73,7 @@ files = [ name = "aiohttp" version = "3.8.5" description = "Async http client/server framework (asyncio)" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -178,6 +182,7 @@ speedups = ["Brotli", "aiodns", "cchardet"] name = "aioitertools" version = "0.11.0" description = "itertools and builtins for AsyncIO and mixed iterables" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -189,6 +194,7 @@ files = [ name = "aiolimiter" version = "1.1.0" description = "asyncio rate limiter, a leaky bucket implementation" +category = "main" optional = true python-versions = ">=3.7,<4.0" files = [ @@ -200,6 +206,7 @@ files = [ name = "aioresponses" version = "0.7.4" description = "Mock out requests made by ClientSession from aiohttp package" +category = "dev" optional = false python-versions = "*" files = [ @@ -214,6 +221,7 @@ aiohttp = ">=2.0.0,<4.0.0" name = "aiosignal" version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -228,6 +236,7 @@ frozenlist = ">=1.1.0" name = "arrow" version = "1.3.0" description = "Better dates & times for Python" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -241,12 +250,13 @@ types-python-dateutil = ">=2.8.10" [package.extras] doc = ["doc8", "sphinx (>=7.0.0)", "sphinx-autobuild", "sphinx-autodoc-typehints", "sphinx_rtd_theme (>=1.3.0)"] -test = ["dateparser (==1.*)", "pre-commit", "pytest", "pytest-cov", "pytest-mock", "pytz (==2021.1)", "simplejson (==3.*)"] +test = ["dateparser (>=1.0.0,<2.0.0)", "pre-commit", "pytest", "pytest-cov", "pytest-mock", "pytz (==2021.1)", "simplejson (>=3.0.0,<4.0.0)"] [[package]] name = "astroid" version = "3.0.0" description = "An abstract syntax tree for Python with inference support." +category = "dev" optional = false python-versions = ">=3.8.0" files = [ @@ -261,6 +271,7 @@ typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} name = "async-timeout" version = "4.0.3" description = "Timeout context manager for asyncio programs" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -272,6 +283,7 @@ files = [ name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -290,6 +302,7 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "binaryornot" version = "0.4.4" description = "Ultra-lightweight pure Python package to check if a file is binary or text." +category = "main" optional = false python-versions = "*" files = [ @@ -304,6 +317,7 @@ chardet = ">=3.0.2" name = "black" version = "23.9.1" description = "The uncompromising code formatter." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -350,6 +364,7 @@ uvloop = ["uvloop (>=0.15.2)"] name = "boto3" version = "1.28.17" description = "The AWS SDK for Python" +category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -369,6 +384,7 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] name = "botocore" version = "1.31.17" description = "Low-level, data-driven core of boto 3." +category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -388,6 +404,7 @@ crt = ["awscrt (==0.16.26)"] name = "certifi" version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -399,6 +416,7 @@ files = [ name = "cfgv" version = "3.4.0" description = "Validate configuration and produce human readable error messages." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -410,6 +428,7 @@ files = [ name = "chardet" version = "5.2.0" description = "Universal encoding detector for Python 3" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -421,6 +440,7 @@ files = [ name = "charset-normalizer" version = "3.3.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -520,6 +540,7 @@ files = [ name = "click" version = "8.1.7" description = "Composable command line interface toolkit" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -534,6 +555,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -545,6 +567,7 @@ files = [ name = "commonmark" version = "0.9.1" description = "Python parser for the CommonMark Markdown spec" +category = "main" optional = false python-versions = "*" files = [ @@ -559,6 +582,7 @@ test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] name = "cookiecutter" version = "2.4.0" description = "A command-line utility that creates projects from project templates, e.g. creating a Python package project from a Python package project template." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -580,6 +604,7 @@ rich = "*" name = "coverage" version = "7.3.2" description = "Code coverage measurement for Python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -647,6 +672,7 @@ toml = ["tomli"] name = "dill" version = "0.3.7" description = "serialize all of Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -661,6 +687,7 @@ graph = ["objgraph (>=1.7.2)"] name = "distlib" version = "0.3.7" description = "Distribution utilities" +category = "dev" optional = false python-versions = "*" files = [ @@ -672,6 +699,7 @@ files = [ name = "exceptiongroup" version = "1.1.3" description = "Backport of PEP 654 (exception groups)" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -686,6 +714,7 @@ test = ["pytest (>=6)"] name = "execnet" version = "2.0.2" description = "execnet: rapid multi-Python deployment" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -700,6 +729,7 @@ testing = ["hatch", "pre-commit", "pytest", "tox"] name = "faker" version = "19.6.2" description = "Faker is a Python package that generates fake data for you." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -714,6 +744,7 @@ python-dateutil = ">=2.4" name = "filelock" version = "3.12.4" description = "A platform independent file lock." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -730,6 +761,7 @@ typing = ["typing-extensions (>=4.7.1)"] name = "frozenlist" version = "1.4.0" description = "A list-like structure which implements collections.abc.MutableSequence" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -800,6 +832,7 @@ files = [ name = "identify" version = "2.5.30" description = "File identification library for Python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -814,6 +847,7 @@ license = ["ukkonen"] name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -825,6 +859,7 @@ files = [ name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -836,6 +871,7 @@ files = [ name = "isort" version = "5.12.0" description = "A Python utility / library to sort Python imports." +category = "dev" optional = false python-versions = ">=3.8.0" files = [ @@ -853,6 +889,7 @@ requirements-deprecated-finder = ["pip-api", "pipreqs"] name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -870,6 +907,7 @@ i18n = ["Babel (>=2.7)"] name = "jmespath" version = "1.0.1" description = "JSON Matching Expressions" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -881,6 +919,7 @@ files = [ name = "loguru" version = "0.7.2" description = "Python logging made (stupidly) simple" +category = "main" optional = true python-versions = ">=3.5" files = [ @@ -899,6 +938,7 @@ dev = ["Sphinx (==7.2.5)", "colorama (==0.4.5)", "colorama (==0.4.6)", "exceptio name = "lxml" version = "4.9.3" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" files = [ @@ -1006,6 +1046,7 @@ source = ["Cython (>=0.29.35)"] name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1065,6 +1106,7 @@ files = [ name = "mccabe" version = "0.7.0" description = "McCabe checker, plugin for flake8" +category = "dev" optional = false python-versions = ">=3.6" files = [ @@ -1076,6 +1118,7 @@ files = [ name = "multidict" version = "6.0.4" description = "multidict implementation" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1159,6 +1202,7 @@ files = [ name = "mypy" version = "1.5.1" description = "Optional static typing for Python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1205,6 +1249,7 @@ reports = ["lxml"] name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -1216,6 +1261,7 @@ files = [ name = "nodeenv" version = "1.8.0" description = "Node.js virtual environment builder" +category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" files = [ @@ -1230,6 +1276,7 @@ setuptools = "*" name = "orjson" version = "3.9.7" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1299,6 +1346,7 @@ files = [ name = "packaging" version = "23.2" description = "Core utilities for Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1310,6 +1358,7 @@ files = [ name = "pathspec" version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1321,6 +1370,7 @@ files = [ name = "platformdirs" version = "3.11.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1336,6 +1386,7 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co name = "pluggy" version = "1.3.0" description = "plugin and hook calling mechanisms for python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1351,6 +1402,7 @@ testing = ["pytest", "pytest-benchmark"] name = "pre-commit" version = "3.4.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1369,6 +1421,7 @@ virtualenv = ">=20.10.0" name = "prometheus-client" version = "0.16.0" description = "Python client for the Prometheus monitoring system." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1383,6 +1436,7 @@ twisted = ["twisted"] name = "pydantic" version = "1.10.13" description = "Data validation and settings management using python type hints" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1435,6 +1489,7 @@ email = ["email-validator (>=1.0.3)"] name = "pygments" version = "2.16.1" description = "Pygments is a syntax highlighting package written in Python." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1449,6 +1504,7 @@ plugins = ["importlib-metadata"] name = "pylint" version = "3.0.0" description = "python code static checker" +category = "dev" optional = false python-versions = ">=3.8.0" files = [ @@ -1477,6 +1533,7 @@ testutils = ["gitpython (>3)"] name = "pytest" version = "7.4.2" description = "pytest: simple powerful testing with Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1499,6 +1556,7 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no name = "pytest-asyncio" version = "0.21.1" description = "Pytest support for asyncio" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1517,6 +1575,7 @@ testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy name = "pytest-cov" version = "4.1.0" description = "Pytest plugin for measuring coverage." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1535,6 +1594,7 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale name = "pytest-env" version = "1.0.1" description = "py.test plugin that allows you to add environment variables." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1552,6 +1612,7 @@ test = ["coverage (>=7.2.7)", "pytest-mock (>=3.10)"] name = "pytest-xdist" version = "3.3.1" description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1572,6 +1633,7 @@ testing = ["filelock"] name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -1586,6 +1648,7 @@ six = ">=1.5" name = "python-slugify" version = "5.0.2" description = "A Python Slugify application that handles Unicode" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1603,6 +1666,7 @@ unidecode = ["Unidecode (>=1.1.1)"] name = "pyyaml" version = "6.0.1" description = "YAML parser and emitter for Python" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1611,7 +1675,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1619,15 +1682,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1644,7 +1700,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1652,7 +1707,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1662,6 +1716,7 @@ files = [ name = "requests" version = "2.31.0" description = "Python HTTP for Humans." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1683,6 +1738,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "requests-mock" version = "1.11.0" description = "Mock out responses from the requests package" +category = "dev" optional = false python-versions = "*" files = [ @@ -1702,6 +1758,7 @@ test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "tes name = "rich" version = "12.6.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "main" optional = false python-versions = ">=3.6.3,<4.0.0" files = [ @@ -1720,6 +1777,7 @@ jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] name = "ruff" version = "0.0.292" description = "An extremely fast Python linter, written in Rust." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1746,6 +1804,7 @@ files = [ name = "s3path" version = "0.5.0" description = "" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1761,6 +1820,7 @@ smart-open = "*" name = "s3transfer" version = "0.6.2" description = "An Amazon S3 Transfer Manager" +category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -1778,6 +1838,7 @@ crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] name = "sentry-sdk" version = "1.31.0" description = "Python client for Sentry (https://sentry.io)" +category = "main" optional = false python-versions = "*" files = [ @@ -1823,6 +1884,7 @@ tornado = ["tornado (>=5)"] name = "setuptools" version = "68.2.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1839,6 +1901,7 @@ testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jar name = "shellingham" version = "1.5.3" description = "Tool to Detect Surrounding Shell" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1850,6 +1913,7 @@ files = [ name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1861,6 +1925,7 @@ files = [ name = "smart-open" version = "6.4.0" description = "Utils for streaming large files (S3, HDFS, GCS, Azure Blob Storage, gzip, bz2...)" +category = "main" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -1882,6 +1947,7 @@ webhdfs = ["requests"] name = "tenacity" version = "8.2.3" description = "Retry code until it succeeds" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1896,6 +1962,7 @@ doc = ["reno", "sphinx", "tornado (>=4.5)"] name = "text-unidecode" version = "1.3" description = "The most basic Text::Unidecode port" +category = "main" optional = false python-versions = "*" files = [ @@ -1907,6 +1974,7 @@ files = [ name = "tomli" version = "2.0.1" description = "A lil' TOML parser" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1918,6 +1986,7 @@ files = [ name = "tomlkit" version = "0.12.1" description = "Style preserving TOML library" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1929,6 +1998,7 @@ files = [ name = "typer" version = "0.7.0" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1952,6 +2022,7 @@ test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6. name = "types-aiofiles" version = "23.2.0.0" description = "Typing stubs for aiofiles" +category = "dev" optional = false python-versions = "*" files = [ @@ -1963,6 +2034,7 @@ files = [ name = "types-python-dateutil" version = "2.8.19.14" description = "Typing stubs for python-dateutil" +category = "main" optional = false python-versions = "*" files = [ @@ -1974,6 +2046,7 @@ files = [ name = "types-python-slugify" version = "8.0.0.3" description = "Typing stubs for python-slugify" +category = "dev" optional = false python-versions = "*" files = [ @@ -1985,6 +2058,7 @@ files = [ name = "types-pyyaml" version = "6.0.12.12" description = "Typing stubs for PyYAML" +category = "dev" optional = false python-versions = "*" files = [ @@ -1996,6 +2070,7 @@ files = [ name = "types-requests" version = "2.31.0.6" description = "Typing stubs for requests" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2010,6 +2085,7 @@ types-urllib3 = "*" name = "types-urllib3" version = "1.26.25.14" description = "Typing stubs for urllib3" +category = "dev" optional = false python-versions = "*" files = [ @@ -2021,6 +2097,7 @@ files = [ name = "typing-extensions" version = "4.8.0" description = "Backported and Experimental Type Hints for Python 3.8+" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -2032,6 +2109,7 @@ files = [ name = "unittest-xml-reporting" version = "3.2.0" description = "unittest-based test runner with Ant/JUnit like XML reporting." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2046,6 +2124,7 @@ lxml = "*" name = "urllib3" version = "1.26.17" description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -2062,6 +2141,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "virtualenv" version = "20.24.5" description = "Virtual Python Environment builder" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2082,6 +2162,7 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess name = "win32-setctime" version = "1.1.0" description = "A small Python utility to set file creation time on Windows" +category = "main" optional = true python-versions = ">=3.5" files = [ @@ -2096,6 +2177,7 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] name = "wrapt" version = "1.15.0" description = "Module for decorators, wrappers and monkey patching." +category = "main" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" files = [ @@ -2180,6 +2262,7 @@ files = [ name = "yarl" version = "1.9.2" description = "Yet another URL library" +category = "main" optional = false python-versions = ">=3.7" files = [ diff --git a/sekoia_automation/connector/__init__.py b/sekoia_automation/connector/__init__.py index 1f3cfb2..ac5a3be 100644 --- a/sekoia_automation/connector/__init__.py +++ b/sekoia_automation/connector/__init__.py @@ -11,12 +11,24 @@ import orjson import requests +import sentry_sdk from pydantic import BaseModel from requests import Response -from tenacity import Retrying, stop_after_delay, wait_exponential +from tenacity import ( + Retrying, + stop_after_delay, + wait_exponential, +) from sekoia_automation.constants import CHUNK_BYTES_MAX_SIZE, EVENT_BYTES_MAX_SIZE +from sekoia_automation.exceptions import ( + TriggerConfigurationError, +) from sekoia_automation.trigger import Trigger +from sekoia_automation.utils import ( + get_annotation_for, + get_as_model, +) # Connector are a kind of trigger that fetch events from remote sources. # We should add the content of push_events_to_intakes @@ -29,9 +41,42 @@ class DefaultConnectorConfiguration(BaseModel): class Connector(Trigger, ABC): + CONNECTOR_CONFIGURATION_FILE_NAME = "connector-configuration" + seconds_without_events = 3600 * 6 + + # Required for Pydantic to correctly type the configuration object configuration: DefaultConnectorConfiguration - seconds_without_events = 3600 * 6 + @property # type: ignore[override, no-redef] + def configuration(self) -> DefaultConnectorConfiguration: + if self._configuration is None: + try: + self.configuration = self.module.load_config( + self.CONNECTOR_CONFIGURATION_FILE_NAME, "json" + ) + except FileNotFoundError: + return super().configuration # type: ignore[return-value] + return self._configuration # type: ignore[return-value] + + @configuration.setter + def configuration(self, configuration: dict) -> None: + """ + Set the connector configuration. + + Args: + configuration: dict + """ + try: + self._configuration = get_as_model( + get_annotation_for(self.__class__, "configuration"), configuration + ) + except Exception as e: + raise TriggerConfigurationError(str(e)) + + if isinstance(self._configuration, BaseModel): + sentry_sdk.set_context( + "connector_configuration", self._configuration.dict() + ) def __init__(self, *args, **kwargs): executor_max_worker = kwargs.pop("executor_max_worker", 4) @@ -64,7 +109,10 @@ def _send_chunk( collect_ids: dict[int, list[str]], ): try: - request_body = {"intake_key": self.configuration.intake_key, "jsons": chunk} + request_body = { + "intake_key": self.configuration.intake_key, + "jsons": chunk, + } for attempt in self._retry(): with attempt: diff --git a/sekoia_automation/module.py b/sekoia_automation/module.py index ece91b6..3c3eeda 100644 --- a/sekoia_automation/module.py +++ b/sekoia_automation/module.py @@ -3,6 +3,7 @@ import sys import time from abc import ABC, abstractmethod +from functools import cached_property from pathlib import Path from typing import Any, cast @@ -32,6 +33,7 @@ class Module: PLAYBOOK_RUN_UUID_FILE_NAME = "playbook_run_uuid" NODE_RUN_UUID_FILE_NAME = "node_run_uuid" TRIGGER_CONFIGURATION_UUID_FILE_NAME = "trigger_configuration_uuid" + CONNECTOR_CONFIGURATION_UUID_FILE_NAME = "connector_configuration_uuid" SENTRY_FILE_NAME = "sentry_dsn" ENVIRONMENT_FILE_NAME = "environment" @@ -46,6 +48,7 @@ def __init__(self): self._playbook_run_uuid: str | None = None self._node_run_uuid: str | None = None self._trigger_configuration_uuid: str | None = None + self._connector_configuration_uuid: str | None = None self._name = None self.init_sentry() @@ -247,6 +250,15 @@ def trigger_configuration_uuid(self) -> str | None: return self._trigger_configuration_uuid + @property + def connector_configuration_uuid(self) -> str | None: + if self._connector_configuration_uuid is None: + self._connector_configuration_uuid = self.load_config( + self.CONNECTOR_CONFIGURATION_UUID_FILE_NAME, non_exist_ok=True + ) + + return self._connector_configuration_uuid + def load_config(self, file_name: str, type_: str = "str", non_exist_ok=False): return load_config(file_name, type_, non_exist_ok=non_exist_ok) @@ -287,6 +299,10 @@ def init_sentry(self): sentry_sdk.set_tag( "trigger_configuration_uuid", self.trigger_configuration_uuid ) + if self.connector_configuration_uuid: + sentry_sdk.set_tag( + "connector_configuration_uuid", self.connector_configuration_uuid + ) def _load_sentry_dsn(self) -> str | None: try: @@ -304,6 +320,8 @@ def _load_environment(self) -> str | None: class ModuleItem(ABC): TOKEN_FILE_NAME = "token" CALLBACK_URL_FILE_NAME = "url_callback" + SECRETS_URL_FILE_NAME = "url_secrets" + LOGS_URL_FILE_NAME = "url_logs" name: str | None = None description: str | None = None @@ -315,7 +333,6 @@ def __init__(self, module: Module | None = None, data_path: Path | None = None): self.module: Module = module or Module() self._token: str | None = None - self._callback_url: str | None = None # Name may be set by the action/trigger class or the module during the register # Worse case we use the class name @@ -384,12 +401,23 @@ def log_exception(self, exception: Exception, **kwargs): scope.set_extra(key, value) sentry_sdk.capture_exception(exception) - @property + @cached_property def callback_url(self) -> str: - if self._callback_url is None: - self._callback_url = self.module.load_config(self.CALLBACK_URL_FILE_NAME) + return self.module.load_config(self.CALLBACK_URL_FILE_NAME) + + @cached_property + def logs_url(self) -> str: + try: + return self.module.load_config(self.LOGS_URL_FILE_NAME) + except FileNotFoundError: + return self.callback_url.replace("/callback", "/logs") - return self._callback_url + @cached_property + def secrets_url(self) -> str: + try: + return self.module.load_config(self.SECRETS_URL_FILE_NAME) + except FileNotFoundError: + return self.callback_url.replace("/callback", "/secrets") @property def _headers(self) -> dict[str, str]: diff --git a/sekoia_automation/scripts/files_generator.py b/sekoia_automation/scripts/files_generator.py index 858cc28..3bb926b 100644 --- a/sekoia_automation/scripts/files_generator.py +++ b/sekoia_automation/scripts/files_generator.py @@ -12,6 +12,7 @@ from rich import print from sekoia_automation.action import Action +from sekoia_automation.connector import Connector from sekoia_automation.module import Module from sekoia_automation.trigger import Trigger from sekoia_automation.utils import get_annotation_for @@ -70,6 +71,7 @@ def execute(self): self.generate_main(module, actions, triggers) self.generate_action_manifests(actions) self.generate_trigger_manifests(triggers) + self.generate_connector_manifests(triggers) self.update_module_manifest(module) sys.path = _old_path @@ -164,6 +166,33 @@ def generate_trigger_manifests(self, triggers: set[type[Trigger]]): print(f"[green][+][/green] Generated {filepath}") + def generate_connector_manifests(self, connectors: set[type[Connector]]): + for connector in connectors: + name = connector.name or connector.__name__ + filepath = ( + self.base_path / f"connector_{name.lower().replace(' ', '_')}.json" + ) + + manifest: dict[str, str | dict | None] = { + "name": name, + "description": connector.description, + "uuid": str(uuid5(self.module_uuid, name)), + "docker_parameters": connector.__name__, + "arguments": {}, + "results": {}, + } + + if connector.results_model: + manifest["results"] = connector.results_model.schema() + + if configuration_model := get_annotation_for(connector, "configuration"): + manifest["arguments"] = configuration_model.schema() + + with filepath.open("w") as out: + out.write(json.dumps(manifest, indent=2)) + + print(f"[green][+][/green] Generated {filepath}") + def update_module_manifest(self, module: type[Module]): configuration_model = get_annotation_for(module, "configuration") diff --git a/sekoia_automation/scripts/sync_library.py b/sekoia_automation/scripts/sync_library.py index 799202b..c192dc6 100755 --- a/sekoia_automation/scripts/sync_library.py +++ b/sekoia_automation/scripts/sync_library.py @@ -227,6 +227,27 @@ def load_triggers(self, module_path: Path) -> list: return triggers + def load_connectors(self, module_path: Path) -> list: + """Load JSON files representing the connectors linked to a module + + Args: + module_path (Path): Path of the parent module + + Returns: + list: List of connectors related to the parent module + """ + connectors = [] + + for filename in module_path.iterdir(): + if filename.name.endswith(".json") and filename.name.startswith( + "connector_" + ): + connector_path = module_path / filename + with connector_path.open() as fd: + connectors.append(json.load(fd)) + + return connectors + def set_docker(self, manifests: list, module: dict) -> list: """Loops over the Docker name of objets linked to a module and adds the Docker version if missing @@ -334,6 +355,7 @@ def load_module(self, module_path: Path): raise typer.Exit(code=1) triggers = self.set_docker(self.load_triggers(module_path), module_info) + connectors = self.set_docker(self.load_connectors(module_path), module_info) actions = self.set_docker(self.load_actions(module_path), module_info) module_uuid: str = module_info["uuid"] @@ -360,6 +382,14 @@ def load_module(self, module_path: Path): name="action", ) print() + if connectors: + self.sync_list( + module_name=module_name, + module_uuid=module_uuid, + list_objects=connectors, + name="connector", + ) + print() def load(self, library_path: Path): """Lods all modules that can be found in a given library diff --git a/sekoia_automation/trigger.py b/sekoia_automation/trigger.py index e768170..659a77f 100644 --- a/sekoia_automation/trigger.py +++ b/sekoia_automation/trigger.py @@ -4,7 +4,6 @@ from collections.abc import Generator from contextlib import contextmanager from datetime import datetime, timedelta -from functools import cached_property from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path from threading import Event, Thread @@ -90,7 +89,7 @@ def _get_secrets_from_server(self) -> dict[str, Any]: if self.module.has_secrets(): try: response = requests.get( - self.callback_url.replace("/callback", "/secrets"), + self.secrets_url, headers=self._headers, timeout=30, ) @@ -287,10 +286,6 @@ def send_event( remove_directory, ) - @cached_property - def _log_url(self): - return self.callback_url.replace("/callback", "/logs") - # Try to send the log record to the API # If it can't be done, give up after 10 attempts and capture the logging error @@ -332,7 +327,7 @@ def _send_logs_to_api(self): return data = {"logs": self._logs} response = requests.request( - "POST", self._log_url, json=data, headers=self._headers, timeout=30 + "POST", self.logs_url, json=data, headers=self._headers, timeout=30 ) response.raise_for_status() self._logs = [] diff --git a/tests/conftest.py b/tests/conftest.py index 26b95da..2f548fd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -77,11 +77,23 @@ def tls_storage(): @pytest.fixture def mocked_trigger_logs(): with patch.object( + Trigger, + "logs_url", + new_callable=PropertyMock, + return_value="http://sekoia-playbooks/logs", + ), patch.object( + Trigger, + "secrets_url", + new_callable=PropertyMock, + return_value="http://sekoia-playbooks/secrets", + ), patch.object( Trigger, "callback_url", new_callable=PropertyMock, return_value="http://sekoia-playbooks/callback", - ), patch.object(Trigger, "token", return_value="secure_token"): + ), patch.object( + Trigger, "token", return_value="secure_token" + ): with requests_mock.Mocker() as mock: mock.post("http://sekoia-playbooks/logs") diff --git a/tests/connectors/test_connector.py b/tests/connectors/test_connector.py index 0e04c6a..d027aa4 100644 --- a/tests/connectors/test_connector.py +++ b/tests/connectors/test_connector.py @@ -1,11 +1,13 @@ -from unittest.mock import Mock, patch +from unittest.mock import Mock, PropertyMock, patch import pytest from tenacity import Retrying, stop_after_attempt, wait_none -from sekoia_automation.connector import Connector +from sekoia_automation.connector import Connector, DefaultConnectorConfiguration from sekoia_automation.constants import CHUNK_BYTES_MAX_SIZE, EVENT_BYTES_MAX_SIZE from sekoia_automation.exceptions import TriggerConfigurationError +from sekoia_automation.module import Module +from sekoia_automation.trigger import Trigger from tests.utils import match_events EVENTS = ["foo", "bar"] @@ -194,3 +196,27 @@ def test_query_exception_api(test_connector, requests_mock): ) test_connector._retry = lambda: Retrying(reraise=True) assert test_connector.push_events_to_intakes(EVENTS) == ["001", "002"] + + +def test_connector_configuration(test_connector): + test_connector._configuration = None + config = DefaultConnectorConfiguration(intake_key="foo") + with patch.object( + Module, "load_config", return_value=config + ) as mock_load_config, patch("sentry_sdk.set_context") as mock_set_sentry_context: + assert test_connector.configuration == config + mock_set_sentry_context.assert_called_with( + "connector_configuration", config.dict() + ) + mock_load_config.assert_called_with( + test_connector.CONNECTOR_CONFIGURATION_FILE_NAME, "json" + ) + + +def test_connector_configuration_file_not_found(test_connector): + test_connector._configuration = None + config = "foo" + with patch.object( + Trigger, "configuration", new_callable=PropertyMock, return_value=config + ): + assert test_connector.configuration == config diff --git a/tests/data/sample_module/connector_test.json b/tests/data/sample_module/connector_test.json new file mode 100644 index 0000000..fe78793 --- /dev/null +++ b/tests/data/sample_module/connector_test.json @@ -0,0 +1,26 @@ +{ + "arguments": { + "$schema": "https://json-schema.org/draft-07/schema#", + "properties": { + "rule_filter": { + "type": "string", + "description": "Some description" + } + }, + "type": "object", + "title": "Connector configuration" + }, + "description": "Sekoia: connecting... things", + "docker_parameters": "connector", + "name": "Connector", + "results": { + "$schema": "https://json-schema.org/draft-07/schema#", + "properties": { + }, + "required": [ + ], + "title": "Results", + "type": "object" + }, + "uuid": "667f7e89-d907-4086-a578-a2324c9a277a" +} diff --git a/tests/scripts/test_sync_library.py b/tests/scripts/test_sync_library.py index 0de0350..1b69b2b 100644 --- a/tests/scripts/test_sync_library.py +++ b/tests/scripts/test_sync_library.py @@ -37,8 +37,15 @@ def trigger(): return trigger +@pytest.fixture +def connector(): + with open("tests/data/sample_module/connector_test.json") as f: + connector = json.load(f) + return connector + + @requests_mock.Mocker(kw="m") -def test_no_module_success(module, action, trigger, **kwargs): +def test_no_module_success(module, action, trigger, connector, **kwargs): kwargs["m"].register_uri( "GET", re.compile(f"{SYMPOHNY_URL}.*"), status_code=200, json={} ) @@ -47,7 +54,7 @@ def test_no_module_success(module, action, trigger, **kwargs): sync_lib.execute() history = kwargs["m"].request_history - assert len(history) == 6 + assert len(history) == 8 assert history[0].method == "GET" assert history[0].url == f"{SYMPOHNY_URL}/modules/{module['uuid']}" assert history[0].headers["Authorization"] == f"Bearer {API_KEY}" @@ -66,10 +73,16 @@ def test_no_module_success(module, action, trigger, **kwargs): assert history[5].method == "PATCH" assert history[5].url == f"{SYMPOHNY_URL}/actions/{action['uuid']}" assert history[5].headers["Authorization"] == f"Bearer {API_KEY}" + assert history[6].method == "GET" + assert history[6].url == f"{SYMPOHNY_URL}/connectors/{connector['uuid']}" + assert history[6].headers["Authorization"] == f"Bearer {API_KEY}" + assert history[7].method == "PATCH" + assert history[7].url == f"{SYMPOHNY_URL}/connectors/{connector['uuid']}" + assert history[7].headers["Authorization"] == f"Bearer {API_KEY}" @requests_mock.Mocker(kw="m") -def test_no_module_404(module, action, trigger, **kwargs): +def test_no_module_404(module, action, trigger, connector, **kwargs): kwargs["m"].register_uri( "GET", re.compile(f"{SYMPOHNY_URL}.*"), status_code=404, json={} ) @@ -78,7 +91,7 @@ def test_no_module_404(module, action, trigger, **kwargs): sync_lib.execute() history = kwargs["m"].request_history - assert len(history) == 6 + assert len(history) == 8 assert history[0].method == "GET" assert history[0].url == f"{SYMPOHNY_URL}/modules/{module['uuid']}" assert history[0].headers["Authorization"] == f"Bearer {API_KEY}" @@ -97,10 +110,16 @@ def test_no_module_404(module, action, trigger, **kwargs): assert history[5].method == "POST" assert history[5].url == f"{SYMPOHNY_URL}/actions" assert history[5].headers["Authorization"] == f"Bearer {API_KEY}" + assert history[6].method == "GET" + assert history[6].url == f"{SYMPOHNY_URL}/connectors/{connector['uuid']}" + assert history[6].headers["Authorization"] == f"Bearer {API_KEY}" + assert history[7].method == "POST" + assert history[7].url == f"{SYMPOHNY_URL}/connectors" + assert history[7].headers["Authorization"] == f"Bearer {API_KEY}" @requests_mock.Mocker(kw="m") -def test_no_module_other_code(module, action, trigger, **kwargs): +def test_no_module_other_code(module, action, trigger, connector, **kwargs): kwargs["m"].register_uri( "GET", re.compile(f"{SYMPOHNY_URL}.*"), status_code=418, json={} ) @@ -109,7 +128,7 @@ def test_no_module_other_code(module, action, trigger, **kwargs): sync_lib.execute() history = kwargs["m"].request_history - assert len(history) == 3 + assert len(history) == 4 assert history[0].method == "GET" assert history[0].url == f"{SYMPOHNY_URL}/modules/{module['uuid']}" assert history[0].headers["Authorization"] == f"Bearer {API_KEY}" @@ -119,10 +138,13 @@ def test_no_module_other_code(module, action, trigger, **kwargs): assert history[2].method == "GET" assert history[2].url == f"{SYMPOHNY_URL}/actions/{action['uuid']}" assert history[2].headers["Authorization"] == f"Bearer {API_KEY}" + assert history[3].method == "GET" + assert history[3].url == f"{SYMPOHNY_URL}/connectors/{connector['uuid']}" + assert history[3].headers["Authorization"] == f"Bearer {API_KEY}" @requests_mock.Mocker(kw="m") -def test_with_module(module, action, trigger, **kwargs): +def test_with_module(module, action, trigger, connector, **kwargs): kwargs["m"].register_uri( "GET", re.compile(f"{SYMPOHNY_URL}.*"), status_code=200, json={} ) @@ -133,7 +155,7 @@ def test_with_module(module, action, trigger, **kwargs): sync_lib.execute() history = kwargs["m"].request_history - assert len(history) == 6 + assert len(history) == 8 assert history[0].method == "GET" assert history[0].url == f"{SYMPOHNY_URL}/modules/{module['uuid']}" assert history[0].headers["Authorization"] == f"Bearer {API_KEY}" @@ -152,6 +174,12 @@ def test_with_module(module, action, trigger, **kwargs): assert history[5].method == "PATCH" assert history[5].url == f"{SYMPOHNY_URL}/actions/{action['uuid']}" assert history[5].headers["Authorization"] == f"Bearer {API_KEY}" + assert history[6].method == "GET" + assert history[6].url == f"{SYMPOHNY_URL}/connectors/{connector['uuid']}" + assert history[6].headers["Authorization"] == f"Bearer {API_KEY}" + assert history[7].method == "PATCH" + assert history[7].url == f"{SYMPOHNY_URL}/connectors/{connector['uuid']}" + assert history[7].headers["Authorization"] == f"Bearer {API_KEY}" def test_with_module_invalid_name(): diff --git a/tests/test_trigger.py b/tests/test_trigger.py index 8276dc4..8a5b51f 100644 --- a/tests/test_trigger.py +++ b/tests/test_trigger.py @@ -54,6 +54,20 @@ def test_callback_url(): mock.assert_called_with(trigger.CALLBACK_URL_FILE_NAME) +def test_secrets_url(): + trigger = DummyTrigger() + with patch.object(Module, "load_config", return_value="secrets") as mock: + assert trigger.secrets_url == "secrets" + mock.assert_called_with(trigger.SECRETS_URL_FILE_NAME) + + +def test_logs_url(): + trigger = DummyTrigger() + with patch.object(Module, "load_config", return_value="logs") as mock: + assert trigger.logs_url == "logs" + mock.assert_called_with(trigger.LOGS_URL_FILE_NAME) + + def test_trigger_configuration(): trigger = DummyTrigger() with patch.object( @@ -401,9 +415,9 @@ def test_trigger_log_critical_only_once(mocked_trigger_logs): @patch.object(Module, "has_secrets", return_value=True) @patch.object( Trigger, - "callback_url", + "secrets_url", new_callable=PropertyMock, - return_value="http://sekoia-playbooks/callback", + return_value="http://sekoia-playbooks/secrets", ) @patch.object(Trigger, "token", return_value="secure_token") def test_get_secrets(_, __, ___):