From 8670557453625c58ad39ccdd82e53e2cd7b5a317 Mon Sep 17 00:00:00 2001 From: Raphael Cohen Date: Wed, 13 Sep 2023 12:39:41 +0200 Subject: [PATCH] feat: Retry errors during storage access --- CHANGELOG.md | 6 +++ poetry.lock | 87 ++++++++++++++++++++++-------------- pyproject.toml | 6 +-- sekoia_automation/module.py | 11 ++++- sekoia_automation/storage.py | 39 +++++++++++++++- sekoia_automation/trigger.py | 7 +++ tests/test_trigger.py | 1 + 7 files changed, 116 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0bf510..fc013bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.4.1] - 2023-09-13 + +### Added + +- Add retry when accessing the storage + ## [1.4.0] - 2023-09-09 ### Added diff --git a/poetry.lock b/poetry.lock index d16f92d..84eeb05 100644 --- a/poetry.lock +++ b/poetry.lock @@ -49,13 +49,33 @@ chardet = ">=3.0.2" [[package]] name = "black" -version = "23.9.0" +version = "23.9.1" description = "The uncompromising code formatter." optional = false python-versions = ">=3.8" files = [ - {file = "black-23.9.0-py3-none-any.whl", hash = "sha256:9366c1f898981f09eb8da076716c02fd021f5a0e63581c66501d68a2e4eab844"}, - {file = "black-23.9.0.tar.gz", hash = "sha256:3511c8a7e22ce653f89ae90dfddaf94f3bb7e2587a245246572d3b9c92adf066"}, + {file = "black-23.9.1-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:d6bc09188020c9ac2555a498949401ab35bb6bf76d4e0f8ee251694664df6301"}, + {file = "black-23.9.1-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:13ef033794029b85dfea8032c9d3b92b42b526f1ff4bf13b2182ce4e917f5100"}, + {file = "black-23.9.1-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:75a2dc41b183d4872d3a500d2b9c9016e67ed95738a3624f4751a0cb4818fe71"}, + {file = "black-23.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13a2e4a93bb8ca74a749b6974925c27219bb3df4d42fc45e948a5d9feb5122b7"}, + {file = "black-23.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:adc3e4442eef57f99b5590b245a328aad19c99552e0bdc7f0b04db6656debd80"}, + {file = "black-23.9.1-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:8431445bf62d2a914b541da7ab3e2b4f3bc052d2ccbf157ebad18ea126efb91f"}, + {file = "black-23.9.1-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:8fc1ddcf83f996247505db6b715294eba56ea9372e107fd54963c7553f2b6dfe"}, + {file = "black-23.9.1-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:7d30ec46de88091e4316b17ae58bbbfc12b2de05e069030f6b747dfc649ad186"}, + {file = "black-23.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:031e8c69f3d3b09e1aa471a926a1eeb0b9071f80b17689a655f7885ac9325a6f"}, + {file = "black-23.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:538efb451cd50f43aba394e9ec7ad55a37598faae3348d723b59ea8e91616300"}, + {file = "black-23.9.1-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:638619a559280de0c2aa4d76f504891c9860bb8fa214267358f0a20f27c12948"}, + {file = "black-23.9.1-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:a732b82747235e0542c03bf352c126052c0fbc458d8a239a94701175b17d4855"}, + {file = "black-23.9.1-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:cf3a4d00e4cdb6734b64bf23cd4341421e8953615cba6b3670453737a72ec204"}, + {file = "black-23.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf99f3de8b3273a8317681d8194ea222f10e0133a24a7548c73ce44ea1679377"}, + {file = "black-23.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:14f04c990259576acd093871e7e9b14918eb28f1866f91968ff5524293f9c573"}, + {file = "black-23.9.1-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:c619f063c2d68f19b2d7270f4cf3192cb81c9ec5bc5ba02df91471d0b88c4c5c"}, + {file = "black-23.9.1-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:6a3b50e4b93f43b34a9d3ef00d9b6728b4a722c997c99ab09102fd5efdb88325"}, + {file = "black-23.9.1-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:c46767e8df1b7beefb0899c4a95fb43058fa8500b6db144f4ff3ca38eb2f6393"}, + {file = "black-23.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50254ebfa56aa46a9fdd5d651f9637485068a1adf42270148cd101cdf56e0ad9"}, + {file = "black-23.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:403397c033adbc45c2bd41747da1f7fc7eaa44efbee256b53842470d4ac5a70f"}, + {file = "black-23.9.1-py3-none-any.whl", hash = "sha256:6ccd59584cc834b6d127628713e4b6b968e5f79572da66284532525a042549f9"}, + {file = "black-23.9.1.tar.gz", hash = "sha256:24b6b3ff5c6d9ea08a8888f6977eae858e1f340d7260cf56d70a49823236b62d"}, ] [package.dependencies] @@ -75,17 +95,17 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.28.44" +version = "1.28.46" description = "The AWS SDK for Python" optional = false python-versions = ">= 3.7" files = [ - {file = "boto3-1.28.44-py3-none-any.whl", hash = "sha256:c53c92dfe22489ba31e918c2e7b59ff43e2e778bd3d3559e62351a739382bb5c"}, - {file = "boto3-1.28.44.tar.gz", hash = "sha256:eea3b07e0f28c9f92bccab972af24a3b0dd951c69d93da75227b8ecd3e18f6c4"}, + {file = "boto3-1.28.46-py3-none-any.whl", hash = "sha256:04445d70127c25fad69e2cab7e3f5cb219c8d6e60463af3657f20e29ac517957"}, + {file = "boto3-1.28.46.tar.gz", hash = "sha256:2ca2852f7b7c1bc2e56f10f968d4c8483c8228b935ecd89a444ae8292ad0dc24"}, ] [package.dependencies] -botocore = ">=1.31.44,<1.32.0" +botocore = ">=1.31.46,<1.32.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -94,13 +114,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.31.44" +version = "1.31.46" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">= 3.7" files = [ - {file = "botocore-1.31.44-py3-none-any.whl", hash = "sha256:83d61c1ca781e6ede19fcc4d5dd73004eee3825a2b220f0d7727e32069209d98"}, - {file = "botocore-1.31.44.tar.gz", hash = "sha256:84f90919fecb4a4f417fd10145c8a87ff2c4b14d6381cd34d9babf02110b3315"}, + {file = "botocore-1.31.46-py3-none-any.whl", hash = "sha256:ac0c1258b1782cde42950bd00138fdce6bd7d04855296af8c326d5844a426473"}, + {file = "botocore-1.31.46.tar.gz", hash = "sha256:6c30be3371624a80d6a881d9c7771a80e0eb82697ee374aaf522cd59b76e14dd"}, ] [package.dependencies] @@ -1137,44 +1157,43 @@ jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] [[package]] name = "ruff" -version = "0.0.287" +version = "0.0.289" description = "An extremely fast Python linter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.0.287-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:1e0f9ee4c3191444eefeda97d7084721d9b8e29017f67997a20c153457f2eafd"}, - {file = "ruff-0.0.287-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:e9843e5704d4fb44e1a8161b0d31c1a38819723f0942639dfeb53d553be9bfb5"}, - {file = "ruff-0.0.287-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ca1ed11d759a29695aed2bfc7f914b39bcadfe2ef08d98ff69c873f639ad3a8"}, - {file = "ruff-0.0.287-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1cf4d5ad3073af10f186ea22ce24bc5a8afa46151f6896f35c586e40148ba20b"}, - {file = "ruff-0.0.287-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66d9d58bcb29afd72d2afe67120afcc7d240efc69a235853813ad556443dc922"}, - {file = "ruff-0.0.287-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:06ac5df7dd3ba8bf83bba1490a72f97f1b9b21c7cbcba8406a09de1a83f36083"}, - {file = "ruff-0.0.287-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2bfb478e1146a60aa740ab9ebe448b1f9e3c0dfb54be3cc58713310eef059c30"}, - {file = "ruff-0.0.287-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:00d579a011949108c4b4fa04c4f1ee066dab536a9ba94114e8e580c96be2aeb4"}, - {file = "ruff-0.0.287-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3a810a79b8029cc92d06c36ea1f10be5298d2323d9024e1d21aedbf0a1a13e5"}, - {file = "ruff-0.0.287-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:150007028ad4976ce9a7704f635ead6d0e767f73354ce0137e3e44f3a6c0963b"}, - {file = "ruff-0.0.287-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a24a280db71b0fa2e0de0312b4aecb8e6d08081d1b0b3c641846a9af8e35b4a7"}, - {file = "ruff-0.0.287-py3-none-musllinux_1_2_i686.whl", hash = "sha256:2918cb7885fa1611d542de1530bea3fbd63762da793751cc8c8d6e4ba234c3d8"}, - {file = "ruff-0.0.287-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:33d7b251afb60bec02a64572b0fd56594b1923ee77585bee1e7e1daf675e7ae7"}, - {file = "ruff-0.0.287-py3-none-win32.whl", hash = "sha256:022f8bed2dcb5e5429339b7c326155e968a06c42825912481e10be15dafb424b"}, - {file = "ruff-0.0.287-py3-none-win_amd64.whl", hash = "sha256:26bd0041d135a883bd6ab3e0b29c42470781fb504cf514e4c17e970e33411d90"}, - {file = "ruff-0.0.287-py3-none-win_arm64.whl", hash = "sha256:44bceb3310ac04f0e59d4851e6227f7b1404f753997c7859192e41dbee9f5c8d"}, - {file = "ruff-0.0.287.tar.gz", hash = "sha256:02dc4f5bf53ef136e459d467f3ce3e04844d509bc46c025a05b018feb37bbc39"}, + {file = "ruff-0.0.289-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:c9a89d748e90c840bac9c37afe90cf13a5bfd460ca02ea93dad9d7bee3af03b4"}, + {file = "ruff-0.0.289-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:7f7396c6ea01ba332a6ad9d47642bac25d16bd2076aaa595b001f58b2f32ff05"}, + {file = "ruff-0.0.289-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7180de86c8ecd39624dec1699136f941c07e723201b4ce979bec9e7c67b40ad2"}, + {file = "ruff-0.0.289-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:73f37c65508203dd01a539926375a10243769c20d4fcab3fa6359cd3fbfc54b7"}, + {file = "ruff-0.0.289-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c14abcd7563b5c80be2dd809eeab20e4aa716bf849860b60a22d87ddf19eb88"}, + {file = "ruff-0.0.289-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:91b6d63b6b46d4707916472c91baa87aa0592e73f62a80ff55efdf6c0668cfd6"}, + {file = "ruff-0.0.289-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6479b8c4be3c36046c6c92054762b276fa0fddb03f6b9a310fbbf4c4951267fd"}, + {file = "ruff-0.0.289-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c5424318c254bcb091cb67e140ec9b9f7122074e100b06236f252923fb41e767"}, + {file = "ruff-0.0.289-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4daa90865796aedcedf0d8897fdd4cd09bf0ddd3504529a4ccf211edcaff3c7d"}, + {file = "ruff-0.0.289-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:8057e8ab0016c13b9419bad119e854f881e687bd96bc5e2d52c8baac0f278a44"}, + {file = "ruff-0.0.289-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:7eebfab2e6a6991908ff1bf82f2dc1e5095fc7e316848e62124526837b445f4d"}, + {file = "ruff-0.0.289-py3-none-musllinux_1_2_i686.whl", hash = "sha256:ebc7af550018001a7fb39ca22cdce20e1a0de4388ea4a007eb5c822f6188c297"}, + {file = "ruff-0.0.289-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:6e4e6eccb753efe760ba354fc8e9f783f6bba71aa9f592756f5bd0d78db898ed"}, + {file = "ruff-0.0.289-py3-none-win32.whl", hash = "sha256:bbb3044f931c09cf17dbe5b339896eece0d6ac10c9a86e172540fcdb1974f2b7"}, + {file = "ruff-0.0.289-py3-none-win_amd64.whl", hash = "sha256:6d043c5456b792be2615a52f16056c3cf6c40506ce1f2d6f9d3083cfcb9eeab6"}, + {file = "ruff-0.0.289-py3-none-win_arm64.whl", hash = "sha256:04a720bcca5e987426bb14ad8b9c6f55e259ea774da1cbeafe71569744cfd20a"}, + {file = "ruff-0.0.289.tar.gz", hash = "sha256:2513f853b0fc42f0339b7ab0d2751b63ce7a50a0032d2689b54b2931b3b866d7"}, ] [[package]] name = "s3path" -version = "0.4.2" +version = "0.5.0" description = "" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "s3path-0.4.2-py3-none-any.whl", hash = "sha256:abbd723ced0567aabc10b4f7b435016d9678330365703aae7d3f905451efbaeb"}, - {file = "s3path-0.4.2.tar.gz", hash = "sha256:04a2ba953555c0191122d87aa90cde27ac5e68253817831f17c538c96f3e58cf"}, + {file = "s3path-0.5.0-py3-none-any.whl", hash = "sha256:94b0add85ce6be5c38c2bf109e27bef938850ce8d0c731e54f4527007710acbb"}, + {file = "s3path-0.5.0.tar.gz", hash = "sha256:ccf2ab37d5c68eabcd10f8c6d2f7ed1ab4911a955cbd617de46b3478db70d1ac"}, ] [package.dependencies] boto3 = ">=1.16.35" -packaging = "*" smart-open = "*" [[package]] @@ -1478,4 +1497,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.12" -content-hash = "c4d25dcac41c1a67832aea7d54ad9480ec6a225098c446be29d5c8bb5f857e24" +content-hash = "49cdb80a9a37e60fc7a1f31c5477d560d6a7ba5bb417ae6fb1d997d6a7602bbd" diff --git a/pyproject.toml b/pyproject.toml index bf151f7..2e95f18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "sekoia-automation-sdk" -version = "1.4.0" +version = "1.4.1" description = "SDK to create Sekoia.io playbook modules" license = "MIT" readme = "README.md" @@ -33,8 +33,8 @@ python = ">=3.10,<3.12" requests = "^2.25" sentry-sdk = "*" tenacity = "*" -boto3 = "^1.26" -s3path = "^0.4" +boto3 = "^1.28" +s3path = "^0.5" orjson = "^3.8" pydantic = "^1.10" typer = { extras = ["all"], version = "^0.7"} diff --git a/sekoia_automation/module.py b/sekoia_automation/module.py index ff21837..76a0e8d 100644 --- a/sekoia_automation/module.py +++ b/sekoia_automation/module.py @@ -1,6 +1,7 @@ import json import logging import sys +import time from abc import ABC, abstractmethod from pathlib import Path from typing import Any, cast @@ -17,7 +18,10 @@ SendEventError, ) from sekoia_automation.storage import get_data_path -from sekoia_automation.utils import get_annotation_for, get_as_model +from sekoia_automation.utils import ( + get_annotation_for, + get_as_model, +) class Module: @@ -304,6 +308,8 @@ class ModuleItem(ABC): description: str | None = None results_model: type[BaseModel] | None = None + _http_error_base_sleep = 0.5 + def __init__(self, module: Module | None = None, data_path: Path | None = None): self.module: Module = module or Module() @@ -391,7 +397,7 @@ def _send_request(self, data: dict, verb: str = "POST", attempt=1) -> Response: return response except HTTPError as exception: self._log_request_error(exception) - if attempt == 3: + if attempt == 5: status_code = ( exception.response.status_code if isinstance(exception.response, Response) @@ -408,6 +414,7 @@ def _send_request(self, data: dict, verb: str = "POST", attempt=1) -> Response: "Impossible to send event to Sekoia.io API", status_code=exception.response.status_code, ) + time.sleep(attempt * self._http_error_base_sleep) return self._send_request(data, verb, attempt + 1) def _log_request_error(self, exception: HTTPError): diff --git a/sekoia_automation/storage.py b/sekoia_automation/storage.py index 14906e2..a10b651 100644 --- a/sekoia_automation/storage.py +++ b/sekoia_automation/storage.py @@ -7,14 +7,22 @@ import orjson from botocore.config import Config from s3path import S3Path, register_configuration_parameter +from tenacity import retry, stop_after_attempt, wait_exponential from sekoia_automation import constants from sekoia_automation.config import VOLUME_PATH, load_config +from sekoia_automation.utils import capture_retry_error FilePath = Path | str @lru_cache +@retry( + reraise=True, + wait=wait_exponential(max=6), + stop=stop_after_attempt(10), + retry_error_callback=capture_retry_error, +) def get_s3_data_path() -> Path: ca_path, cert_path, key_path = _get_tls_client_credentials() config = Config() @@ -133,6 +141,12 @@ def __init__(self, filepath: FilePath, data_path: Path | None = None): self._data: dict = {} + @retry( + reraise=True, + wait=wait_exponential(max=6), + stop=stop_after_attempt(10), + retry_error_callback=capture_retry_error, + ) def load(self): if not self._data and self._filepath.is_file(): with self._filepath.open("r") as fd: @@ -142,16 +156,31 @@ def load(self): # The content is not valid json self._data = {} + @retry( + reraise=True, + wait=wait_exponential(max=6), + stop=stop_after_attempt(10), + retry_error_callback=capture_retry_error, + ) + def dump(self): + with self._filepath.open("w") as out: + out.write(orjson.dumps(self._data).decode("utf-8")) + def __enter__(self): self.load() return self._data def __exit__(self, _, __, ___): - with self._filepath.open("w") as out: - out.write(orjson.dumps(self._data).decode("utf-8")) + self.dump() +@retry( + reraise=True, + wait=wait_exponential(max=6), + stop=stop_after_attempt(10), + retry_error_callback=capture_retry_error, +) def temp_directory(data_path: Path | None = None) -> str: """Create a temporary directory inside data storage. @@ -166,6 +195,12 @@ def temp_directory(data_path: Path | None = None) -> str: return name +@retry( + reraise=True, + wait=wait_exponential(max=6), + stop=stop_after_attempt(10), + retry_error_callback=capture_retry_error, +) def write( filename: str, content: dict | str, diff --git a/sekoia_automation/trigger.py b/sekoia_automation/trigger.py index a1ab9bb..4a6e0bb 100644 --- a/sekoia_automation/trigger.py +++ b/sekoia_automation/trigger.py @@ -70,6 +70,12 @@ def __init__(self, module: Module | None = None, data_path: Path | None = None): self._liveness_server = None self._exporter = None + @retry( + reraise=True, + wait=wait_exponential(max=10), + stop=stop_after_attempt(10), + retry_error_callback=capture_retry_error, + ) def _get_secrets_from_server(self) -> dict[str, Any]: """Calls the API to fetch this trigger's secrets @@ -278,6 +284,7 @@ def log(self, message: str, level: str = "info", *args, **kwargs) -> None: self._critical_log_sent = True @retry( + reraise=True, wait=wait_exponential(max=10), stop=stop_after_attempt(10), retry_error_callback=capture_retry_error, diff --git a/tests/test_trigger.py b/tests/test_trigger.py index 54a0b17..8276dc4 100644 --- a/tests/test_trigger.py +++ b/tests/test_trigger.py @@ -103,6 +103,7 @@ def test_send_event_4xx_error(mocked_trigger_logs): def test_send_event_too_many_failures(mocked_trigger_logs): trigger = DummyTrigger() + trigger._http_error_base_sleep = 0 mocked_trigger_logs.post( "http://sekoia-playbooks/callback",