From 0bd946c3d4a1d953830a8cd1ee6c131c50dcd3ef Mon Sep 17 00:00:00 2001 From: BryanFauble <17128019+BryanFauble@users.noreply.github.com> Date: Thu, 24 Oct 2024 14:07:15 -0700 Subject: [PATCH] [FDS-2386] Synapse entity tracking and code concurrency updates (#1505) * [FDS-2386] Synapse entity tracking and code concurrency updates --- .gitignore | 3 +- main.py | 3 - poetry.lock | 1045 +++++++---------- pyproject.toml | 54 +- pytest.ini | 12 +- schematic/__init__.py | 112 ++ schematic/manifest/commands.py | 33 +- schematic/manifest/generator.py | 3 +- schematic/models/GE_Helpers.py | 57 +- schematic/models/metadata.py | 4 - schematic/models/validate_attribute.py | 2 +- schematic/models/validate_manifest.py | 87 +- schematic/schemas/data_model_parser.py | 12 +- schematic/store/__init__.py | 2 - schematic/store/database/README.md | 18 + schematic/store/database/synapse_database.py | 138 +++ .../database/synapse_database_wrapper.py | 156 +++ schematic/store/synapse.py | 588 +++++++--- schematic/store/synapse_tracker.py | 147 +++ schematic/utils/general.py | 30 +- schematic/utils/io_utils.py | 43 +- schematic/utils/validate_utils.py | 18 +- .../visualization/attributes_explorer.py | 6 +- schematic_api/api/__init__.py | 29 +- schematic_api/api/routes.py | 106 +- tests/conftest.py | 34 +- tests/integration/test_commands.py | 97 ++ tests/integration/test_store_synapse.py | 1 - tests/test_api.py | 233 +++- tests/test_cli.py | 9 +- tests/test_ge_helpers.py | 36 +- tests/test_store.py | 203 ++-- tests/test_utils.py | 54 +- tests/test_validation.py | 4 - tests/unit/test_io_utils.py | 96 ++ 35 files changed, 2169 insertions(+), 1306 deletions(-) create mode 100644 schematic/store/database/README.md create mode 100644 schematic/store/database/synapse_database.py create mode 100644 schematic/store/database/synapse_database_wrapper.py create mode 100644 schematic/store/synapse_tracker.py create mode 100644 tests/integration/test_commands.py create mode 100644 tests/unit/test_io_utils.py diff --git a/.gitignore b/.gitignore index 6d00e45d3..91f6d7c64 100644 --- a/.gitignore +++ b/.gitignore @@ -164,7 +164,7 @@ clean.sh # Intermediate files data/json_schema_logs/json_schema_log.json great_expectations/checkpoints/manifest_checkpoint.yml -great_expectations/expectations/Manifest_test_suite.json +great_expectations/expectations/Manifest_test_suite*.json tests/data/example.MockComponent.schema.json tests/data/mock_manifests/Invalid_Test_Manifest_censored.csv @@ -177,6 +177,7 @@ tests/data/schema.gpickle # Created during testting Example* manifests/* +https:* # schematic config file config.yml \ No newline at end of file diff --git a/main.py b/main.py index 8081a7578..f5b51bcac 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,4 @@ import os -import connexion -from schematic import CONFIG -from flask_cors import CORS from schematic_api.api import app diff --git a/poetry.lock b/poetry.lock index b6b193d77..0b87e58a1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -35,13 +35,13 @@ dev = ["black", "docutils", "flake8", "ipython", "m2r", "mistune (<2.0.0)", "pyt [[package]] name = "anyio" -version = "4.4.0" +version = "4.5.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" optional = false python-versions = ">=3.8" files = [ - {file = "anyio-4.4.0-py3-none-any.whl", hash = "sha256:c1b2d8f46a8a812513012e1107cb0e68c17159a7a594208005a57dc776e1bdc7"}, - {file = "anyio-4.4.0.tar.gz", hash = "sha256:5aadc6a1bbb7cdb0bede386cac5e2940f5e2ff3aa20277e991cf028e0585ce94"}, + {file = "anyio-4.5.0-py3-none-any.whl", hash = "sha256:fdeb095b7cc5a5563175eedd926ec4ae55413bb4be5770c424af0ba46ccb4a78"}, + {file = "anyio-4.5.0.tar.gz", hash = "sha256:c5a275fe5ca0afd788001f58fca1e69e29ce706d746e317d660e21f70c530ef9"}, ] [package.dependencies] @@ -51,9 +51,9 @@ sniffio = ">=1.1" typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} [package.extras] -doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] -test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] -trio = ["trio (>=0.23)"] +doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.21.0b1)"] +trio = ["trio (>=0.26.1)"] [[package]] name = "appnope" @@ -250,17 +250,6 @@ files = [ [package.extras] dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] -[[package]] -name = "backoff" -version = "2.2.1" -description = "Function decoration for backoff and retry" -optional = false -python-versions = ">=3.7,<4.0" -files = [ - {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, - {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, -] - [[package]] name = "beautifulsoup4" version = "4.12.3" @@ -897,20 +886,6 @@ wrapt = ">=1.10,<2" [package.extras] dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] -[[package]] -name = "deprecation" -version = "2.1.0" -description = "A library to handle automated deprecations" -optional = false -python-versions = "*" -files = [ - {file = "deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a"}, - {file = "deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff"}, -] - -[package.dependencies] -packaging = "*" - [[package]] name = "dill" version = "0.3.8" @@ -1028,19 +1003,19 @@ devel = ["colorama", "json-spec", "jsonschema", "pylint", "pytest", "pytest-benc [[package]] name = "filelock" -version = "3.15.4" +version = "3.16.1" description = "A platform independent file lock." optional = false python-versions = ">=3.8" files = [ - {file = "filelock-3.15.4-py3-none-any.whl", hash = "sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7"}, - {file = "filelock-3.15.4.tar.gz", hash = "sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb"}, + {file = "filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0"}, + {file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"}, ] [package.extras] -docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8.0.1)", "pytest (>=7.4.3)", "pytest-asyncio (>=0.21)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)", "virtualenv (>=20.26.2)"] -typing = ["typing-extensions (>=4.8)"] +docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4.1)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.2)", "pytest (>=8.3.3)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.4)"] +typing = ["typing-extensions (>=4.12.2)"] [[package]] name = "flake8" @@ -1095,24 +1070,6 @@ files = [ Flask = ">=0.9" Six = "*" -[[package]] -name = "flask-opentracing" -version = "2.0.0" -description = "OpenTracing support for Flask applications" -optional = true -python-versions = "*" -files = [ - {file = "Flask-OpenTracing-2.0.0.tar.gz", hash = "sha256:4de9db3d4f0d2b506ce3874fc721278d41b2e8b0125ea567164be0100df502fe"}, - {file = "Flask_OpenTracing-2.0.0-py3-none-any.whl", hash = "sha256:e7086ffb3531a518c6e3bf2b365af4a51e56a0922fdd5ebe91c9ddeeda632e70"}, -] - -[package.dependencies] -Flask = "*" -opentracing = ">=2.0,<3" - -[package.extras] -tests = ["flake8", "flake8-quotes", "mock", "pytest", "pytest-cov", "tox"] - [[package]] name = "fqdn" version = "1.5.1" @@ -1126,13 +1083,13 @@ files = [ [[package]] name = "google-api-core" -version = "2.19.2" +version = "2.20.0" description = "Google API client core library" optional = false python-versions = ">=3.7" files = [ - {file = "google_api_core-2.19.2-py3-none-any.whl", hash = "sha256:53ec0258f2837dd53bbd3d3df50f5359281b3cc13f800c941dd15a9b5a415af4"}, - {file = "google_api_core-2.19.2.tar.gz", hash = "sha256:ca07de7e8aa1c98a8bfca9321890ad2340ef7f2eb136e558cee68f24b94b0a8f"}, + {file = "google_api_core-2.20.0-py3-none-any.whl", hash = "sha256:ef0591ef03c30bb83f79b3d0575c3f31219001fc9c5cf37024d08310aeffed8a"}, + {file = "google_api_core-2.20.0.tar.gz", hash = "sha256:f74dff1889ba291a4b76c5079df0711810e2d9da81abfdc99957bc961c1eb28f"}, ] [package.dependencies] @@ -1167,13 +1124,13 @@ uritemplate = ">=3.0.1,<5" [[package]] name = "google-auth" -version = "2.34.0" +version = "2.35.0" description = "Google Authentication Library" optional = false python-versions = ">=3.7" files = [ - {file = "google_auth-2.34.0-py2.py3-none-any.whl", hash = "sha256:72fd4733b80b6d777dcde515628a9eb4a577339437012874ea286bca7261ee65"}, - {file = "google_auth-2.34.0.tar.gz", hash = "sha256:8eb87396435c19b20d32abd2f984e31c191a15284af72eb922f10e5bde9c04cc"}, + {file = "google_auth-2.35.0-py2.py3-none-any.whl", hash = "sha256:25df55f327ef021de8be50bad0dfd4a916ad0de96da86cd05661c9297723ad3f"}, + {file = "google_auth-2.35.0.tar.gz", hash = "sha256:f4c64ed4e01e8e8b646ef34c018f8bf3338df0c8e37d8b3bba40e7f574a3278a"}, ] [package.dependencies] @@ -1328,77 +1285,6 @@ test = ["black[jupyter] (==22.3.0)", "boto3 (==1.17.106)", "docstring-parser (== trino = ["sqlalchemy (>=1.3.18,<2.0.0)", "trino (>=0.310.0,!=0.316.0)"] vertica = ["sqlalchemy (>=1.3.18,<2.0.0)", "sqlalchemy-vertica-python (>=0.5.10)"] -[[package]] -name = "greenlet" -version = "3.0.3" -description = "Lightweight in-process concurrent programming" -optional = false -python-versions = ">=3.7" -files = [ - {file = "greenlet-3.0.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9da2bd29ed9e4f15955dd1595ad7bc9320308a3b766ef7f837e23ad4b4aac31a"}, - {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d353cadd6083fdb056bb46ed07e4340b0869c305c8ca54ef9da3421acbdf6881"}, - {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dca1e2f3ca00b84a396bc1bce13dd21f680f035314d2379c4160c98153b2059b"}, - {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ed7fb269f15dc662787f4119ec300ad0702fa1b19d2135a37c2c4de6fadfd4a"}, - {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd4f49ae60e10adbc94b45c0b5e6a179acc1736cf7a90160b404076ee283cf83"}, - {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73a411ef564e0e097dbe7e866bb2dda0f027e072b04da387282b02c308807405"}, - {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7f362975f2d179f9e26928c5b517524e89dd48530a0202570d55ad6ca5d8a56f"}, - {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:649dde7de1a5eceb258f9cb00bdf50e978c9db1b996964cd80703614c86495eb"}, - {file = "greenlet-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:68834da854554926fbedd38c76e60c4a2e3198c6fbed520b106a8986445caaf9"}, - {file = "greenlet-3.0.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b1b5667cced97081bf57b8fa1d6bfca67814b0afd38208d52538316e9422fc61"}, - {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52f59dd9c96ad2fc0d5724107444f76eb20aaccb675bf825df6435acb7703559"}, - {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:afaff6cf5200befd5cec055b07d1c0a5a06c040fe5ad148abcd11ba6ab9b114e"}, - {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"}, - {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2797aa5aedac23af156bbb5a6aa2cd3427ada2972c828244eb7d1b9255846379"}, - {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7f009caad047246ed379e1c4dbcb8b020f0a390667ea74d2387be2998f58a22"}, - {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c5e1536de2aad7bf62e27baf79225d0d64360d4168cf2e6becb91baf1ed074f3"}, - {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:894393ce10ceac937e56ec00bb71c4c2f8209ad516e96033e4b3b1de270e200d"}, - {file = "greenlet-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:1ea188d4f49089fc6fb283845ab18a2518d279c7cd9da1065d7a84e991748728"}, - {file = "greenlet-3.0.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:70fb482fdf2c707765ab5f0b6655e9cfcf3780d8d87355a063547b41177599be"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4d1ac74f5c0c0524e4a24335350edad7e5f03b9532da7ea4d3c54d527784f2e"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149e94a2dd82d19838fe4b2259f1b6b9957d5ba1b25640d2380bea9c5df37676"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15d79dd26056573940fcb8c7413d84118086f2ec1a8acdfa854631084393efcc"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b7db1ebff4ba09aaaeae6aa491daeb226c8150fc20e836ad00041bcb11230"}, - {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fcd2469d6a2cf298f198f0487e0a5b1a47a42ca0fa4dfd1b6862c999f018ebbf"}, - {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1f672519db1796ca0d8753f9e78ec02355e862d0998193038c7073045899f305"}, - {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2516a9957eed41dd8f1ec0c604f1cdc86758b587d964668b5b196a9db5bfcde6"}, - {file = "greenlet-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:bba5387a6975598857d86de9eac14210a49d554a77eb8261cc68b7d082f78ce2"}, - {file = "greenlet-3.0.3-cp37-cp37m-macosx_11_0_universal2.whl", hash = "sha256:5b51e85cb5ceda94e79d019ed36b35386e8c37d22f07d6a751cb659b180d5274"}, - {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:daf3cb43b7cf2ba96d614252ce1684c1bccee6b2183a01328c98d36fcd7d5cb0"}, - {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99bf650dc5d69546e076f413a87481ee1d2d09aaaaaca058c9251b6d8c14783f"}, - {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dd6e660effd852586b6a8478a1d244b8dc90ab5b1321751d2ea15deb49ed414"}, - {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3391d1e16e2a5a1507d83e4a8b100f4ee626e8eca43cf2cadb543de69827c4c"}, - {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1f145462f1fa6e4a4ae3c0f782e580ce44d57c8f2c7aae1b6fa88c0b2efdb41"}, - {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1a7191e42732df52cb5f39d3527217e7ab73cae2cb3694d241e18f53d84ea9a7"}, - {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0448abc479fab28b00cb472d278828b3ccca164531daab4e970a0458786055d6"}, - {file = "greenlet-3.0.3-cp37-cp37m-win32.whl", hash = "sha256:b542be2440edc2d48547b5923c408cbe0fc94afb9f18741faa6ae970dbcb9b6d"}, - {file = "greenlet-3.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:01bc7ea167cf943b4c802068e178bbf70ae2e8c080467070d01bfa02f337ee67"}, - {file = "greenlet-3.0.3-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:1996cb9306c8595335bb157d133daf5cf9f693ef413e7673cb07e3e5871379ca"}, - {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ddc0f794e6ad661e321caa8d2f0a55ce01213c74722587256fb6566049a8b04"}, - {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9db1c18f0eaad2f804728c67d6c610778456e3e1cc4ab4bbd5eeb8e6053c6fc"}, - {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7170375bcc99f1a2fbd9c306f5be8764eaf3ac6b5cb968862cad4c7057756506"}, - {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b66c9c1e7ccabad3a7d037b2bcb740122a7b17a53734b7d72a344ce39882a1b"}, - {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:098d86f528c855ead3479afe84b49242e174ed262456c342d70fc7f972bc13c4"}, - {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:81bb9c6d52e8321f09c3d165b2a78c680506d9af285bfccbad9fb7ad5a5da3e5"}, - {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da"}, - {file = "greenlet-3.0.3-cp38-cp38-win32.whl", hash = "sha256:d46677c85c5ba00a9cb6f7a00b2bfa6f812192d2c9f7d9c4f6a55b60216712f3"}, - {file = "greenlet-3.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:419b386f84949bf0e7c73e6032e3457b82a787c1ab4a0e43732898a761cc9dbf"}, - {file = "greenlet-3.0.3-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:da70d4d51c8b306bb7a031d5cff6cc25ad253affe89b70352af5f1cb68e74b53"}, - {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086152f8fbc5955df88382e8a75984e2bb1c892ad2e3c80a2508954e52295257"}, - {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d73a9fe764d77f87f8ec26a0c85144d6a951a6c438dfe50487df5595c6373eac"}, - {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7dcbe92cc99f08c8dd11f930de4d99ef756c3591a5377d1d9cd7dd5e896da71"}, - {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1551a8195c0d4a68fac7a4325efac0d541b48def35feb49d803674ac32582f61"}, - {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64d7675ad83578e3fc149b617a444fab8efdafc9385471f868eb5ff83e446b8b"}, - {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b37eef18ea55f2ffd8f00ff8fe7c8d3818abd3e25fb73fae2ca3b672e333a7a6"}, - {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:77457465d89b8263bca14759d7c1684df840b6811b2499838cc5b040a8b5b113"}, - {file = "greenlet-3.0.3-cp39-cp39-win32.whl", hash = "sha256:57e8974f23e47dac22b83436bdcf23080ade568ce77df33159e019d161ce1d1e"}, - {file = "greenlet-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:c5ee858cfe08f34712f548c3c363e807e7186f03ad7a5039ebadb29e8c6be067"}, - {file = "greenlet-3.0.3.tar.gz", hash = "sha256:43374442353259554ce33599da8b692d5aa96f8976d567d4badf263371fbe491"}, -] - -[package.extras] -docs = ["Sphinx", "furo"] -test = ["objgraph", "psutil"] - [[package]] name = "grpcio" version = "1.66.1" @@ -1530,13 +1416,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "identify" -version = "2.6.0" +version = "2.6.1" description = "File identification library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "identify-2.6.0-py2.py3-none-any.whl", hash = "sha256:e79ae4406387a9d300332b5fd366d8994f1525e8414984e1a59e058b2eda2dd0"}, - {file = "identify-2.6.0.tar.gz", hash = "sha256:cb171c685bdc31bcc4c1734698736a7d5b6c8bf2e0c15117f4d469c8640ae5cf"}, + {file = "identify-2.6.1-py2.py3-none-any.whl", hash = "sha256:53863bcac7caf8d2ed85bd20312ea5dcfc22226800f6d6881f232d861db5a8f0"}, + {file = "identify-2.6.1.tar.gz", hash = "sha256:91478c5fb7c3aac5ff7bf9b4344f803843dc586832d5f110d672b19aa1984c98"}, ] [package.extras] @@ -1544,15 +1430,18 @@ license = ["ukkonen"] [[package]] name = "idna" -version = "3.8" +version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" files = [ - {file = "idna-3.8-py3-none-any.whl", hash = "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac"}, - {file = "idna-3.8.tar.gz", hash = "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603"}, + {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, + {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, ] +[package.extras] +all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] + [[package]] name = "imagesize" version = "1.4.1" @@ -1566,22 +1455,22 @@ files = [ [[package]] name = "importlib-metadata" -version = "6.11.0" +version = "8.4.0" description = "Read metadata from Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "importlib_metadata-6.11.0-py3-none-any.whl", hash = "sha256:f0afba6205ad8f8947c7d338b5342d5db2afbfd82f9cbef7879a9539cc12eb9b"}, - {file = "importlib_metadata-6.11.0.tar.gz", hash = "sha256:1231cf92d825c9e03cfc4da076a16de6422c863558229ea0b22b675657463443"}, + {file = "importlib_metadata-8.4.0-py3-none-any.whl", hash = "sha256:66f342cc6ac9818fc6ff340576acd24d65ba0b3efabb2b4ac08b598965a4a2f1"}, + {file = "importlib_metadata-8.4.0.tar.gz", hash = "sha256:9a547d3bc3608b025f93d403fdd1aae741c24fbb8314df4b155675742ce303c5"}, ] [package.dependencies] zipp = ">=0.5" [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] perf = ["ipython"] -testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] +test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] [[package]] name = "inflection" @@ -1605,31 +1494,6 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] -[[package]] -name = "interrogate" -version = "1.7.0" -description = "Interrogate a codebase for docstring coverage." -optional = false -python-versions = ">=3.8" -files = [ - {file = "interrogate-1.7.0-py3-none-any.whl", hash = "sha256:b13ff4dd8403369670e2efe684066de9fcb868ad9d7f2b4095d8112142dc9d12"}, - {file = "interrogate-1.7.0.tar.gz", hash = "sha256:a320d6ec644dfd887cc58247a345054fc4d9f981100c45184470068f4b3719b0"}, -] - -[package.dependencies] -attrs = "*" -click = ">=7.1" -colorama = "*" -py = "*" -tabulate = "*" -tomli = {version = "*", markers = "python_version < \"3.11\""} - -[package.extras] -dev = ["cairosvg", "coverage[toml]", "pre-commit", "pytest", "pytest-cov", "pytest-mock", "sphinx", "sphinx-autobuild", "wheel"] -docs = ["sphinx", "sphinx-autobuild"] -png = ["cairosvg"] -tests = ["coverage[toml]", "pytest", "pytest-cov", "pytest-mock"] - [[package]] name = "ipykernel" version = "6.29.5" @@ -1774,25 +1638,6 @@ files = [ {file = "itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173"}, ] -[[package]] -name = "jaeger-client" -version = "4.8.0" -description = "Jaeger Python OpenTracing Tracer implementation" -optional = true -python-versions = ">=3.7" -files = [ - {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, -] - -[package.dependencies] -opentracing = ">=2.1,<3.0" -threadloop = ">=1,<2" -thrift = "*" -tornado = ">=4.3" - -[package.extras] -tests = ["codecov", "coverage", "flake8", "flake8-quotes", "flake8-typing-imports", "mock", "mypy", "opentracing_instrumentation (>=3,<4)", "prometheus_client (==0.11.0)", "pycurl", "pytest", "pytest-benchmark[histogram]", "pytest-cov", "pytest-localserver", "pytest-timeout", "pytest-tornado", "tchannel (==2.1.0)"] - [[package]] name = "jedi" version = "0.19.1" @@ -1910,13 +1755,13 @@ referencing = ">=0.31.0" [[package]] name = "jupyter-client" -version = "8.6.2" +version = "8.6.3" description = "Jupyter protocol implementation and client libraries" optional = false python-versions = ">=3.8" files = [ - {file = "jupyter_client-8.6.2-py3-none-any.whl", hash = "sha256:50cbc5c66fd1b8f65ecb66bc490ab73217993632809b6e505687de18e9dea39f"}, - {file = "jupyter_client-8.6.2.tar.gz", hash = "sha256:2bda14d55ee5ba58552a8c53ae43d215ad9868853489213f37da060ced54d8df"}, + {file = "jupyter_client-8.6.3-py3-none-any.whl", hash = "sha256:e8a19cc986cc45905ac3362915f410f3af85424b4c0905e94fa5f2cb08e8f23f"}, + {file = "jupyter_client-8.6.3.tar.gz", hash = "sha256:35b3a0947c4a6e9d589eb97d7d4cd5e90f910ee73101611f01283732bd6d9419"}, ] [package.dependencies] @@ -2462,21 +2307,21 @@ files = [ [[package]] name = "networkx" -version = "2.8.8" +version = "3.2.1" description = "Python package for creating and manipulating graphs and networks" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "networkx-2.8.8-py3-none-any.whl", hash = "sha256:e435dfa75b1d7195c7b8378c3859f0445cd88c6b0375c181ed66823a9ceb7524"}, - {file = "networkx-2.8.8.tar.gz", hash = "sha256:230d388117af870fce5647a3c52401fcf753e94720e6ea6b4197a5355648885e"}, + {file = "networkx-3.2.1-py3-none-any.whl", hash = "sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2"}, + {file = "networkx-3.2.1.tar.gz", hash = "sha256:9f1bb5cf3409bf324e0a722c20bdb4c20ee39bf1c30ce8ae499c8502b0b5e0c6"}, ] [package.extras] -default = ["matplotlib (>=3.4)", "numpy (>=1.19)", "pandas (>=1.3)", "scipy (>=1.8)"] -developer = ["mypy (>=0.982)", "pre-commit (>=2.20)"] -doc = ["nb2plots (>=0.6)", "numpydoc (>=1.5)", "pillow (>=9.2)", "pydata-sphinx-theme (>=0.11)", "sphinx (>=5.2)", "sphinx-gallery (>=0.11)", "texext (>=0.6.6)"] -extra = ["lxml (>=4.6)", "pydot (>=1.4.2)", "pygraphviz (>=1.9)", "sympy (>=1.10)"] -test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] +default = ["matplotlib (>=3.5)", "numpy (>=1.22)", "pandas (>=1.4)", "scipy (>=1.9,!=1.11.0,!=1.11.1)"] +developer = ["changelist (==0.4)", "mypy (>=1.1)", "pre-commit (>=3.2)", "rtoml"] +doc = ["nb2plots (>=0.7)", "nbconvert (<7.9)", "numpydoc (>=1.6)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.14)", "sphinx (>=7)", "sphinx-gallery (>=0.14)", "texext (>=0.6.7)"] +extra = ["lxml (>=4.6)", "pydot (>=1.4.2)", "pygraphviz (>=1.11)", "sympy (>=1.10)"] +test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] [[package]] name = "nodeenv" @@ -2624,91 +2469,211 @@ et-xmlfile = "*" [[package]] name = "opentelemetry-api" -version = "1.21.0" +version = "1.27.0" description = "OpenTelemetry Python API" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "opentelemetry_api-1.21.0-py3-none-any.whl", hash = "sha256:4bb86b28627b7e41098f0e93280fe4892a1abed1b79a19aec6f928f39b17dffb"}, - {file = "opentelemetry_api-1.21.0.tar.gz", hash = "sha256:d6185fd5043e000075d921822fd2d26b953eba8ca21b1e2fa360dd46a7686316"}, + {file = "opentelemetry_api-1.27.0-py3-none-any.whl", hash = "sha256:953d5871815e7c30c81b56d910c707588000fff7a3ca1c73e6531911d53065e7"}, + {file = "opentelemetry_api-1.27.0.tar.gz", hash = "sha256:ed673583eaa5f81b5ce5e86ef7cdaf622f88ef65f0b9aab40b843dcae5bef342"}, ] [package.dependencies] deprecated = ">=1.2.6" -importlib-metadata = ">=6.0,<7.0" +importlib-metadata = ">=6.0,<=8.4.0" [[package]] name = "opentelemetry-exporter-otlp-proto-common" -version = "1.21.0" +version = "1.27.0" description = "OpenTelemetry Protobuf encoding" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "opentelemetry_exporter_otlp_proto_common-1.21.0-py3-none-any.whl", hash = "sha256:97b1022b38270ec65d11fbfa348e0cd49d12006485c2321ea3b1b7037d42b6ec"}, - {file = "opentelemetry_exporter_otlp_proto_common-1.21.0.tar.gz", hash = "sha256:61db274d8a68d636fb2ec2a0f281922949361cdd8236e25ff5539edf942b3226"}, + {file = "opentelemetry_exporter_otlp_proto_common-1.27.0-py3-none-any.whl", hash = "sha256:675db7fffcb60946f3a5c43e17d1168a3307a94a930ecf8d2ea1f286f3d4f79a"}, + {file = "opentelemetry_exporter_otlp_proto_common-1.27.0.tar.gz", hash = "sha256:159d27cf49f359e3798c4c3eb8da6ef4020e292571bd8c5604a2a573231dd5c8"}, ] [package.dependencies] -backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""} -opentelemetry-proto = "1.21.0" +opentelemetry-proto = "1.27.0" [[package]] name = "opentelemetry-exporter-otlp-proto-grpc" -version = "1.21.0" +version = "1.27.0" description = "OpenTelemetry Collector Protobuf over gRPC Exporter" optional = true -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "opentelemetry_exporter_otlp_proto_grpc-1.21.0-py3-none-any.whl", hash = "sha256:ab37c63d6cb58d6506f76d71d07018eb1f561d83e642a8f5aa53dddf306087a4"}, - {file = "opentelemetry_exporter_otlp_proto_grpc-1.21.0.tar.gz", hash = "sha256:a497c5611245a2d17d9aa1e1cbb7ab567843d53231dcc844a62cea9f0924ffa7"}, + {file = "opentelemetry_exporter_otlp_proto_grpc-1.27.0-py3-none-any.whl", hash = "sha256:56b5bbd5d61aab05e300d9d62a6b3c134827bbd28d0b12f2649c2da368006c9e"}, + {file = "opentelemetry_exporter_otlp_proto_grpc-1.27.0.tar.gz", hash = "sha256:af6f72f76bcf425dfb5ad11c1a6d6eca2863b91e63575f89bb7b4b55099d968f"}, ] [package.dependencies] -backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""} deprecated = ">=1.2.6" googleapis-common-protos = ">=1.52,<2.0" grpcio = ">=1.0.0,<2.0.0" opentelemetry-api = ">=1.15,<2.0" -opentelemetry-exporter-otlp-proto-common = "1.21.0" -opentelemetry-proto = "1.21.0" -opentelemetry-sdk = ">=1.21.0,<1.22.0" - -[package.extras] -test = ["pytest-grpc"] +opentelemetry-exporter-otlp-proto-common = "1.27.0" +opentelemetry-proto = "1.27.0" +opentelemetry-sdk = ">=1.27.0,<1.28.0" [[package]] name = "opentelemetry-exporter-otlp-proto-http" -version = "1.21.0" +version = "1.27.0" description = "OpenTelemetry Collector Protobuf over HTTP Exporter" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "opentelemetry_exporter_otlp_proto_http-1.21.0-py3-none-any.whl", hash = "sha256:56837773de6fb2714c01fc4895caebe876f6397bbc4d16afddf89e1299a55ee2"}, - {file = "opentelemetry_exporter_otlp_proto_http-1.21.0.tar.gz", hash = "sha256:19d60afa4ae8597f7ef61ad75c8b6c6b7ef8cb73a33fb4aed4dbc86d5c8d3301"}, + {file = "opentelemetry_exporter_otlp_proto_http-1.27.0-py3-none-any.whl", hash = "sha256:688027575c9da42e179a69fe17e2d1eba9b14d81de8d13553a21d3114f3b4d75"}, + {file = "opentelemetry_exporter_otlp_proto_http-1.27.0.tar.gz", hash = "sha256:2103479092d8eb18f61f3fbff084f67cc7f2d4a7d37e75304b8b56c1d09ebef5"}, ] [package.dependencies] -backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""} deprecated = ">=1.2.6" googleapis-common-protos = ">=1.52,<2.0" opentelemetry-api = ">=1.15,<2.0" -opentelemetry-exporter-otlp-proto-common = "1.21.0" -opentelemetry-proto = "1.21.0" -opentelemetry-sdk = ">=1.21.0,<1.22.0" +opentelemetry-exporter-otlp-proto-common = "1.27.0" +opentelemetry-proto = "1.27.0" +opentelemetry-sdk = ">=1.27.0,<1.28.0" requests = ">=2.7,<3.0" +[[package]] +name = "opentelemetry-instrumentation" +version = "0.48b0" +description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_instrumentation-0.48b0-py3-none-any.whl", hash = "sha256:a69750dc4ba6a5c3eb67986a337185a25b739966d80479befe37b546fc870b44"}, + {file = "opentelemetry_instrumentation-0.48b0.tar.gz", hash = "sha256:94929685d906380743a71c3970f76b5f07476eea1834abd5dd9d17abfe23cc35"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.4,<2.0" +setuptools = ">=16.0" +wrapt = ">=1.0.0,<2.0.0" + +[[package]] +name = "opentelemetry-instrumentation-flask" +version = "0.48b0" +description = "Flask instrumentation for OpenTelemetry" +optional = true +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_instrumentation_flask-0.48b0-py3-none-any.whl", hash = "sha256:26b045420b9d76e85493b1c23fcf27517972423480dc6cf78fd6924248ba5808"}, + {file = "opentelemetry_instrumentation_flask-0.48b0.tar.gz", hash = "sha256:e03a34428071aebf4864ea6c6a564acef64f88c13eb3818e64ea90da61266c3d"}, +] + +[package.dependencies] +importlib-metadata = ">=4.0" +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-instrumentation = "0.48b0" +opentelemetry-instrumentation-wsgi = "0.48b0" +opentelemetry-semantic-conventions = "0.48b0" +opentelemetry-util-http = "0.48b0" +packaging = ">=21.0" + +[package.extras] +instruments = ["flask (>=1.0)"] + +[[package]] +name = "opentelemetry-instrumentation-httpx" +version = "0.48b0" +description = "OpenTelemetry HTTPX Instrumentation" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_instrumentation_httpx-0.48b0-py3-none-any.whl", hash = "sha256:d94f9d612c82d09fe22944d1904a30a464c19bea2ba76be656c99a28ad8be8e5"}, + {file = "opentelemetry_instrumentation_httpx-0.48b0.tar.gz", hash = "sha256:ee977479e10398931921fb995ac27ccdeea2e14e392cb27ef012fc549089b60a"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-instrumentation = "0.48b0" +opentelemetry-semantic-conventions = "0.48b0" +opentelemetry-util-http = "0.48b0" + +[package.extras] +instruments = ["httpx (>=0.18.0)"] + +[[package]] +name = "opentelemetry-instrumentation-requests" +version = "0.48b0" +description = "OpenTelemetry requests instrumentation" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_instrumentation_requests-0.48b0-py3-none-any.whl", hash = "sha256:d4f01852121d0bd4c22f14f429654a735611d4f7bf3cf93f244bdf1489b2233d"}, + {file = "opentelemetry_instrumentation_requests-0.48b0.tar.gz", hash = "sha256:67ab9bd877a0352ee0db4616c8b4ae59736ddd700c598ed907482d44f4c9a2b3"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-instrumentation = "0.48b0" +opentelemetry-semantic-conventions = "0.48b0" +opentelemetry-util-http = "0.48b0" + [package.extras] -test = ["responses (==0.22.0)"] +instruments = ["requests (>=2.0,<3.0)"] + +[[package]] +name = "opentelemetry-instrumentation-threading" +version = "0.48b0" +description = "Thread context propagation support for OpenTelemetry" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_instrumentation_threading-0.48b0-py3-none-any.whl", hash = "sha256:e81cb3a5342bbbc3f40b4c3f5180629905d504e2f364dc436ecb1123491f4080"}, + {file = "opentelemetry_instrumentation_threading-0.48b0.tar.gz", hash = "sha256:daef8a6fd06aa8b35594582d96ffb30954c4a9ae1ffdace7b00d0904fd650d2e"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-instrumentation = "0.48b0" +wrapt = ">=1.0.0,<2.0.0" + +[[package]] +name = "opentelemetry-instrumentation-urllib" +version = "0.48b0" +description = "OpenTelemetry urllib instrumentation" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_instrumentation_urllib-0.48b0-py3-none-any.whl", hash = "sha256:8115399fc786f5a46f30b158ab32a9cc77a248d421dcb0d411da657250388915"}, + {file = "opentelemetry_instrumentation_urllib-0.48b0.tar.gz", hash = "sha256:a9db839b4248efc9b01628dc8aa886c1269a81cec84bc375d344239037823d48"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-instrumentation = "0.48b0" +opentelemetry-semantic-conventions = "0.48b0" +opentelemetry-util-http = "0.48b0" + +[[package]] +name = "opentelemetry-instrumentation-wsgi" +version = "0.48b0" +description = "WSGI Middleware for OpenTelemetry" +optional = true +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_instrumentation_wsgi-0.48b0-py3-none-any.whl", hash = "sha256:c6051124d741972090fe94b2fa302555e1e2a22e9cdda32dd39ed49a5b34e0c6"}, + {file = "opentelemetry_instrumentation_wsgi-0.48b0.tar.gz", hash = "sha256:1a1e752367b0df4397e0b835839225ef5c2c3c053743a261551af13434fc4d51"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-instrumentation = "0.48b0" +opentelemetry-semantic-conventions = "0.48b0" +opentelemetry-util-http = "0.48b0" [[package]] name = "opentelemetry-proto" -version = "1.21.0" +version = "1.27.0" description = "OpenTelemetry Python Proto" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "opentelemetry_proto-1.21.0-py3-none-any.whl", hash = "sha256:32fc4248e83eebd80994e13963e683f25f3b443226336bb12b5b6d53638f50ba"}, - {file = "opentelemetry_proto-1.21.0.tar.gz", hash = "sha256:7d5172c29ed1b525b5ecf4ebe758c7138a9224441b3cfe683d0a237c33b1941f"}, + {file = "opentelemetry_proto-1.27.0-py3-none-any.whl", hash = "sha256:b133873de5581a50063e1e4b29cdcf0c5e253a8c2d8dc1229add20a4c3830ace"}, + {file = "opentelemetry_proto-1.27.0.tar.gz", hash = "sha256:33c9345d91dafd8a74fc3d7576c5a38f18b7fdf8d02983ac67485386132aedd6"}, ] [package.dependencies] @@ -2716,44 +2681,46 @@ protobuf = ">=3.19,<5.0" [[package]] name = "opentelemetry-sdk" -version = "1.21.0" +version = "1.27.0" description = "OpenTelemetry Python SDK" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "opentelemetry_sdk-1.21.0-py3-none-any.whl", hash = "sha256:9fe633243a8c655fedace3a0b89ccdfc654c0290ea2d8e839bd5db3131186f73"}, - {file = "opentelemetry_sdk-1.21.0.tar.gz", hash = "sha256:3ec8cd3020328d6bc5c9991ccaf9ae820ccb6395a5648d9a95d3ec88275b8879"}, + {file = "opentelemetry_sdk-1.27.0-py3-none-any.whl", hash = "sha256:365f5e32f920faf0fd9e14fdfd92c086e317eaa5f860edba9cdc17a380d9197d"}, + {file = "opentelemetry_sdk-1.27.0.tar.gz", hash = "sha256:d525017dea0ccce9ba4e0245100ec46ecdc043f2d7b8315d56b19aff0904fa6f"}, ] [package.dependencies] -opentelemetry-api = "1.21.0" -opentelemetry-semantic-conventions = "0.42b0" +opentelemetry-api = "1.27.0" +opentelemetry-semantic-conventions = "0.48b0" typing-extensions = ">=3.7.4" [[package]] name = "opentelemetry-semantic-conventions" -version = "0.42b0" +version = "0.48b0" description = "OpenTelemetry Semantic Conventions" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "opentelemetry_semantic_conventions-0.42b0-py3-none-any.whl", hash = "sha256:5cd719cbfec448af658860796c5d0fcea2fdf0945a2bed2363f42cb1ee39f526"}, - {file = "opentelemetry_semantic_conventions-0.42b0.tar.gz", hash = "sha256:44ae67a0a3252a05072877857e5cc1242c98d4cf12870159f1a94bec800d38ec"}, + {file = "opentelemetry_semantic_conventions-0.48b0-py3-none-any.whl", hash = "sha256:a0de9f45c413a8669788a38569c7e0a11ce6ce97861a628cca785deecdc32a1f"}, + {file = "opentelemetry_semantic_conventions-0.48b0.tar.gz", hash = "sha256:12d74983783b6878162208be57c9effcb89dc88691c64992d70bb89dc00daa1a"}, ] +[package.dependencies] +deprecated = ">=1.2.6" +opentelemetry-api = "1.27.0" + [[package]] -name = "opentracing" -version = "2.4.0" -description = "OpenTracing API for Python. See documentation at http://opentracing.io" -optional = true -python-versions = "*" +name = "opentelemetry-util-http" +version = "0.48b0" +description = "Web util for OpenTelemetry" +optional = false +python-versions = ">=3.8" files = [ - {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, + {file = "opentelemetry_util_http-0.48b0-py3-none-any.whl", hash = "sha256:76f598af93aab50328d2a69c786beaedc8b6a7770f7a818cc307eb353debfffb"}, + {file = "opentelemetry_util_http-0.48b0.tar.gz", hash = "sha256:60312015153580cc20f322e5cdc3d3ecad80a71743235bdb77716e742814623c"}, ] -[package.extras] -tests = ["Sphinx", "doubles", "flake8", "flake8-quotes", "gevent", "mock", "pytest", "pytest-cov", "pytest-mock", "six (>=1.10.0,<2.0)", "sphinx_rtd_theme", "tornado"] - [[package]] name = "overrides" version = "7.7.0" @@ -2797,40 +2764,53 @@ doc = ["mkdocs-material"] [[package]] name = "pandas" -version = "2.2.2" +version = "2.2.3" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.9" files = [ - {file = "pandas-2.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce"}, - {file = "pandas-2.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c7adfc142dac335d8c1e0dcbd37eb8617eac386596eb9e1a1b77791cf2498238"}, - {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4abfe0be0d7221be4f12552995e58723c7422c80a659da13ca382697de830c08"}, - {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8635c16bf3d99040fdf3ca3db669a7250ddf49c55dc4aa8fe0ae0fa8d6dcc1f0"}, - {file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:40ae1dffb3967a52203105a077415a86044a2bea011b5f321c6aa64b379a3f51"}, - {file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8e5a0b00e1e56a842f922e7fae8ae4077aee4af0acb5ae3622bd4b4c30aedf99"}, - {file = "pandas-2.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:ddf818e4e6c7c6f4f7c8a12709696d193976b591cc7dc50588d3d1a6b5dc8772"}, - {file = "pandas-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:696039430f7a562b74fa45f540aca068ea85fa34c244d0deee539cb6d70aa288"}, - {file = "pandas-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8e90497254aacacbc4ea6ae5e7a8cd75629d6ad2b30025a4a8b09aa4faf55151"}, - {file = "pandas-2.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58b84b91b0b9f4bafac2a0ac55002280c094dfc6402402332c0913a59654ab2b"}, - {file = "pandas-2.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2123dc9ad6a814bcdea0f099885276b31b24f7edf40f6cdbc0912672e22eee"}, - {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2925720037f06e89af896c70bca73459d7e6a4be96f9de79e2d440bd499fe0db"}, - {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1"}, - {file = "pandas-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24"}, - {file = "pandas-2.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef"}, - {file = "pandas-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce"}, - {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad"}, - {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad"}, - {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76"}, - {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32"}, - {file = "pandas-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23"}, - {file = "pandas-2.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0ca6377b8fca51815f382bd0b697a0814c8bda55115678cbc94c30aacbb6eff2"}, - {file = "pandas-2.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9057e6aa78a584bc93a13f0a9bf7e753a5e9770a30b4d758b8d5f2a62a9433cd"}, - {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:001910ad31abc7bf06f49dcc903755d2f7f3a9186c0c040b827e522e9cef0863"}, - {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66b479b0bd07204e37583c191535505410daa8df638fd8e75ae1b383851fe921"}, - {file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a77e9d1c386196879aa5eb712e77461aaee433e54c68cf253053a73b7e49c33a"}, - {file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:92fd6b027924a7e178ac202cfbe25e53368db90d56872d20ffae94b96c7acc57"}, - {file = "pandas-2.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:640cef9aa381b60e296db324337a554aeeb883ead99dc8f6c18e81a93942f5f4"}, - {file = "pandas-2.2.2.tar.gz", hash = "sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54"}, + {file = "pandas-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1948ddde24197a0f7add2bdc4ca83bf2b1ef84a1bc8ccffd95eda17fd836ecb5"}, + {file = "pandas-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:381175499d3802cde0eabbaf6324cce0c4f5d52ca6f8c377c29ad442f50f6348"}, + {file = "pandas-2.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d9c45366def9a3dd85a6454c0e7908f2b3b8e9c138f5dc38fed7ce720d8453ed"}, + {file = "pandas-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86976a1c5b25ae3f8ccae3a5306e443569ee3c3faf444dfd0f41cda24667ad57"}, + {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b8661b0238a69d7aafe156b7fa86c44b881387509653fdf857bebc5e4008ad42"}, + {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37e0aced3e8f539eccf2e099f65cdb9c8aa85109b0be6e93e2baff94264bdc6f"}, + {file = "pandas-2.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:56534ce0746a58afaf7942ba4863e0ef81c9c50d3f0ae93e9497d6a41a057645"}, + {file = "pandas-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039"}, + {file = "pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd"}, + {file = "pandas-2.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd8d0c3be0515c12fed0bdbae072551c8b54b7192c7b1fda0ba56059a0179698"}, + {file = "pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc"}, + {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:63cc132e40a2e084cf01adf0775b15ac515ba905d7dcca47e9a251819c575ef3"}, + {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32"}, + {file = "pandas-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5"}, + {file = "pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9"}, + {file = "pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4"}, + {file = "pandas-2.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3"}, + {file = "pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319"}, + {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8"}, + {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a"}, + {file = "pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13"}, + {file = "pandas-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f00d1345d84d8c86a63e476bb4955e46458b304b9575dcf71102b5c705320015"}, + {file = "pandas-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3508d914817e153ad359d7e069d752cdd736a247c322d932eb89e6bc84217f28"}, + {file = "pandas-2.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22a9d949bfc9a502d320aa04e5d02feab689d61da4e7764b62c30b991c42c5f0"}, + {file = "pandas-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3a255b2c19987fbbe62a9dfd6cff7ff2aa9ccab3fc75218fd4b7530f01efa24"}, + {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:800250ecdadb6d9c78eae4990da62743b857b470883fa27f652db8bdde7f6659"}, + {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6374c452ff3ec675a8f46fd9ab25c4ad0ba590b71cf0656f8b6daa5202bca3fb"}, + {file = "pandas-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:61c5ad4043f791b61dd4752191d9f07f0ae412515d59ba8f005832a532f8736d"}, + {file = "pandas-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3b71f27954685ee685317063bf13c7709a7ba74fc996b84fc6821c59b0f06468"}, + {file = "pandas-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:38cf8125c40dae9d5acc10fa66af8ea6fdf760b2714ee482ca691fc66e6fcb18"}, + {file = "pandas-2.2.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba96630bc17c875161df3818780af30e43be9b166ce51c9a18c1feae342906c2"}, + {file = "pandas-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db71525a1538b30142094edb9adc10be3f3e176748cd7acc2240c2f2e5aa3a4"}, + {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15c0e1e02e93116177d29ff83e8b1619c93ddc9c49083f237d4312337a61165d"}, + {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ad5b65698ab28ed8d7f18790a0dc58005c7629f227be9ecc1072aa74c0c1d43a"}, + {file = "pandas-2.2.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc6b93f9b966093cb0fd62ff1a7e4c09e6d546ad7c1de191767baffc57628f39"}, + {file = "pandas-2.2.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5dbca4c1acd72e8eeef4753eeca07de9b1db4f398669d5994086f788a5d7cc30"}, + {file = "pandas-2.2.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8cd6d7cc958a3910f934ea8dbdf17b2364827bb4dafc38ce6eef6bb3d65ff09c"}, + {file = "pandas-2.2.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99df71520d25fade9db7c1076ac94eb994f4d2673ef2aa2e86ee039b6746d20c"}, + {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:31d0ced62d4ea3e231a9f228366919a5ea0b07440d9d4dac345376fd8e1477ea"}, + {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7eee9e7cea6adf3e3d24e304ac6b8300646e2a5d1cd3a3c2abed9101b0846761"}, + {file = "pandas-2.2.3-cp39-cp39-win_amd64.whl", hash = "sha256:4850ba03528b6dd51d6c5d273c46f183f39a9baf3f0143e566b89450965b105e"}, + {file = "pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667"}, ] [package.dependencies] @@ -2903,13 +2883,13 @@ files = [ [[package]] name = "pdoc" -version = "14.6.0" +version = "14.7.0" description = "API Documentation for Python Projects" optional = false python-versions = ">=3.8" files = [ - {file = "pdoc-14.6.0-py3-none-any.whl", hash = "sha256:36c42c546a317d8e3e8c0b39645f24161374de0c7066ccaae76628d721e49ba5"}, - {file = "pdoc-14.6.0.tar.gz", hash = "sha256:6e98a24c5e0ca5d188397969cf82581836eaef13f172fc3820047bfe15c61c9a"}, + {file = "pdoc-14.7.0-py3-none-any.whl", hash = "sha256:72377a907efc6b2c5b3c56b717ef34f11d93621dced3b663f3aede0b844c0ad2"}, + {file = "pdoc-14.7.0.tar.gz", hash = "sha256:2d28af9c0acc39180744ad0543e4bbc3223ecba0d1302db315ec521c51f71f93"}, ] [package.dependencies] @@ -2936,19 +2916,19 @@ ptyprocess = ">=0.5" [[package]] name = "platformdirs" -version = "4.2.2" +version = "4.3.6" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" files = [ - {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, - {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, + {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"}, + {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"}, ] [package.extras] -docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] -type = ["mypy (>=1.8)"] +docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"] +type = ["mypy (>=1.11.2)"] [[package]] name = "pluggy" @@ -2985,13 +2965,13 @@ virtualenv = ">=20.10.0" [[package]] name = "prometheus-client" -version = "0.20.0" +version = "0.21.0" description = "Python client for the Prometheus monitoring system." optional = false python-versions = ">=3.8" files = [ - {file = "prometheus_client-0.20.0-py3-none-any.whl", hash = "sha256:cde524a85bce83ca359cc837f28b8c0db5cac7aa653a588fd7e84ba061c329e7"}, - {file = "prometheus_client-0.20.0.tar.gz", hash = "sha256:287629d00b147a32dcb2be0b9df905da599b2d82f80377083ec8463309a4bb89"}, + {file = "prometheus_client-0.21.0-py3-none-any.whl", hash = "sha256:4fa6b4dd0ac16d58bb587c04b1caae65b8c5043e85f778f42f5f632f6af2e166"}, + {file = "prometheus_client-0.21.0.tar.gz", hash = "sha256:96c83c606b71ff2b0a433c98889d275f51ffec6c5e267de37c7a2b5c9aa9233e"}, ] [package.extras] @@ -3030,22 +3010,22 @@ testing = ["google-api-core (>=1.31.5)"] [[package]] name = "protobuf" -version = "4.25.4" +version = "4.25.5" description = "" optional = false python-versions = ">=3.8" files = [ - {file = "protobuf-4.25.4-cp310-abi3-win32.whl", hash = "sha256:db9fd45183e1a67722cafa5c1da3e85c6492a5383f127c86c4c4aa4845867dc4"}, - {file = "protobuf-4.25.4-cp310-abi3-win_amd64.whl", hash = "sha256:ba3d8504116a921af46499471c63a85260c1a5fc23333154a427a310e015d26d"}, - {file = "protobuf-4.25.4-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:eecd41bfc0e4b1bd3fa7909ed93dd14dd5567b98c941d6c1ad08fdcab3d6884b"}, - {file = "protobuf-4.25.4-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:4c8a70fdcb995dcf6c8966cfa3a29101916f7225e9afe3ced4395359955d3835"}, - {file = "protobuf-4.25.4-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:3319e073562e2515c6ddc643eb92ce20809f5d8f10fead3332f71c63be6a7040"}, - {file = "protobuf-4.25.4-cp38-cp38-win32.whl", hash = "sha256:7e372cbbda66a63ebca18f8ffaa6948455dfecc4e9c1029312f6c2edcd86c4e1"}, - {file = "protobuf-4.25.4-cp38-cp38-win_amd64.whl", hash = "sha256:051e97ce9fa6067a4546e75cb14f90cf0232dcb3e3d508c448b8d0e4265b61c1"}, - {file = "protobuf-4.25.4-cp39-cp39-win32.whl", hash = "sha256:90bf6fd378494eb698805bbbe7afe6c5d12c8e17fca817a646cd6a1818c696ca"}, - {file = "protobuf-4.25.4-cp39-cp39-win_amd64.whl", hash = "sha256:ac79a48d6b99dfed2729ccccee547b34a1d3d63289c71cef056653a846a2240f"}, - {file = "protobuf-4.25.4-py3-none-any.whl", hash = "sha256:bfbebc1c8e4793cfd58589acfb8a1026be0003e852b9da7db5a4285bde996978"}, - {file = "protobuf-4.25.4.tar.gz", hash = "sha256:0dc4a62cc4052a036ee2204d26fe4d835c62827c855c8a03f29fe6da146b380d"}, + {file = "protobuf-4.25.5-cp310-abi3-win32.whl", hash = "sha256:5e61fd921603f58d2f5acb2806a929b4675f8874ff5f330b7d6f7e2e784bbcd8"}, + {file = "protobuf-4.25.5-cp310-abi3-win_amd64.whl", hash = "sha256:4be0571adcbe712b282a330c6e89eae24281344429ae95c6d85e79e84780f5ea"}, + {file = "protobuf-4.25.5-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:b2fde3d805354df675ea4c7c6338c1aecd254dfc9925e88c6d31a2bcb97eb173"}, + {file = "protobuf-4.25.5-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:919ad92d9b0310070f8356c24b855c98df2b8bd207ebc1c0c6fcc9ab1e007f3d"}, + {file = "protobuf-4.25.5-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fe14e16c22be926d3abfcb500e60cab068baf10b542b8c858fa27e098123e331"}, + {file = "protobuf-4.25.5-cp38-cp38-win32.whl", hash = "sha256:98d8d8aa50de6a2747efd9cceba361c9034050ecce3e09136f90de37ddba66e1"}, + {file = "protobuf-4.25.5-cp38-cp38-win_amd64.whl", hash = "sha256:b0234dd5a03049e4ddd94b93400b67803c823cfc405689688f59b34e0742381a"}, + {file = "protobuf-4.25.5-cp39-cp39-win32.whl", hash = "sha256:abe32aad8561aa7cc94fc7ba4fdef646e576983edb94a73381b03c53728a626f"}, + {file = "protobuf-4.25.5-cp39-cp39-win_amd64.whl", hash = "sha256:7a183f592dc80aa7c8da7ad9e55091c4ffc9497b3054452d629bb85fa27c2a45"}, + {file = "protobuf-4.25.5-py3-none-any.whl", hash = "sha256:0aebecb809cae990f8129ada5ca273d9d670b76d9bfc9b1809f0a9c02b7dbf41"}, + {file = "protobuf-4.25.5.tar.gz", hash = "sha256:7f8249476b4a9473645db7f8ab42b02fe1488cbe5fb72fddd445e0665afd8584"}, ] [[package]] @@ -3101,37 +3081,26 @@ files = [ [package.extras] tests = ["pytest"] -[[package]] -name = "py" -version = "1.11.0" -description = "library with cross-python path, ini-parsing, io, code, log facilities" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, - {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, -] - [[package]] name = "pyasn1" -version = "0.6.0" +version = "0.6.1" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = false python-versions = ">=3.8" files = [ - {file = "pyasn1-0.6.0-py2.py3-none-any.whl", hash = "sha256:cca4bb0f2df5504f02f6f8a775b6e416ff9b0b3b16f7ee80b5a3153d9b804473"}, - {file = "pyasn1-0.6.0.tar.gz", hash = "sha256:3a35ab2c4b5ef98e17dfdec8ab074046fbda76e281c5a706ccd82328cfc8f64c"}, + {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"}, + {file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"}, ] [[package]] name = "pyasn1-modules" -version = "0.4.0" +version = "0.4.1" description = "A collection of ASN.1-based protocols modules" optional = false python-versions = ">=3.8" files = [ - {file = "pyasn1_modules-0.4.0-py3-none-any.whl", hash = "sha256:be04f15b66c206eed667e0bb5ab27e2b1855ea54a842e5037738099e8ca4ae0b"}, - {file = "pyasn1_modules-0.4.0.tar.gz", hash = "sha256:831dbcea1b177b28c9baddf4c6d1013c24c3accd14a1873fffaa6a2e905f17b6"}, + {file = "pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd"}, + {file = "pyasn1_modules-0.4.1.tar.gz", hash = "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c"}, ] [package.dependencies] @@ -3338,13 +3307,13 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" -version = "8.3.2" +version = "8.3.3" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.3.2-py3-none-any.whl", hash = "sha256:4ba08f9ae7dcf84ded419494d229b48d0903ea6407b030eaec46df5e6a73bba5"}, - {file = "pytest-8.3.2.tar.gz", hash = "sha256:c132345d12ce551242c87269de812483f5bcc87cdbb4722e48487ba194f9fdce"}, + {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"}, + {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"}, ] [package.dependencies] @@ -3487,13 +3456,13 @@ files = [ [[package]] name = "pytz" -version = "2024.1" +version = "2024.2" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" files = [ - {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, - {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, + {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, + {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, ] [[package]] @@ -3755,90 +3724,105 @@ rpds-py = ">=0.7.0" [[package]] name = "regex" -version = "2024.7.24" +version = "2024.9.11" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.8" files = [ - {file = "regex-2024.7.24-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b0d3f567fafa0633aee87f08b9276c7062da9616931382993c03808bb68ce"}, - {file = "regex-2024.7.24-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3426de3b91d1bc73249042742f45c2148803c111d1175b283270177fdf669024"}, - {file = "regex-2024.7.24-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f273674b445bcb6e4409bf8d1be67bc4b58e8b46fd0d560055d515b8830063cd"}, - {file = "regex-2024.7.24-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23acc72f0f4e1a9e6e9843d6328177ae3074b4182167e34119ec7233dfeccf53"}, - {file = "regex-2024.7.24-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65fd3d2e228cae024c411c5ccdffae4c315271eee4a8b839291f84f796b34eca"}, - {file = "regex-2024.7.24-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c414cbda77dbf13c3bc88b073a1a9f375c7b0cb5e115e15d4b73ec3a2fbc6f59"}, - {file = "regex-2024.7.24-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf7a89eef64b5455835f5ed30254ec19bf41f7541cd94f266ab7cbd463f00c41"}, - {file = "regex-2024.7.24-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19c65b00d42804e3fbea9708f0937d157e53429a39b7c61253ff15670ff62cb5"}, - {file = "regex-2024.7.24-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7a5486ca56c8869070a966321d5ab416ff0f83f30e0e2da1ab48815c8d165d46"}, - {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6f51f9556785e5a203713f5efd9c085b4a45aecd2a42573e2b5041881b588d1f"}, - {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:a4997716674d36a82eab3e86f8fa77080a5d8d96a389a61ea1d0e3a94a582cf7"}, - {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:c0abb5e4e8ce71a61d9446040c1e86d4e6d23f9097275c5bd49ed978755ff0fe"}, - {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:18300a1d78cf1290fa583cd8b7cde26ecb73e9f5916690cf9d42de569c89b1ce"}, - {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:416c0e4f56308f34cdb18c3f59849479dde5b19febdcd6e6fa4d04b6c31c9faa"}, - {file = "regex-2024.7.24-cp310-cp310-win32.whl", hash = "sha256:fb168b5924bef397b5ba13aabd8cf5df7d3d93f10218d7b925e360d436863f66"}, - {file = "regex-2024.7.24-cp310-cp310-win_amd64.whl", hash = "sha256:6b9fc7e9cc983e75e2518496ba1afc524227c163e43d706688a6bb9eca41617e"}, - {file = "regex-2024.7.24-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:382281306e3adaaa7b8b9ebbb3ffb43358a7bbf585fa93821300a418bb975281"}, - {file = "regex-2024.7.24-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4fdd1384619f406ad9037fe6b6eaa3de2749e2e12084abc80169e8e075377d3b"}, - {file = "regex-2024.7.24-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3d974d24edb231446f708c455fd08f94c41c1ff4f04bcf06e5f36df5ef50b95a"}, - {file = "regex-2024.7.24-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a2ec4419a3fe6cf8a4795752596dfe0adb4aea40d3683a132bae9c30b81e8d73"}, - {file = "regex-2024.7.24-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb563dd3aea54c797adf513eeec819c4213d7dbfc311874eb4fd28d10f2ff0f2"}, - {file = "regex-2024.7.24-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:45104baae8b9f67569f0f1dca5e1f1ed77a54ae1cd8b0b07aba89272710db61e"}, - {file = "regex-2024.7.24-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:994448ee01864501912abf2bad9203bffc34158e80fe8bfb5b031f4f8e16da51"}, - {file = "regex-2024.7.24-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3fac296f99283ac232d8125be932c5cd7644084a30748fda013028c815ba3364"}, - {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7e37e809b9303ec3a179085415cb5f418ecf65ec98cdfe34f6a078b46ef823ee"}, - {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:01b689e887f612610c869421241e075c02f2e3d1ae93a037cb14f88ab6a8934c"}, - {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f6442f0f0ff81775eaa5b05af8a0ffa1dda36e9cf6ec1e0d3d245e8564b684ce"}, - {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:871e3ab2838fbcb4e0865a6e01233975df3a15e6fce93b6f99d75cacbd9862d1"}, - {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c918b7a1e26b4ab40409820ddccc5d49871a82329640f5005f73572d5eaa9b5e"}, - {file = "regex-2024.7.24-cp311-cp311-win32.whl", hash = "sha256:2dfbb8baf8ba2c2b9aa2807f44ed272f0913eeeba002478c4577b8d29cde215c"}, - {file = "regex-2024.7.24-cp311-cp311-win_amd64.whl", hash = "sha256:538d30cd96ed7d1416d3956f94d54e426a8daf7c14527f6e0d6d425fcb4cca52"}, - {file = "regex-2024.7.24-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:fe4ebef608553aff8deb845c7f4f1d0740ff76fa672c011cc0bacb2a00fbde86"}, - {file = "regex-2024.7.24-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:74007a5b25b7a678459f06559504f1eec2f0f17bca218c9d56f6a0a12bfffdad"}, - {file = "regex-2024.7.24-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7df9ea48641da022c2a3c9c641650cd09f0cd15e8908bf931ad538f5ca7919c9"}, - {file = "regex-2024.7.24-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a1141a1dcc32904c47f6846b040275c6e5de0bf73f17d7a409035d55b76f289"}, - {file = "regex-2024.7.24-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80c811cfcb5c331237d9bad3bea2c391114588cf4131707e84d9493064d267f9"}, - {file = "regex-2024.7.24-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7214477bf9bd195894cf24005b1e7b496f46833337b5dedb7b2a6e33f66d962c"}, - {file = "regex-2024.7.24-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d55588cba7553f0b6ec33130bc3e114b355570b45785cebdc9daed8c637dd440"}, - {file = "regex-2024.7.24-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:558a57cfc32adcf19d3f791f62b5ff564922942e389e3cfdb538a23d65a6b610"}, - {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a512eed9dfd4117110b1881ba9a59b31433caed0c4101b361f768e7bcbaf93c5"}, - {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:86b17ba823ea76256b1885652e3a141a99a5c4422f4a869189db328321b73799"}, - {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5eefee9bfe23f6df09ffb6dfb23809f4d74a78acef004aa904dc7c88b9944b05"}, - {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:731fcd76bbdbf225e2eb85b7c38da9633ad3073822f5ab32379381e8c3c12e94"}, - {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eaef80eac3b4cfbdd6de53c6e108b4c534c21ae055d1dbea2de6b3b8ff3def38"}, - {file = "regex-2024.7.24-cp312-cp312-win32.whl", hash = "sha256:185e029368d6f89f36e526764cf12bf8d6f0e3a2a7737da625a76f594bdfcbfc"}, - {file = "regex-2024.7.24-cp312-cp312-win_amd64.whl", hash = "sha256:2f1baff13cc2521bea83ab2528e7a80cbe0ebb2c6f0bfad15be7da3aed443908"}, - {file = "regex-2024.7.24-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:66b4c0731a5c81921e938dcf1a88e978264e26e6ac4ec96a4d21ae0354581ae0"}, - {file = "regex-2024.7.24-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:88ecc3afd7e776967fa16c80f974cb79399ee8dc6c96423321d6f7d4b881c92b"}, - {file = "regex-2024.7.24-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:64bd50cf16bcc54b274e20235bf8edbb64184a30e1e53873ff8d444e7ac656b2"}, - {file = "regex-2024.7.24-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb462f0e346fcf41a901a126b50f8781e9a474d3927930f3490f38a6e73b6950"}, - {file = "regex-2024.7.24-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a82465ebbc9b1c5c50738536fdfa7cab639a261a99b469c9d4c7dcbb2b3f1e57"}, - {file = "regex-2024.7.24-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:68a8f8c046c6466ac61a36b65bb2395c74451df2ffb8458492ef49900efed293"}, - {file = "regex-2024.7.24-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dac8e84fff5d27420f3c1e879ce9929108e873667ec87e0c8eeb413a5311adfe"}, - {file = "regex-2024.7.24-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba2537ef2163db9e6ccdbeb6f6424282ae4dea43177402152c67ef869cf3978b"}, - {file = "regex-2024.7.24-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:43affe33137fcd679bdae93fb25924979517e011f9dea99163f80b82eadc7e53"}, - {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:c9bb87fdf2ab2370f21e4d5636e5317775e5d51ff32ebff2cf389f71b9b13750"}, - {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:945352286a541406f99b2655c973852da7911b3f4264e010218bbc1cc73168f2"}, - {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:8bc593dcce679206b60a538c302d03c29b18e3d862609317cb560e18b66d10cf"}, - {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:3f3b6ca8eae6d6c75a6cff525c8530c60e909a71a15e1b731723233331de4169"}, - {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c51edc3541e11fbe83f0c4d9412ef6c79f664a3745fab261457e84465ec9d5a8"}, - {file = "regex-2024.7.24-cp38-cp38-win32.whl", hash = "sha256:d0a07763776188b4db4c9c7fb1b8c494049f84659bb387b71c73bbc07f189e96"}, - {file = "regex-2024.7.24-cp38-cp38-win_amd64.whl", hash = "sha256:8fd5afd101dcf86a270d254364e0e8dddedebe6bd1ab9d5f732f274fa00499a5"}, - {file = "regex-2024.7.24-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0ffe3f9d430cd37d8fa5632ff6fb36d5b24818c5c986893063b4e5bdb84cdf24"}, - {file = "regex-2024.7.24-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:25419b70ba00a16abc90ee5fce061228206173231f004437730b67ac77323f0d"}, - {file = "regex-2024.7.24-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:33e2614a7ce627f0cdf2ad104797d1f68342d967de3695678c0cb84f530709f8"}, - {file = "regex-2024.7.24-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d33a0021893ede5969876052796165bab6006559ab845fd7b515a30abdd990dc"}, - {file = "regex-2024.7.24-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:04ce29e2c5fedf296b1a1b0acc1724ba93a36fb14031f3abfb7abda2806c1535"}, - {file = "regex-2024.7.24-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b16582783f44fbca6fcf46f61347340c787d7530d88b4d590a397a47583f31dd"}, - {file = "regex-2024.7.24-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:836d3cc225b3e8a943d0b02633fb2f28a66e281290302a79df0e1eaa984ff7c1"}, - {file = "regex-2024.7.24-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:438d9f0f4bc64e8dea78274caa5af971ceff0f8771e1a2333620969936ba10be"}, - {file = "regex-2024.7.24-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:973335b1624859cb0e52f96062a28aa18f3a5fc77a96e4a3d6d76e29811a0e6e"}, - {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c5e69fd3eb0b409432b537fe3c6f44ac089c458ab6b78dcec14478422879ec5f"}, - {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:fbf8c2f00904eaf63ff37718eb13acf8e178cb940520e47b2f05027f5bb34ce3"}, - {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ae2757ace61bc4061b69af19e4689fa4416e1a04840f33b441034202b5cd02d4"}, - {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:44fc61b99035fd9b3b9453f1713234e5a7c92a04f3577252b45feefe1b327759"}, - {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:84c312cdf839e8b579f504afcd7b65f35d60b6285d892b19adea16355e8343c9"}, - {file = "regex-2024.7.24-cp39-cp39-win32.whl", hash = "sha256:ca5b2028c2f7af4e13fb9fc29b28d0ce767c38c7facdf64f6c2cd040413055f1"}, - {file = "regex-2024.7.24-cp39-cp39-win_amd64.whl", hash = "sha256:7c479f5ae937ec9985ecaf42e2e10631551d909f203e31308c12d703922742f9"}, - {file = "regex-2024.7.24.tar.gz", hash = "sha256:9cfd009eed1a46b27c14039ad5bbc5e71b6367c5b2e6d5f5da0ea91600817506"}, + {file = "regex-2024.9.11-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1494fa8725c285a81d01dc8c06b55287a1ee5e0e382d8413adc0a9197aac6408"}, + {file = "regex-2024.9.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0e12c481ad92d129c78f13a2a3662317e46ee7ef96c94fd332e1c29131875b7d"}, + {file = "regex-2024.9.11-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:16e13a7929791ac1216afde26f712802e3df7bf0360b32e4914dca3ab8baeea5"}, + {file = "regex-2024.9.11-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46989629904bad940bbec2106528140a218b4a36bb3042d8406980be1941429c"}, + {file = "regex-2024.9.11-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a906ed5e47a0ce5f04b2c981af1c9acf9e8696066900bf03b9d7879a6f679fc8"}, + {file = "regex-2024.9.11-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9a091b0550b3b0207784a7d6d0f1a00d1d1c8a11699c1a4d93db3fbefc3ad35"}, + {file = "regex-2024.9.11-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ddcd9a179c0a6fa8add279a4444015acddcd7f232a49071ae57fa6e278f1f71"}, + {file = "regex-2024.9.11-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6b41e1adc61fa347662b09398e31ad446afadff932a24807d3ceb955ed865cc8"}, + {file = "regex-2024.9.11-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ced479f601cd2f8ca1fd7b23925a7e0ad512a56d6e9476f79b8f381d9d37090a"}, + {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:635a1d96665f84b292e401c3d62775851aedc31d4f8784117b3c68c4fcd4118d"}, + {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:c0256beda696edcf7d97ef16b2a33a8e5a875affd6fa6567b54f7c577b30a137"}, + {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:3ce4f1185db3fbde8ed8aa223fc9620f276c58de8b0d4f8cc86fd1360829edb6"}, + {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:09d77559e80dcc9d24570da3745ab859a9cf91953062e4ab126ba9d5993688ca"}, + {file = "regex-2024.9.11-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7a22ccefd4db3f12b526eccb129390942fe874a3a9fdbdd24cf55773a1faab1a"}, + {file = "regex-2024.9.11-cp310-cp310-win32.whl", hash = "sha256:f745ec09bc1b0bd15cfc73df6fa4f726dcc26bb16c23a03f9e3367d357eeedd0"}, + {file = "regex-2024.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:01c2acb51f8a7d6494c8c5eafe3d8e06d76563d8a8a4643b37e9b2dd8a2ff623"}, + {file = "regex-2024.9.11-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2cce2449e5927a0bf084d346da6cd5eb016b2beca10d0013ab50e3c226ffc0df"}, + {file = "regex-2024.9.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b37fa423beefa44919e009745ccbf353d8c981516e807995b2bd11c2c77d268"}, + {file = "regex-2024.9.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:64ce2799bd75039b480cc0360907c4fb2f50022f030bf9e7a8705b636e408fad"}, + {file = "regex-2024.9.11-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a4cc92bb6db56ab0c1cbd17294e14f5e9224f0cc6521167ef388332604e92679"}, + {file = "regex-2024.9.11-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d05ac6fa06959c4172eccd99a222e1fbf17b5670c4d596cb1e5cde99600674c4"}, + {file = "regex-2024.9.11-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040562757795eeea356394a7fb13076ad4f99d3c62ab0f8bdfb21f99a1f85664"}, + {file = "regex-2024.9.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6113c008a7780792efc80f9dfe10ba0cd043cbf8dc9a76ef757850f51b4edc50"}, + {file = "regex-2024.9.11-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8e5fb5f77c8745a60105403a774fe2c1759b71d3e7b4ca237a5e67ad066c7199"}, + {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:54d9ff35d4515debf14bc27f1e3b38bfc453eff3220f5bce159642fa762fe5d4"}, + {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:df5cbb1fbc74a8305b6065d4ade43b993be03dbe0f8b30032cced0d7740994bd"}, + {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7fb89ee5d106e4a7a51bce305ac4efb981536301895f7bdcf93ec92ae0d91c7f"}, + {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:a738b937d512b30bf75995c0159c0ddf9eec0775c9d72ac0202076c72f24aa96"}, + {file = "regex-2024.9.11-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e28f9faeb14b6f23ac55bfbbfd3643f5c7c18ede093977f1df249f73fd22c7b1"}, + {file = "regex-2024.9.11-cp311-cp311-win32.whl", hash = "sha256:18e707ce6c92d7282dfce370cd205098384b8ee21544e7cb29b8aab955b66fa9"}, + {file = "regex-2024.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:313ea15e5ff2a8cbbad96ccef6be638393041b0a7863183c2d31e0c6116688cf"}, + {file = "regex-2024.9.11-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b0d0a6c64fcc4ef9c69bd5b3b3626cc3776520a1637d8abaa62b9edc147a58f7"}, + {file = "regex-2024.9.11-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:49b0e06786ea663f933f3710a51e9385ce0cba0ea56b67107fd841a55d56a231"}, + {file = "regex-2024.9.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5b513b6997a0b2f10e4fd3a1313568e373926e8c252bd76c960f96fd039cd28d"}, + {file = "regex-2024.9.11-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee439691d8c23e76f9802c42a95cfeebf9d47cf4ffd06f18489122dbb0a7ad64"}, + {file = "regex-2024.9.11-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a8f877c89719d759e52783f7fe6e1c67121076b87b40542966c02de5503ace42"}, + {file = "regex-2024.9.11-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:23b30c62d0f16827f2ae9f2bb87619bc4fba2044911e2e6c2eb1af0161cdb766"}, + {file = "regex-2024.9.11-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85ab7824093d8f10d44330fe1e6493f756f252d145323dd17ab6b48733ff6c0a"}, + {file = "regex-2024.9.11-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8dee5b4810a89447151999428fe096977346cf2f29f4d5e29609d2e19e0199c9"}, + {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:98eeee2f2e63edae2181c886d7911ce502e1292794f4c5ee71e60e23e8d26b5d"}, + {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:57fdd2e0b2694ce6fc2e5ccf189789c3e2962916fb38779d3e3521ff8fe7a822"}, + {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:d552c78411f60b1fdaafd117a1fca2f02e562e309223b9d44b7de8be451ec5e0"}, + {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a0b2b80321c2ed3fcf0385ec9e51a12253c50f146fddb2abbb10f033fe3d049a"}, + {file = "regex-2024.9.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:18406efb2f5a0e57e3a5881cd9354c1512d3bb4f5c45d96d110a66114d84d23a"}, + {file = "regex-2024.9.11-cp312-cp312-win32.whl", hash = "sha256:e464b467f1588e2c42d26814231edecbcfe77f5ac414d92cbf4e7b55b2c2a776"}, + {file = "regex-2024.9.11-cp312-cp312-win_amd64.whl", hash = "sha256:9e8719792ca63c6b8340380352c24dcb8cd7ec49dae36e963742a275dfae6009"}, + {file = "regex-2024.9.11-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c157bb447303070f256e084668b702073db99bbb61d44f85d811025fcf38f784"}, + {file = "regex-2024.9.11-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4db21ece84dfeefc5d8a3863f101995de646c6cb0536952c321a2650aa202c36"}, + {file = "regex-2024.9.11-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:220e92a30b426daf23bb67a7962900ed4613589bab80382be09b48896d211e92"}, + {file = "regex-2024.9.11-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb1ae19e64c14c7ec1995f40bd932448713d3c73509e82d8cd7744dc00e29e86"}, + {file = "regex-2024.9.11-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f47cd43a5bfa48f86925fe26fbdd0a488ff15b62468abb5d2a1e092a4fb10e85"}, + {file = "regex-2024.9.11-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9d4a76b96f398697fe01117093613166e6aa8195d63f1b4ec3f21ab637632963"}, + {file = "regex-2024.9.11-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ea51dcc0835eea2ea31d66456210a4e01a076d820e9039b04ae8d17ac11dee6"}, + {file = "regex-2024.9.11-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7aaa315101c6567a9a45d2839322c51c8d6e81f67683d529512f5bcfb99c802"}, + {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c57d08ad67aba97af57a7263c2d9006d5c404d721c5f7542f077f109ec2a4a29"}, + {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f8404bf61298bb6f8224bb9176c1424548ee1181130818fcd2cbffddc768bed8"}, + {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:dd4490a33eb909ef5078ab20f5f000087afa2a4daa27b4c072ccb3cb3050ad84"}, + {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:eee9130eaad130649fd73e5cd92f60e55708952260ede70da64de420cdcad554"}, + {file = "regex-2024.9.11-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6a2644a93da36c784e546de579ec1806bfd2763ef47babc1b03d765fe560c9f8"}, + {file = "regex-2024.9.11-cp313-cp313-win32.whl", hash = "sha256:e997fd30430c57138adc06bba4c7c2968fb13d101e57dd5bb9355bf8ce3fa7e8"}, + {file = "regex-2024.9.11-cp313-cp313-win_amd64.whl", hash = "sha256:042c55879cfeb21a8adacc84ea347721d3d83a159da6acdf1116859e2427c43f"}, + {file = "regex-2024.9.11-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:35f4a6f96aa6cb3f2f7247027b07b15a374f0d5b912c0001418d1d55024d5cb4"}, + {file = "regex-2024.9.11-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:55b96e7ce3a69a8449a66984c268062fbaa0d8ae437b285428e12797baefce7e"}, + {file = "regex-2024.9.11-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cb130fccd1a37ed894824b8c046321540263013da72745d755f2d35114b81a60"}, + {file = "regex-2024.9.11-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:323c1f04be6b2968944d730e5c2091c8c89767903ecaa135203eec4565ed2b2b"}, + {file = "regex-2024.9.11-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be1c8ed48c4c4065ecb19d882a0ce1afe0745dfad8ce48c49586b90a55f02366"}, + {file = "regex-2024.9.11-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b5b029322e6e7b94fff16cd120ab35a253236a5f99a79fb04fda7ae71ca20ae8"}, + {file = "regex-2024.9.11-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6fff13ef6b5f29221d6904aa816c34701462956aa72a77f1f151a8ec4f56aeb"}, + {file = "regex-2024.9.11-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:587d4af3979376652010e400accc30404e6c16b7df574048ab1f581af82065e4"}, + {file = "regex-2024.9.11-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:079400a8269544b955ffa9e31f186f01d96829110a3bf79dc338e9910f794fca"}, + {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f9268774428ec173654985ce55fc6caf4c6d11ade0f6f914d48ef4719eb05ebb"}, + {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:23f9985c8784e544d53fc2930fc1ac1a7319f5d5332d228437acc9f418f2f168"}, + {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:ae2941333154baff9838e88aa71c1d84f4438189ecc6021a12c7573728b5838e"}, + {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:e93f1c331ca8e86fe877a48ad64e77882c0c4da0097f2212873a69bbfea95d0c"}, + {file = "regex-2024.9.11-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:846bc79ee753acf93aef4184c040d709940c9d001029ceb7b7a52747b80ed2dd"}, + {file = "regex-2024.9.11-cp38-cp38-win32.whl", hash = "sha256:c94bb0a9f1db10a1d16c00880bdebd5f9faf267273b8f5bd1878126e0fbde771"}, + {file = "regex-2024.9.11-cp38-cp38-win_amd64.whl", hash = "sha256:2b08fce89fbd45664d3df6ad93e554b6c16933ffa9d55cb7e01182baaf971508"}, + {file = "regex-2024.9.11-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:07f45f287469039ffc2c53caf6803cd506eb5f5f637f1d4acb37a738f71dd066"}, + {file = "regex-2024.9.11-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4838e24ee015101d9f901988001038f7f0d90dc0c3b115541a1365fb439add62"}, + {file = "regex-2024.9.11-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6edd623bae6a737f10ce853ea076f56f507fd7726bee96a41ee3d68d347e4d16"}, + {file = "regex-2024.9.11-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c69ada171c2d0e97a4b5aa78fbb835e0ffbb6b13fc5da968c09811346564f0d3"}, + {file = "regex-2024.9.11-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:02087ea0a03b4af1ed6ebab2c54d7118127fee8d71b26398e8e4b05b78963199"}, + {file = "regex-2024.9.11-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:69dee6a020693d12a3cf892aba4808fe168d2a4cef368eb9bf74f5398bfd4ee8"}, + {file = "regex-2024.9.11-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:297f54910247508e6e5cae669f2bc308985c60540a4edd1c77203ef19bfa63ca"}, + {file = "regex-2024.9.11-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ecea58b43a67b1b79805f1a0255730edaf5191ecef84dbc4cc85eb30bc8b63b9"}, + {file = "regex-2024.9.11-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:eab4bb380f15e189d1313195b062a6aa908f5bd687a0ceccd47c8211e9cf0d4a"}, + {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0cbff728659ce4bbf4c30b2a1be040faafaa9eca6ecde40aaff86f7889f4ab39"}, + {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:54c4a097b8bc5bb0dfc83ae498061d53ad7b5762e00f4adaa23bee22b012e6ba"}, + {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:73d6d2f64f4d894c96626a75578b0bf7d9e56dcda8c3d037a2118fdfe9b1c664"}, + {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:e53b5fbab5d675aec9f0c501274c467c0f9a5d23696cfc94247e1fb56501ed89"}, + {file = "regex-2024.9.11-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0ffbcf9221e04502fc35e54d1ce9567541979c3fdfb93d2c554f0ca583a19b35"}, + {file = "regex-2024.9.11-cp39-cp39-win32.whl", hash = "sha256:e4c22e1ac1f1ec1e09f72e6c44d8f2244173db7eb9629cc3a346a8d7ccc31142"}, + {file = "regex-2024.9.11-cp39-cp39-win_amd64.whl", hash = "sha256:faa3c142464efec496967359ca99696c896c591c56c53506bac1ad465f66e919"}, + {file = "regex-2024.9.11.tar.gz", hash = "sha256:6c188c307e8433bcb63dc1915022deb553b4203a70722fc542c363bf120a01fd"}, ] [[package]] @@ -4108,36 +4092,6 @@ files = [ {file = "ruamel.yaml.clib-0.2.8.tar.gz", hash = "sha256:beb2e0404003de9a4cab9753a8805a8fe9320ee6673136ed7f04255fe60bb512"}, ] -[[package]] -name = "schematic-db" -version = "0.0.41" -description = "" -optional = false -python-versions = ">=3.9,<4.0" -files = [ - {file = "schematic_db-0.0.41-py3-none-any.whl", hash = "sha256:bf8e8a73fb06113431a89a25df15f3eefbe7b40c2cfe149c4e9afa6e6b33fd5b"}, - {file = "schematic_db-0.0.41.tar.gz", hash = "sha256:cd5ec936cdb4fca203de57aa0c771b2b251c5eec7e0af719c388cad70d8d9f6d"}, -] - -[package.dependencies] -deprecation = ">=2.1.0,<3.0.0" -interrogate = ">=1.5.0,<2.0.0" -networkx = ">=2.8.6,<3.0.0" -pandas = ">=2.0.0,<3.0.0" -pydantic = ">=1.10.7,<2.0.0" -PyYAML = ">=6.0,<7.0" -requests = ">=2.28.1,<3.0.0" -SQLAlchemy = ">=2.0.19,<3.0.0" -SQLAlchemy-Utils = ">=0.41.1,<0.42.0" -synapseclient = {version = ">=4.0.0,<5.0.0", optional = true, markers = "extra == \"synapse\""} -tenacity = ">=8.1.0,<9.0.0" -validators = ">=0.20.0,<0.21.0" - -[package.extras] -mysql = ["mysqlclient (>=2.1.1,<3.0.0)"] -postgres = ["psycopg2-binary (>=2.9.5,<3.0.0)"] -synapse = ["synapseclient (>=4.0.0,<5.0.0)"] - [[package]] name = "scipy" version = "1.13.1" @@ -4402,121 +4356,6 @@ lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] standalone = ["Sphinx (>=5)"] test = ["pytest"] -[[package]] -name = "sqlalchemy" -version = "2.0.34" -description = "Database Abstraction Library" -optional = false -python-versions = ">=3.7" -files = [ - {file = "SQLAlchemy-2.0.34-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:95d0b2cf8791ab5fb9e3aa3d9a79a0d5d51f55b6357eecf532a120ba3b5524db"}, - {file = "SQLAlchemy-2.0.34-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:243f92596f4fd4c8bd30ab8e8dd5965afe226363d75cab2468f2c707f64cd83b"}, - {file = "SQLAlchemy-2.0.34-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ea54f7300553af0a2a7235e9b85f4204e1fc21848f917a3213b0e0818de9a24"}, - {file = "SQLAlchemy-2.0.34-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:173f5f122d2e1bff8fbd9f7811b7942bead1f5e9f371cdf9e670b327e6703ebd"}, - {file = "SQLAlchemy-2.0.34-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:196958cde924a00488e3e83ff917be3b73cd4ed8352bbc0f2989333176d1c54d"}, - {file = "SQLAlchemy-2.0.34-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bd90c221ed4e60ac9d476db967f436cfcecbd4ef744537c0f2d5291439848768"}, - {file = "SQLAlchemy-2.0.34-cp310-cp310-win32.whl", hash = "sha256:3166dfff2d16fe9be3241ee60ece6fcb01cf8e74dd7c5e0b64f8e19fab44911b"}, - {file = "SQLAlchemy-2.0.34-cp310-cp310-win_amd64.whl", hash = "sha256:6831a78bbd3c40f909b3e5233f87341f12d0b34a58f14115c9e94b4cdaf726d3"}, - {file = "SQLAlchemy-2.0.34-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7db3db284a0edaebe87f8f6642c2b2c27ed85c3e70064b84d1c9e4ec06d5d84"}, - {file = "SQLAlchemy-2.0.34-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:430093fce0efc7941d911d34f75a70084f12f6ca5c15d19595c18753edb7c33b"}, - {file = "SQLAlchemy-2.0.34-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79cb400c360c7c210097b147c16a9e4c14688a6402445ac848f296ade6283bbc"}, - {file = "SQLAlchemy-2.0.34-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb1b30f31a36c7f3fee848391ff77eebdd3af5750bf95fbf9b8b5323edfdb4ec"}, - {file = "SQLAlchemy-2.0.34-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8fddde2368e777ea2a4891a3fb4341e910a056be0bb15303bf1b92f073b80c02"}, - {file = "SQLAlchemy-2.0.34-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:80bd73ea335203b125cf1d8e50fef06be709619eb6ab9e7b891ea34b5baa2287"}, - {file = "SQLAlchemy-2.0.34-cp311-cp311-win32.whl", hash = "sha256:6daeb8382d0df526372abd9cb795c992e18eed25ef2c43afe518c73f8cccb721"}, - {file = "SQLAlchemy-2.0.34-cp311-cp311-win_amd64.whl", hash = "sha256:5bc08e75ed11693ecb648b7a0a4ed80da6d10845e44be0c98c03f2f880b68ff4"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:53e68b091492c8ed2bd0141e00ad3089bcc6bf0e6ec4142ad6505b4afe64163e"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bcd18441a49499bf5528deaa9dee1f5c01ca491fc2791b13604e8f972877f812"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:165bbe0b376541092bf49542bd9827b048357f4623486096fc9aaa6d4e7c59a2"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3330415cd387d2b88600e8e26b510d0370db9b7eaf984354a43e19c40df2e2b"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:97b850f73f8abbffb66ccbab6e55a195a0eb655e5dc74624d15cff4bfb35bd74"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee4c6917857fd6121ed84f56d1dc78eb1d0e87f845ab5a568aba73e78adf83"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-win32.whl", hash = "sha256:fbb034f565ecbe6c530dff948239377ba859420d146d5f62f0271407ffb8c580"}, - {file = "SQLAlchemy-2.0.34-cp312-cp312-win_amd64.whl", hash = "sha256:707c8f44931a4facd4149b52b75b80544a8d824162602b8cd2fe788207307f9a"}, - {file = "SQLAlchemy-2.0.34-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:24af3dc43568f3780b7e1e57c49b41d98b2d940c1fd2e62d65d3928b6f95f021"}, - {file = "SQLAlchemy-2.0.34-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e60ed6ef0a35c6b76b7640fe452d0e47acc832ccbb8475de549a5cc5f90c2c06"}, - {file = "SQLAlchemy-2.0.34-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:413c85cd0177c23e32dee6898c67a5f49296640041d98fddb2c40888fe4daa2e"}, - {file = "SQLAlchemy-2.0.34-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:25691f4adfb9d5e796fd48bf1432272f95f4bbe5f89c475a788f31232ea6afba"}, - {file = "SQLAlchemy-2.0.34-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:526ce723265643dbc4c7efb54f56648cc30e7abe20f387d763364b3ce7506c82"}, - {file = "SQLAlchemy-2.0.34-cp37-cp37m-win32.whl", hash = "sha256:13be2cc683b76977a700948411a94c67ad8faf542fa7da2a4b167f2244781cf3"}, - {file = "SQLAlchemy-2.0.34-cp37-cp37m-win_amd64.whl", hash = "sha256:e54ef33ea80d464c3dcfe881eb00ad5921b60f8115ea1a30d781653edc2fd6a2"}, - {file = "SQLAlchemy-2.0.34-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:43f28005141165edd11fbbf1541c920bd29e167b8bbc1fb410d4fe2269c1667a"}, - {file = "SQLAlchemy-2.0.34-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b68094b165a9e930aedef90725a8fcfafe9ef95370cbb54abc0464062dbf808f"}, - {file = "SQLAlchemy-2.0.34-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a1e03db964e9d32f112bae36f0cc1dcd1988d096cfd75d6a588a3c3def9ab2b"}, - {file = "SQLAlchemy-2.0.34-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:203d46bddeaa7982f9c3cc693e5bc93db476ab5de9d4b4640d5c99ff219bee8c"}, - {file = "SQLAlchemy-2.0.34-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:ae92bebca3b1e6bd203494e5ef919a60fb6dfe4d9a47ed2453211d3bd451b9f5"}, - {file = "SQLAlchemy-2.0.34-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:9661268415f450c95f72f0ac1217cc6f10256f860eed85c2ae32e75b60278ad8"}, - {file = "SQLAlchemy-2.0.34-cp38-cp38-win32.whl", hash = "sha256:895184dfef8708e15f7516bd930bda7e50ead069280d2ce09ba11781b630a434"}, - {file = "SQLAlchemy-2.0.34-cp38-cp38-win_amd64.whl", hash = "sha256:6e7cde3a2221aa89247944cafb1b26616380e30c63e37ed19ff0bba5e968688d"}, - {file = "SQLAlchemy-2.0.34-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dbcdf987f3aceef9763b6d7b1fd3e4ee210ddd26cac421d78b3c206d07b2700b"}, - {file = "SQLAlchemy-2.0.34-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ce119fc4ce0d64124d37f66a6f2a584fddc3c5001755f8a49f1ca0a177ef9796"}, - {file = "SQLAlchemy-2.0.34-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a17d8fac6df9835d8e2b4c5523666e7051d0897a93756518a1fe101c7f47f2f0"}, - {file = "SQLAlchemy-2.0.34-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ebc11c54c6ecdd07bb4efbfa1554538982f5432dfb8456958b6d46b9f834bb7"}, - {file = "SQLAlchemy-2.0.34-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2e6965346fc1491a566e019a4a1d3dfc081ce7ac1a736536367ca305da6472a8"}, - {file = "SQLAlchemy-2.0.34-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:220574e78ad986aea8e81ac68821e47ea9202b7e44f251b7ed8c66d9ae3f4278"}, - {file = "SQLAlchemy-2.0.34-cp39-cp39-win32.whl", hash = "sha256:b75b00083e7fe6621ce13cfce9d4469c4774e55e8e9d38c305b37f13cf1e874c"}, - {file = "SQLAlchemy-2.0.34-cp39-cp39-win_amd64.whl", hash = "sha256:c29d03e0adf3cc1a8c3ec62d176824972ae29b67a66cbb18daff3062acc6faa8"}, - {file = "SQLAlchemy-2.0.34-py3-none-any.whl", hash = "sha256:7286c353ee6475613d8beff83167374006c6b3e3f0e6491bfe8ca610eb1dec0f"}, - {file = "sqlalchemy-2.0.34.tar.gz", hash = "sha256:10d8f36990dd929690666679b0f42235c159a7051534adb135728ee52828dd22"}, -] - -[package.dependencies] -greenlet = {version = "!=0.4.17", markers = "python_version < \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} -typing-extensions = ">=4.6.0" - -[package.extras] -aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"] -aioodbc = ["aioodbc", "greenlet (!=0.4.17)"] -aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] -asyncio = ["greenlet (!=0.4.17)"] -asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"] -mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"] -mssql = ["pyodbc"] -mssql-pymssql = ["pymssql"] -mssql-pyodbc = ["pyodbc"] -mypy = ["mypy (>=0.910)"] -mysql = ["mysqlclient (>=1.4.0)"] -mysql-connector = ["mysql-connector-python"] -oracle = ["cx_oracle (>=8)"] -oracle-oracledb = ["oracledb (>=1.0.1)"] -postgresql = ["psycopg2 (>=2.7)"] -postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] -postgresql-pg8000 = ["pg8000 (>=1.29.1)"] -postgresql-psycopg = ["psycopg (>=3.0.7)"] -postgresql-psycopg2binary = ["psycopg2-binary"] -postgresql-psycopg2cffi = ["psycopg2cffi"] -postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"] -pymysql = ["pymysql"] -sqlcipher = ["sqlcipher3_binary"] - -[[package]] -name = "sqlalchemy-utils" -version = "0.41.2" -description = "Various utility functions for SQLAlchemy." -optional = false -python-versions = ">=3.7" -files = [ - {file = "SQLAlchemy-Utils-0.41.2.tar.gz", hash = "sha256:bc599c8c3b3319e53ce6c5c3c471120bd325d0071fb6f38a10e924e3d07b9990"}, - {file = "SQLAlchemy_Utils-0.41.2-py3-none-any.whl", hash = "sha256:85cf3842da2bf060760f955f8467b87983fb2e30f1764fd0e24a48307dc8ec6e"}, -] - -[package.dependencies] -SQLAlchemy = ">=1.3" - -[package.extras] -arrow = ["arrow (>=0.3.4)"] -babel = ["Babel (>=1.3)"] -color = ["colour (>=0.0.4)"] -encrypted = ["cryptography (>=0.6)"] -intervals = ["intervals (>=0.7.1)"] -password = ["passlib (>=1.6,<2.0)"] -pendulum = ["pendulum (>=2.0.5)"] -phone = ["phonenumbers (>=5.9.2)"] -test = ["Jinja2 (>=2.3)", "Pygments (>=1.2)", "backports.zoneinfo", "docutils (>=0.10)", "flake8 (>=2.4.0)", "flexmock (>=0.9.7)", "isort (>=4.2.2)", "pg8000 (>=1.12.4)", "psycopg (>=3.1.8)", "psycopg2 (>=2.5.1)", "psycopg2cffi (>=2.8.1)", "pymysql", "pyodbc", "pytest (==7.4.4)", "python-dateutil (>=2.6)", "pytz (>=2014.2)"] -test-all = ["Babel (>=1.3)", "Jinja2 (>=2.3)", "Pygments (>=1.2)", "arrow (>=0.3.4)", "backports.zoneinfo", "colour (>=0.0.4)", "cryptography (>=0.6)", "docutils (>=0.10)", "flake8 (>=2.4.0)", "flexmock (>=0.9.7)", "furl (>=0.4.1)", "intervals (>=0.7.1)", "isort (>=4.2.2)", "passlib (>=1.6,<2.0)", "pendulum (>=2.0.5)", "pg8000 (>=1.12.4)", "phonenumbers (>=5.9.2)", "psycopg (>=3.1.8)", "psycopg2 (>=2.5.1)", "psycopg2cffi (>=2.8.1)", "pymysql", "pyodbc", "pytest (==7.4.4)", "python-dateutil", "python-dateutil (>=2.6)", "pytz (>=2014.2)"] -timezone = ["python-dateutil"] -url = ["furl (>=0.4.1)"] - [[package]] name = "stack-data" version = "0.6.3" @@ -4552,13 +4391,13 @@ Jinja2 = ">=2.0" [[package]] name = "synapseclient" -version = "4.4.1" +version = "4.5.1" description = "A client for Synapse, a collaborative, open-source research platform that allows teams to share data, track analyses, and collaborate." optional = false python-versions = ">=3.8" files = [ - {file = "synapseclient-4.4.1-py3-none-any.whl", hash = "sha256:fe5716f234184ad0290c930f98383ce87bbf687221365ef477de826831c73994"}, - {file = "synapseclient-4.4.1.tar.gz", hash = "sha256:fc6ec5a0fd49edf2b05ecd7f69316784a4b813dd0fd259785932c0786d480629"}, + {file = "synapseclient-4.5.1-py3-none-any.whl", hash = "sha256:527d06bb1804b797356564056f4be970daafe235b049b790d59cb69496928210"}, + {file = "synapseclient-4.5.1.tar.gz", hash = "sha256:d259eec60de536198851883d2e1232a8219290712eadf88fc85e469b5d1fb35a"}, ] [package.dependencies] @@ -4567,9 +4406,13 @@ asyncio-atexit = ">=1.0.1,<1.1.0" deprecated = ">=1.2.4,<2.0" httpx = ">=0.27.0,<0.28.0" nest-asyncio = ">=1.6.0,<1.7.0" -opentelemetry-api = ">=1.21.0,<1.22.0" -opentelemetry-exporter-otlp-proto-http = ">=1.21.0,<1.22.0" -opentelemetry-sdk = ">=1.21.0,<1.22.0" +opentelemetry-api = ">=1.21.0" +opentelemetry-exporter-otlp-proto-http = ">=1.21.0" +opentelemetry-instrumentation-httpx = ">=0.48b0" +opentelemetry-instrumentation-requests = ">=0.48b0" +opentelemetry-instrumentation-threading = ">=0.48b0" +opentelemetry-instrumentation-urllib = ">=0.48b0" +opentelemetry-sdk = ">=1.21.0" psutil = ">=5.9.8,<5.10.0" requests = ">=2.22.0,<3.0" tqdm = ">=4.66.2,<5.0" @@ -4579,24 +4422,10 @@ urllib3 = ">=1.26.18,<2" boto3 = ["boto3 (>=1.7.0,<2.0)"] dev = ["black", "flake8 (>=3.7.0,<4.0)", "func-timeout (>=4.3,<5.0)", "pandas (>=1.5,<3.0)", "pre-commit", "pytest (>=7.0.0,<8.0)", "pytest-asyncio (>=0.23.6,<1.0)", "pytest-cov (>=4.1.0,<4.2.0)", "pytest-mock (>=3.0,<4.0)", "pytest-rerunfailures (>=12.0,<13.0)", "pytest-socket (>=0.6.0,<0.7.0)", "pytest-xdist[psutil] (>=2.2,<3.0.0)"] docs = ["markdown-include (>=0.8.1,<0.9.0)", "mkdocs (>=1.5.3)", "mkdocs-material (>=9.4.14)", "mkdocs-open-in-new-tab (>=1.0.3,<1.1.0)", "mkdocstrings (>=0.24.0)", "mkdocstrings-python (>=1.7.5)", "termynal (>=0.11.1)"] -pandas = ["pandas (>=1.5,<3.0)"] +pandas = ["numpy (<2.0.0)", "pandas (>=1.5,<3.0)"] pysftp = ["pysftp (>=0.2.8,<0.3)"] tests = ["flake8 (>=3.7.0,<4.0)", "func-timeout (>=4.3,<5.0)", "pandas (>=1.5,<3.0)", "pytest (>=7.0.0,<8.0)", "pytest-asyncio (>=0.23.6,<1.0)", "pytest-cov (>=4.1.0,<4.2.0)", "pytest-mock (>=3.0,<4.0)", "pytest-rerunfailures (>=12.0,<13.0)", "pytest-socket (>=0.6.0,<0.7.0)", "pytest-xdist[psutil] (>=2.2,<3.0.0)"] -[[package]] -name = "tabulate" -version = "0.9.0" -description = "Pretty-print tabular data" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"}, - {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"}, -] - -[package.extras] -widechars = ["wcwidth"] - [[package]] name = "tenacity" version = "8.5.0" @@ -4633,38 +4462,6 @@ docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] test = ["pre-commit", "pytest (>=7.0)", "pytest-timeout"] typing = ["mypy (>=1.6,<2.0)", "traitlets (>=5.11.1)"] -[[package]] -name = "threadloop" -version = "1.0.2" -description = "Tornado IOLoop Backed Concurrent Futures" -optional = true -python-versions = "*" -files = [ - {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, - {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, -] - -[package.dependencies] -tornado = "*" - -[[package]] -name = "thrift" -version = "0.20.0" -description = "Python bindings for the Apache Thrift RPC system" -optional = true -python-versions = "*" -files = [ - {file = "thrift-0.20.0.tar.gz", hash = "sha256:4dd662eadf6b8aebe8a41729527bd69adf6ceaa2a8681cbef64d1273b3e8feba"}, -] - -[package.dependencies] -six = ">=1.7.2" - -[package.extras] -all = ["tornado (>=4.0)", "twisted"] -tornado = ["tornado (>=4.0)"] -twisted = ["twisted"] - [[package]] name = "tinycss2" version = "1.3.0" @@ -4898,31 +4695,15 @@ files = [ {file = "uwsgi-2.0.26.tar.gz", hash = "sha256:86e6bfcd4dc20529665f5b7777193cdc48622fb2c59f0a7f1e3dc32b3882e7f9"}, ] -[[package]] -name = "validators" -version = "0.20.0" -description = "Python Data Validation for Humans™." -optional = false -python-versions = ">=3.4" -files = [ - {file = "validators-0.20.0.tar.gz", hash = "sha256:24148ce4e64100a2d5e267233e23e7afeb55316b47d30faae7eb6e7292bc226a"}, -] - -[package.dependencies] -decorator = ">=3.4.0" - -[package.extras] -test = ["flake8 (>=2.4.0)", "isort (>=4.2.2)", "pytest (>=2.2.3)"] - [[package]] name = "virtualenv" -version = "20.26.3" +version = "20.26.5" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.26.3-py3-none-any.whl", hash = "sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589"}, - {file = "virtualenv-20.26.3.tar.gz", hash = "sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a"}, + {file = "virtualenv-20.26.5-py3-none-any.whl", hash = "sha256:4f3ac17b81fba3ce3bd6f4ead2749a72da5929c01774948e243db9ba41df4ff6"}, + {file = "virtualenv-20.26.5.tar.gz", hash = "sha256:ce489cac131aa58f4b25e321d6d186171f78e6cb13fafbf32a840cee67733ff4"}, ] [package.dependencies] @@ -5096,13 +4877,13 @@ files = [ [[package]] name = "zipp" -version = "3.20.1" +version = "3.20.2" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" files = [ - {file = "zipp-3.20.1-py3-none-any.whl", hash = "sha256:9960cd8967c8f85a56f920d5d507274e74f9ff813a0ab8889a5b5be2daf44064"}, - {file = "zipp-3.20.1.tar.gz", hash = "sha256:c22b14cc4763c5a5b04134207736c107db42e9d3ef2d9779d465f5f1bcba572b"}, + {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, + {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, ] [package.extras] @@ -5114,10 +4895,10 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", type = ["pytest-mypy"] [extras] -api = ["Flask", "Flask-Cors", "Jinja2", "connexion", "flask-opentracing", "jaeger-client", "opentelemetry-exporter-otlp-proto-grpc", "pyopenssl"] +api = ["Flask", "Flask-Cors", "Jinja2", "connexion", "opentelemetry-api", "opentelemetry-exporter-otlp-proto-grpc", "opentelemetry-instrumentation-flask", "opentelemetry-sdk", "pyopenssl"] aws = ["uWSGI"] [metadata] lock-version = "2.0" python-versions = ">=3.9.0,<3.11" -content-hash = "f814725d68db731c704f4ebcd169ae71e4031f0d939c3ce789145ddcb5f196eb" +content-hash = "e72a6816f0534115e5b8cd1c0e55e1e35778e4a059caebfc0b7c2bae96c65a59" diff --git a/pyproject.toml b/pyproject.toml index f7045473c..cccda5718 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ pygsheets = "^2.0.4" PyYAML = "^6.0.0" rdflib = "^6.0.0" setuptools = "^66.0.0" -synapseclient = "4.4.1" +synapseclient = "4.5.1" tenacity = "^8.0.1" toml = "^0.10.2" great-expectations = "^0.15.0" @@ -65,7 +65,6 @@ openpyxl = "^3.0.9" pdoc = "^14.0.0" dateparser = "^1.1.4" pandarallel = "^1.6.4" -schematic-db = {version = "0.0.41", extras = ["synapse"]} pyopenssl = {version = "^23.0.0", optional = true} dataclasses-json = "^0.6.1" pydantic = "^1.10.4" @@ -75,13 +74,14 @@ Flask-Cors = {version = "^3.0.10", optional = true} uWSGI = {version = "^2.0.21", optional = true} Jinja2 = {version = ">2.11.3", optional = true} asyncio = "^3.4.3" -jaeger-client = {version = "^4.8.0", optional = true} -flask-opentracing = {version="^2.0.0", optional = true} PyJWT = "^2.9.0" +opentelemetry-api = {version = ">=1.21.0", optional = true} +opentelemetry-sdk = {version = ">=1.21.0", optional = true} opentelemetry-exporter-otlp-proto-grpc = {version="^1.0.0", optional = true} +opentelemetry-instrumentation-flask = {version=">=0.48b0 ", optional = true} [tool.poetry.extras] -api = ["connexion", "Flask", "Flask-Cors", "Jinja2", "pyopenssl", "jaeger-client", "flask-opentracing", "opentelemetry-exporter-otlp-proto-grpc"] +api = ["connexion", "Flask", "Flask-Cors", "Jinja2", "pyopenssl", "opentelemetry-api", "opentelemetry-sdk", "opentelemetry-exporter-otlp-proto-grpc", "opentelemetry-instrumentation-flask"] aws = ["uWSGI"] @@ -99,12 +99,6 @@ pylint = "^2.16.1" pytest-xdist = "^3.5.0" pre-commit = "^3.6.2" -[tool.poetry.group.aws] -optional = true - -[tool.poetry.group.aws.dependencies] - - [tool.black] line-length = 88 include = '\.pyi?$' @@ -135,41 +129,3 @@ testpaths = [ filterwarnings = [ "ignore::DeprecationWarning" ] -markers = [ - """\ - google_credentials_needed: marks tests requiring \ - Google credentials (skipped on GitHub CI) \ - """, - """\ - submission: tests that involve submitting manifests - """, - """\ - not_windows: tests that don't work on on windows machine - """, - """\ - schematic_api: marks tests covering \ - API functionality (skipped on regular GitHub CI test suite) - """, - """\ - rule_combos: marks tests covering \ - combinations of rules that aren't always necessary \ - and can add significantly to CI runtime (skipped on GitHub CI unless prompted to run in commit message) - """, - """\ - table_operations: marks tests covering \ - table operations that pass locally \ - but fail on CI due to interactions with Synapse (skipped on GitHub CI) - """, - """\ - rule_benchmark: marks tests covering \ - validation rule benchmarking - """, - """\ - synapse_credentials_needed: marks api tests that \ - require synapse credentials to run - """, - """\ - empty_token: marks api tests that \ - send empty credentials in the request - """ -] \ No newline at end of file diff --git a/pytest.ini b/pytest.ini index 982e6ef86..8cc4b91be 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,4 +3,14 @@ python_files = test_*.py asyncio_mode = auto asyncio_default_fixture_loop_scope = session log_cli = False -log_cli_level = INFO \ No newline at end of file +log_cli_level = INFO +markers = + google_credentials_needed: marks tests requiring Google credentials (skipped on GitHub CI) + submission: tests that involve submitting manifests + not_windows: tests that dont work on on windows machine + schematic_api: marks tests covering API functionality (skipped on regular GitHub CI test suite) + rule_combos: marks tests covering combinations of rules that arent always necessary and can add significantly to CI runtime (skipped on GitHub CI unless prompted to run in commit message) + table_operations: marks tests covering table operations that pass locally but fail on CI due to interactions with Synapse (skipped on GitHub CI) + rule_benchmark: marks tests covering validation rule benchmarking + synapse_credentials_needed: marks api tests that require synapse credentials to run + empty_token: marks api tests that send empty credentials in the request \ No newline at end of file diff --git a/schematic/__init__.py b/schematic/__init__.py index 1b4ec14fe..b336be701 100644 --- a/schematic/__init__.py +++ b/schematic/__init__.py @@ -1,2 +1,114 @@ +import logging +import os +import time +from typing import Dict, List + +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.flask import FlaskInstrumentor +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ( + BatchSpanProcessor, + ConsoleSpanExporter, + SimpleSpanProcessor, + Span, +) +from opentelemetry.sdk.trace.sampling import ALWAYS_OFF +from synapseclient import Synapse +from werkzeug import Request + from schematic.configuration.configuration import CONFIG from schematic.loader import LOADER +from schematic_api.api.security_controller import info_from_bearer_auth + +Synapse.allow_client_caching(False) +logger = logging.getLogger(__name__) + + +# borrowed from: https://github.com/Sage-Bionetworks/synapsePythonClient/blob/develop/tests/integration/conftest.py +class FileSpanExporter(ConsoleSpanExporter): + """Create an exporter for OTEL data to a file.""" + + def __init__(self, file_path: str) -> None: + """Init with a path.""" + self.file_path = file_path + + def export(self, spans: List[Span]) -> None: + """Export the spans to the file.""" + with open(self.file_path, "a", encoding="utf-8") as f: + for span in spans: + span_json_one_line = span.to_json().replace("\n", "") + "\n" + f.write(span_json_one_line) + + +def set_up_tracing() -> None: + """Set up tracing for the API.""" + tracing_export = os.environ.get("TRACING_EXPORT_FORMAT", None) + if tracing_export is not None and tracing_export: + Synapse.enable_open_telemetry(True) + tracing_service_name = os.environ.get("TRACING_SERVICE_NAME", "schematic-api") + + trace.set_tracer_provider( + TracerProvider( + resource=Resource(attributes={SERVICE_NAME: tracing_service_name}) + ) + ) + FlaskInstrumentor().instrument( + request_hook=request_hook, response_hook=response_hook + ) + + if tracing_export == "otlp": + trace.get_tracer_provider().add_span_processor( + BatchSpanProcessor(OTLPSpanExporter()) + ) + elif tracing_export == "file": + timestamp_millis = int(time.time() * 1000) + file_name = f"otel_spans_integration_testing_{timestamp_millis}.ndjson" + file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), file_name) + processor = SimpleSpanProcessor(FileSpanExporter(file_path)) + trace.get_tracer_provider().add_span_processor(processor) + else: + trace.set_tracer_provider(TracerProvider(sampler=ALWAYS_OFF)) + + +def request_hook(span: Span, environ: Dict) -> None: + """ + Request hook for the flask server to handle setting attributes in the span. If + anything here happens to fail we do not want to stop the request from being + processed so we catch all exceptions and log them. + + Args: + span: The span object to set attributes in + environ: The environment variables from the request + """ + if not span or not span.is_recording(): + return + try: + if auth_header := environ.get("HTTP_AUTHORIZATION", None): + split_headers = auth_header.split(" ") + if len(split_headers) > 1: + token = auth_header.split(" ")[1] + user_info = info_from_bearer_auth(token) + if user_info: + span.set_attribute("user.id", user_info.get("sub")) + except Exception: + logger.exception("Failed to set user info in span") + + try: + if (request := environ.get("werkzeug.request", None)) and isinstance( + request, Request + ): + for arg in request.args: + span.set_attribute(key=f"schematic.{arg}", value=request.args[arg]) + except Exception: + logger.exception("Failed to set request info in span") + + +def response_hook(span: Span, status: str, response_headers: List) -> None: + """Nothing is implemented here yet, but it follows the same pattern as the + request hook.""" + pass + + +set_up_tracing() diff --git a/schematic/manifest/commands.py b/schematic/manifest/commands.py index 6b212239f..bc68cd03c 100644 --- a/schematic/manifest/commands.py +++ b/schematic/manifest/commands.py @@ -1,25 +1,21 @@ -import json -import os -import pandas as pd import logging -from pathlib import Path +import os import sys -from typing import get_args, List +from pathlib import Path +from typing import List, get_args + import click import click_log -from schematic.schemas.data_model_parser import DataModelParser -from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer -from schematic.manifest.generator import ManifestGenerator - -from schematic.utils.schema_utils import DisplayLabelType -from schematic.utils.cli_utils import log_value_from_config, query_dict, parse_syn_ids -from schematic.utils.google_api_utils import export_manifest_csv - +from schematic.configuration.configuration import CONFIG from schematic.help import manifest_commands - +from schematic.manifest.generator import ManifestGenerator +from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer +from schematic.schemas.data_model_parser import DataModelParser from schematic.store.synapse import SynapseStorage -from schematic.configuration.configuration import CONFIG +from schematic.utils.cli_utils import log_value_from_config, parse_syn_ids, query_dict +from schematic.utils.google_api_utils import export_manifest_csv +from schematic.utils.schema_utils import DisplayLabelType logger = logging.getLogger("schematic") click_log.basic_config(logger) @@ -343,14 +339,15 @@ def migrate_manifests( ) @click.pass_obj def download_manifest(ctx, dataset_id, new_manifest_name): - master_fileview = CONFIG["synapse"]["master_fileview"] - # use Synapse Storage store = SynapseStorage() # download existing file manifest_data = store.getDatasetManifest( - datasetId=dataset_id, downloadFile=True, newManifestName=new_manifest_name + datasetId=dataset_id, + downloadFile=True, + newManifestName=new_manifest_name, + use_temporary_folder=False, ) if not manifest_data: diff --git a/schematic/manifest/generator.py b/schematic/manifest/generator.py index 47acad4b4..69b86e136 100644 --- a/schematic/manifest/generator.py +++ b/schematic/manifest/generator.py @@ -3,8 +3,7 @@ import os from collections import OrderedDict from pathlib import Path -from tempfile import NamedTemporaryFile -from typing import Any, BinaryIO, Dict, List, Literal, Optional, Tuple, Union +from typing import Dict, List, Literal, Optional, Tuple, Union import networkx as nx import pandas as pd diff --git a/schematic/models/GE_Helpers.py b/schematic/models/GE_Helpers.py index 9eda117a8..bc4075cc2 100644 --- a/schematic/models/GE_Helpers.py +++ b/schematic/models/GE_Helpers.py @@ -1,21 +1,16 @@ import logging import os -import re -from statistics import mode -from tabnanny import check +import uuid # allows specifying explicit variable types -from typing import Any, Dict, List, Optional, Text -from urllib import error -from urllib.parse import urlparse -from urllib.request import HTTPDefaultErrorHandler, OpenerDirector, Request, urlopen +from typing import Dict, List import numpy as np -from attr import attr from great_expectations.core import ExpectationSuite from great_expectations.core.expectation_configuration import ExpectationConfiguration from great_expectations.data_context import BaseDataContext from great_expectations.data_context.types.base import ( + AnonymizedUsageStatisticsConfig, DataContextConfig, DatasourceConfig, FilesystemStoreBackendDefaults, @@ -24,7 +19,6 @@ ExpectationSuiteIdentifier, ) from great_expectations.exceptions.exceptions import GreatExpectationsError -from ruamel import yaml import great_expectations as ge from schematic.models.validate_attribute import GenerateError @@ -119,6 +113,9 @@ def build_context(self): }, } + # Setting this to False prevents extra data from leaving schematic + anonymous_usage_statistics = AnonymizedUsageStatisticsConfig(enabled=False) + # create data context configuration data_context_config = DataContextConfig( datasources={ @@ -136,6 +133,7 @@ def build_context(self): store_backend_defaults=FilesystemStoreBackendDefaults( root_directory=os.path.join(os.getcwd(), "great_expectations") ), + anonymous_usage_statistics=anonymous_usage_statistics, ) # build context and add data source @@ -151,30 +149,17 @@ def add_expectation_suite_if_not_exists(self) -> ExpectationSuite: Returns: saves expectation suite and identifier to self """ - self.expectation_suite_name = "Manifest_test_suite" - # Get a list of all expectation suites - suite_names = self.context.list_expectation_suite_names() - # Get a list of all checkpoints - all_checkpoints = self.context.list_checkpoints() - - # if the suite exists, delete it - if self.expectation_suite_name in suite_names: - self.context.delete_expectation_suite(self.expectation_suite_name) - - # also delete all the checkpoints associated with the suite - if all_checkpoints: - for checkpoint_name in all_checkpoints: - self.context.delete_checkpoint(checkpoint_name) - - self.suite = self.context.add_expectation_suite( + self.expectation_suite_name = f"Manifest_test_suite_{uuid.uuid4()}" + expectation_suite = self.context.add_expectation_suite( expectation_suite_name=self.expectation_suite_name, ) + self.suite = expectation_suite return self.suite def build_expectation_suite( self, - ): + ) -> None: """ Purpose: Construct an expectation suite to validate columns with rules that have expectations @@ -374,9 +359,11 @@ def build_expectation_suite( suite_identifier = ExpectationSuiteIdentifier( expectation_suite_name=self.expectation_suite_name ) - self.context.build_data_docs(resource_identifiers=[suite_identifier]) - ##Webpage DataDocs opened here: - # self.context.open_data_docs(resource_identifier=suite_identifier) + + if logger.isEnabledFor(logging.DEBUG): + self.context.build_data_docs(resource_identifiers=[suite_identifier]) + ##Webpage DataDocs opened here: + # self.context.open_data_docs(resource_identifier=suite_identifier) def add_expectation( self, @@ -421,7 +408,7 @@ def build_checkpoint(self): adds checkpoint to self """ # create manifest checkpoint - self.checkpoint_name = "manifest_checkpoint" + self.checkpoint_name = f"manifest_checkpoint_{uuid.uuid4()}" checkpoint_config = { "name": self.checkpoint_name, "config_version": 1, @@ -486,6 +473,8 @@ def generate_errors( if ( "exception_info" in result_dict.keys() + # This changes in 0.18.x of GE, details on this: + # https://docs.greatexpectations.io/docs/0.18/reference/learn/terms/validation_result/ and result_dict["exception_info"]["exception_message"] ): raise GreatExpectationsError( @@ -501,6 +490,14 @@ def generate_errors( # because type validation is column aggregate expectation and not column map expectation when columns are not of object type, # indices and values cannot be returned else: + # This changes in 0.17.x of GE, refactored code: + # for i, item in enumerate(self.manifest[errColumn]): + # observed_type = result_dict.get("result", {}).get("observed_value", None) + # is_instance_type = observed_type is not None and isinstance( + # item, type_dict[observed_type] + # ) + # indices.append(i) if is_instance_type else indices + # values.append(item) if is_instance_type else values for i, item in enumerate(self.manifest[errColumn]): observed_type = result_dict["result"]["observed_value"] indices.append(i) if isinstance( diff --git a/schematic/models/metadata.py b/schematic/models/metadata.py index 582a00168..1e44c13e0 100644 --- a/schematic/models/metadata.py +++ b/schematic/models/metadata.py @@ -273,10 +273,6 @@ def validateModelManifest( return errors, warnings - # check if suite has been created. If so, delete it - if os.path.exists("great_expectations/expectations/Manifest_test_suite.json"): - os.remove("great_expectations/expectations/Manifest_test_suite.json") - errors, warnings, manifest = validate_all( self, errors=errors, diff --git a/schematic/models/validate_attribute.py b/schematic/models/validate_attribute.py index e196bbe14..55a75a5f3 100644 --- a/schematic/models/validate_attribute.py +++ b/schematic/models/validate_attribute.py @@ -873,7 +873,7 @@ def get_target_manifests( project_scope: Optional[list[str]], access_token: Optional[str] = None, ) -> tuple[list[str], list[str]]: - """Gets a list of synapse ids of mainfests to check against + """Gets a list of synapse ids of manifests to check against Args: target_component (str): Manifet ids are gotten fo this type diff --git a/schematic/models/validate_manifest.py b/schematic/models/validate_manifest.py index 3b85b1414..403d0d506 100644 --- a/schematic/models/validate_manifest.py +++ b/schematic/models/validate_manifest.py @@ -1,27 +1,19 @@ import json import logging -import os -import re -import sys +import uuid from numbers import Number -from statistics import mode -from tabnanny import check from time import perf_counter # allows specifying explicit variable types -from typing import Any, Dict, List, Optional, Text -from urllib import error -from urllib.parse import urlparse -from urllib.request import HTTPDefaultErrorHandler, OpenerDirector, Request, urlopen +from typing import List, Optional, Tuple import numpy as np import pandas as pd -from jsonschema import Draft7Validator, ValidationError, exceptions +from jsonschema import Draft7Validator, exceptions from schematic.models.GE_Helpers import GreatExpectationsHelpers from schematic.models.validate_attribute import GenerateError, ValidateAttribute from schematic.schemas.data_model_graph import DataModelGraphExplorer -from schematic.store.synapse import SynapseStorage from schematic.utils.schema_utils import extract_component_validation_rules from schematic.utils.validate_rules_utils import validation_rule_info from schematic.utils.validate_utils import ( @@ -69,13 +61,13 @@ def get_multiple_types_error( def check_max_rule_num( self, validation_rules: list[str], - col: pd.core.series.Series, + col: pd.Series, errors: list[list[str]], ) -> list[list[str]]: """Check that user isnt applying more rule combinations than allowed. Do not consider certain rules as a part of this rule limit. Args: validation_rules, list: Validation rules for current manifest column/attribute being evaluated - col, pd.core.series.Series: the current manifest column being evaluated + col, pd.Series: the current manifest column being evaluated errors, list[list[str]]: list of errors being compiled. Returns: errors, list[list[str]]: list of errors being compiled, with additional error list being appended if appropriate @@ -101,25 +93,25 @@ def check_max_rule_num( def validate_manifest_rules( self, - manifest: pd.core.frame.DataFrame, + manifest: pd.DataFrame, dmge: DataModelGraphExplorer, restrict_rules: bool, project_scope: list[str], dataset_scope: Optional[str] = None, access_token: Optional[str] = None, - ) -> (pd.core.frame.DataFrame, list[list[str]]): + ) -> Tuple[pd.DataFrame, list[list[str]]]: """ Purpose: Take validation rules set for a particular attribute and validate manifest entries based on these rules. Input: - manifest: pd.core.frame.DataFrame + manifest: pd.DataFrame imported from models/metadata.py contains metadata input from user for each attribute. dmge: DataModelGraphExplorer initialized within models/metadata.py Returns: - manifest: pd.core.frame.DataFrame + manifest: pd.DataFrame If a 'list' validatior is run, the manifest needs to be updated to change the attribute column values to a list. In this case the manifest will be updated then exported. @@ -139,12 +131,6 @@ def validate_manifest_rules( validation_types = validation_rule_info() - type_dict = { - "float64": float, - "int64": int, - "str": str, - } - unimplemented_expectations = [ "url", "list", @@ -174,7 +160,8 @@ def validate_manifest_rules( warnings = [] if not restrict_rules: - t_GE = perf_counter() + if logger.isEnabledFor(logging.DEBUG): + t_GE = perf_counter() # operations necessary to set up and run ge suite validation ge_helpers = GreatExpectationsHelpers( dmge=dmge, @@ -193,14 +180,16 @@ def validate_manifest_rules( checkpoint_name=ge_helpers.checkpoint_name, batch_request={ "runtime_parameters": {"batch_data": manifest}, - "batch_identifiers": {"default_identifier_name": "manifestID"}, + "batch_identifiers": { + "default_identifier_name": f"manifestID_{uuid.uuid4()}" + }, }, result_format={"result_format": "COMPLETE"}, ) finally: - ge_helpers.context.delete_checkpoint(ge_helpers.checkpoint_name) + ge_helpers.context.delete_checkpoint(name=ge_helpers.checkpoint_name) ge_helpers.context.delete_expectation_suite( - ge_helpers.expectation_suite_name + expectation_suite_name=ge_helpers.expectation_suite_name ) validation_results = results.list_validation_results() @@ -213,12 +202,13 @@ def validate_manifest_rules( validation_types=validation_types, dmge=dmge, ) - logger.debug(f"GE elapsed time {perf_counter()-t_GE}") + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"GE elapsed time {perf_counter()-t_GE}") else: logger.info("Great Expetations suite will not be utilized.") - t_err = perf_counter() - regex_re = re.compile("regex.*") + if logger.isEnabledFor(logging.DEBUG): + t_err = perf_counter() # Instantiate Validate Attribute validate_attribute = ValidateAttribute(dmge=dmge) @@ -255,7 +245,8 @@ def validate_manifest_rules( ) continue - t_indiv_rule = perf_counter() + if logger.isEnabledFor(logging.DEBUG): + t_indiv_rule = perf_counter() # Validate for each individual validation rule. validation_method = getattr( validate_attribute, validation_types[validation_type]["type"] @@ -289,10 +280,14 @@ def validate_manifest_rules( errors.extend(vr_errors) if vr_warnings: warnings.extend(vr_warnings) - logger.debug( - f"Rule {rule} elapsed time: {perf_counter()-t_indiv_rule}" - ) - logger.debug(f"In House validation elapsed time {perf_counter()-t_err}") + + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + f"Rule {rule} elapsed time: {perf_counter()-t_indiv_rule}" + ) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"In House validation elapsed time {perf_counter()-t_err}") return manifest, errors, warnings def validate_manifest_values( @@ -300,12 +295,11 @@ def validate_manifest_values( manifest, jsonSchema, dmge, - ) -> (List[List[str]], List[List[str]]): + ) -> Tuple[List[List[str]], List[List[str]]]: t_json_schema = perf_counter() errors = [] warnings = [] - col_attr = {} # save the mapping between column index and attribute name manifest = convert_nan_entries_to_empty_strings(manifest=manifest) @@ -321,12 +315,21 @@ def validate_manifest_values( annotations = json.loads(manifest.to_json(orient="records")) for i, annotation in enumerate(annotations): v = Draft7Validator(jsonSchema) - for error in sorted(v.iter_errors(annotation), key=exceptions.relevance): + for sorted_error in sorted( + v.iter_errors(annotation), key=exceptions.relevance + ): errorRow = str(i + 2) - errorCol = error.path[-1] if len(error.path) > 0 else "Wrong schema" - errorColName = error.path[0] if len(error.path) > 0 else "Wrong schema" - errorMsg = error.message[0:500] - errorVal = error.instance if len(error.path) > 0 else "Wrong schema" + errorColName = ( + sorted_error.path[0] + if len(sorted_error.path) > 0 + else "Wrong schema" + ) + errorMsg = sorted_error.message[0:500] + errorVal = ( + sorted_error.instance + if len(sorted_error.path) > 0 + else "Wrong schema" + ) val_errors, val_warnings = GenerateError.generate_schema_error( row_num=errorRow, diff --git a/schematic/schemas/data_model_parser.py b/schematic/schemas/data_model_parser.py index 0da26e933..6434d5b70 100644 --- a/schematic/schemas/data_model_parser.py +++ b/schematic/schemas/data_model_parser.py @@ -1,20 +1,17 @@ "Data Model Parser" +import logging import pathlib -from typing import Any, Union, Optional +from typing import Any, Optional, Union -import logging import pandas as pd from opentelemetry import trace +from schematic.schemas.data_model_relationships import DataModelRelationships from schematic.utils.df_utils import load_df from schematic.utils.io_utils import load_json from schematic.utils.schema_utils import attr_dict_template -from schematic.schemas.data_model_relationships import DataModelRelationships - -from schematic import LOADER - logger = logging.getLogger("Schemas") tracer = trace.get_tracer("Schematic") @@ -49,6 +46,9 @@ def _get_base_schema_path(self, base_schema: Optional[str] = None) -> str: Returns: base_schema_path: Path to base schema based on provided argument. """ + # Lazy import to avoid circular imports + from schematic import LOADER # pylint: disable=import-outside-toplevel + biothings_schema_path = LOADER.filename("data_models/biothings.model.jsonld") self.base_schema_path = ( biothings_schema_path if base_schema is None else base_schema diff --git a/schematic/store/__init__.py b/schematic/store/__init__.py index 60d8d876a..e69de29bb 100644 --- a/schematic/store/__init__.py +++ b/schematic/store/__init__.py @@ -1,2 +0,0 @@ -from schematic.store.base import BaseStorage -from schematic.store.synapse import SynapseStorage diff --git a/schematic/store/database/README.md b/schematic/store/database/README.md new file mode 100644 index 000000000..347f0cafd --- /dev/null +++ b/schematic/store/database/README.md @@ -0,0 +1,18 @@ +The python scripts stored here are sourced from . +This logic was extracted out of `schematic_db` as there were a new of required +dependency updates that prevented using the updated `schematic_db` code. Those +dependency updates included: + +- Great expectations +- Pydantic +- tenacity +- Discontinuing python 3.9 + +As such the following considerations were made: + +- Extract the required functionality out of `schematic_db` such that `schematic` can +continue to function with the current dependencies, but, updates to the dependent code +may still occur. +- Functionality that exists within this extracted code should be split between +application (schematic) specific business logic, and core (SYNPY) logic. This will start +to come to fruition with SYNPY-1418 where table functionality is going to be expanded. diff --git a/schematic/store/database/synapse_database.py b/schematic/store/database/synapse_database.py new file mode 100644 index 000000000..9f61f94ee --- /dev/null +++ b/schematic/store/database/synapse_database.py @@ -0,0 +1,138 @@ +"""SynapseDatabase""" + +import pandas as pd +import synapseclient as sc # type: ignore + +from schematic.store.database.synapse_database_wrapper import Synapse +from schematic.store.synapse_tracker import SynapseEntityTracker + + +class SynapseDatabaseMissingTableAnnotationsError(Exception): + """Raised when a table is missing expected annotations""" + + def __init__(self, message: str, table_name: str) -> None: + self.message = message + self.table_name = table_name + super().__init__(self.message) + + def __str__(self) -> str: + return f"{self.message}; " f"name: {self.table_name};" + + +class InputDataframeMissingColumn(Exception): + """Raised when an input dataframe is missing a needed column(s)""" + + def __init__( + self, message: str, table_columns: list[str], missing_columns: list[str] + ) -> None: + self.message = message + self.table_columns = table_columns + self.missing_columns = missing_columns + super().__init__(self.message) + + def __str__(self) -> str: + return ( + f"{self.message}; " + f"table_columns: {self.table_columns}; " + f"missing_columns: {self.missing_columns}" + ) + + +class SynapseDatabase: + """Represents a database stored as Synapse tables""" + + def __init__( + self, + auth_token: str, + project_id: str, + synapse_entity_tracker: SynapseEntityTracker = None, + syn: sc.Synapse = None, + ) -> None: + """Init + + Args: + auth_token (str): A Synapse auth_token + project_id (str): A Synapse id for a project + synapse_entity_tracker: Tracker for a pull-through cache of Synapse entities + """ + self.synapse = Synapse( + auth_token=auth_token, + project_id=project_id, + synapse_entity_tracker=synapse_entity_tracker, + syn=syn, + ) + + def upsert_table_rows(self, table_name: str, data: pd.DataFrame) -> None: + """Upserts rows into the given table + + Args: + table_name (str): The name of the table to be upserted into. + data (pd.DataFrame): The table the rows will come from + + Raises: + SynapseDatabaseMissingTableAnnotationsError: Raised when the table has no + primary key annotation. + """ + table_id = self.synapse.get_synapse_id_from_table_name(table_name) + annotations = self.synapse.get_entity_annotations(table_id) + if "primary_key" not in annotations: + raise SynapseDatabaseMissingTableAnnotationsError( + "Table has no primary_key annotation", table_name + ) + primary_key = annotations["primary_key"][0] + self._upsert_table_rows(table_id, data, primary_key) + + def _upsert_table_rows( + self, table_id: str, data: pd.DataFrame, primary_key: str + ) -> None: + """Upserts rows into the given table + + Args: + table_id (str): The Synapse id of the table to be upserted into. + data (pd.DataFrame): The table the rows will come from + primary_key (str): The primary key of the table used to identify + which rows to update + + Raises: + InputDataframeMissingColumn: Raised when the input dataframe has + no column that matches the primary key argument. + """ + if primary_key not in list(data.columns): + raise InputDataframeMissingColumn( + "Input dataframe missing primary key column.", + list(data.columns), + [primary_key], + ) + + table = self._create_primary_key_table(table_id, primary_key) + merged_table = pd.merge( + data, table, how="left", on=primary_key, validate="one_to_one" + ) + self.synapse.upsert_table_rows(table_id, merged_table) + + def _create_primary_key_table( + self, table_id: str, primary_key: str + ) -> pd.DataFrame: + """Creates a dataframe with just the primary key of the table + + Args: + table_id (str): The id of the table to query + primary_key (str): The name of the primary key + + Returns: + pd.DataFrame: The table in pandas.DataFrame form with the primary key, ROW_ID, and + ROW_VERSION columns + + Raises: + InputDataframeMissingColumn: Raised when the synapse table has no column that + matches the primary key argument. + """ + table = self.synapse.query_table(table_id, include_row_data=True) + if primary_key not in list(table.columns): + raise InputDataframeMissingColumn( + "Synapse table missing primary key column", + list(table.columns), + [primary_key], + ) + table = table[["ROW_ID", "ROW_VERSION", primary_key]] + return table diff --git a/schematic/store/database/synapse_database_wrapper.py b/schematic/store/database/synapse_database_wrapper.py new file mode 100644 index 000000000..52bf2d4d8 --- /dev/null +++ b/schematic/store/database/synapse_database_wrapper.py @@ -0,0 +1,156 @@ +"""Wrapper class for interacting with Synapse database objects. Eventually this will +be replaced with a more database/table class that exists within the SYNPY project.""" + +from typing import Optional + +import pandas # type: ignore +import synapseclient # type: ignore + +from schematic.store.synapse_tracker import SynapseEntityTracker + + +class SynapseTableNameError(Exception): + """SynapseTableNameError""" + + def __init__(self, message: str, table_name: str) -> None: + """ + Args: + message (str): A message describing the error + table_name (str): The name of the table + """ + self.message = message + self.table_name = table_name + super().__init__(self.message) + + def __str__(self) -> str: + return f"{self.message}:{self.table_name}" + + +class Synapse: # pylint: disable=too-many-public-methods + """ + The Synapse class handles interactions with a project in Synapse. + """ + + def __init__( + self, + auth_token: str, + project_id: str, + cache_root_dir: Optional[str] = None, + synapse_entity_tracker: SynapseEntityTracker = None, + syn: synapseclient.Synapse = None, + ) -> None: + """Init + + Args: + auth_token (str): A Synapse auth_token + project_id (str): A Synapse id for a project + cache_root_dir( str ): Where the directory of the synapse cache should be located + synapse_entity_tracker: Tracker for a pull-through cache of Synapse entities + """ + self.project_id = project_id + if syn: + self.syn = syn + else: + syn = synapseclient.Synapse(cache_root_dir=cache_root_dir) + syn.login(authToken=auth_token, silent=True) + self.syn = syn + self.synapse_entity_tracker = synapse_entity_tracker or SynapseEntityTracker() + + def get_synapse_id_from_table_name(self, table_name: str) -> str: + """Gets the synapse id from the table name + + Args: + table_name (str): The name of the table + + Raises: + SynapseTableNameError: When no tables match the name + SynapseTableNameError: When multiple tables match the name + + Returns: + str: A synapse id + """ + matching_table_id = self.syn.findEntityId( + name=table_name, parent=self.project_id + ) + if matching_table_id is None: + raise SynapseTableNameError("No matching tables with name:", table_name) + return matching_table_id + + def query_table( + self, synapse_id: str, include_row_data: bool = False + ) -> pandas.DataFrame: + """Queries a whole table + + Args: + synapse_id (str): The Synapse id of the table to delete + include_row_data (bool): Include row_id and row_etag. Defaults to False. + + Returns: + pandas.DataFrame: The queried table + """ + query = f"SELECT * FROM {synapse_id}" + return self.execute_sql_query(query, include_row_data) + + def execute_sql_query( + self, query: str, include_row_data: bool = False + ) -> pandas.DataFrame: + """Execute a Sql query + + Args: + query (str): A SQL statement that can be run by Synapse + include_row_data (bool): Include row_id and row_etag. Defaults to False. + + Returns: + pandas.DataFrame: The queried table + """ + result = self.execute_sql_statement(query, include_row_data) + table = pandas.read_csv(result.filepath) + return table + + def execute_sql_statement( + self, statement: str, include_row_data: bool = False + ) -> synapseclient.table.CsvFileTable: + """Execute a SQL statement + + Args: + statement (str): A SQL statement that can be run by Synapse + include_row_data (bool): Include row_id and row_etag. Defaults to False. + + Returns: + synapseclient.table.CsvFileTable: The synapse table result from + the provided statement + """ + table = self.syn.tableQuery( + statement, includeRowIdAndRowVersion=include_row_data + ) + assert isinstance(table, synapseclient.table.CsvFileTable) + return table + + def upsert_table_rows(self, synapse_id: str, data: pandas.DataFrame) -> None: + """Upserts rows from the given table + + Args: + synapse_id (str): The Synapse ID fo the table to be upserted into + data (pandas.DataFrame): The table the rows will come from + """ + self.syn.store(synapseclient.Table(synapse_id, data)) + # Commented out until https://sagebionetworks.jira.com/browse/PLFM-8605 is resolved + # storage_result = self.syn.store(synapseclient.Table(synapse_id, data)) + # self.synapse_entity_tracker.add(synapse_id=storage_result.schema.id, entity=storage_result.schema) + self.synapse_entity_tracker.remove(synapse_id=synapse_id) + + def get_entity_annotations(self, synapse_id: str) -> synapseclient.Annotations: + """Gets the annotations for the Synapse entity + + Args: + synapse_id (str): The Synapse id of the entity + + Returns: + synapseclient.Annotations: The annotations of the Synapse entity in dict form. + """ + entity = self.synapse_entity_tracker.get( + synapse_id=synapse_id, syn=self.syn, download_file=False + ) + return synapseclient.Annotations( + id=entity.id, etag=entity.etag, values=entity.annotations + ) diff --git a/schematic/store/synapse.py b/schematic/store/synapse.py index 861789374..9993038b5 100644 --- a/schematic/store/synapse.py +++ b/schematic/store/synapse.py @@ -7,9 +7,10 @@ import re import secrets import shutil +import time import uuid # used to generate unique names for entities from copy import deepcopy -from dataclasses import asdict, dataclass +from dataclasses import dataclass, field from time import sleep # allows specifying explicit variable types @@ -20,7 +21,7 @@ import synapseclient import synapseutils from opentelemetry import trace -from schematic_db.rdb.synapse_database import SynapseDatabase +from synapseclient import Annotations as OldAnnotations from synapseclient import ( Column, Entity, @@ -33,14 +34,14 @@ Table, as_table_columns, ) -from synapseclient.api import get_entity_id_bundle2 +from synapseclient.annotations import _convert_to_annotations_list +from synapseclient.api import get_config_file, get_entity_id_bundle2 from synapseclient.core.constants.concrete_types import PROJECT_ENTITY from synapseclient.core.exceptions import ( SynapseAuthenticationError, SynapseHTTPError, SynapseUnmetAccessRestrictions, ) -from synapseclient.entity import File from synapseclient.models.annotations import Annotations from synapseclient.table import CsvFileTable, Schema, build_table from tenacity import ( @@ -55,6 +56,8 @@ from schematic.exceptions import AccessCredentialsError from schematic.schemas.data_model_graph import DataModelGraphExplorer from schematic.store.base import BaseStorage +from schematic.store.database.synapse_database import SynapseDatabase +from schematic.store.synapse_tracker import SynapseEntityTracker from schematic.utils.df_utils import col_in_dataframe, load_df, update_df # entity_type_mapping, get_dir_size, create_temp_folder, check_synapse_cache_size, and clear_synapse_cache functions are used for AWS deployment @@ -66,6 +69,7 @@ entity_type_mapping, get_dir_size, ) +from schematic.utils.io_utils import cleanup_temporary_storage from schematic.utils.schema_utils import get_class_label_from_display_name from schematic.utils.validate_utils import comma_separated_list_regex, rule_in_rule_list @@ -79,35 +83,98 @@ class ManifestDownload(object): """ syn: an object of type synapseclient. manifest_id: id of a manifest + synapse_entity_tracker: Tracker for a pull-through cache of Synapse entities """ syn: synapseclient.Synapse manifest_id: str + synapse_entity_tracker: SynapseEntityTracker = field( + default_factory=SynapseEntityTracker + ) - def _download_manifest_to_folder(self) -> File: + def _download_manifest_to_folder(self, use_temporary_folder: bool = True) -> File: """ - try downloading a manifest to local cache or a given folder - manifest + Try downloading a manifest to a specific folder (temporary or not). When the + `use_temporary_folder` is set to True, the manifest will be downloaded to a + temporary folder. This is useful for when the code is running as an API server + where multiple requests are being made at the same time. This will prevent + multiple requests from overwriting the same manifest file. When the + `use_temporary_folder` is set to False, the manifest will be downloaded to the + default manifest folder. + + Args: + use_temporary_folder: boolean argument indicating if a temporary folder + should be used to store the manifest file. This is useful when running + this code as an API server where multiple requests could be made at the + same time. This is set to False when the code is being used from the + CLI. Defaults to True. + Return: manifest_data: A Synapse file entity of the downloaded manifest """ + manifest_data = self.synapse_entity_tracker.get( + synapse_id=self.manifest_id, + syn=self.syn, + download_file=False, + retrieve_if_not_present=False, + ) + + if manifest_data and manifest_data.path: + return manifest_data + if "SECRETS_MANAGER_SECRETS" in os.environ: temporary_manifest_storage = "/var/tmp/temp_manifest_download" - # clear out all the existing manifests - if os.path.exists(temporary_manifest_storage): - shutil.rmtree(temporary_manifest_storage) + cleanup_temporary_storage( + temporary_manifest_storage, time_delta_seconds=3600 + ) # create a new directory to store manifest if not os.path.exists(temporary_manifest_storage): os.mkdir(temporary_manifest_storage) # create temporary folders for storing manifests - download_location = create_temp_folder(temporary_manifest_storage) + download_location = create_temp_folder( + path=temporary_manifest_storage, + prefix=f"{self.manifest_id}-{time.time()}-", + ) else: - download_location = CONFIG.manifest_folder - manifest_data = self.syn.get( - self.manifest_id, - downloadLocation=download_location, - ifcollision="overwrite.local", + if use_temporary_folder: + download_location = create_temp_folder( + path=CONFIG.manifest_folder, + prefix=f"{self.manifest_id}-{time.time()}-", + ) + else: + download_location = CONFIG.manifest_folder + + manifest_data = self.synapse_entity_tracker.get( + synapse_id=self.manifest_id, + syn=self.syn, + download_file=True, + retrieve_if_not_present=True, + download_location=download_location, ) + + # This is doing a rename of the downloaded file. The reason this is important + # is that if we are re-using a file that was previously downloaded, but the + # file had been renamed. The file downloaded from the Synapse client is just + # a direct copy of that renamed file. This code will set the name of the file + # to the original name that was used to download the file. Note: An MD5 checksum + # of the file will still be performed so if the file has changed, it will be + # downloaded again. + filename = manifest_data._file_handle.fileName + if filename != os.path.basename(manifest_data.path): + parent_folder = os.path.dirname(manifest_data.path) + manifest_original_name_and_path = os.path.join(parent_folder, filename) + + self.syn.cache.remove( + file_handle_id=manifest_data.dataFileHandleId, path=manifest_data.path + ) + os.rename(manifest_data.path, manifest_original_name_and_path) + manifest_data.path = manifest_original_name_and_path + self.syn.cache.add( + file_handle_id=manifest_data.dataFileHandleId, + path=manifest_original_name_and_path, + md5=manifest_data._file_handle.contentMd5, + ) + return manifest_data def _entity_type_checking(self) -> str: @@ -117,15 +184,21 @@ def _entity_type_checking(self) -> str: if the entity type is wrong, raise an error """ # check the type of entity - entity_type = entity_type_mapping(self.syn, self.manifest_id) + entity_type = entity_type_mapping( + syn=self.syn, + entity_id=self.manifest_id, + synapse_entity_tracker=self.synapse_entity_tracker, + ) if entity_type != "file": logger.error( f"You are using entity type: {entity_type}. Please provide a file ID" ) - @staticmethod def download_manifest( - self, newManifestName: str = "", manifest_df: pd.DataFrame = pd.DataFrame() + self, + newManifestName: str = "", + manifest_df: pd.DataFrame = pd.DataFrame(), + use_temporary_folder: bool = True, ) -> Union[str, File]: """ Download a manifest based on a given manifest id. @@ -145,7 +218,9 @@ def download_manifest( # download a manifest try: - manifest_data = self._download_manifest_to_folder() + manifest_data = self._download_manifest_to_folder( + use_temporary_folder=use_temporary_folder + ) except (SynapseUnmetAccessRestrictions, SynapseAuthenticationError): # if there's an error getting an uncensored manifest, try getting the censored manifest if not manifest_df.empty: @@ -154,7 +229,9 @@ def download_manifest( new_manifest_id = manifest_df[censored]["id"][0] self.manifest_id = new_manifest_id try: - manifest_data = self._download_manifest_to_folder() + manifest_data = self._download_manifest_to_folder( + use_temporary_folder=use_temporary_folder + ) except ( SynapseUnmetAccessRestrictions, SynapseAuthenticationError, @@ -175,12 +252,25 @@ def download_manifest( parent_folder = os.path.dirname(manifest_data.get("path")) new_manifest_path_name = os.path.join(parent_folder, new_manifest_filename) - os.rename(manifest_data["path"], new_manifest_path_name) + + # Copy file to new location. The purpose of using a copy instead of a rename + # is to avoid any potential issues with the file being used in another + # process. This avoids any potential race or code cocurrency conditions. + shutil.copyfile(src=manifest_data["path"], dst=new_manifest_path_name) + + # Adding this to cache will allow us to re-use the already downloaded + # manifest file for up to 1 hour. + self.syn.cache.add( + file_handle_id=manifest_data.dataFileHandleId, + path=new_manifest_path_name, + md5=manifest_data._file_handle.contentMd5, + ) # Update file names/paths in manifest_data manifest_data["name"] = new_manifest_filename manifest_data["filename"] = new_manifest_filename manifest_data["path"] = new_manifest_path_name + return manifest_data @@ -223,9 +313,15 @@ def __init__( self.storageFileview = CONFIG.synapse_master_fileview_id self.manifest = CONFIG.synapse_manifest_basename self.root_synapse_cache = self.syn.cache.cache_root_dir + self.synapse_entity_tracker = SynapseEntityTracker() if perform_query: self.query_fileview(columns=columns, where_clauses=where_clauses) + # TODO: When moving this over to a regular cron-job the following logic should be + # out of `manifest_download`: + # if "SECRETS_MANAGER_SECRETS" in os.environ: + # temporary_manifest_storage = "/var/tmp/temp_manifest_download" + # cleanup_temporary_storage(temporary_manifest_storage, time_delta_seconds=3600) @tracer.start_as_current_span("SynapseStorage::_purge_synapse_cache") def _purge_synapse_cache( self, maximum_storage_allowed_cache_gb: int = 1, minute_buffer: int = 15 @@ -273,9 +369,6 @@ def query_fileview( """ self._purge_synapse_cache() - self.storageFileview = CONFIG.synapse_master_fileview_id - self.manifest = CONFIG.synapse_manifest_basename - # Initialize to assume that the new fileview query will be different from what may already be stored. Initializes to True because generally one will not have already been performed self.new_query_different = True @@ -348,6 +441,7 @@ def _build_query( return @staticmethod + @tracer.start_as_current_span("SynapseStorage::login") def login( synapse_cache_path: Optional[str] = None, access_token: Optional[str] = None, @@ -371,7 +465,12 @@ def login( # login using a token if access_token: try: - syn = synapseclient.Synapse(cache_root_dir=synapse_cache_path) + syn = synapseclient.Synapse( + cache_root_dir=synapse_cache_path, + debug=False, + skip_checks=True, + cache_client=False, + ) syn.login(authToken=access_token, silent=True) except SynapseHTTPError as exc: raise ValueError( @@ -382,6 +481,9 @@ def login( syn = synapseclient.Synapse( configPath=CONFIG.synapse_configuration_path, cache_root_dir=synapse_cache_path, + debug=False, + skip_checks=True, + cache_client=False, ) syn.login(silent=True) return syn @@ -462,23 +564,20 @@ def getStorageProjects(self, project_scope: List = None) -> list[tuple[str, str] storageProjects = self.storageFileviewTable["projectId"].unique() # get the set of storage Synapse project accessible for this user - - # get current user name and user ID - currentUser = self.syn.getUserProfile() - currentUserName = currentUser.userName - currentUserId = currentUser.ownerId - # get a list of projects from Synapse - currentUserProjects = self.getPaginatedRestResults(currentUserId) - - # prune results json filtering project id - currentUserProjects = [ - currentUserProject.get("id") - for currentUserProject in currentUserProjects["results"] - ] + current_user_project_headers = self.synapse_entity_tracker.get_project_headers( + current_user_id=self.syn.credentials.owner_id, syn=self.syn + ) + project_id_to_name_dict = {} + current_user_projects = [] + for project_header in current_user_project_headers: + project_id_to_name_dict[project_header.get("id")] = project_header.get( + "name" + ) + current_user_projects.append(project_header.get("id")) # find set of user projects that are also in this pipeline's storage projects set - storageProjects = list(set(storageProjects) & set(currentUserProjects)) + storageProjects = list(set(storageProjects) & set(current_user_projects)) # Limit projects to scope if specified if project_scope: @@ -492,8 +591,8 @@ def getStorageProjects(self, project_scope: List = None) -> list[tuple[str, str] # prepare a return list of project IDs and names projects = [] for projectId in storageProjects: - projectName = self.syn.get(projectId, downloadFile=False).name - projects.append((projectId, projectName)) + project_name_from_project_header = project_id_to_name_dict.get(projectId) + projects.append((projectId, project_name_from_project_header)) sorted_projects_list = sorted(projects, key=lambda tup: tup[0]) @@ -513,13 +612,11 @@ def getStorageDatasetsInProject(self, projectId: str) -> list[tuple[str, str]]: # select all folders and fetch their names from within the storage project; # if folder content type is defined, only select folders that contain datasets - areDatasets = False if "contentType" in self.storageFileviewTable.columns: foldersTable = self.storageFileviewTable[ (self.storageFileviewTable["contentType"] == "dataset") & (self.storageFileviewTable["projectId"] == projectId) ] - areDatasets = True else: foldersTable = self.storageFileviewTable[ (self.storageFileviewTable["type"] == "folder") @@ -568,7 +665,9 @@ def getFilesInStorageDataset( self.syn, datasetId, includeTypes=["folder", "file"] ) - current_entity_location = self.syn.get(entity=datasetId, downloadFile=False) + current_entity_location = self.synapse_entity_tracker.get( + synapse_id=datasetId, syn=self.syn, download_file=False + ) def walk_back_to_project( current_location: Entity, location_prefix: str, skip_entry: bool @@ -605,8 +704,13 @@ def walk_back_to_project( and current_location["concreteType"] == PROJECT_ENTITY ): return updated_prefix + current_location = self.synapse_entity_tracker.get( + synapse_id=current_location["parentId"], + syn=self.syn, + download_file=False, + ) return walk_back_to_project( - current_location=self.syn.get(entity=current_location["parentId"]), + current_location=current_location, location_prefix=updated_prefix, skip_entry=False, ) @@ -617,8 +721,11 @@ def walk_back_to_project( skip_entry=True, ) - project = self.getDatasetProject(datasetId) - project_name = self.syn.get(project, downloadFile=False).name + project_id = self.getDatasetProject(datasetId) + project = self.synapse_entity_tracker.get( + synapse_id=project_id, syn=self.syn, download_file=False + ) + project_name = project.name file_list = [] # iterate over all results @@ -685,6 +792,7 @@ def getDatasetManifest( datasetId: str, downloadFile: bool = False, newManifestName: str = "", + use_temporary_folder: bool = True, ) -> Union[str, File]: """Gets the manifest associated with a given dataset. @@ -692,6 +800,11 @@ def getDatasetManifest( datasetId: synapse ID of a storage dataset. downloadFile: boolean argument indicating if manifest file in dataset should be downloaded or not. newManifestName: new name of a manifest that gets downloaded + use_temporary_folder: boolean argument indicating if a temporary folder + should be used to store the manifest file. This is useful when running + this code as an API server where multiple requests could be made at the + same time. This is set to False when the code is being used from the + CLI. Defaults to True. Returns: manifest_syn_id (String): Synapse ID of exisiting manifest file. @@ -726,9 +839,15 @@ def getDatasetManifest( else: manifest_syn_id = self._get_manifest_id(manifest) if downloadFile: - md = ManifestDownload(self.syn, manifest_id=manifest_syn_id) - manifest_data = ManifestDownload.download_manifest( - md, newManifestName=newManifestName, manifest_df=manifest + md = ManifestDownload( + self.syn, + manifest_id=manifest_syn_id, + synapse_entity_tracker=self.synapse_entity_tracker, + ) + manifest_data = md.download_manifest( + newManifestName=newManifestName, + manifest_df=manifest, + use_temporary_folder=use_temporary_folder, ) # TO DO: revisit how downstream code handle manifest_data. If the downstream code would break when manifest_data is an empty string, # then we should catch the error here without returning an empty string. @@ -745,7 +864,10 @@ def getDataTypeFromManifest(self, manifestId: str): manifestId: synapse ID of a manifest """ # get manifest file path - manifest_filepath = self.syn.get(manifestId).path + manifest_entity = self.synapse_entity_tracker.get( + synapse_id=manifestId, syn=self.syn, download_file=True + ) + manifest_filepath = manifest_entity.path # load manifest dataframe manifest = load_df( @@ -923,7 +1045,11 @@ def updateDatasetManifestFiles( if not manifest_id: return None - manifest_filepath = self.syn.get(manifest_id).path + manifest_entity = self.synapse_entity_tracker.get( + synapse_id=manifest_id, syn=self.syn, download_file=True + ) + manifest_filepath = manifest_entity.path + manifest = load_df(manifest_filepath) manifest_is_file_based = "Filename" in manifest.columns @@ -1024,7 +1150,9 @@ def getProjectManifests( # If manifest has annotations specifying component, use that if annotations and "Component" in annotations: component = annotations["Component"] - entity = self.syn.get(manifestId, downloadFile=False) + entity = self.synapse_entity_tracker.get( + synapse_id=manifestId, syn=self.syn, download_file=False + ) manifest_name = entity["properties"]["name"] # otherwise download the manifest and parse for information @@ -1174,7 +1302,7 @@ def upload_annotated_project_manifests_to_synapse( ("", ""), ) if not dry_run: - manifest_syn_id = self.associateMetadataWithFiles( + self.associateMetadataWithFiles( dmge, manifest_path, datasetId, manifest_record_type="table" ) manifest_loaded.append(manifest) @@ -1226,7 +1354,10 @@ def move_entities_to_new_project( if returnEntities: for entityId in annotation_entities: if not dry_run: - self.syn.move(entityId, datasetId) + moved_entity = self.syn.move(entityId, datasetId) + self.synapse_entity_tracker.add( + synapse_id=moved_entity.id, entity=moved_entity + ) else: logging.info( f"{entityId} will be moved to folder {datasetId}." @@ -1237,6 +1368,10 @@ def move_entities_to_new_project( projectId + "_archive", parent=newProjectId ) archive_project_folder = self.syn.store(archive_project_folder) + self.synapse_entity_tracker.add( + synapse_id=archive_project_folder.id, + entity=archive_project_folder, + ) # generate dataset folder dataset_archive_folder = Folder( @@ -1244,11 +1379,20 @@ def move_entities_to_new_project( parent=archive_project_folder.id, ) dataset_archive_folder = self.syn.store(dataset_archive_folder) + self.synapse_entity_tracker.add( + synapse_id=dataset_archive_folder.id, + entity=dataset_archive_folder, + ) for entityId in annotation_entities: # move entities to folder if not dry_run: - self.syn.move(entityId, dataset_archive_folder.id) + moved_entity = self.syn.move( + entityId, dataset_archive_folder.id + ) + self.synapse_entity_tracker.add( + synapse_id=moved_entity.id, entity=moved_entity + ) else: logging.info( f"{entityId} will be moved to folder {dataset_archive_folder.id}." @@ -1272,27 +1416,6 @@ def get_synapse_table(self, synapse_id: str) -> Tuple[pd.DataFrame, CsvFileTable return df, results - @tracer.start_as_current_span("SynapseStorage::_get_tables") - def _get_tables(self, datasetId: str = None, projectId: str = None) -> List[Table]: - if projectId: - project = projectId - elif datasetId: - project = self.syn.get(self.getDatasetProject(datasetId)) - - return list(self.syn.getChildren(project, includeTypes=["table"])) - - def get_table_info(self, datasetId: str = None, projectId: str = None) -> List[str]: - """Gets the names of the tables in the schema - Can pass in a synID for a dataset or project - Returns: - list[str]: A list of table names - """ - tables = self._get_tables(datasetId=datasetId, projectId=projectId) - if tables: - return {table["name"]: table["id"] for table in tables} - else: - return {None: None} - @missing_entity_handler @tracer.start_as_current_span("SynapseStorage::uploadDB") def uploadDB( @@ -1432,34 +1555,27 @@ def buildDB( manifest_table_id: synID of the uploaded table """ - table_info = self.get_table_info(datasetId=datasetId) - # Put table manifest onto synapse - schema = Schema( - name=table_name, - columns=col_schema, - parent=self.getDatasetProject(datasetId), + table_parent_id = self.getDatasetProject(datasetId=datasetId) + existing_table_id = self.syn.findEntityId( + name=table_name, parent=table_parent_id ) - if table_name in table_info: - existingTableId = table_info[table_name] - else: - existingTableId = None - tableOps = TableOperations( synStore=self, tableToLoad=table_manifest, tableName=table_name, datasetId=datasetId, - existingTableId=existingTableId, + existingTableId=existing_table_id, restrict=restrict, + synapse_entity_tracker=self.synapse_entity_tracker, ) - if not table_manipulation or table_name not in table_info.keys(): + if not table_manipulation or existing_table_id is None: manifest_table_id = tableOps.createTable( columnTypeDict=col_schema, specifySchema=True, ) - elif table_name in table_info.keys() and table_info[table_name]: + elif existing_table_id is not None: if table_manipulation.lower() == "replace": manifest_table_id = tableOps.replaceTable( specifySchema=True, @@ -1473,11 +1589,20 @@ def buildDB( manifest_table_id = tableOps.updateTable() if table_manipulation and table_manipulation.lower() == "upsert": - existing_tables = self.get_table_info(datasetId=datasetId) - tableId = existing_tables[table_name] - annos = self.syn.get_annotations(tableId) + table_entity = self.synapse_entity_tracker.get( + synapse_id=existing_table_id or manifest_table_id, + syn=self.syn, + download_file=False, + ) + annos = OldAnnotations( + id=table_entity.id, + etag=table_entity.etag, + values=table_entity.annotations, + ) annos["primary_key"] = table_manifest["Component"][0] + "_id" annos = self.syn.set_annotations(annos) + table_entity.etag = annos.etag + table_entity.annotations = annos return manifest_table_id @@ -1517,24 +1642,89 @@ def upload_manifest_file( + file_extension ) - manifestSynapseFile = File( - metadataManifestPath, - description="Manifest for dataset " + datasetId, - parent=datasetId, - name=file_name_new, + manifest_synapse_file = None + try: + # Rename the file to file_name_new then revert + # This is to maintain the original file name in-case other code is + # expecting that the file exists with the original name + original_file_path = metadataManifestPath + new_file_path = os.path.join( + os.path.dirname(metadataManifestPath), file_name_new + ) + os.rename(original_file_path, new_file_path) + + manifest_synapse_file = self._store_file_for_manifest_upload( + new_file_path=new_file_path, + dataset_id=datasetId, + existing_file_name=file_name_full, + file_name_new=file_name_new, + restrict_manifest=restrict_manifest, + ) + manifest_synapse_file_id = manifest_synapse_file.id + + finally: + # Revert the file name back to the original + os.rename(new_file_path, original_file_path) + + if manifest_synapse_file: + manifest_synapse_file.path = original_file_path + + return manifest_synapse_file_id + + def _store_file_for_manifest_upload( + self, + new_file_path: str, + dataset_id: str, + existing_file_name: str, + file_name_new: str, + restrict_manifest: bool, + ) -> File: + """Handles a create or update of a manifest file that is going to be uploaded. + If we already have a copy of the Entity in memory we will update that instance, + otherwise create a new File instance to be created in Synapse. Once stored + this will add the file to the `synapse_entity_tracker` for future reference. + + Args: + new_file_path (str): The path to the new manifest file + dataset_id (str): The Synapse ID of the dataset the manifest is associated with + existing_file_name (str): The name of the existing file + file_name_new (str): The name of the new file + restrict_manifest (bool): Whether the manifest should be restricted + + Returns: + File: The stored manifest file + """ + local_tracked_file_instance = ( + self.synapse_entity_tracker.search_local_by_parent_and_name( + name=existing_file_name, parent_id=dataset_id + ) + or self.synapse_entity_tracker.search_local_by_parent_and_name( + name=file_name_new, parent_id=dataset_id + ) ) - manifest_synapse_file_id = self.syn.store( - manifestSynapseFile, isRestricted=restrict_manifest - ).id - synapseutils.copy_functions.changeFileMetaData( - syn=self.syn, - entity=manifest_synapse_file_id, - downloadAs=file_name_new, - forceVersion=False, + if local_tracked_file_instance: + local_tracked_file_instance.path = new_file_path + local_tracked_file_instance.description = ( + "Manifest for dataset " + dataset_id + ) + manifest_synapse_file = local_tracked_file_instance + else: + manifest_synapse_file = File( + path=new_file_path, + description="Manifest for dataset " + dataset_id, + parent=dataset_id, + name=file_name_new, + ) + + manifest_synapse_file = self.syn.store( + manifest_synapse_file, isRestricted=restrict_manifest ) - return manifest_synapse_file_id + self.synapse_entity_tracker.add( + synapse_id=manifest_synapse_file.id, entity=manifest_synapse_file + ) + return manifest_synapse_file async def get_async_annotation(self, synapse_id: str) -> Dict[str, Any]: """get annotations asynchronously @@ -1569,7 +1759,19 @@ async def store_async_annotation(self, annotation_dict: dict) -> Annotations: etag=annotation_dict["annotations"]["etag"], id=annotation_dict["annotations"]["id"], ) - return await annotation_class.store_async(synapse_client=self.syn) + annotation_storage_result = await annotation_class.store_async( + synapse_client=self.syn + ) + local_entity = self.synapse_entity_tracker.get( + synapse_id=annotation_dict["annotations"]["id"], + syn=self.syn, + download_file=False, + retrieve_if_not_present=False, + ) + if local_entity: + local_entity.etag = annotation_storage_result.etag + local_entity.annotations = annotation_storage_result + return annotation_storage_result def process_row_annotations( self, @@ -1705,9 +1907,31 @@ async def format_row_annotations( v = v[0:472] + "[truncatedByDataCuratorApp]" metadataSyn[keySyn] = v - # set annotation(s) for the various objects/items in a dataset on Synapse - annos = await self.get_async_annotation(entityId) + # This will first check if the entity is already in memory, and if so, that + # instance is used. Unfortunately, the expected return format needs to match + # the Synapse API, so we need to convert the annotations to the expected format. + entity = self.synapse_entity_tracker.get( + synapse_id=entityId, + syn=self.syn, + download_file=False, + retrieve_if_not_present=False, + ) + if entity is not None: + synapse_annotations = _convert_to_annotations_list( + annotations=entity.annotations + ) + annos = { + "annotations": { + "id": entity.id, + "etag": entity.etag, + "annotations": synapse_annotations, + } + } + else: + annos = await self.get_async_annotation(entityId) + + # set annotation(s) for the various objects/items in a dataset on Synapse csv_list_regex = comma_separated_list_regex() annos = self.process_row_annotations( @@ -1729,7 +1953,9 @@ def format_manifest_annotations(self, manifest, manifest_synapse_id): For now just getting the Component. """ - entity = self.syn.get(manifest_synapse_id, downloadFile=False) + entity = self.synapse_entity_tracker.get( + synapse_id=manifest_synapse_id, syn=self.syn, download_file=False + ) is_file = entity.concreteType.endswith(".FileEntity") is_table = entity.concreteType.endswith(".TableEntity") @@ -1758,7 +1984,9 @@ def format_manifest_annotations(self, manifest, manifest_synapse_id): metadata = self.getTableAnnotations(manifest_synapse_id) # Get annotations - annos = self.syn.get_annotations(manifest_synapse_id) + annos = OldAnnotations( + id=entity.id, etag=entity.etag, values=entity.annotations + ) # Add metadata to the annotations for annos_k, annos_v in metadata.items(): @@ -1949,6 +2177,7 @@ def _create_entity_id(self, idx, row, manifest, datasetId): rowEntity = Folder(str(uuid.uuid4()), parent=datasetId) rowEntity = self.syn.store(rowEntity) entityId = rowEntity["id"] + self.synapse_entity_tracker.add(synapse_id=entityId, entity=rowEntity) row["entityId"] = entityId manifest.loc[idx, "entityId"] = entityId return manifest, entityId @@ -1973,18 +2202,11 @@ async def _process_store_annos(self, requests: Set[asyncio.Task]) -> None: annos = completed_task.result() if isinstance(annos, Annotations): - annos_dict = asdict(annos) - normalized_annos = {k.lower(): v for k, v in annos_dict.items()} - entity_id = normalized_annos["id"] - logger.info(f"Successfully stored annotations for {entity_id}") + logger.info(f"Successfully stored annotations for {annos.id}") else: # store annotations if they are not None if annos: - normalized_annos = { - k.lower(): v - for k, v in annos["annotations"]["annotations"].items() - } - entity_id = normalized_annos["entityid"] + entity_id = annos["annotations"]["id"] logger.info( f"Obtained and processed annotations for {entity_id} entity" ) @@ -2132,22 +2354,28 @@ def upload_manifest_as_table( ) # Load manifest to synapse as a CSV File manifest_synapse_file_id = self.upload_manifest_file( - manifest, - metadataManifestPath, - datasetId, - restrict, + manifest=manifest, + metadataManifestPath=metadataManifestPath, + datasetId=datasetId, + restrict_manifest=restrict, component_name=component_name, ) # Set annotations for the file manifest. manifest_annotations = self.format_manifest_annotations( - manifest, manifest_synapse_file_id + manifest=manifest, manifest_synapse_id=manifest_synapse_file_id ) - self.syn.set_annotations(manifest_annotations) + annos = self.syn.set_annotations(annotations=manifest_annotations) + manifest_entity = self.synapse_entity_tracker.get( + synapse_id=manifest_synapse_file_id, syn=self.syn, download_file=False + ) + manifest_entity.annotations = annos + manifest_entity.etag = annos.etag + logger.info("Associated manifest file with dataset on Synapse.") # Update manifest Synapse table with new entity id column. - manifest_synapse_table_id, manifest, table_manifest = self.uploadDB( + manifest_synapse_table_id, manifest, _ = self.uploadDB( dmge=dmge, manifest=manifest, datasetId=datasetId, @@ -2159,9 +2387,17 @@ def upload_manifest_as_table( # Set annotations for the table manifest manifest_annotations = self.format_manifest_annotations( - manifest, manifest_synapse_table_id + manifest=manifest, manifest_synapse_id=manifest_synapse_table_id ) - self.syn.set_annotations(manifest_annotations) + annotations_manifest_table = self.syn.set_annotations( + annotations=manifest_annotations + ) + manifest_table_entity = self.synapse_entity_tracker.get( + synapse_id=manifest_synapse_table_id, syn=self.syn, download_file=False + ) + manifest_table_entity.annotations = annotations_manifest_table + manifest_table_entity.etag = annotations_manifest_table.etag + return manifest_synapse_file_id @tracer.start_as_current_span("SynapseStorage::upload_manifest_as_csv") @@ -2219,7 +2455,12 @@ def upload_manifest_as_csv( manifest_annotations = self.format_manifest_annotations( manifest, manifest_synapse_file_id ) - self.syn.set_annotations(manifest_annotations) + annos = self.syn.set_annotations(manifest_annotations) + manifest_entity = self.synapse_entity_tracker.get( + synapse_id=manifest_synapse_file_id, syn=self.syn, download_file=False + ) + manifest_entity.annotations = annos + manifest_entity.etag = annos.etag logger.info("Associated manifest file with dataset on Synapse.") @@ -2296,7 +2537,12 @@ def upload_manifest_combo( manifest_annotations = self.format_manifest_annotations( manifest, manifest_synapse_file_id ) - self.syn.set_annotations(manifest_annotations) + file_manifest_annoations = self.syn.set_annotations(manifest_annotations) + manifest_entity = self.synapse_entity_tracker.get( + synapse_id=manifest_synapse_file_id, syn=self.syn, download_file=False + ) + manifest_entity.annotations = file_manifest_annoations + manifest_entity.etag = file_manifest_annoations.etag logger.info("Associated manifest file with dataset on Synapse.") # Update manifest Synapse table with new entity id column. @@ -2314,7 +2560,12 @@ def upload_manifest_combo( manifest_annotations = self.format_manifest_annotations( manifest, manifest_synapse_table_id ) - self.syn.set_annotations(manifest_annotations) + table_manifest_annotations = self.syn.set_annotations(manifest_annotations) + manifest_entity = self.synapse_entity_tracker.get( + synapse_id=manifest_synapse_table_id, syn=self.syn, download_file=False + ) + manifest_entity.annotations = table_manifest_annotations + manifest_entity.etag = table_manifest_annotations.etag return manifest_synapse_file_id @tracer.start_as_current_span("SynapseStorage::associateMetadataWithFiles") @@ -2444,7 +2695,9 @@ def getTableAnnotations(self, table_id: str): dict: Annotations as comma-separated strings. """ try: - entity = self.syn.get(table_id, downloadFile=False) + entity = self.synapse_entity_tracker.get( + synapse_id=table_id, syn=self.syn, download_file=False + ) is_table = entity.concreteType.endswith(".TableEntity") annotations_raw = entity.annotations except SynapseHTTPError: @@ -2476,7 +2729,9 @@ def getFileAnnotations(self, fileId: str) -> Dict[str, str]: # Get entity metadata, including annotations try: - entity = self.syn.get(fileId, downloadFile=False) + entity = self.synapse_entity_tracker.get( + synapse_id=fileId, syn=self.syn, download_file=False + ) is_file = entity.concreteType.endswith(".FileEntity") is_folder = entity.concreteType.endswith(".Folder") annotations_raw = entity.annotations @@ -2641,7 +2896,9 @@ def getDatasetProject(self, datasetId: str) -> str: # Otherwise, check if already project itself try: - syn_object = self.syn.get(datasetId) + syn_object = self.synapse_entity_tracker.get( + synapse_id=datasetId, syn=self.syn, download_file=False + ) if syn_object.properties["concreteType"].endswith("Project"): return datasetId except SynapseHTTPError: @@ -2717,6 +2974,7 @@ def __init__( datasetId: str = None, existingTableId: str = None, restrict: bool = False, + synapse_entity_tracker: SynapseEntityTracker = None, ): """ Class governing table operations (creation, replacement, upserts, updates) in schematic @@ -2726,6 +2984,7 @@ def __init__( datasetId: synID of the dataset for the manifest existingTableId: synId of the table currently exising on synapse (if there is one) restrict: bool, whether or not the manifest contains sensitive data that will need additional access restrictions + synapse_entity_tracker: Tracker for a pull-through cache of Synapse entities """ self.synStore = synStore @@ -2734,6 +2993,7 @@ def __init__( self.datasetId = datasetId self.existingTableId = existingTableId self.restrict = restrict + self.synapse_entity_tracker = synapse_entity_tracker or SynapseEntityTracker() @tracer.start_as_current_span("TableOperations::createTable") def createTable( @@ -2751,8 +3011,9 @@ def createTable( Returns: table.schema.id: synID of the newly created table """ - - datasetEntity = self.synStore.syn.get(self.datasetId, downloadFile=False) + datasetEntity = self.synapse_entity_tracker.get( + synapse_id=self.datasetId, syn=self.synStore.syn, download_file=False + ) datasetName = datasetEntity.name table_schema_by_cname = self.synStore._get_table_schema_by_cname(columnTypeDict) @@ -2796,12 +3057,18 @@ def createTable( ) table = Table(schema, self.tableToLoad) table = self.synStore.syn.store(table, isRestricted=self.restrict) + # Commented out until https://sagebionetworks.jira.com/browse/PLFM-8605 is resolved + # self.synapse_entity_tracker.add(synapse_id=table.schema.id, entity=table.schema) + self.synapse_entity_tracker.remove(synapse_id=table.schema.id) return table.schema.id else: # For just uploading the tables to synapse using default # column types. table = build_table(self.tableName, datasetParentProject, self.tableToLoad) table = self.synStore.syn.store(table, isRestricted=self.restrict) + # Commented out until https://sagebionetworks.jira.com/browse/PLFM-8605 is resolved + # self.synapse_entity_tracker.add(synapse_id=table.schema.id, entity=table.schema) + self.synapse_entity_tracker.remove(synapse_id=table.schema.id) return table.schema.id @tracer.start_as_current_span("TableOperations::replaceTable") @@ -2820,7 +3087,10 @@ def replaceTable( Returns: existingTableId: synID of the already existing table that had its metadata replaced """ - datasetEntity = self.synStore.syn.get(self.datasetId, downloadFile=False) + datasetEntity = self.synapse_entity_tracker.get( + synapse_id=self.datasetId, syn=self.synStore.syn, download_file=False + ) + datasetName = datasetEntity.name table_schema_by_cname = self.synStore._get_table_schema_by_cname(columnTypeDict) existing_table, existing_results = self.synStore.get_synapse_table( @@ -2828,11 +3098,16 @@ def replaceTable( ) # remove rows self.synStore.syn.delete(existing_results) + # Data changes such as removing all rows causes the eTag to change. + self.synapse_entity_tracker.remove(synapse_id=self.existingTableId) # wait for row deletion to finish on synapse before getting empty table sleep(10) # removes all current columns - current_table = self.synStore.syn.get(self.existingTableId) + current_table = self.synapse_entity_tracker.get( + synapse_id=self.existingTableId, syn=self.synStore.syn, download_file=False + ) + current_columns = self.synStore.syn.getTableColumns(current_table) for col in current_columns: current_table.removeColumn(col) @@ -2880,7 +3155,12 @@ def replaceTable( # adds new columns to schema for col in cols: current_table.addColumn(col) - self.synStore.syn.store(current_table, isRestricted=self.restrict) + table_result = self.synStore.syn.store( + current_table, isRestricted=self.restrict + ) + # Commented out until https://sagebionetworks.jira.com/browse/PLFM-8605 is resolved + # self.synapse_entity_tracker.add(synapse_id=table_result.schema.id, entity=table_result.schema) + self.synapse_entity_tracker.remove(synapse_id=table_result.id) # wait for synapse store to finish sleep(1) @@ -2892,6 +3172,9 @@ def replaceTable( schema.id = self.existingTableId table = Table(schema, self.tableToLoad, etag=existing_results.etag) table = self.synStore.syn.store(table, isRestricted=self.restrict) + # Commented out until https://sagebionetworks.jira.com/browse/PLFM-8605 is resolved + # self.synapse_entity_tracker.add(synapse_id=table.schema.id, entity=table.schema) + self.synapse_entity_tracker.remove(synapse_id=table.schema.id) else: logging.error("Must specify a schema for table replacements") @@ -2929,7 +3212,7 @@ def _get_auth_token( # Try getting creds from .synapseConfig file if it exists # Primarily useful for local users. Seems to correlate with credentials stored in synaspe object when logged in if os.path.exists(CONFIG.synapse_configuration_path): - config = self.synStore.syn.getConfigFile(CONFIG.synapse_configuration_path) + config = get_config_file(CONFIG.synapse_configuration_path) # check which credentials are provided in file if config.has_option("authentication", "authtoken"): @@ -2964,6 +3247,8 @@ def upsertTable(self, dmge: DataModelGraphExplorer): synapseDB = SynapseDatabase( auth_token=authtoken, project_id=self.synStore.getDatasetProject(self.datasetId), + syn=self.synStore.syn, + synapse_entity_tracker=self.synapse_entity_tracker, ) try: @@ -3000,7 +3285,10 @@ def _update_table_uuid_column( """ # Get the columns of the schema - schema = self.synStore.syn.get(self.existingTableId) + schema = self.synapse_entity_tracker.get( + synapse_id=self.existingTableId, syn=self.synStore.syn, download_file=False + ) + cols = self.synStore.syn.getTableColumns(schema) # Iterate through columns until `Uuid` column is found @@ -3017,6 +3305,9 @@ def _update_table_uuid_column( new_col = Column(columnType="STRING", maximumSize=64, name="Id") schema.addColumn(new_col) schema = self.synStore.syn.store(schema) + # self.synapse_entity_tracker.add(synapse_id=schema.id, entity=schema) + # Commented out until https://sagebionetworks.jira.com/browse/PLFM-8605 is resolved + self.synapse_entity_tracker.remove(synapse_id=schema.id) # If there is not, then use the old `Uuid` column as a basis for the new `Id` column else: # Build ColumnModel that will be used for new column @@ -3070,10 +3361,15 @@ def updateTable( self.tableToLoad = update_df(existing_table, self.tableToLoad, update_col) # store table with existing etag data and impose restrictions as appropriate - self.synStore.syn.store( + table_result = self.synStore.syn.store( Table(self.existingTableId, self.tableToLoad, etag=existing_results.etag), isRestricted=self.restrict, ) + # We cannot store the Table to the `synapse_entity_tracker` because there is + # not `Schema` on the table object. The above `.store()` function call would + # also update the ETag of the entity within Synapse. Remove it from the tracker + # and re-retrieve it later on if needed again. + self.synapse_entity_tracker.remove(synapse_id=table_result.tableId) return self.existingTableId diff --git a/schematic/store/synapse_tracker.py b/schematic/store/synapse_tracker.py new file mode 100644 index 000000000..6d163c1e0 --- /dev/null +++ b/schematic/store/synapse_tracker.py @@ -0,0 +1,147 @@ +"""This script is responsible for creating a 'pull through cache' class that can be +added through composition to any class where Synapse entities might be used. The idea +behind this class is to provide a mechanism such that if a Synapse entity is requested +multiple times, the entity is only downloaded once. This is useful for preventing +multiple downloads of the same entity, which can be time consuming.""" +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Union + +import synapseclient +from synapseclient import Entity, File, Folder, Project, Schema + + +@dataclass +class SynapseEntityTracker: + """The SynapseEntityTracker class handles tracking synapse entities throughout the + lifecycle of a request to schematic. It is used to prevent multiple downloads of + the same entity.""" + + synapse_entities: Dict[str, Union[Entity, Project, File, Folder, Schema]] = field( + default_factory=dict + ) + project_headers: Dict[str, List[Dict[str, str]]] = field(default_factory=dict) + """A dictionary of project headers for each user requested.""" + + def get( + self, + synapse_id: str, + syn: synapseclient.Synapse, + download_file: bool = False, + retrieve_if_not_present: bool = True, + download_location: str = None, + if_collision: str = None, + ) -> Optional[Union[Entity, Project, File, Folder, Schema]]: + """Retrieves a Synapse entity from the cache if it exists, otherwise downloads + the entity from Synapse and adds it to the cache. + + Args: + synapse_id: The Synapse ID of the entity to retrieve. + syn: A Synapse object. + download_file: If True, download the file. + retrieve_if_not_present: If True, retrieve the entity if it is not present + in the cache. If not found in the cache, and this is False, return None. + download_location: The location to download the file to. + if_collision: The action to take if there is a collision when downloading + the file. May be "overwrite.local", "keep.local", or "keep.both". A + collision occurs when a file with the same name already exists at the + download location. + + Returns: + The Synapse entity if found. When retrieve_if_not_present is False and the + entity is not found in the local cache, returns None. If + retrieve_if_not_present is True and the entity is not found in the local + cache, retrieve the entity from Synapse and add it to the cache. + """ + entity = self.synapse_entities.get(synapse_id, None) + + if entity is None or (download_file and not entity.path): + if not retrieve_if_not_present: + return None + entity = syn.get( + synapse_id, + downloadFile=download_file, + downloadLocation=download_location, + ifcollision=if_collision, + ) + self.synapse_entities.update({synapse_id: entity}) + return entity + + def add( + self, synapse_id: str, entity: Union[Entity, Project, File, Folder, Schema] + ) -> None: + """Adds a Synapse entity to the cache. + + Args: + synapse_id: The Synapse ID of the entity to add. + entity: The Synapse entity to add. + """ + self.synapse_entities.update({synapse_id: entity}) + + def remove(self, synapse_id: str) -> None: + """Removes a Synapse entity from the cache. + + Args: + synapse_id: The Synapse ID of the entity to remove. + """ + self.synapse_entities.pop(synapse_id, None) + + def search_local_by_parent_and_name( + self, name: str, parent_id: str + ) -> Union[Entity, Project, File, Folder, Schema, None]: + """ + Searches the local cache for an entity with the given name and parent_id. The + is useful in situations where we might not have the ID of the resource, but we + do have the name and parent ID. + + Args: + name: The name of the entity to search for. + parent_id: The parent ID of the entity to search for. + + Returns: + The entity if it exists, otherwise None. + """ + for entity in self.synapse_entities.values(): + if entity.name == name and entity.parentId == parent_id: + return entity + return None + + def get_project_headers( + self, syn: synapseclient.Synapse, current_user_id: str + ) -> List[Dict[str, str]]: + """Gets the paginated results of the REST call to Synapse to check what projects the current user has access to. + + Args: + syn: A Synapse object + current_user_id: profile id for the user whose projects we want to get. + + Returns: + A list of dictionaries matching + """ + project_headers = self.project_headers.get(current_user_id, None) + if project_headers: + return project_headers + + all_results = syn.restGET( + "/projects/user/{principalId}".format(principalId=current_user_id) + ) + + while ( + "nextPageToken" in all_results + ): # iterate over next page token in results while there is any + results_token = syn.restGET( + "/projects/user/{principalId}?nextPageToken={nextPageToken}".format( + principalId=current_user_id, + nextPageToken=all_results["nextPageToken"], + ) + ) + all_results["results"].extend(results_token["results"]) + + if "nextPageToken" in results_token: + all_results["nextPageToken"] = results_token["nextPageToken"] + else: + del all_results["nextPageToken"] + + results = all_results["results"] + self.project_headers.update({current_user_id: results}) + + return results diff --git a/schematic/utils/general.py b/schematic/utils/general.py index 974805043..0bb932aa3 100644 --- a/schematic/utils/general.py +++ b/schematic/utils/general.py @@ -10,13 +10,15 @@ from cProfile import Profile from datetime import datetime, timedelta from functools import wraps -from typing import Union, TypeVar, Any, Optional, Sequence, Callable +from typing import Any, Callable, Optional, Sequence, TypeVar, Union +from synapseclient import Synapse # type: ignore +from synapseclient.core import cache # type: ignore from synapseclient.core.exceptions import SynapseHTTPError # type: ignore from synapseclient.entity import File, Folder, Project # type: ignore from synapseclient.table import EntityViewSchema # type: ignore -from synapseclient.core import cache # type: ignore -from synapseclient import Synapse # type: ignore + +from schematic.store.synapse_tracker import SynapseEntityTracker logger = logging.getLogger(__name__) @@ -180,12 +182,17 @@ def clear_synapse_cache(synapse_cache: cache.Cache, minutes: int) -> int: return num_of_deleted_files -def entity_type_mapping(syn: Synapse, entity_id: str) -> str: +def entity_type_mapping( + syn: Synapse, + entity_id: str, + synapse_entity_tracker: Optional[SynapseEntityTracker] = None, +) -> str: """Return the entity type of manifest Args: syn (Synapse): Synapse object entity_id (str): id of an entity + synapse_entity_tracker: Tracker for a pull-through cache of Synapse entities Raises: SynapseHTTPError: Re-raised SynapseHTTPError @@ -195,7 +202,11 @@ def entity_type_mapping(syn: Synapse, entity_id: str) -> str: """ # check the type of entity try: - entity = syn.get(entity_id, downloadFile=False) + if not synapse_entity_tracker: + synapse_entity_tracker = SynapseEntityTracker() + entity = synapse_entity_tracker.get( + synapse_id=entity_id, syn=syn, download_file=False + ) except SynapseHTTPError as exc: logger.error( f"cannot get {entity_id} from asset store. Please make sure that {entity_id} exists" @@ -213,19 +224,24 @@ def entity_type_mapping(syn: Synapse, entity_id: str) -> str: elif isinstance(entity, Project): entity_type = "project" else: + assert entity is not None # if there's no matching type, return concreteType entity_type = entity.concreteType return entity_type -def create_temp_folder(path: str) -> str: +def create_temp_folder(path: str, prefix: Optional[str] = None) -> str: """This function creates a temporary directory in the specified directory Args: path(str): a directory path where all the temporary files will live + prefix(str): a prefix to be added to the temporary directory name Returns: returns the absolute pathname of the new directory. """ + if not os.path.exists(path): + os.makedirs(path, exist_ok=True) + # Create a temporary directory in the specified directory - path = tempfile.mkdtemp(dir=path) + path = tempfile.mkdtemp(dir=path, prefix=prefix) return path diff --git a/schematic/utils/io_utils.py b/schematic/utils/io_utils.py index 1651d085e..a0bb9d241 100644 --- a/schematic/utils/io_utils.py +++ b/schematic/utils/io_utils.py @@ -1,9 +1,10 @@ """io utils""" -from typing import Any import json +import os +import time import urllib.request -from schematic import LOADER +from typing import Any def load_json(file_path: str) -> Any: @@ -31,6 +32,9 @@ def export_json(json_doc: Any, file_path: str) -> None: def load_default() -> Any: """Load biolink vocabulary""" data_path = "data_models/biothings.model.jsonld" + # Lazy import to avoid circular imports + from schematic import LOADER # pylint: disable=import-outside-toplevel + biothings_path = LOADER.filename(data_path) return load_json(biothings_path) @@ -38,5 +42,40 @@ def load_default() -> Any: def load_schemaorg() -> Any: """Load SchemaOrg vocabulary""" data_path = "data_models/schema_org.model.jsonld" + # Lazy import to avoid circular imports + from schematic import LOADER # pylint: disable=import-outside-toplevel + schema_org_path = LOADER.filename(data_path) return load_json(schema_org_path) + + +def cleanup_temporary_storage( + temporary_storage_directory: str, time_delta_seconds: int +) -> None: + """Handles cleanup of temporary storage directory. The usage of the + `time_delta_seconds` parameter is to prevent deleting files that are currently + being used by other requests. In production we will be deleting those files + which have not been modified for more than 1 hour. + + Args: + temporary_storage_directory: Path to the temporary storage directory. + time_delta_seconds: The time delta in seconds used to determine which files + should be deleted. + """ + if os.path.exists(temporary_storage_directory): + for root, all_dirs, files in os.walk( + temporary_storage_directory, topdown=False + ): + # Delete files older than the specified time delta + for file in files: + file_path = os.path.join(root, file) + if os.path.isfile(file_path) and os.path.getmtime(file_path) < ( + time.time() - time_delta_seconds + ): + os.remove(file_path) + + # Delete empty directories + for all_dir in all_dirs: + dir_path = os.path.join(root, all_dir) + if not os.listdir(dir_path): + os.rmdir(dir_path) diff --git a/schematic/utils/validate_utils.py b/schematic/utils/validate_utils.py index 5f50dfb02..faaf7e23a 100644 --- a/schematic/utils/validate_utils.py +++ b/schematic/utils/validate_utils.py @@ -2,16 +2,17 @@ # pylint: disable = anomalous-backslash-in-string +import logging import re from collections.abc import Mapping -import logging -from typing import Pattern, Union, Iterable, Any, Optional from numbers import Number -from jsonschema import validate +from typing import Any, Iterable, Optional, Pattern, Union + import numpy as np import pandas as pd +from jsonschema import validate + from schematic.utils.io_utils import load_json -from schematic import LOADER logger = logging.getLogger(__name__) @@ -19,6 +20,9 @@ def validate_schema(schema: Union[Mapping, bool]) -> None: """Validate schema against schema.org standard""" data_path = "validation_schemas/model.schema.json" + # Lazy import to avoid circular imports + from schematic import LOADER # pylint: disable=import-outside-toplevel + json_schema_path = LOADER.filename(data_path) json_schema = load_json(json_schema_path) return validate(schema, json_schema) @@ -27,6 +31,9 @@ def validate_schema(schema: Union[Mapping, bool]) -> None: def validate_property_schema(schema: Union[Mapping, bool]) -> None: """Validate schema against SchemaORG property definition standard""" data_path = "validation_schemas/property.schema.json" + # Lazy import to avoid circular imports + from schematic import LOADER # pylint: disable=import-outside-toplevel + json_schema_path = LOADER.filename(data_path) json_schema = load_json(json_schema_path) return validate(schema, json_schema) @@ -35,6 +42,9 @@ def validate_property_schema(schema: Union[Mapping, bool]) -> None: def validate_class_schema(schema: Union[Mapping, bool]) -> None: """Validate schema against SchemaORG class definition standard""" data_path = "validation_schemas/class.schema.json" + # Lazy import to avoid circular imports + from schematic import LOADER # pylint: disable=import-outside-toplevel + json_schema_path = LOADER.filename(data_path) json_schema = load_json(json_schema_path) return validate(schema, json_schema) diff --git a/schematic/visualization/attributes_explorer.py b/schematic/visualization/attributes_explorer.py index 9691932e7..668ea1374 100644 --- a/schematic/visualization/attributes_explorer.py +++ b/schematic/visualization/attributes_explorer.py @@ -3,14 +3,15 @@ import logging import os from typing import Optional, no_type_check + import numpy as np import pandas as pd -from schematic.schemas.data_model_parser import DataModelParser from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer from schematic.schemas.data_model_json_schema import DataModelJSONSchema -from schematic.utils.schema_utils import DisplayLabelType +from schematic.schemas.data_model_parser import DataModelParser from schematic.utils.io_utils import load_json +from schematic.utils.schema_utils import DisplayLabelType logger = logging.getLogger(__name__) @@ -40,6 +41,7 @@ def __init__( # Instantiate DataModelGraph if not data_model_grapher: + assert parsed_data_model is not None data_model_grapher = DataModelGraph(parsed_data_model, data_model_labels) # Generate graph diff --git a/schematic_api/api/__init__.py b/schematic_api/api/__init__.py index a65398ee5..299353935 100644 --- a/schematic_api/api/__init__.py +++ b/schematic_api/api/__init__.py @@ -1,30 +1,14 @@ import os +import traceback +from typing import Tuple import connexion -from typing import Tuple +from synapseclient.core.exceptions import SynapseAuthenticationError -import traceback -from synapseclient.core.exceptions import ( - SynapseAuthenticationError, -) from schematic.exceptions import AccessCredentialsError -from schematic import CONFIG -from jaeger_client import Config -from flask_opentracing import FlaskTracer -config = Config( - config={ - "enabled": True, - "sampler": {"type": "const", "param": 1}, - "logging": True, - }, - service_name="schema-api", -) -jaeger_tracer = config.initialize_tracer - - -def create_app(): +def create_app() -> None: connexionapp = connexion.FlaskApp(__name__, specification_dir="openapi/") connexionapp.add_api( "api.yaml", arguments={"title": "Schematic REST API"}, pythonic_params=True @@ -71,11 +55,6 @@ def handle_synapse_access_error(e: Exception) -> Tuple[str, int]: app = create_app() -flask_tracer = FlaskTracer( - jaeger_tracer, True, app, ["url", "url_rule", "environ.HTTP_X_REAL_IP", "path"] -) - - # def route_code(): # import flask_schematic as sc # sc.method1() diff --git a/schematic_api/api/routes.py b/schematic_api/api/routes.py index e977e480d..0cda1f4ac 100644 --- a/schematic_api/api/routes.py +++ b/schematic_api/api/routes.py @@ -4,9 +4,7 @@ import pickle import shutil import tempfile -import time import urllib.request -from functools import wraps from typing import List, Tuple import connexion @@ -15,16 +13,6 @@ from flask import request, send_from_directory from flask_cors import cross_origin from opentelemetry import trace -from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter -from opentelemetry.sdk.resources import SERVICE_NAME, Resource -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import ( - BatchSpanProcessor, - ConsoleSpanExporter, - SimpleSpanProcessor, - Span, -) -from opentelemetry.sdk.trace.sampling import ALWAYS_OFF from schematic.configuration.configuration import CONFIG from schematic.manifest.generator import ManifestGenerator @@ -32,7 +20,7 @@ from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer from schematic.schemas.data_model_parser import DataModelParser from schematic.store.synapse import ManifestDownload, SynapseStorage -from schematic.utils.general import entity_type_mapping +from schematic.utils.general import create_temp_folder, entity_type_mapping from schematic.utils.schema_utils import ( DisplayLabelType, get_property_label_from_display_name, @@ -43,77 +31,10 @@ logger = logging.getLogger(__name__) logging.basicConfig(level=logging.DEBUG) -tracing_service_name = os.environ.get("TRACING_SERVICE_NAME", "schematic-api") - -trace.set_tracer_provider( - TracerProvider(resource=Resource(attributes={SERVICE_NAME: tracing_service_name})) -) - - -# borrowed from: https://github.com/Sage-Bionetworks/synapsePythonClient/blob/develop/tests/integration/conftest.py -class FileSpanExporter(ConsoleSpanExporter): - """Create an exporter for OTEL data to a file.""" - - def __init__(self, file_path: str) -> None: - """Init with a path.""" - self.file_path = file_path - - def export(self, spans: List[Span]) -> None: - """Export the spans to the file.""" - with open(self.file_path, "a", encoding="utf-8") as f: - for span in spans: - span_json_one_line = span.to_json().replace("\n", "") + "\n" - f.write(span_json_one_line) - - -def set_up_tracing() -> None: - """Set up tracing for the API.""" - tracing_export = os.environ.get("TRACING_EXPORT_FORMAT", None) - if tracing_export == "otlp": - trace.get_tracer_provider().add_span_processor( - BatchSpanProcessor(OTLPSpanExporter()) - ) - elif tracing_export == "file": - timestamp_millis = int(time.time() * 1000) - file_name = f"otel_spans_integration_testing_{timestamp_millis}.ndjson" - file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), file_name) - processor = SimpleSpanProcessor(FileSpanExporter(file_path)) - trace.get_tracer_provider().add_span_processor(processor) - else: - trace.set_tracer_provider(TracerProvider(sampler=ALWAYS_OFF)) - -set_up_tracing() tracer = trace.get_tracer("Schematic") -def trace_function_params(): - """capture all the parameters of API requests""" - - def decorator(func): - """create a decorator""" - - @wraps(func) - def wrapper(*args, **kwargs): - """create a wrapper function. Any number of positional arguments and keyword arguments can be passed here.""" - tracer = trace.get_tracer(__name__) - # Start a new span with the function's name - with tracer.start_as_current_span(func.__name__) as span: - # Set values of parameters as tags - for i, arg in enumerate(args): - span.set_attribute(f"arg{i}", arg) - - for name, value in kwargs.items(): - span.set_attribute(name, value) - # Call the actual function - result = func(*args, **kwargs) - return result - - return wrapper - - return decorator - - def config_handler(asset_view: str = None): # check if path to config is provided path_to_config = app.config["SCHEMATIC_CONFIG"] @@ -182,7 +103,7 @@ def convert_df_to_csv(self, df, file_name): """ # convert dataframe to a temporary csv file - temp_dir = tempfile.gettempdir() + temp_dir = create_temp_folder(path=tempfile.gettempdir()) temp_path = os.path.join(temp_dir, file_name) df.to_csv(temp_path, encoding="utf-8", index=False) return temp_path @@ -271,7 +192,7 @@ def save_file(file_key="csv_file"): manifest_file = connexion.request.files[file_key] # save contents of incoming manifest CSV file to temp file - temp_dir = tempfile.gettempdir() + temp_dir = create_temp_folder(path=tempfile.gettempdir()) # path to temp file where manifest file contents will be saved temp_path = os.path.join(temp_dir, manifest_file.filename) # save content @@ -296,7 +217,9 @@ def get_temp_jsonld(schema_url): # retrieve a JSON-LD via URL and store it in a temporary location with urllib.request.urlopen(schema_url) as response: with tempfile.NamedTemporaryFile( - delete=False, suffix=".model.jsonld" + delete=False, + suffix=".model.jsonld", + dir=create_temp_folder(path=tempfile.gettempdir()), ) as tmp_file: shutil.copyfileobj(response, tmp_file) @@ -307,13 +230,18 @@ def get_temp_jsonld(schema_url): def get_temp_csv(schema_url): # retrieve a CSV via URL and store it in a temporary location with urllib.request.urlopen(schema_url) as response: - with tempfile.NamedTemporaryFile(delete=False, suffix=".model.csv") as tmp_file: + with tempfile.NamedTemporaryFile( + delete=False, + suffix=".model.csv", + dir=create_temp_folder(path=tempfile.gettempdir()), + ) as tmp_file: shutil.copyfileobj(response, tmp_file) # get path to temporary csv file return tmp_file.name +@tracer.start_as_current_span("routes::get_temp_model_path") def get_temp_model_path(schema_url): # Get model type: model_extension = pathlib.Path(schema_url).suffix.replace(".", "").upper() @@ -329,7 +257,6 @@ def get_temp_model_path(schema_url): # @before_request -@trace_function_params() def get_manifest_route( schema_url: str, use_annotations: bool, @@ -392,7 +319,6 @@ def get_manifest_route( return all_results -@trace_function_params() def validate_manifest_route( schema_url, data_type, @@ -451,7 +377,6 @@ def validate_manifest_route( # profile validate manifest route function -@trace_function_params() def submit_manifest_route( schema_url, data_model_labels: str, @@ -496,9 +421,6 @@ def submit_manifest_route( else: validate_component = data_type - # get path to temp data model file (csv or jsonld) as appropriate - data_model = get_temp_model_path(schema_url) - if not table_column_names: table_column_names = "class_label" @@ -638,7 +560,7 @@ def check_entity_type(entity_id): config_handler() syn = SynapseStorage.login(access_token=access_token) - entity_type = entity_type_mapping(syn, entity_id) + entity_type = entity_type_mapping(syn=syn, entity_id=entity_id) return entity_type @@ -738,7 +660,7 @@ def download_manifest(manifest_id, new_manifest_name="", as_json=True): syn = SynapseStorage.login(access_token=access_token) try: md = ManifestDownload(syn, manifest_id) - manifest_data = ManifestDownload.download_manifest(md, new_manifest_name) + manifest_data = md.download_manifest(newManifestName=new_manifest_name) # return local file path manifest_local_file_path = manifest_data["path"] except TypeError as e: diff --git a/tests/conftest.py b/tests/conftest.py index e6382ed40..3e07bbe86 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,14 +11,9 @@ from opentelemetry import trace from opentelemetry._logs import set_logger_provider from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter -from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler from opentelemetry.sdk._logs.export import BatchLogRecordProcessor -from opentelemetry.sdk.resources import SERVICE_NAME, Resource -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor -from opentelemetry.sdk.trace.sampling import ALWAYS_OFF -from pytest_asyncio import is_async_test +from opentelemetry.sdk.resources import Resource from schematic.configuration.configuration import CONFIG, Configuration from schematic.models.metadata import MetadataModel @@ -227,36 +222,11 @@ def cleanup_scheduled_items() -> None: return _append_cleanup -active_span_processors = [] - - -@pytest.fixture(scope="session", autouse=True) -def set_up_tracing() -> None: - """Set up tracing for the API.""" - tracing_export = os.environ.get("TRACING_EXPORT_FORMAT", None) - tracing_service_name = os.environ.get("TRACING_SERVICE_NAME", "schematic-tests") - if tracing_export == "otlp": - trace.set_tracer_provider( - TracerProvider( - resource=Resource(attributes={SERVICE_NAME: tracing_service_name}) - ) - ) - processor = BatchSpanProcessor(OTLPSpanExporter()) - active_span_processors.append(processor) - trace.get_tracer_provider().add_span_processor(processor) - else: - trace.set_tracer_provider(TracerProvider(sampler=ALWAYS_OFF)) - - @pytest.fixture(autouse=True, scope="function") def wrap_with_otel(request): """Start a new OTEL Span for each test function.""" with tracer.start_as_current_span(request.node.name): - try: - yield - finally: - for processor in active_span_processors: - processor.force_flush() + yield @pytest.fixture(scope="session", autouse=True) diff --git a/tests/integration/test_commands.py b/tests/integration/test_commands.py new file mode 100644 index 000000000..2968d3163 --- /dev/null +++ b/tests/integration/test_commands.py @@ -0,0 +1,97 @@ +import os +import uuid + +import pytest +from click.testing import CliRunner + +from schematic.configuration.configuration import Configuration +from schematic.manifest.commands import manifest + + +@pytest.fixture +def runner() -> CliRunner: + """Fixture for invoking command-line interfaces.""" + + return CliRunner() + + +class TestDownloadManifest: + """Tests the command line interface for downloading a manifest""" + + def test_download_manifest_found( + self, + runner: CliRunner, + config: Configuration, + ) -> None: + # GIVEN a manifest name to download as + manifest_name = f"{uuid.uuid4()}" + + # AND a dataset id + dataset_id = "syn23643250" + + # AND a configuration file + config.load_config("config_example.yml") + + # WHEN the download command is run + result = runner.invoke( + cli=manifest, + args=[ + "--config", + config.config_path, + "download", + "--new_manifest_name", + manifest_name, + "--dataset_id", + dataset_id, + ], + ) + + # THEN the command should run successfully + assert result.exit_code == 0 + + # AND the manifest file should be created + expected_manifest_file = os.path.join( + config.manifest_folder, f"{manifest_name}.csv" + ) + assert os.path.exists(expected_manifest_file) + try: + os.remove(expected_manifest_file) + except Exception: + pass + + def test_download_manifest_not_found( + self, + runner: CliRunner, + config: Configuration, + ) -> None: + # GIVEN a manifest name to download as + manifest_name = f"{uuid.uuid4()}" + + # AND a dataset id that does not exist + dataset_id = "syn1234" + + # AND a configuration file + config.load_config("config_example.yml") + + # WHEN the download command is run + result = runner.invoke( + cli=manifest, + args=[ + "--config", + config.config_path, + "download", + "--new_manifest_name", + manifest_name, + "--dataset_id", + dataset_id, + ], + ) + + # THEN the command should not run successfully + assert result.exit_code == 1 + + # AND the manifest file should not be created + expected_manifest_file = os.path.join( + config.manifest_folder, f"{manifest_name}.csv" + ) + assert not os.path.exists(expected_manifest_file) diff --git a/tests/integration/test_store_synapse.py b/tests/integration/test_store_synapse.py index 0bf23c1b1..085a48fc3 100644 --- a/tests/integration/test_store_synapse.py +++ b/tests/integration/test_store_synapse.py @@ -6,7 +6,6 @@ from schematic.schemas.data_model_graph import DataModelGraphExplorer from schematic.store.synapse import SynapseStorage from schematic.utils.validate_utils import comma_separated_list_regex -from tests.conftest import Helpers class TestStoreSynapse: diff --git a/tests/test_api.py b/tests/test_api.py index 0a27b5c73..76bfe8301 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -2,18 +2,22 @@ import logging import os import re +import uuid from math import ceil from time import perf_counter from typing import Dict, Generator, List, Tuple, Union +from unittest.mock import patch import flask import pandas as pd # third party library import import pytest from flask.testing import FlaskClient +from opentelemetry import trace from schematic.configuration.configuration import Configuration from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer from schematic.schemas.data_model_parser import DataModelParser +from schematic.utils.general import create_temp_folder from schematic_api.api import create_app logging.basicConfig(level=logging.INFO) @@ -120,15 +124,40 @@ def get_MockComponent_attribute() -> Generator[str, None, None]: yield MockComponent_attribute +def get_traceparent() -> str: + """Create and format the `traceparent` to used in the header of the request. This + is used by opentelemetry to attach the context that was started outside of the + flask server to the request. The purpose is so that we can propagate the trace + context across services.""" + current_span = trace.get_current_span() + span_context = current_span.get_span_context() + trace_id = format(span_context.trace_id, "032x") + span_id = format(span_context.span_id, "016x") + trace_flags = format(span_context.trace_flags, "02x") + + traceparent = f"00-{trace_id}-{span_id}-{trace_flags}" + + return traceparent + + @pytest.fixture def request_headers(syn_token: str) -> Dict[str, str]: - headers = {"Authorization": "Bearer " + syn_token} + headers = {"Authorization": "Bearer " + syn_token, "traceparent": get_traceparent()} + return headers + + +@pytest.fixture +def request_headers_trace() -> Dict[str, str]: + headers = {"traceparent": get_traceparent()} return headers @pytest.fixture def request_invalid_headers() -> Dict[str, str]: - headers = {"Authorization": "Bearer invalid headers"} + headers = { + "Authorization": "Bearer invalid headers", + "traceparent": get_traceparent(), + } return headers @@ -338,7 +367,9 @@ def test_if_in_assetview( @pytest.mark.schematic_api class TestMetadataModelOperation: @pytest.mark.parametrize("as_graph", [True, False]) - def test_component_requirement(self, client: FlaskClient, as_graph: bool) -> None: + def test_component_requirement( + self, client: FlaskClient, as_graph: bool, request_headers_trace: Dict[str, str] + ) -> None: params = { "schema_url": DATA_MODEL_JSON_LD, "source_component": "BulkRNA-seqAssay", @@ -346,7 +377,9 @@ def test_component_requirement(self, client: FlaskClient, as_graph: bool) -> Non } response = client.get( - "http://localhost:3001/v1/model/component-requirements", query_string=params + "http://localhost:3001/v1/model/component-requirements", + query_string=params, + headers=request_headers_trace, ) assert response.status_code == 200 @@ -366,7 +399,10 @@ def test_component_requirement(self, client: FlaskClient, as_graph: bool) -> Non class TestUtilsOperation: @pytest.mark.parametrize("strict_camel_case", [True, False]) def test_get_property_label_from_display_name( - self, client: FlaskClient, strict_camel_case: bool + self, + client: FlaskClient, + strict_camel_case: bool, + request_headers_trace: Dict[str, str], ) -> None: params = { "display_name": "mocular entity", @@ -376,6 +412,7 @@ def test_get_property_label_from_display_name( response = client.get( "http://localhost:3001/v1/utils/get_property_label_from_display_name", query_string=params, + headers=request_headers_trace, ) assert response.status_code == 200 @@ -389,10 +426,14 @@ def test_get_property_label_from_display_name( @pytest.mark.schematic_api class TestDataModelGraphExplorerOperation: - def test_get_schema(self, client: FlaskClient) -> None: + def test_get_schema( + self, client: FlaskClient, request_headers_trace: Dict[str, str] + ) -> None: params = {"schema_url": DATA_MODEL_JSON_LD, "data_model_labels": "class_label"} response = client.get( - "http://localhost:3001/v1/schemas/get/schema", query_string=params + "http://localhost:3001/v1/schemas/get/schema", + query_string=params, + headers=request_headers_trace, ) response_dt = response.data @@ -403,7 +444,9 @@ def test_get_schema(self, client: FlaskClient) -> None: if os.path.exists(response_dt): os.remove(response_dt) - def test_if_node_required(test, client: FlaskClient) -> None: + def test_if_node_required( + test, client: FlaskClient, request_headers_trace: Dict[str, str] + ) -> None: params = { "schema_url": DATA_MODEL_JSON_LD, "node_display_name": "FamilyHistory", @@ -411,13 +454,17 @@ def test_if_node_required(test, client: FlaskClient) -> None: } response = client.get( - "http://localhost:3001/v1/schemas/is_node_required", query_string=params + "http://localhost:3001/v1/schemas/is_node_required", + query_string=params, + headers=request_headers_trace, ) response_dta = json.loads(response.data) assert response.status_code == 200 assert response_dta == True - def test_get_node_validation_rules(test, client: FlaskClient) -> None: + def test_get_node_validation_rules( + test, client: FlaskClient, request_headers_trace: Dict[str, str] + ) -> None: params = { "schema_url": DATA_MODEL_JSON_LD, "node_display_name": "CheckRegexList", @@ -425,13 +472,16 @@ def test_get_node_validation_rules(test, client: FlaskClient) -> None: response = client.get( "http://localhost:3001/v1/schemas/get_node_validation_rules", query_string=params, + headers=request_headers_trace, ) response_dta = json.loads(response.data) assert response.status_code == 200 assert "list" in response_dta assert "regex match [a-f]" in response_dta - def test_get_nodes_display_names(test, client: FlaskClient) -> None: + def test_get_nodes_display_names( + test, client: FlaskClient, request_headers_trace: Dict[str, str] + ) -> None: params = { "schema_url": DATA_MODEL_JSON_LD, "node_list": ["FamilyHistory", "Biospecimen"], @@ -439,6 +489,7 @@ def test_get_nodes_display_names(test, client: FlaskClient) -> None: response = client.get( "http://localhost:3001/v1/schemas/get_nodes_display_names", query_string=params, + headers=request_headers_trace, ) response_dta = json.loads(response.data) assert response.status_code == 200 @@ -447,19 +498,29 @@ def test_get_nodes_display_names(test, client: FlaskClient) -> None: @pytest.mark.parametrize( "relationship", ["parentOf", "requiresDependency", "rangeValue", "domainValue"] ) - def test_get_subgraph_by_edge(self, client: FlaskClient, relationship: str) -> None: + def test_get_subgraph_by_edge( + self, + client: FlaskClient, + relationship: str, + request_headers_trace: Dict[str, str], + ) -> None: params = {"schema_url": DATA_MODEL_JSON_LD, "relationship": relationship} response = client.get( "http://localhost:3001/v1/schemas/get/graph_by_edge_type", query_string=params, + headers=request_headers_trace, ) assert response.status_code == 200 @pytest.mark.parametrize("return_display_names", [True, False]) @pytest.mark.parametrize("node_label", ["FamilyHistory", "TissueStatus"]) def test_get_node_range( - self, client: FlaskClient, return_display_names: bool, node_label: str + self, + client: FlaskClient, + return_display_names: bool, + node_label: str, + request_headers_trace: Dict[str, str], ) -> None: params = { "schema_url": DATA_MODEL_JSON_LD, @@ -468,7 +529,9 @@ def test_get_node_range( } response = client.get( - "http://localhost:3001/v1/schemas/get_node_range", query_string=params + "http://localhost:3001/v1/schemas/get_node_range", + query_string=params, + headers=request_headers_trace, ) response_dt = json.loads(response.data) assert response.status_code == 200 @@ -490,6 +553,7 @@ def test_node_dependencies( source_node: str, return_display_names: Union[bool, None], return_schema_ordered: Union[bool, None], + request_headers_trace: Dict[str, str], ) -> None: return_display_names = True return_schema_ordered = False @@ -504,6 +568,7 @@ def test_node_dependencies( response = client.get( "http://localhost:3001/v1/schemas/get_node_dependencies", query_string=params, + headers=request_headers_trace, ) response_dt = json.loads(response.data) assert response.status_code == 200 @@ -748,7 +813,11 @@ def test_generate_new_manifest( ], ) def test_generate_manifest_file_based_annotations( - self, client: FlaskClient, use_annotations: bool, expected: list[str] + self, + client: FlaskClient, + use_annotations: bool, + expected: list[str], + request_headers_trace: Dict[str, str], ) -> None: params = { "schema_url": DATA_MODEL_JSON_LD, @@ -760,7 +829,9 @@ def test_generate_manifest_file_based_annotations( } response = client.get( - "http://localhost:3001/v1/manifest/generate", query_string=params + "http://localhost:3001/v1/manifest/generate", + query_string=params, + headers=request_headers_trace, ) assert response.status_code == 200 @@ -798,7 +869,7 @@ def test_generate_manifest_file_based_annotations( # test case: generate a manifest with annotations when use_annotations is set to True for a component that is not file-based # the dataset folder does not contain an existing manifest def test_generate_manifest_not_file_based_with_annotations( - self, client: FlaskClient + self, client: FlaskClient, request_headers_trace: Dict[str, str] ) -> None: params = { "schema_url": DATA_MODEL_JSON_LD, @@ -809,7 +880,9 @@ def test_generate_manifest_not_file_based_with_annotations( "use_annotations": False, } response = client.get( - "http://localhost:3001/v1/manifest/generate", query_string=params + "http://localhost:3001/v1/manifest/generate", + query_string=params, + headers=request_headers_trace, ) assert response.status_code == 200 @@ -833,21 +906,28 @@ def test_generate_manifest_not_file_based_with_annotations( ] ) - def test_generate_manifest_data_type_not_found(self, client: FlaskClient) -> None: + def test_generate_manifest_data_type_not_found( + self, client: FlaskClient, request_headers_trace: Dict[str, str] + ) -> None: params = { "schema_url": DATA_MODEL_JSON_LD, "data_type": "wrong data type", "use_annotations": False, } response = client.get( - "http://localhost:3001/v1/manifest/generate", query_string=params + "http://localhost:3001/v1/manifest/generate", + query_string=params, + headers=request_headers_trace, ) assert response.status_code == 500 assert "LookupError" in str(response.data) def test_populate_manifest( - self, client: FlaskClient, valid_test_manifest_csv: str + self, + client: FlaskClient, + valid_test_manifest_csv: str, + request_headers_trace: Dict[str, str], ) -> None: # test manifest test_manifest_data = open(valid_test_manifest_csv, "rb") @@ -860,7 +940,9 @@ def test_populate_manifest( } response = client.get( - "http://localhost:3001/v1/manifest/generate", query_string=params + "http://localhost:3001/v1/manifest/generate", + query_string=params, + headers=request_headers_trace, ) assert response.status_code == 200 @@ -925,7 +1007,12 @@ def test_validate_manifest( data = None if test_manifest_fixture: test_manifest_path = request.getfixturevalue(test_manifest_fixture) - data = {"file_name": (open(test_manifest_path, "rb"), "test.csv")} + data = { + "file_name": ( + open(test_manifest_path, "rb"), + f"test_{uuid.uuid4()}.csv", + ) + } # AND the appropriate headers for the test if update_headers: @@ -1001,33 +1088,39 @@ def test_manifest_download( "new_manifest_name": new_manifest_name, "as_json": as_json, } - - response = client.get( - "http://localhost:3001/v1/manifest/download", - query_string=params, - headers=request_headers, + temp_manifest_folder = create_temp_folder( + path=config.manifest_folder, prefix=str(uuid.uuid4()) ) + with patch( + "schematic.store.synapse.create_temp_folder", + return_value=temp_manifest_folder, + ) as mock_create_temp_folder: + response = client.get( + "http://localhost:3001/v1/manifest/download", + query_string=params, + headers=request_headers, + ) + mock_create_temp_folder.assert_called_once() assert response.status_code == 200 # if as_json is set to True or as_json is not defined, then a json gets returned if as_json or as_json is None: - response_dta = json.loads(response.data) + response_data = json.loads(response.data) # check if the correct manifest gets downloaded - assert response_dta[0]["Component"] == expected_component - - current_work_dir = os.getcwd() - folder_test_manifests = config.manifest_folder - folder_dir = os.path.join(current_work_dir, folder_test_manifests) + assert response_data[0]["Component"] == expected_component + assert temp_manifest_folder is not None # if a manfiest gets renamed, get new manifest file path if new_manifest_name: manifest_file_path = os.path.join( - folder_dir, new_manifest_name + "." + "csv" + temp_manifest_folder, new_manifest_name + "." + "csv" ) # if a manifest does not get renamed, get existing manifest file path else: - manifest_file_path = os.path.join(folder_dir, expected_file_name) + manifest_file_path = os.path.join( + temp_manifest_folder, expected_file_name + ) else: # manifest file path gets returned @@ -1090,6 +1183,7 @@ def test_dataset_manifest_download( assert response.status_code == 200 response_dt = response.data + # TODO: Check assertions if as_json: response_json = json.loads(response_dt) assert response_json[0]["Component"] == "BulkRNA-seqAssay" @@ -1128,7 +1222,12 @@ def test_submit_manifest_table_and_file_replace( response_csv = client.post( "http://localhost:3001/v1/model/submit", query_string=params, - data={"file_name": (open(test_manifest_submit, "rb"), "test.csv")}, + data={ + "file_name": ( + open(test_manifest_submit, "rb"), + f"test_{uuid.uuid4()}.csv", + ) + }, headers=request_headers, ) assert response_csv.status_code == 200 @@ -1188,7 +1287,7 @@ def test_submit_manifest_file_only_replace( response_csv = client.post( "http://localhost:3001/v1/model/submit", query_string=params, - data={"file_name": (open(manifest_path, "rb"), "test.csv")}, + data={"file_name": (open(manifest_path, "rb"), f"test_{uuid.uuid4()}.csv")}, headers=request_headers, ) assert response_csv.status_code == 200 @@ -1246,7 +1345,12 @@ def test_submit_manifest_w_file_and_entities( response_csv = client.post( "http://localhost:3001/v1/model/submit", query_string=params, - data={"file_name": (open(test_manifest_submit, "rb"), "test.csv")}, + data={ + "file_name": ( + open(test_manifest_submit, "rb"), + f"test_{uuid.uuid4()}.csv", + ) + }, headers=request_headers, ) assert response_csv.status_code == 200 @@ -1276,7 +1380,12 @@ def test_submit_manifest_table_and_file_upsert( response_csv = client.post( "http://localhost:3001/v1/model/submit", query_string=params, - data={"file_name": (open(test_upsert_manifest_csv, "rb"), "test.csv")}, + data={ + "file_name": ( + open(test_upsert_manifest_csv, "rb"), + f"test_{uuid.uuid4()}.csv", + ) + }, headers=request_headers, ) assert response_csv.status_code == 200 @@ -1307,7 +1416,12 @@ def test_submit_and_validate_filebased_manifest( response_csv = client.post( "http://localhost:3001/v1/model/submit", query_string=params, - data={"file_name": (open(valid_filename_manifest_csv, "rb"), "test.csv")}, + data={ + "file_name": ( + open(valid_filename_manifest_csv, "rb"), + f"test_{uuid.uuid4()}.csv", + ) + }, headers=request_headers, ) @@ -1317,18 +1431,25 @@ def test_submit_and_validate_filebased_manifest( @pytest.mark.schematic_api class TestSchemaVisualization: - def test_visualize_attributes(self, client: FlaskClient) -> None: + def test_visualize_attributes( + self, client: FlaskClient, request_headers_trace: Dict[str, str] + ) -> None: params = {"schema_url": DATA_MODEL_JSON_LD} response = client.get( - "http://localhost:3001/v1/visualize/attributes", query_string=params + "http://localhost:3001/v1/visualize/attributes", + query_string=params, + headers=request_headers_trace, ) assert response.status_code == 200 @pytest.mark.parametrize("figure_type", ["component", "dependency"]) def test_visualize_tangled_tree_layers( - self, client: FlaskClient, figure_type: str + self, + client: FlaskClient, + figure_type: str, + request_headers_trace: Dict[str, str], ) -> None: # TODO: Determine a 2nd data model to use for this test, test both models sequentially, add checks for content of response params = {"schema_url": DATA_MODEL_JSON_LD, "figure_type": figure_type} @@ -1336,6 +1457,7 @@ def test_visualize_tangled_tree_layers( response = client.get( "http://localhost:3001/v1/visualize/tangled_tree/layers", query_string=params, + headers=request_headers_trace, ) assert response.status_code == 200 @@ -1436,7 +1558,11 @@ def test_visualize_tangled_tree_layers( ], ) def test_visualize_component( - self, client: FlaskClient, component: str, response_text: str + self, + client: FlaskClient, + component: str, + response_text: str, + request_headers_trace: Dict[str, str], ) -> None: params = { "schema_url": DATA_MODEL_JSON_LD, @@ -1446,7 +1572,9 @@ def test_visualize_component( } response = client.get( - "http://localhost:3001/v1/visualize/component", query_string=params + "http://localhost:3001/v1/visualize/component", + query_string=params, + headers=request_headers_trace, ) assert response.status_code == 200 @@ -1487,7 +1615,11 @@ def test_validation_performance( "schema_url": BENCHMARK_DATA_MODEL_JSON_LD, "data_type": "MockComponent", } - headers = {"Content-Type": "multipart/form-data", "Accept": "application/json"} + headers = { + "Content-Type": "multipart/form-data", + "Accept": "application/json", + "traceparent": get_traceparent(), + } # Enforce error rate when possible if MockComponent_attribute == "Check Ages": @@ -1521,7 +1653,12 @@ def test_validation_performance( response = client.post( endpoint_url, query_string=params, - data={"file_name": (open(large_manifest_path, "rb"), "large_test.csv")}, + data={ + "file_name": ( + open(large_manifest_path, "rb"), + f"large_test_{uuid.uuid4()}.csv", + ) + }, headers=headers, ) response_time = perf_counter() - t_start diff --git a/tests/test_cli.py b/tests/test_cli.py index 308f9c73f..a6e4e8ef7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,14 +1,13 @@ import os - -import pytest from unittest.mock import patch +import pytest from click.testing import CliRunner -from schematic.schemas.commands import schema +from schematic.configuration.configuration import Configuration from schematic.manifest.commands import manifest from schematic.models.commands import model -from schematic.configuration.configuration import Configuration +from schematic.schemas.commands import schema from tests.conftest import Helpers @@ -156,9 +155,9 @@ def test_submit_file_based_manifest( runner: CliRunner, helpers: Helpers, with_annotations: bool, - config: Configuration, ) -> None: manifest_path = helpers.get_data_path("mock_manifests/test_BulkRNAseq.csv") + config = Configuration() config.load_config("config_example.yml") config.synapse_master_fileview_id = "syn1234" diff --git a/tests/test_ge_helpers.py b/tests/test_ge_helpers.py index 6e0b3e01c..8ecaac2e9 100644 --- a/tests/test_ge_helpers.py +++ b/tests/test_ge_helpers.py @@ -8,7 +8,7 @@ from tests.conftest import Helpers -@pytest.fixture(scope="class") +@pytest.fixture(scope="function") def mock_ge_helpers( helpers: Helpers, ) -> Generator[GreatExpectationsHelpers, None, None]: @@ -34,39 +34,9 @@ def test_add_expectation_suite_if_not_exists_does_not_exist( """test add_expectation_suite_if_not_exists method when the expectation suite does not exists""" # mock context provided by ge_helpers mock_ge_helpers.context = MagicMock() - mock_ge_helpers.context.list_expectation_suite_names.return_value = [] # Call the method - result = mock_ge_helpers.add_expectation_suite_if_not_exists() + mock_ge_helpers.add_expectation_suite_if_not_exists() # Make sure the method of creating expectation suites if it doesn't exist - mock_ge_helpers.context.list_expectation_suite_names.assert_called_once() - mock_ge_helpers.context.add_expectation_suite.assert_called_once_with( - expectation_suite_name="Manifest_test_suite" - ) - - def test_add_expectation_suite_if_not_exists_does_exist( - self, mock_ge_helpers: Generator[GreatExpectationsHelpers, None, None] - ) -> None: - """test add_expectation_suite_if_not_exists method when the expectation suite does exists""" - # mock context provided by ge_helpers - mock_ge_helpers.context = MagicMock() - mock_ge_helpers.context.list_expectation_suite_names.return_value = [ - "Manifest_test_suite" - ] - mock_ge_helpers.context.list_checkpoints.return_value = ["test_checkpoint"] - - # Call the method - result = mock_ge_helpers.add_expectation_suite_if_not_exists() - - # Make sure the method of deleting suites gets called - mock_ge_helpers.context.list_expectation_suite_names.assert_called_once() - mock_ge_helpers.context.delete_expectation_suite.assert_called_once_with( - "Manifest_test_suite" - ) - mock_ge_helpers.context.add_expectation_suite.assert_called_once_with( - expectation_suite_name="Manifest_test_suite" - ) - mock_ge_helpers.context.delete_checkpoint.assert_called_once_with( - "test_checkpoint" - ) + mock_ge_helpers.context.add_expectation_suite.assert_called_once() diff --git a/tests/test_store.py b/tests/test_store.py index 717b4542e..af00ba301 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -27,7 +27,7 @@ from schematic.schemas.data_model_parser import DataModelParser from schematic.store.base import BaseStorage from schematic.store.synapse import DatasetFileView, ManifestDownload, SynapseStorage -from schematic.utils.general import check_synapse_cache_size +from schematic.utils.general import check_synapse_cache_size, create_temp_folder from tests.conftest import Helpers from tests.utils import CleanupItem @@ -124,6 +124,11 @@ def dmge( yield dmge +@pytest.fixture +def mock_file() -> File: + return File(parentId="syn123", id="syn456", name="mock_file") + + @pytest.fixture(scope="module") def synapse_store_special_scope(): yield SynapseStorage(perform_query=False) @@ -1020,20 +1025,41 @@ async def test_create_table( # AND a copy of all the folders in the manifest. Added to the dataset directory for easy cleanup manifest = helpers.get_data_frame(manifest_path) - for index, row in manifest.iterrows(): + + async def copy_folder_and_update_manifest( + row: pd.Series, + index: int, + datasetId: str, + synapse_store: SynapseStorage, + manifest: pd.DataFrame, + schedule_for_cleanup: Callable[[CleanupItem], None], + ) -> None: + """Internal function to copy a folder and update the manifest.""" folder_id = row["entityId"] - folder_copy = FolderModel(id=folder_id).copy( + folder_copy = await FolderModel(id=folder_id).copy_async( parent_id=datasetId, synapse_client=synapse_store.syn ) schedule_for_cleanup(CleanupItem(synapse_id=folder_copy.id)) manifest.at[index, "entityId"] = folder_copy.id + tasks = [] + + for index, row in manifest.iterrows(): + tasks.append( + copy_folder_and_update_manifest( + row, index, datasetId, synapse_store, manifest, schedule_for_cleanup + ) + ) + await asyncio.gather(*tasks) + with patch.object( synapse_store, "_generate_table_name", return_value=(table_name, "followup") ), patch.object( synapse_store, "getDatasetProject", return_value=projectId ), tempfile.NamedTemporaryFile( - delete=True, suffix=".csv" + delete=True, + suffix=".csv", + dir=create_temp_folder(path=tempfile.gettempdir()), ) as tmp_file: # Write the DF to a temporary file to prevent modifying the original manifest.to_csv(tmp_file.name, index=False) @@ -1053,10 +1079,12 @@ async def test_create_table( schedule_for_cleanup(CleanupItem(synapse_id=manifest_id)) # THEN the table should exist - existing_tables = synapse_store.get_table_info(projectId=projectId) + existing_table_id = synapse_store.syn.findEntityId( + name=table_name, parent=projectId + ) # assert table exists - assert table_name in existing_tables.keys() + assert existing_table_id is not None @pytest.mark.parametrize( "table_column_names", @@ -1090,24 +1118,42 @@ async def test_replace_table( # AND a copy of all the folders in the manifest. Added to the dataset directory for easy cleanup manifest = helpers.get_data_frame(manifest_path) replacement_manifest = helpers.get_data_frame(replacement_manifest_path) - for index, row in manifest.iterrows(): + + async def copy_folder_and_update_manifest( + row: pd.Series, + index: int, + datasetId: str, + synapse_store: SynapseStorage, + manifest: pd.DataFrame, + schedule_for_cleanup: Callable[[CleanupItem], None], + ) -> None: + """Internal function to copy a folder and update the manifest.""" folder_id = row["entityId"] - folder_copy = FolderModel(id=folder_id).copy( + folder_copy = await FolderModel(id=folder_id).copy_async( parent_id=datasetId, synapse_client=synapse_store.syn ) schedule_for_cleanup(CleanupItem(synapse_id=folder_copy.id)) manifest.at[index, "entityId"] = folder_copy.id replacement_manifest.at[index, "entityId"] = folder_copy.id - # Check if FollowUp table exists if so delete - existing_tables = synapse_store.get_table_info(projectId=projectId) + tasks = [] + for index, row in manifest.iterrows(): + tasks.append( + copy_folder_and_update_manifest( + row, index, datasetId, synapse_store, manifest, schedule_for_cleanup + ) + ) + + await asyncio.gather(*tasks) with patch.object( synapse_store, "_generate_table_name", return_value=(table_name, "followup") ), patch.object( synapse_store, "getDatasetProject", return_value=projectId ), tempfile.NamedTemporaryFile( - delete=True, suffix=".csv" + delete=True, + suffix=".csv", + dir=create_temp_folder(path=tempfile.gettempdir()), ) as tmp_file: # Write the DF to a temporary file to prevent modifying the original manifest.to_csv(tmp_file.name, index=False) @@ -1125,10 +1171,9 @@ async def test_replace_table( annotation_keys=annotation_keys, ) schedule_for_cleanup(CleanupItem(synapse_id=manifest_id)) - existing_tables = synapse_store.get_table_info(projectId=projectId) # Query table for DaystoFollowUp column - table_id = existing_tables[table_name] + table_id = synapse_store.syn.findEntityId(name=table_name, parent=projectId) days_to_follow_up = ( synapse_store.syn.tableQuery(f"SELECT {column_of_interest} FROM {table_id}") .asDataFrame() @@ -1143,7 +1188,9 @@ async def test_replace_table( ), patch.object( synapse_store, "getDatasetProject", return_value=projectId ), tempfile.NamedTemporaryFile( - delete=True, suffix=".csv" + delete=True, + suffix=".csv", + dir=create_temp_folder(path=tempfile.gettempdir()), ) as tmp_file: # Write the DF to a temporary file to prevent modifying the original replacement_manifest.to_csv(tmp_file.name, index=False) @@ -1161,10 +1208,9 @@ async def test_replace_table( annotation_keys=annotation_keys, ) schedule_for_cleanup(CleanupItem(synapse_id=manifest_id)) - existing_tables = synapse_store.get_table_info(projectId=projectId) # Query table for DaystoFollowUp column - table_id = existing_tables[table_name] + table_id = synapse_store.syn.findEntityId(name=table_name, parent=projectId) days_to_follow_up = ( synapse_store.syn.tableQuery(f"SELECT {column_of_interest} FROM {table_id}") .asDataFrame() @@ -1202,7 +1248,9 @@ async def test_upsert_table( ), patch.object( synapse_store, "getDatasetProject", return_value=projectId ), tempfile.NamedTemporaryFile( - delete=True, suffix=".csv" + delete=True, + suffix=".csv", + dir=create_temp_folder(path=tempfile.gettempdir()), ) as tmp_file: # Copy to a temporary file to prevent modifying the original shutil.copyfile(helpers.get_data_path(manifest_path), tmp_file.name) @@ -1220,10 +1268,9 @@ async def test_upsert_table( annotation_keys=annotation_keys, ) schedule_for_cleanup(CleanupItem(synapse_id=manifest_id)) - existing_tables = synapse_store.get_table_info(projectId=projectId) # set primary key annotation for uploaded table - table_id = existing_tables[table_name] + table_id = synapse_store.syn.findEntityId(name=table_name, parent=projectId) # Query table for DaystoFollowUp column table_query = ( @@ -1242,7 +1289,9 @@ async def test_upsert_table( ), patch.object( synapse_store, "getDatasetProject", return_value=projectId ), tempfile.NamedTemporaryFile( - delete=True, suffix=".csv" + delete=True, + suffix=".csv", + dir=create_temp_folder(path=tempfile.gettempdir()), ) as tmp_file: # Copy to a temporary file to prevent modifying the original shutil.copyfile( @@ -1262,10 +1311,9 @@ async def test_upsert_table( annotation_keys=annotation_keys, ) schedule_for_cleanup(CleanupItem(synapse_id=manifest_id)) - existing_tables = synapse_store.get_table_info(projectId=projectId) # Query table for DaystoFollowUp column - table_id = existing_tables[table_name] + table_id = synapse_store.syn.findEntityId(name=table_name, parent=projectId) table_query = ( synapse_store.syn.tableQuery(f"SELECT {column_of_interest} FROM {table_id}") .asDataFrame() @@ -1321,9 +1369,7 @@ def test_get_manifest_id(self, synapse_store, datasetFileView): @pytest.mark.parametrize("newManifestName", ["", "Example"]) def test_download_manifest(self, mock_manifest_download, newManifestName): # test the download function by downloading a manifest - manifest_data = mock_manifest_download.download_manifest( - mock_manifest_download, newManifestName - ) + manifest_data = mock_manifest_download.download_manifest(newManifestName) assert os.path.exists(manifest_data["path"]) if not newManifestName: @@ -1338,7 +1384,7 @@ def test_download_access_restricted_manifest(self, synapse_store): # attempt to download an uncensored manifest that has access restriction. # if the code works correctly, the censored manifest that does not have access restriction would get downloaded (see: syn29862066) md = ManifestDownload(synapse_store.syn, "syn29862066") - manifest_data = md.download_manifest(md) + manifest_data = md.download_manifest() assert os.path.exists(manifest_data["path"]) @@ -1348,7 +1394,7 @@ def test_download_access_restricted_manifest(self, synapse_store): def test_download_manifest_on_aws(self, mock_manifest_download, monkeypatch): # mock AWS environment by providing SECRETS_MANAGER_SECRETS environment variable and attempt to download a manifest monkeypatch.setenv("SECRETS_MANAGER_SECRETS", "mock_value") - manifest_data = mock_manifest_download.download_manifest(mock_manifest_download) + manifest_data = mock_manifest_download.download_manifest() assert os.path.exists(manifest_data["path"]) # clean up @@ -1359,11 +1405,10 @@ def test_entity_type_checking(self, synapse_store, entity_id, caplog): md = ManifestDownload(synapse_store.syn, entity_id) md._entity_type_checking() if entity_id == "syn27600053": - for record in caplog.records: - assert ( - "You are using entity type: folder. Please provide a file ID" - in record.message - ) + assert ( + "You are using entity type: folder. Please provide a file ID" + in caplog.text + ) class TestManifestUpload: @@ -1427,6 +1472,7 @@ async def test_add_annotations_to_entities_files( files_in_dataset: str, expected_filenames: list[str], expected_entity_ids: list[str], + mock_file: File, ) -> None: """test adding annotations to entities files @@ -1449,39 +1495,39 @@ async def mock_process_store_annos(requests): with patch( "schematic.store.synapse.SynapseStorage.getFilesInStorageDataset", return_value=files_in_dataset, + ), patch( + "schematic.store.synapse.SynapseStorage.format_row_annotations", + return_value=mock_format_row_annos, + new_callable=AsyncMock, + ) as mock_format_row, patch( + "schematic.store.synapse.SynapseStorage._process_store_annos", + return_value=mock_process_store_annos, + new_callable=AsyncMock, + ) as mock_process_store, patch.object( + synapse_store.synapse_entity_tracker, "get", return_value=mock_file ): - with patch( - "schematic.store.synapse.SynapseStorage.format_row_annotations", - return_value=mock_format_row_annos, - new_callable=AsyncMock, - ) as mock_format_row: - with patch( - "schematic.store.synapse.SynapseStorage._process_store_annos", - return_value=mock_process_store_annos, - new_callable=AsyncMock, - ) as mock_process_store: - manifest_df = pd.DataFrame(original_manifest) - - new_df = await synapse_store.add_annotations_to_entities_files( - dmge, - manifest_df, - manifest_record_type="entity", - datasetId="mock id", - hideBlanks=True, - ) + manifest_df = pd.DataFrame(original_manifest) + + new_df = await synapse_store.add_annotations_to_entities_files( + dmge, + manifest_df, + manifest_record_type="entity", + datasetId="mock id", + hideBlanks=True, + ) - file_names_lst = new_df["Filename"].tolist() - entity_ids_lst = new_df["entityId"].tolist() + file_names_lst = new_df["Filename"].tolist() + entity_ids_lst = new_df["entityId"].tolist() - # test entityId and Id columns get added - assert "entityId" in new_df.columns - assert "Id" in new_df.columns - assert file_names_lst == expected_filenames - assert entity_ids_lst == expected_entity_ids + # test entityId and Id columns get added + assert "entityId" in new_df.columns + assert "Id" in new_df.columns + assert file_names_lst == expected_filenames + assert entity_ids_lst == expected_entity_ids - # make sure async function gets called as expected - assert mock_format_row.call_count == len(expected_entity_ids) - assert mock_process_store.call_count == 1 + # make sure async function gets called as expected + assert mock_format_row.call_count == len(expected_entity_ids) + assert mock_process_store.call_count == 1 @pytest.mark.parametrize( "mock_manifest_file_path", @@ -1495,6 +1541,7 @@ def test_upload_manifest_file( helpers: Helpers, synapse_store: SynapseStorage, mock_manifest_file_path: str, + mock_file: File, ) -> None: """test upload manifest file function @@ -1523,9 +1570,9 @@ def test_upload_manifest_file( "entityId": {0: "syn1224", 1: "syn1225", 2: "syn1226"}, } ) - with patch("synapseclient.Synapse.store") as syn_store_mock, patch( - "schematic.store.synapse.synapseutils.copy_functions.changeFileMetaData" - ) as mock_change_file_metadata: + with patch("synapseclient.Synapse.store") as syn_store_mock, patch.object( + synapse_store.synapse_entity_tracker, "get", return_value=mock_file + ): syn_store_mock.return_value.id = "mock manifest id" mock_component_name = "BulkRNA-seqAssay" mock_file_path = helpers.get_data_path(mock_manifest_file_path) @@ -1536,20 +1583,8 @@ def test_upload_manifest_file( restrict_manifest=True, component_name=mock_component_name, ) - if "censored" in mock_manifest_file_path: - file_name = ( - f"synapse_storage_manifest_{mock_component_name}_censored.csv" - ) - else: - file_name = f"synapse_storage_manifest_{mock_component_name}.csv" assert mock_manifest_synapse_file_id == "mock manifest id" - mock_change_file_metadata.assert_called_once_with( - forceVersion=False, - syn=synapse_store.syn, - entity=syn_store_mock.return_value.id, - downloadAs=file_name, - ) @pytest.mark.parametrize("file_annotations_upload", [True, False]) @pytest.mark.parametrize("hide_blanks", [True, False]) @@ -1564,6 +1599,7 @@ def test_upload_manifest_as_csv( manifest_record_type: str, hide_blanks: bool, restrict: bool, + mock_file: File, ) -> None: async def mock_add_annotations_to_entities_files(): return @@ -1582,6 +1618,9 @@ async def mock_add_annotations_to_entities_files(): "schematic.store.synapse.SynapseStorage.format_manifest_annotations" ) as format_manifest_anno_mock, patch.object(synapse_store.syn, "set_annotations"), + patch.object( + synapse_store.synapse_entity_tracker, "get", return_value=mock_file + ), ): manifest_path = helpers.get_data_path("mock_manifests/test_BulkRNAseq.csv") manifest_df = helpers.get_data_frame(manifest_path) @@ -1618,6 +1657,7 @@ def test_upload_manifest_as_table( hide_blanks: bool, restrict: bool, manifest_record_type: str, + mock_file: File, ) -> None: mock_df = pd.DataFrame() @@ -1642,6 +1682,9 @@ async def mock_add_annotations_to_entities_files(): patch( "schematic.store.synapse.SynapseStorage.format_manifest_annotations" ) as format_manifest_anno_mock, + patch.object( + synapse_store.synapse_entity_tracker, "get", return_value=mock_file + ), ): manifest_path = helpers.get_data_path("mock_manifests/test_BulkRNAseq.csv") manifest_df = helpers.get_data_frame(manifest_path) @@ -1682,6 +1725,7 @@ def test_upload_manifest_combo( hide_blanks: bool, restrict: bool, manifest_record_type: str, + mock_file: File, ) -> None: mock_df = pd.DataFrame() manifest_path = helpers.get_data_path("mock_manifests/test_BulkRNAseq.csv") @@ -1708,6 +1752,9 @@ async def mock_add_annotations_to_entities_files(): patch( "schematic.store.synapse.SynapseStorage.format_manifest_annotations" ) as format_manifest_anno_mock, + patch.object( + synapse_store.synapse_entity_tracker, "get", return_value=mock_file + ), ): synapse_store.upload_manifest_combo( dmge, @@ -1756,6 +1803,7 @@ def test_associate_metadata_with_files( expected: str, file_annotations_upload: bool, dmge: DataModelGraphExplorer, + mock_file: File, ) -> None: with ( patch( @@ -1770,6 +1818,9 @@ def test_associate_metadata_with_files( "schematic.store.synapse.SynapseStorage.upload_manifest_combo", return_value="mock_id_entities", ), + patch.object( + synapse_store.synapse_entity_tracker, "get", return_value=mock_file + ), ): manifest_path = "mock_manifests/test_BulkRNAseq.csv" manifest_id = synapse_store.associateMetadataWithFiles( diff --git a/tests/test_utils.py b/tests/test_utils.py index 5b37abe6e..5883ef4af 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,50 +2,30 @@ import json import logging import os -import shutil import tempfile import time from datetime import datetime -from unittest import mock from pathlib import Path -from typing import Union, Generator -from _pytest.fixtures import FixtureRequest - +from typing import Generator, Union import numpy as np import pandas as pd import pytest -import synapseclient import synapseclient.core.cache as cache +from _pytest.fixtures import FixtureRequest from pandas.testing import assert_frame_equal from synapseclient.core.exceptions import SynapseHTTPError -from schematic.models.validate_manifest import ValidateManifest from schematic.models.metadata import MetadataModel - -from schematic.schemas.data_model_parser import DataModelParser -from schematic.schemas.data_model_graph import DataModelGraph, DataModelGraphExplorer +from schematic.models.validate_manifest import ValidateManifest +from schematic.schemas.data_model_graph import DataModelGraph +from schematic.schemas.data_model_json_schema import DataModelJSONSchema from schematic.schemas.data_model_jsonld import ( - DataModelJsonLD, - BaseTemplate, - PropertyTemplate, ClassTemplate, + PropertyTemplate, + convert_graph_to_jsonld, ) -from schematic.schemas.data_model_json_schema import DataModelJSONSchema - -from schematic.schemas.data_model_relationships import DataModelRelationships -from schematic.schemas.data_model_jsonld import DataModelJsonLD, convert_graph_to_jsonld - -from schematic.exceptions import ( - MissingConfigValueError, - MissingConfigAndArgumentValueError, -) -from schematic import LOADER -from schematic.exceptions import ( - MissingConfigAndArgumentValueError, - MissingConfigValueError, -) - +from schematic.schemas.data_model_parser import DataModelParser from schematic.utils import cli_utils, df_utils, general, io_utils, validate_utils from schematic.utils.df_utils import load_df from schematic.utils.general import ( @@ -55,25 +35,23 @@ entity_type_mapping, ) from schematic.utils.schema_utils import ( + check_for_duplicate_components, + check_if_display_name_is_valid_label, export_schema, - get_property_label_from_display_name, + extract_component_validation_rules, get_class_label_from_display_name, - strip_context, + get_component_name_rules, + get_individual_rules, + get_json_schema_log_file_path, get_label_from_display_name, + get_property_label_from_display_name, get_schema_label, get_stripped_label, - check_if_display_name_is_valid_label, - get_individual_rules, - get_component_name_rules, - parse_component_validation_rules, parse_single_set_validation_rules, parse_validation_rules, - extract_component_validation_rules, - check_for_duplicate_components, - get_json_schema_log_file_path, + strip_context, ) - logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) diff --git a/tests/test_validation.py b/tests/test_validation.py index cdd6766c0..c16a95bb7 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -36,10 +36,6 @@ def get_rule_combinations(): class TestManifestValidation: - # check if suite has been created. If so, delete it - if os.path.exists("great_expectations/expectations/Manifest_test_suite.json"): - os.remove("great_expectations/expectations/Manifest_test_suite.json") - @pytest.mark.parametrize( ("model_name", "manifest_name", "root_node"), [ diff --git a/tests/unit/test_io_utils.py b/tests/unit/test_io_utils.py new file mode 100644 index 000000000..ce868aac3 --- /dev/null +++ b/tests/unit/test_io_utils.py @@ -0,0 +1,96 @@ +import asyncio +import os +import tempfile + +from schematic.utils.general import create_temp_folder +from schematic.utils.io_utils import cleanup_temporary_storage + + +class TestCleanup: + async def test_cleanup_temporary_storage_nothing_to_cleanup(self) -> None: + # GIVEN a temporary folder that has a file that is not older than the time delta + temp_folder = create_temp_folder(path=tempfile.gettempdir()) + + # AND A File that is not older than the time delta + with open(os.path.join(temp_folder, "file.txt"), "w") as f: + f.write("hello world") + + assert os.path.exists(temp_folder) + assert os.path.exists(os.path.join(temp_folder, "file.txt")) + + time_delta_seconds = 3600 + + # WHEN I call the cleanup function + cleanup_temporary_storage( + temporary_storage_directory=temp_folder, + time_delta_seconds=time_delta_seconds, + ) + + # THEN the folder should still exist + assert os.path.exists(temp_folder) + + # AND the file should still exist + assert os.path.exists(os.path.join(temp_folder, "file.txt")) + + async def test_cleanup_temporary_storage_file_to_cleanup(self) -> None: + # GIVEN a temporary folder that has a file that will be older than the time delta + temp_folder = create_temp_folder(path=tempfile.gettempdir()) + + # AND A File that is older than the time delta + with open(os.path.join(temp_folder, "file.txt"), "w") as f: + f.write("hello world") + + assert os.path.exists(temp_folder) + assert os.path.exists(os.path.join(temp_folder, "file.txt")) + + time_delta_seconds = 1 + + # AND I wait for the time delta + await asyncio.sleep(time_delta_seconds) + + # WHEN I call the cleanup function + cleanup_temporary_storage( + temporary_storage_directory=temp_folder, + time_delta_seconds=time_delta_seconds, + ) + + # THEN the folder should still exist + assert os.path.exists(temp_folder) + + # AND the file should not exist + assert not os.path.exists(os.path.join(temp_folder, "file.txt")) + + async def test_cleanup_temporary_storage_nested_file_to_cleanup(self) -> None: + # GIVEN a temporary folder that has a file that will be older than the time delta + temp_folder = create_temp_folder(path=tempfile.gettempdir()) + + # AND a nested temporary folder + temp_folder_2 = create_temp_folder(path=temp_folder) + + # AND A File that is older than the time delta + with open(os.path.join(temp_folder_2, "file.txt"), "w") as f: + f.write("hello world") + + assert os.path.exists(temp_folder) + assert os.path.exists(temp_folder_2) + assert os.path.exists(os.path.join(temp_folder_2, "file.txt")) + + time_delta_seconds = 1 + + # AND I wait for the time delta + await asyncio.sleep(time_delta_seconds) + + # WHEN I call the cleanup function + cleanup_temporary_storage( + temporary_storage_directory=temp_folder, + time_delta_seconds=time_delta_seconds, + ) + + # THEN the folder should still exist + assert os.path.exists(temp_folder) + + # AND the nested folder should not exist + assert not os.path.exists(temp_folder_2) + + # AND the file should not exist + assert not os.path.exists(os.path.join(temp_folder_2, "file.txt"))