diff --git a/.devcontainer/scripts/postCreate.sh b/.devcontainer/scripts/postCreate.sh index 4acff5e48..f32f82a27 100644 --- a/.devcontainer/scripts/postCreate.sh +++ b/.devcontainer/scripts/postCreate.sh @@ -53,6 +53,10 @@ op inject -f --in-file=.devcontainer/tpl/dbt_cloud.yml.tpl \ --out-file=env/dbt_cloud.yml && sudo mv -f env/dbt_cloud.yml /home/vscode/.dbt/dbt_cloud.yml +op inject -f --in-file=.devcontainer/tpl/gcloud_teamster_dlt_keyfile.json.tpl \ + --out-file=env/gcloud_teamster_dlt_keyfile.json && + sudo mv -f env/gcloud_teamster_dlt_keyfile.json /etc/secret-volume/gcloud_teamster_dlt_keyfile.json + # install pdm dependencies pdm install --frozen-lockfile diff --git a/.devcontainer/tpl/.env.tpl b/.devcontainer/tpl/.env.tpl index 284836d36..64e8016ff 100644 --- a/.devcontainer/tpl/.env.tpl +++ b/.devcontainer/tpl/.env.tpl @@ -39,6 +39,9 @@ DEANSLIST_SFTP_HOST=op://Data Team/DeansList SFTP/host DEANSLIST_SFTP_PASSWORD=op://Data Team/DeansList SFTP/password DEANSLIST_SFTP_USERNAME=op://Data Team/DeansList SFTP/username DEANSLIST_SUBDOMAIN=op://Data Team/DeansList API/subdomain +DESTINATION__BIGQUERY__CREDENTIALS__CLIENT_EMAIL=op://Data Team/Teamster Service Account - dlt/username +DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY=op://Data Team/Teamster Service Account - dlt/password +DESTINATION__BIGQUERY__CREDENTIALS__PROJECT_ID=op://Data Team/Teamster Service Account - dlt/project id EDPLAN_SFTP_HOST=op://Data Team/edplan SFTP - Camden/host EDPLAN_SFTP_PASSWORD_KIPPCAMDEN=op://Data Team/edplan SFTP - Camden/password EDPLAN_SFTP_PASSWORD_KIPPNEWARK=op://Data Team/edplan SFTP - Newark/password @@ -126,6 +129,12 @@ SCHOOLMINT_GROW_CLIENT_SECRET=op://Data Team/SchooMint Grow API/client secret SCHOOLMINT_GROW_DISTRICT_ID=op://Data Team/SchooMint Grow API/district id SLACK_TOKEN=op://Data Team/Slack API - Teamster/credential SMARTRECRUITERS_SMARTTOKEN=op://Data Team/SmartRecruiters API/smart token +SOURCES__SQL_DATABASE__CREDENTIALS__DATABASE=op://Data Team/Illuminate ODBC/database +SOURCES__SQL_DATABASE__CREDENTIALS__DRIVERNAME=op://Data Team/Illuminate ODBC/driver +SOURCES__SQL_DATABASE__CREDENTIALS__HOST=op://Data Team/Illuminate ODBC/ip +SOURCES__SQL_DATABASE__CREDENTIALS__PASSWORD=op://Data Team/Illuminate ODBC/password +SOURCES__SQL_DATABASE__CREDENTIALS__PORT=op://Data Team/Illuminate ODBC/port +SOURCES__SQL_DATABASE__CREDENTIALS__USERNAME=op://Data Team/Illuminate ODBC/username TABLEAU_PERSONAL_ACCESS_TOKEN=op://Data Team/Tableau Server PAT - Dagster/credential TABLEAU_SERVER_ADDRESS=op://Data Team/Tableau Server PAT - Dagster/hostname TABLEAU_SITE_ID=op://Data Team/Tableau Server PAT - Dagster/site id diff --git a/.devcontainer/tpl/gcloud_teamster_dlt_keyfile.json.tpl b/.devcontainer/tpl/gcloud_teamster_dlt_keyfile.json.tpl new file mode 100644 index 000000000..aa7c6d604 --- /dev/null +++ b/.devcontainer/tpl/gcloud_teamster_dlt_keyfile.json.tpl @@ -0,0 +1 @@ +op://Data Team/Teamster Service Account - dlt/keyfile.json \ No newline at end of file diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml index 5a54f7a91..722be64d5 100644 --- a/.trunk/trunk.yaml +++ b/.trunk/trunk.yaml @@ -41,6 +41,12 @@ lint: linters: - sqlfluff - sqlfmt + - paths: + - src/teamster/libraries/dlt_sources/** + linters: + - pyright + - sqlfluff + - sqlfmt - paths: - src/dbt/** linters: @@ -52,6 +58,7 @@ lint: - markdownlint - osv-scanner - oxipng + - pyright - ruff - shellcheck - shfmt diff --git a/pdm.lock b/pdm.lock index feca829a0..70fbd53e2 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:d2937b7f663664fcd3b7c03db3d4690b8f7b8a78793f1df5248cc957da6f5f4e" +content_hash = "sha256:7558dfb62e381caef94fd84b6145b06dfe1ed7822bd9763d323b9091b0e2f72f" [[metadata.targets]] requires_python = ">=3.12,<3.13" @@ -29,6 +29,66 @@ files = [ {file = "agate-1.9.1.tar.gz", hash = "sha256:bc60880c2ee59636a2a80cd8603d63f995be64526abf3cbba12f00767bcd5b3d"}, ] +[[package]] +name = "aiohappyeyeballs" +version = "2.4.3" +requires_python = ">=3.8" +summary = "Happy Eyeballs for asyncio" +groups = ["default"] +files = [ + {file = "aiohappyeyeballs-2.4.3-py3-none-any.whl", hash = "sha256:8a7a83727b2756f394ab2895ea0765a0a8c475e3c71e98d43d76f22b4b435572"}, + {file = "aiohappyeyeballs-2.4.3.tar.gz", hash = "sha256:75cf88a15106a5002a8eb1dab212525c00d1f4c0fa96e551c9fbe6f09a621586"}, +] + +[[package]] +name = "aiohttp" +version = "3.11.5" +requires_python = ">=3.9" +summary = "Async http client/server framework (asyncio)" +groups = ["default"] +dependencies = [ + "aiohappyeyeballs>=2.3.0", + "aiosignal>=1.1.2", + "async-timeout<6.0,>=4.0; python_version < \"3.11\"", + "attrs>=17.3.0", + "frozenlist>=1.1.1", + "multidict<7.0,>=4.5", + "propcache>=0.2.0", + "yarl<2.0,>=1.17.0", +] +files = [ + {file = "aiohttp-3.11.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3f21c6d1fae17f4466af3796975ab34010db3ac1f0d688272a6ce2f9fa2a4ea5"}, + {file = "aiohttp-3.11.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f2041691ac9a4ac5f3ccda419efdbd97f3b25bcc64c5badf57a85a69b8579268"}, + {file = "aiohttp-3.11.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7ad77209639aa7f8d1bd87bd0aa961cac791658c9dd1d32225cbabee95b70bd4"}, + {file = "aiohttp-3.11.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca5c02fec19113abb7d9df9350471fa1ed25f76ad24be81690c96b3b759da795"}, + {file = "aiohttp-3.11.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:35c429f0f761619ea659cfe5bed5c26bc62c5e09c2da28b5ee86d006b1a1eb4d"}, + {file = "aiohttp-3.11.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:68f0f8213e891b81800812ec70c58bac3899f4828e7ad14ba5997c26dd88aa6f"}, + {file = "aiohttp-3.11.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:381c1d8334fb010968dfc8eb1140ed349c5ade9ba20feb0aee2a047d9af0b7a5"}, + {file = "aiohttp-3.11.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0ea7b22c2569007df2c39dbe72b7c7cf4e6f6424b505545c68fde8495a35bcc9"}, + {file = "aiohttp-3.11.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:50d8784cdc111ed0709debe595be831ebb1f0c536b0840684d02fd12d100a092"}, + {file = "aiohttp-3.11.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0a7a8915129e6e9b43b5e2f13e0533314462f34e8f8589fb388b8f35becb997e"}, + {file = "aiohttp-3.11.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c7e0cdfdc6ea4b974c3d546e683bf5a408a8777886c7ec389a780da58a8aa284"}, + {file = "aiohttp-3.11.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9a23bd19042768281c06858a55ee3d85e572111681e5f5dd68ebd27a6ae1e2af"}, + {file = "aiohttp-3.11.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:304316862900286574e38dbd58c9c5c25dfd52bcfea16514a00dd741f992871e"}, + {file = "aiohttp-3.11.5-cp312-cp312-win32.whl", hash = "sha256:3e0f4119290d432fa7babfc76cbde4f3e21b826240ba51a6d4fdb82935cf82bd"}, + {file = "aiohttp-3.11.5-cp312-cp312-win_amd64.whl", hash = "sha256:1fe98b92f943b00e1831aece85638af6ca6c699f82625f7a6c64a2543b7a9769"}, + {file = "aiohttp-3.11.5.tar.gz", hash = "sha256:7b857fdad5f95d05bbd27c68cdd549889287dea7fe3376265a8a85d554deec1e"}, +] + +[[package]] +name = "aiosignal" +version = "1.3.1" +requires_python = ">=3.7" +summary = "aiosignal: a list of registered asynchronous callbacks" +groups = ["default"] +dependencies = [ + "frozenlist>=1.1.0", +] +files = [ + {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, + {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, +] + [[package]] name = "alembic" version = "1.14.0" @@ -89,6 +149,20 @@ files = [ {file = "argcomplete-3.5.1.tar.gz", hash = "sha256:eb1ee355aa2557bd3d0145de7b06b2a45b0ce461e1e7813f5d066039ab4177b4"}, ] +[[package]] +name = "astunparse" +version = "1.6.3" +summary = "An AST unparser for Python" +groups = ["default"] +dependencies = [ + "six<2.0,>=1.6.1", + "wheel<1.0,>=0.23.0", +] +files = [ + {file = "astunparse-1.6.3-py2.py3-none-any.whl", hash = "sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8"}, + {file = "astunparse-1.6.3.tar.gz", hash = "sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872"}, +] + [[package]] name = "attrs" version = "24.2.0" @@ -556,6 +630,22 @@ files = [ {file = "dagster_dbt-0.25.2-py3-none-any.whl", hash = "sha256:1593f080c3505099617aff13b5ea54533afbab3e203b92ec7b796f127951bccc"}, ] +[[package]] +name = "dagster-embedded-elt" +version = "0.25.2" +requires_python = "<3.13,>=3.9" +summary = "Package for performing ETL/ELT tasks with Dagster." +groups = ["default"] +dependencies = [ + "dagster==1.9.2", + "dlt>=0.4", + "sling>=1.1.5", +] +files = [ + {file = "dagster-embedded-elt-0.25.2.tar.gz", hash = "sha256:431b7f4ed94130b16ac75e66d04dc06d29ae8ae2fdc01b9ae9f7e4ebbe82941e"}, + {file = "dagster_embedded_elt-0.25.2-py3-none-any.whl", hash = "sha256:0dc8eb5acc3eb82d91537a8c4f54ce967386307f2a1406cdea7793e90af2d7c3"}, +] + [[package]] name = "dagster-fivetran" version = "0.25.2" @@ -911,6 +1001,47 @@ files = [ {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, ] +[[package]] +name = "dlt" +version = "1.4.0" +requires_python = "<3.13,>=3.8.1" +summary = "dlt is an open-source python-first scalable data loading library that does not require any backend to run." +groups = ["default"] +dependencies = [ + "PyYAML>=5.4.1", + "aiohttp>=3.9", + "astunparse>=1.6.3", + "click>=7.1", + "fsspec>=2022.4.0", + "gitpython>=3.1.29", + "giturlparse>=0.10.0", + "graphlib-backport; python_version < \"3.9\"", + "hexbytes>=0.2.2", + "humanize>=4.4.0", + "jsonpath-ng>=1.5.3", + "makefun>=1.15.0", + "orjson!=3.10.1,!=3.9.11,!=3.9.12,!=3.9.13,!=3.9.14,<4,>=3.6.7; platform_python_implementation != \"PyPy\"", + "packaging>=21.1", + "pathvalidate>=2.5.2", + "pendulum>=2.1.2", + "pluggy>=1.3.0", + "pytz>=2022.6", + "requests>=2.26.0", + "requirements-parser>=0.5.0", + "semver>=2.13.0", + "setuptools>=65.6.0", + "simplejson>=3.17.5", + "tenacity>=8.0.2", + "tomlkit>=0.11.3", + "typing-extensions>=4.0.0", + "tzdata>=2022.1", + "win-precise-time>=1.4.2; os_name == \"nt\"", +] +files = [ + {file = "dlt-1.4.0-py3-none-any.whl", hash = "sha256:c3a69e4067581bf0335796bec62d58058ff1f11249f16b699d6657544b126247"}, + {file = "dlt-1.4.0.tar.gz", hash = "sha256:75208448dc11dd501cf15d76742368816fef8e1b22fb07417f69d5ceb720b324"}, +] + [[package]] name = "dnspython" version = "2.7.0" @@ -987,13 +1118,38 @@ files = [ {file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"}, ] +[[package]] +name = "frozenlist" +version = "1.5.0" +requires_python = ">=3.8" +summary = "A list-like structure which implements collections.abc.MutableSequence" +groups = ["default"] +files = [ + {file = "frozenlist-1.5.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:31115ba75889723431aa9a4e77d5f398f5cf976eea3bdf61749731f62d4a4a21"}, + {file = "frozenlist-1.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7437601c4d89d070eac8323f121fcf25f88674627505334654fd027b091db09d"}, + {file = "frozenlist-1.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7948140d9f8ece1745be806f2bfdf390127cf1a763b925c4a805c603df5e697e"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:feeb64bc9bcc6b45c6311c9e9b99406660a9c05ca8a5b30d14a78555088b0b3a"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:683173d371daad49cffb8309779e886e59c2f369430ad28fe715f66d08d4ab1a"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7d57d8f702221405a9d9b40f9da8ac2e4a1a8b5285aac6100f3393675f0a85ee"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:30c72000fbcc35b129cb09956836c7d7abf78ab5416595e4857d1cae8d6251a6"}, + {file = "frozenlist-1.5.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:000a77d6034fbad9b6bb880f7ec073027908f1b40254b5d6f26210d2dab1240e"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5d7f5a50342475962eb18b740f3beecc685a15b52c91f7d975257e13e029eca9"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:87f724d055eb4785d9be84e9ebf0f24e392ddfad00b3fe036e43f489fafc9039"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:6e9080bb2fb195a046e5177f10d9d82b8a204c0736a97a153c2466127de87784"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:9b93d7aaa36c966fa42efcaf716e6b3900438632a626fb09c049f6a2f09fc631"}, + {file = "frozenlist-1.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:52ef692a4bc60a6dd57f507429636c2af8b6046db8b31b18dac02cbc8f507f7f"}, + {file = "frozenlist-1.5.0-cp312-cp312-win32.whl", hash = "sha256:29d94c256679247b33a3dc96cce0f93cbc69c23bf75ff715919332fdbb6a32b8"}, + {file = "frozenlist-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:8969190d709e7c48ea386db202d708eb94bdb29207a1f269bab1196ce0dcca1f"}, + {file = "frozenlist-1.5.0-py3-none-any.whl", hash = "sha256:d994863bba198a4a518b467bb971c56e1db3f180a25c6cf7bb1949c267f748c3"}, + {file = "frozenlist-1.5.0.tar.gz", hash = "sha256:81d5af29e61b9c8348e876d442253723928dce6433e0e76cd925cd83f1b4b817"}, +] + [[package]] name = "fsspec" version = "2024.10.0" requires_python = ">=3.8" summary = "File-system specification" groups = ["default", "dev"] -marker = "python_version >= \"3.12\"" files = [ {file = "fsspec-2024.10.0-py3-none-any.whl", hash = "sha256:03b9a6785766a4de40368b88906366755e2819e758b83705c88cd7cb5fe81871"}, {file = "fsspec-2024.10.0.tar.gz", hash = "sha256:eda2d8a4116d4f2429db8550f2457da57279247dd930bb12f821b58391359493"}, @@ -1009,6 +1165,20 @@ files = [ {file = "genson-1.3.0.tar.gz", hash = "sha256:e02db9ac2e3fd29e65b5286f7135762e2cd8a986537c075b06fc5f1517308e37"}, ] +[[package]] +name = "gitdb" +version = "4.0.11" +requires_python = ">=3.7" +summary = "Git Object Database" +groups = ["default"] +dependencies = [ + "smmap<6,>=3.0.1", +] +files = [ + {file = "gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4"}, + {file = "gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b"}, +] + [[package]] name = "github3-py" version = "4.0.1" @@ -1026,6 +1196,32 @@ files = [ {file = "github3.py-4.0.1.tar.gz", hash = "sha256:30d571076753efc389edc7f9aaef338a4fcb24b54d8968d5f39b1342f45ddd36"}, ] +[[package]] +name = "gitpython" +version = "3.1.43" +requires_python = ">=3.7" +summary = "GitPython is a Python library used to interact with Git repositories" +groups = ["default"] +dependencies = [ + "gitdb<5,>=4.0.1", + "typing-extensions>=3.7.4.3; python_version < \"3.8\"", +] +files = [ + {file = "GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff"}, + {file = "GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c"}, +] + +[[package]] +name = "giturlparse" +version = "0.12.0" +requires_python = ">=3.8" +summary = "A Git URL parsing module (supports parsing and rewriting)" +groups = ["default"] +files = [ + {file = "giturlparse-0.12.0-py2.py3-none-any.whl", hash = "sha256:412b74f2855f1da2fefa89fd8dde62df48476077a72fc19b62039554d27360eb"}, + {file = "giturlparse-0.12.0.tar.gz", hash = "sha256:c0fff7c21acc435491b1779566e038757a205c1ffdcb47e4f81ea52ad8c3859a"}, +] + [[package]] name = "google-api-core" version = "2.23.0" @@ -1397,21 +1593,21 @@ files = [ [[package]] name = "grpcio" -version = "1.67.1" +version = "1.68.0" requires_python = ">=3.8" summary = "HTTP/2-based RPC framework" groups = ["default", "dev"] files = [ - {file = "grpcio-1.67.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:267d1745894200e4c604958da5f856da6293f063327cb049a51fe67348e4f953"}, - {file = "grpcio-1.67.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:85f69fdc1d28ce7cff8de3f9c67db2b0ca9ba4449644488c1e0303c146135ddb"}, - {file = "grpcio-1.67.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:f26b0b547eb8d00e195274cdfc63ce64c8fc2d3e2d00b12bf468ece41a0423a0"}, - {file = "grpcio-1.67.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4422581cdc628f77302270ff839a44f4c24fdc57887dc2a45b7e53d8fc2376af"}, - {file = "grpcio-1.67.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d7616d2ded471231c701489190379e0c311ee0a6c756f3c03e6a62b95a7146e"}, - {file = "grpcio-1.67.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8a00efecde9d6fcc3ab00c13f816313c040a28450e5e25739c24f432fc6d3c75"}, - {file = "grpcio-1.67.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:699e964923b70f3101393710793289e42845791ea07565654ada0969522d0a38"}, - {file = "grpcio-1.67.1-cp312-cp312-win32.whl", hash = "sha256:4e7b904484a634a0fff132958dabdb10d63e0927398273917da3ee103e8d1f78"}, - {file = "grpcio-1.67.1-cp312-cp312-win_amd64.whl", hash = "sha256:5721e66a594a6c4204458004852719b38f3d5522082be9061d6510b455c90afc"}, - {file = "grpcio-1.67.1.tar.gz", hash = "sha256:3dc2ed4cabea4dc14d5e708c2b426205956077cc5de419b4d4079315017e9732"}, + {file = "grpcio-1.68.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:8af6137cc4ae8e421690d276e7627cfc726d4293f6607acf9ea7260bd8fc3d7d"}, + {file = "grpcio-1.68.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:4028b8e9a3bff6f377698587d642e24bd221810c06579a18420a17688e421af7"}, + {file = "grpcio-1.68.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:f60fa2adf281fd73ae3a50677572521edca34ba373a45b457b5ebe87c2d01e1d"}, + {file = "grpcio-1.68.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e18589e747c1e70b60fab6767ff99b2d0c359ea1db8a2cb524477f93cdbedf5b"}, + {file = "grpcio-1.68.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0d30f3fee9372796f54d3100b31ee70972eaadcc87314be369360248a3dcffe"}, + {file = "grpcio-1.68.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:7e0a3e72c0e9a1acab77bef14a73a416630b7fd2cbd893c0a873edc47c42c8cd"}, + {file = "grpcio-1.68.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a831dcc343440969aaa812004685ed322cdb526cd197112d0db303b0da1e8659"}, + {file = "grpcio-1.68.0-cp312-cp312-win32.whl", hash = "sha256:5a180328e92b9a0050958ced34dddcb86fec5a8b332f5a229e353dafc16cd332"}, + {file = "grpcio-1.68.0-cp312-cp312-win_amd64.whl", hash = "sha256:2bddd04a790b69f7a7385f6a112f46ea0b34c4746f361ebafe9ca0be567c78e9"}, + {file = "grpcio-1.68.0.tar.gz", hash = "sha256:7e7483d39b4a4fddb9906671e9ea21aaad4f031cdfc349fec76bdfa1e404543a"}, ] [[package]] @@ -1474,6 +1670,17 @@ files = [ {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, ] +[[package]] +name = "hexbytes" +version = "1.2.1" +requires_python = "<4,>=3.8" +summary = "hexbytes: Python `bytes` subclass that decodes hex, with a readable console output" +groups = ["default"] +files = [ + {file = "hexbytes-1.2.1-py3-none-any.whl", hash = "sha256:e64890b203a31f4a23ef11470ecfcca565beaee9198df623047df322b757471a"}, + {file = "hexbytes-1.2.1.tar.gz", hash = "sha256:515f00dddf31053db4d0d7636dd16061c1d896c3109b8e751005db4ca46bcca7"}, +] + [[package]] name = "httplib2" version = "0.22.0" @@ -1491,19 +1698,19 @@ files = [ [[package]] name = "httptools" -version = "0.6.1" +version = "0.6.4" requires_python = ">=3.8.0" summary = "A collection of framework independent HTTP protocol utils." groups = ["dev"] files = [ - {file = "httptools-0.6.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:75c8022dca7935cba14741a42744eee13ba05db00b27a4b940f0d646bd4d56d0"}, - {file = "httptools-0.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:48ed8129cd9a0d62cf4d1575fcf90fb37e3ff7d5654d3a5814eb3d55f36478c2"}, - {file = "httptools-0.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f58e335a1402fb5a650e271e8c2d03cfa7cea46ae124649346d17bd30d59c90"}, - {file = "httptools-0.6.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93ad80d7176aa5788902f207a4e79885f0576134695dfb0fefc15b7a4648d503"}, - {file = "httptools-0.6.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9bb68d3a085c2174c2477eb3ffe84ae9fb4fde8792edb7bcd09a1d8467e30a84"}, - {file = "httptools-0.6.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b512aa728bc02354e5ac086ce76c3ce635b62f5fbc32ab7082b5e582d27867bb"}, - {file = "httptools-0.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:97662ce7fb196c785344d00d638fc9ad69e18ee4bfb4000b35a52efe5adcc949"}, - {file = "httptools-0.6.1.tar.gz", hash = "sha256:c6e26c30455600b95d94b1b836085138e82f177351454ee841c148f93a9bad5a"}, + {file = "httptools-0.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:df017d6c780287d5c80601dafa31f17bddb170232d85c066604d8558683711a2"}, + {file = "httptools-0.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:85071a1e8c2d051b507161f6c3e26155b5c790e4e28d7f236422dbacc2a9cc44"}, + {file = "httptools-0.6.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69422b7f458c5af875922cdb5bd586cc1f1033295aa9ff63ee196a87519ac8e1"}, + {file = "httptools-0.6.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16e603a3bff50db08cd578d54f07032ca1631450ceb972c2f834c2b860c28ea2"}, + {file = "httptools-0.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec4f178901fa1834d4a060320d2f3abc5c9e39766953d038f1458cb885f47e81"}, + {file = "httptools-0.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f9eb89ecf8b290f2e293325c646a211ff1c2493222798bb80a530c5e7502494f"}, + {file = "httptools-0.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:db78cb9ca56b59b016e64b6031eda5653be0589dba2b1b43453f6e8b405a0970"}, + {file = "httptools-0.6.4.tar.gz", hash = "sha256:4e93eee4add6493b59a5c514da98c939b244fce4a0d8879cd3f466562f4b7d5c"}, ] [[package]] @@ -1522,6 +1729,17 @@ files = [ {file = "humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc"}, ] +[[package]] +name = "humanize" +version = "4.11.0" +requires_python = ">=3.9" +summary = "Python humanize utilities" +groups = ["default"] +files = [ + {file = "humanize-4.11.0-py3-none-any.whl", hash = "sha256:b53caaec8532bcb2fff70c8826f904c35943f8cecaca29d272d9df38092736c0"}, + {file = "humanize-4.11.0.tar.gz", hash = "sha256:e66f36020a2d5a974c504bd2555cf770621dbdbb6d82f94a6857c0b1ea2608be"}, +] + [[package]] name = "idna" version = "3.10" @@ -1619,6 +1837,18 @@ files = [ {file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"}, ] +[[package]] +name = "jsonpath-ng" +version = "1.7.0" +summary = "A final implementation of JSONPath for Python that aims to be standard compliant, including arithmetic and binary comparison operators and providing clear AST for metaprogramming." +groups = ["default"] +dependencies = [ + "ply", +] +files = [ + {file = "jsonpath-ng-1.7.0.tar.gz", hash = "sha256:f6f5f7fd4e5ff79c785f1573b394043b39849fb2bb47bcead935d12b00beab3c"}, +] + [[package]] name = "jsonschema" version = "4.23.0" @@ -1708,6 +1938,19 @@ files = [ {file = "Logbook-1.5.3.tar.gz", hash = "sha256:66f454ada0f56eae43066f604a222b09893f98c1adc18df169710761b8f32fe8"}, ] +[[package]] +name = "makefun" +version = "1.15.6" +summary = "Small library to dynamically create python functions." +groups = ["default"] +dependencies = [ + "funcsigs; python_version < \"3.3\"", +] +files = [ + {file = "makefun-1.15.6-py2.py3-none-any.whl", hash = "sha256:e69b870f0bb60304765b1e3db576aaecf2f9b3e5105afe8cfeff8f2afe6ad067"}, + {file = "makefun-1.15.6.tar.gz", hash = "sha256:26bc63442a6182fb75efed8b51741dd2d1db2f176bec8c64e20a586256b8f149"}, +] + [[package]] name = "mako" version = "1.3.6" @@ -1857,7 +2100,7 @@ name = "multidict" version = "6.1.0" requires_python = ">=3.8" summary = "multidict implementation" -groups = ["dev"] +groups = ["default", "dev"] dependencies = [ "typing-extensions>=4.1.0; python_version < \"3.11\"", ] @@ -2073,6 +2316,44 @@ files = [ {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, ] +[[package]] +name = "pathvalidate" +version = "3.2.1" +requires_python = ">=3.7" +summary = "pathvalidate is a Python library to sanitize/validate a string such as filenames/file-paths/etc." +groups = ["default"] +files = [ + {file = "pathvalidate-3.2.1-py3-none-any.whl", hash = "sha256:9a6255eb8f63c9e2135b9be97a5ce08f10230128c4ae7b3e935378b82b22c4c9"}, + {file = "pathvalidate-3.2.1.tar.gz", hash = "sha256:f5d07b1e2374187040612a1fcd2bcb2919f8db180df254c9581bb90bf903377d"}, +] + +[[package]] +name = "pendulum" +version = "3.0.0" +requires_python = ">=3.8" +summary = "Python datetimes made easy" +groups = ["default"] +dependencies = [ + "backports-zoneinfo>=0.2.1; python_version < \"3.9\"", + "importlib-resources>=5.9.0; python_version < \"3.9\"", + "python-dateutil>=2.6", + "time-machine>=2.6.0; implementation_name != \"pypy\"", + "tzdata>=2020.1", +] +files = [ + {file = "pendulum-3.0.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:409e64e41418c49f973d43a28afe5df1df4f1dd87c41c7c90f1a63f61ae0f1f7"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a38ad2121c5ec7c4c190c7334e789c3b4624798859156b138fcc4d92295835dc"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fde4d0b2024b9785f66b7f30ed59281bd60d63d9213cda0eb0910ead777f6d37"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2c5675769fb6d4c11238132962939b960fcb365436b6d623c5864287faa319"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8af95e03e066826f0f4c65811cbee1b3123d4a45a1c3a2b4fc23c4b0dff893b5"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ad5e65b874b5e56bd942546ea7ba9dd1d6a25121db1c517700f1c9de91b28518"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9"}, + {file = "pendulum-3.0.0-cp312-none-win_amd64.whl", hash = "sha256:78f8f4e7efe5066aca24a7a57511b9c2119f5c2b5eb81c46ff9222ce11e0a7a5"}, + {file = "pendulum-3.0.0-cp312-none-win_arm64.whl", hash = "sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f"}, + {file = "pendulum-3.0.0.tar.gz", hash = "sha256:5d034998dea404ec31fae27af6b22cff1708f830a1ed7353be4d1019bb9f584e"}, +] + [[package]] name = "pex" version = "2.24.1" @@ -2100,12 +2381,22 @@ name = "pluggy" version = "1.5.0" requires_python = ">=3.8" summary = "plugin and hook calling mechanisms for python" -groups = ["dev"] +groups = ["default", "dev"] files = [ {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] +[[package]] +name = "ply" +version = "3.11" +summary = "Python Lex & Yacc" +groups = ["default"] +files = [ + {file = "ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce"}, + {file = "ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3"}, +] + [[package]] name = "prompt-toolkit" version = "3.0.36" @@ -2125,7 +2416,7 @@ name = "propcache" version = "0.2.0" requires_python = ">=3.8" summary = "Accelerated property cache" -groups = ["dev"] +groups = ["default", "dev"] files = [ {file = "propcache-0.2.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2ee7606193fb267be4b2e3b32714f2d58cad27217638db98a60f9efb5efeccc2"}, {file = "propcache-0.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:91ee8fc02ca52e24bcb77b234f22afc03288e1dafbb1f88fe24db308910c4ac7"}, @@ -2195,6 +2486,74 @@ files = [ {file = "psutil-6.1.0.tar.gz", hash = "sha256:353815f59a7f64cdaca1c0307ee13558a0512f6db064e92fe833784f08539c7a"}, ] +[[package]] +name = "psycopg" +version = "3.2.3" +requires_python = ">=3.8" +summary = "PostgreSQL database adapter for Python" +groups = ["default"] +dependencies = [ + "backports-zoneinfo>=0.2.0; python_version < \"3.9\"", + "typing-extensions>=4.6; python_version < \"3.13\"", + "tzdata; sys_platform == \"win32\"", +] +files = [ + {file = "psycopg-3.2.3-py3-none-any.whl", hash = "sha256:644d3973fe26908c73d4be746074f6e5224b03c1101d302d9a53bf565ad64907"}, + {file = "psycopg-3.2.3.tar.gz", hash = "sha256:a5764f67c27bec8bfac85764d23c534af2c27b893550377e37ce59c12aac47a2"}, +] + +[[package]] +name = "psycopg-binary" +version = "3.2.3" +requires_python = ">=3.8" +summary = "PostgreSQL database adapter for Python -- C optimisation distribution" +groups = ["default"] +marker = "implementation_name != \"pypy\"" +files = [ + {file = "psycopg_binary-3.2.3-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:48f8ca6ee8939bab760225b2ab82934d54330eec10afe4394a92d3f2a0c37dd6"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:5361ea13c241d4f0ec3f95e0bf976c15e2e451e9cc7ef2e5ccfc9d170b197a40"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb987f14af7da7c24f803111dbc7392f5070fd350146af3345103f76ea82e339"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0463a11b1cace5a6aeffaf167920707b912b8986a9c7920341c75e3686277920"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8b7be9a6c06518967b641fb15032b1ed682fd3b0443f64078899c61034a0bca6"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64a607e630d9f4b2797f641884e52b9f8e239d35943f51bef817a384ec1678fe"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:fa33ead69ed133210d96af0c63448b1385df48b9c0247eda735c5896b9e6dbbf"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:1f8b0d0e99d8e19923e6e07379fa00570be5182c201a8c0b5aaa9a4d4a4ea20b"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:709447bd7203b0b2debab1acec23123eb80b386f6c29e7604a5d4326a11e5bd6"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5e37d5027e297a627da3551a1e962316d0f88ee4ada74c768f6c9234e26346d9"}, + {file = "psycopg_binary-3.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:261f0031ee6074765096a19b27ed0f75498a8338c3dcd7f4f0d831e38adf12d1"}, +] + +[[package]] +name = "psycopg-pool" +version = "3.2.3" +requires_python = ">=3.8" +summary = "Connection Pool for Psycopg" +groups = ["default"] +dependencies = [ + "typing-extensions>=4.6", +] +files = [ + {file = "psycopg_pool-3.2.3-py3-none-any.whl", hash = "sha256:53bd8e640625e01b2927b2ad96df8ed8e8f91caea4597d45e7673fc7bbb85eb1"}, + {file = "psycopg_pool-3.2.3.tar.gz", hash = "sha256:bb942f123bef4b7fbe4d55421bd3fb01829903c95c0f33fd42b7e94e5ac9b52a"}, +] + +[[package]] +name = "psycopg" +version = "3.2.3" +extras = ["binary", "pool"] +requires_python = ">=3.8" +summary = "PostgreSQL database adapter for Python" +groups = ["default"] +dependencies = [ + "psycopg-binary==3.2.3; implementation_name != \"pypy\"", + "psycopg-pool", + "psycopg==3.2.3", +] +files = [ + {file = "psycopg-3.2.3-py3-none-any.whl", hash = "sha256:644d3973fe26908c73d4be746074f6e5224b03c1101d302d9a53bf565ad64907"}, + {file = "psycopg-3.2.3.tar.gz", hash = "sha256:a5764f67c27bec8bfac85764d23c534af2c27b893550377e37ce59c12aac47a2"}, +] + [[package]] name = "py-avro-schema" version = "3.8.2" @@ -2358,29 +2717,29 @@ files = [ [[package]] name = "pyjwt" -version = "2.9.0" -requires_python = ">=3.8" +version = "2.10.0" +requires_python = ">=3.9" summary = "JSON Web Token implementation in Python" groups = ["default"] files = [ - {file = "PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850"}, - {file = "pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"}, + {file = "PyJWT-2.10.0-py3-none-any.whl", hash = "sha256:543b77207db656de204372350926bed5a86201c4cbff159f623f79c7bb487a15"}, + {file = "pyjwt-2.10.0.tar.gz", hash = "sha256:7628a7eb7938959ac1b26e819a1df0fd3259505627b575e4bad6d08f76db695c"}, ] [[package]] name = "pyjwt" -version = "2.9.0" +version = "2.10.0" extras = ["crypto"] -requires_python = ">=3.8" +requires_python = ">=3.9" summary = "JSON Web Token implementation in Python" groups = ["default"] dependencies = [ "cryptography>=3.4.0", - "pyjwt==2.9.0", + "pyjwt==2.10.0", ] files = [ - {file = "PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850"}, - {file = "pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"}, + {file = "PyJWT-2.10.0-py3-none-any.whl", hash = "sha256:543b77207db656de204372350926bed5a86201c4cbff159f623f79c7bb487a15"}, + {file = "pyjwt-2.10.0.tar.gz", hash = "sha256:7628a7eb7938959ac1b26e819a1df0fd3259505627b575e4bad6d08f76db695c"}, ] [[package]] @@ -2627,6 +2986,21 @@ files = [ {file = "requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"}, ] +[[package]] +name = "requirements-parser" +version = "0.11.0" +requires_python = "<4.0,>=3.8" +summary = "This is a small Python module for parsing Pip requirement files." +groups = ["default"] +dependencies = [ + "packaging>=23.2", + "types-setuptools>=69.1.0", +] +files = [ + {file = "requirements_parser-0.11.0-py3-none-any.whl", hash = "sha256:50379eb50311834386c2568263ae5225d7b9d0867fb55cf4ecc93959de2c2684"}, + {file = "requirements_parser-0.11.0.tar.gz", hash = "sha256:35f36dc969d14830bf459803da84f314dc3d17c802592e9e970f63d0359e5920"}, +] + [[package]] name = "rich" version = "13.9.4" @@ -2722,6 +3096,17 @@ files = [ {file = "scipy-1.14.1.tar.gz", hash = "sha256:5a275584e726026a5699459aa72f828a610821006228e841b94275c4a7c08417"}, ] +[[package]] +name = "semver" +version = "3.0.2" +requires_python = ">=3.7" +summary = "Python helper for Semantic Versioning (https://semver.org)" +groups = ["default"] +files = [ + {file = "semver-3.0.2-py3-none-any.whl", hash = "sha256:b1ea4686fe70b981f85359eda33199d60c53964284e0cfb4977d243e37cf4bf4"}, + {file = "semver-3.0.2.tar.gz", hash = "sha256:6253adb39c70f6e51afed2fa7152bcd414c411286088fb4b9effb133885ab4cc"}, +] + [[package]] name = "setuptools" version = "75.5.0" @@ -2744,6 +3129,30 @@ files = [ {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, ] +[[package]] +name = "simplejson" +version = "3.19.3" +requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.5" +summary = "Simple, fast, extensible JSON encoder/decoder for Python" +groups = ["default"] +files = [ + {file = "simplejson-3.19.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:66a0399e21c2112acacfebf3d832ebe2884f823b1c7e6d1363f2944f1db31a99"}, + {file = "simplejson-3.19.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6ef9383c5e05f445be60f1735c1816163c874c0b1ede8bb4390aff2ced34f333"}, + {file = "simplejson-3.19.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:42e5acf80d4d971238d4df97811286a044d720693092b20a56d5e56b7dcc5d09"}, + {file = "simplejson-3.19.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0b0efc7279d768db7c74d3d07f0b5c81280d16ae3fb14e9081dc903e8360771"}, + {file = "simplejson-3.19.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0552eb06e7234da892e1d02365cd2b7b2b1f8233aa5aabdb2981587b7cc92ea0"}, + {file = "simplejson-3.19.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5bf6a3b9a7d7191471b464fe38f684df10eb491ec9ea454003edb45a011ab187"}, + {file = "simplejson-3.19.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7017329ca8d4dca94ad5e59f496e5fc77630aecfc39df381ffc1d37fb6b25832"}, + {file = "simplejson-3.19.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:67a20641afebf4cfbcff50061f07daad1eace6e7b31d7622b6fa2c40d43900ba"}, + {file = "simplejson-3.19.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:dd6a7dabcc4c32daf601bc45e01b79175dde4b52548becea4f9545b0a4428169"}, + {file = "simplejson-3.19.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:08f9b443a94e72dd02c87098c96886d35790e79e46b24e67accafbf13b73d43b"}, + {file = "simplejson-3.19.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fa97278ae6614346b5ca41a45a911f37a3261b57dbe4a00602048652c862c28b"}, + {file = "simplejson-3.19.3-cp312-cp312-win32.whl", hash = "sha256:ef28c3b328d29b5e2756903aed888960bc5df39b4c2eab157ae212f70ed5bf74"}, + {file = "simplejson-3.19.3-cp312-cp312-win_amd64.whl", hash = "sha256:1e662336db50ad665777e6548b5076329a94a0c3d4a0472971c588b3ef27de3a"}, + {file = "simplejson-3.19.3-py3-none-any.whl", hash = "sha256:49cc4c7b940d43bd12bf87ec63f28cbc4964fc4e12c031cc8cd01650f43eb94e"}, + {file = "simplejson-3.19.3.tar.gz", hash = "sha256:8e086896c36210ab6050f2f9f095a5f1e03c83fa0e7f296d6cba425411364680"}, +] + [[package]] name = "six" version = "1.16.0" @@ -2757,13 +3166,45 @@ files = [ [[package]] name = "slack-sdk" -version = "3.33.3" +version = "3.33.4" requires_python = ">=3.6" summary = "The Slack API Platform SDK for Python" groups = ["default"] files = [ - {file = "slack_sdk-3.33.3-py2.py3-none-any.whl", hash = "sha256:0515fb93cd03b18de61f876a8304c4c3cef4dd3c2a3bad62d7394d2eb5a3c8e6"}, - {file = "slack_sdk-3.33.3.tar.gz", hash = "sha256:4cc44c9ffe4bb28a01fbe3264c2f466c783b893a4eca62026ab845ec7c176ff1"}, + {file = "slack_sdk-3.33.4-py2.py3-none-any.whl", hash = "sha256:9f30cb3c9c07b441c49d53fc27f9f1837ad1592a7e9d4ca431f53cdad8826cc6"}, + {file = "slack_sdk-3.33.4.tar.gz", hash = "sha256:5e109847f6b6a22d227609226ba4ed936109dc00675bddeb7e0bee502d3ee7e0"}, +] + +[[package]] +name = "sling" +version = "1.2.22" +summary = "Slings data from a source to a target" +groups = ["default"] +dependencies = [ + "sling-linux-amd64==1.2.22", +] +files = [ + {file = "sling-1.2.22.tar.gz", hash = "sha256:13bdf70a0f974b5ad417ec73ff35d39ae5460e48d302e79d0d104208b36d8e1a"}, +] + +[[package]] +name = "sling-linux-amd64" +version = "1.2.22" +summary = "Sling Binary for Linux (AMD64)" +groups = ["default"] +files = [ + {file = "sling-linux-amd64-1.2.22.tar.gz", hash = "sha256:c83c3da996bab735e8003e60ea922ab11ada61f2b7960f10df2a40df39e4edb3"}, +] + +[[package]] +name = "smmap" +version = "5.0.1" +requires_python = ">=3.7" +summary = "A pure Python implementation of a sliding window memory map manager" +groups = ["default"] +files = [ + {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"}, + {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, ] [[package]] @@ -2814,49 +3255,49 @@ files = [ [[package]] name = "sqlglot" -version = "25.30.0" +version = "25.31.4" requires_python = ">=3.7" summary = "An easily customizable SQL parser and transpiler" groups = ["default"] files = [ - {file = "sqlglot-25.30.0-py3-none-any.whl", hash = "sha256:2008e51f0d352d6bb967be71767810abdd396e84f2d07dd7816b75775323246b"}, - {file = "sqlglot-25.30.0.tar.gz", hash = "sha256:f39ad054750f3be164e69ea74c784b5bcbb237c49416695ba95df696cd81c9d5"}, + {file = "sqlglot-25.31.4-py3-none-any.whl", hash = "sha256:fa4f8042e843d7fa57dadf1329e4735601cae4d84392554acc6a2af0a66cf8e8"}, + {file = "sqlglot-25.31.4.tar.gz", hash = "sha256:1cabcbb835fc49c485d4ea84ecdb571f244b6ba78acdfabca9492dbd39b8f091"}, ] [[package]] name = "sqlglot" -version = "25.30.0" +version = "25.31.4" extras = ["rs"] requires_python = ">=3.7" summary = "An easily customizable SQL parser and transpiler" groups = ["default"] dependencies = [ - "sqlglot==25.30.0", - "sqlglotrs==0.2.13", + "sqlglot==25.31.4", + "sqlglotrs==0.2.14", ] files = [ - {file = "sqlglot-25.30.0-py3-none-any.whl", hash = "sha256:2008e51f0d352d6bb967be71767810abdd396e84f2d07dd7816b75775323246b"}, - {file = "sqlglot-25.30.0.tar.gz", hash = "sha256:f39ad054750f3be164e69ea74c784b5bcbb237c49416695ba95df696cd81c9d5"}, + {file = "sqlglot-25.31.4-py3-none-any.whl", hash = "sha256:fa4f8042e843d7fa57dadf1329e4735601cae4d84392554acc6a2af0a66cf8e8"}, + {file = "sqlglot-25.31.4.tar.gz", hash = "sha256:1cabcbb835fc49c485d4ea84ecdb571f244b6ba78acdfabca9492dbd39b8f091"}, ] [[package]] name = "sqlglotrs" -version = "0.2.13" +version = "0.2.14" requires_python = ">=3.7" summary = "An easily customizable SQL parser and transpiler" groups = ["default"] files = [ - {file = "sqlglotrs-0.2.13-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:1bef4dae451f25d7dd7ae5fb76e7a23d9d6951c8a1e6f83558d07d947ea43747"}, - {file = "sqlglotrs-0.2.13-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8aec6c9d5d3db9009c19cdc513c3bb26ffd74010d2446c297afdc9c39abc065f"}, - {file = "sqlglotrs-0.2.13-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cf22beeed2f45a0f8ccd5a6f1e05ccef2ed363628505087809df284bb67fc43"}, - {file = "sqlglotrs-0.2.13-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:26aafec54ce1619c0b192747665c301a5f3e064bd1df17a213ad90cfff9d69d4"}, - {file = "sqlglotrs-0.2.13-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:387dfb3e10ff67c6dc7dd86137e6c182ebf478a4b137b361486d1bb8af7505f9"}, - {file = "sqlglotrs-0.2.13-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e11b5a00ecdbb35f6eccb02e7d901441d5db6bb5e4a655751dc5236e4dd20090"}, - {file = "sqlglotrs-0.2.13-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e848c49dc1cf1530c5e4b5c44a96b272943dbb3fb5d0e59baecc11bfc4d1c6b"}, - {file = "sqlglotrs-0.2.13-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f321ff2cdcb0de32ac1c9ebe4b9b8c0e121541c224459563a67135a9f1410b79"}, - {file = "sqlglotrs-0.2.13-cp312-none-win32.whl", hash = "sha256:a632366a9ba92a5a2d9194febbb5324ecc34346437277247698b8842517091e2"}, - {file = "sqlglotrs-0.2.13-cp312-none-win_amd64.whl", hash = "sha256:9670480c6c5e4260ef5a016edd23db10a376e5db5fe9cdf360bbd61ba920d3c8"}, - {file = "sqlglotrs-0.2.13.tar.gz", hash = "sha256:9bcc16c2264471d049f465dea8efbb946d174251ca41e52950e2a03e85b87cdc"}, + {file = "sqlglotrs-0.2.14-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:e286c2549064fd8e01e66f90c8eccf353ff9355033d9a53a486a8153fe046bd4"}, + {file = "sqlglotrs-0.2.14-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:83aad4d7a659b4f32d30983758f9d095fff04a888834f916ec19d84b80831161"}, + {file = "sqlglotrs-0.2.14-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a28b632abb510a67755fa9a6bda2e901a99dc365955ac1c60d2348f31d1c9636"}, + {file = "sqlglotrs-0.2.14-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:65c4e06aac5fbcf08fa77cbf026c5980e73a045a3910992f8a3117b33372f760"}, + {file = "sqlglotrs-0.2.14-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c3433d4ab25caba64068da6fd2c8b4dbcdb4b60583e52fc10e01ac5df60bf70c"}, + {file = "sqlglotrs-0.2.14-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a279028c1af93f6fdf7ad281ea82a418049395dfa5ca7c177458205b6018b258"}, + {file = "sqlglotrs-0.2.14-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eef4f7ccfef4d3b1bf07cfd3d7e3b1c07ed98e921ab20cda5d009da5194e48b7"}, + {file = "sqlglotrs-0.2.14-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0dc14c4580627da6f6823cef187fd1a4003ed624c1ff1fc9e8efef58098ca5f0"}, + {file = "sqlglotrs-0.2.14-cp312-none-win32.whl", hash = "sha256:35b82c52e178e9ff83f51661c8656a5378583e83688839865d87a17d5e77fc0d"}, + {file = "sqlglotrs-0.2.14-cp312-none-win_amd64.whl", hash = "sha256:266c692265032fd23062b26e5501fb3cd07e818b8b37526788d3719272d002a2"}, + {file = "sqlglotrs-0.2.14.tar.gz", hash = "sha256:16dfc6c40af10d79c507afdbec9cf98e383a8296da677b614239781816c5848b"}, ] [[package]] @@ -2885,7 +3326,7 @@ files = [ [[package]] name = "starlette" -version = "0.41.2" +version = "0.41.3" requires_python = ">=3.8" summary = "The little ASGI library that shines." groups = ["dev"] @@ -2894,8 +3335,8 @@ dependencies = [ "typing-extensions>=3.10.0; python_version < \"3.10\"", ] files = [ - {file = "starlette-0.41.2-py3-none-any.whl", hash = "sha256:fbc189474b4731cf30fcef52f18a8d070e3f3b46c6a04c97579e85e6ffca942d"}, - {file = "starlette-0.41.2.tar.gz", hash = "sha256:9834fd799d1a87fd346deb76158668cfa0b0d56f85caefe8268e2d97c3468b62"}, + {file = "starlette-0.41.3-py3-none-any.whl", hash = "sha256:44cedb2b7c77a9de33a8b74b2b90e9f50d11fcf25d8270ea525ad71a25374ff7"}, + {file = "starlette-0.41.3.tar.gz", hash = "sha256:0e4ab3d16522a255be6b28260b938eae2482f98ce5cc934cb08dce8dc3ba5835"}, ] [[package]] @@ -2970,6 +3411,31 @@ files = [ {file = "threadpoolctl-3.5.0.tar.gz", hash = "sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107"}, ] +[[package]] +name = "time-machine" +version = "2.16.0" +requires_python = ">=3.9" +summary = "Travel through time in your tests." +groups = ["default"] +marker = "implementation_name != \"pypy\"" +dependencies = [ + "python-dateutil", +] +files = [ + {file = "time_machine-2.16.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:84788f4d62a8b1bf5e499bb9b0e23ceceea21c415ad6030be6267ce3d639842f"}, + {file = "time_machine-2.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:15ec236b6571730236a193d9d6c11d472432fc6ab54e85eac1c16d98ddcd71bf"}, + {file = "time_machine-2.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cedc989717c8b44a3881ac3d68ab5a95820448796c550de6a2149ed1525157f0"}, + {file = "time_machine-2.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9d26d79de1c63a8c6586c75967e09b0ff306aa7e944a1eaddb74595c9b1839ca"}, + {file = "time_machine-2.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:317b68b56a9c3731e0cf8886e0f94230727159e375988b36c60edce0ddbcb44a"}, + {file = "time_machine-2.16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:43e1e18279759897be3293a255d53e6b1cb0364b69d9591d0b80c51e461c94b0"}, + {file = "time_machine-2.16.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e43adb22def972a29d2b147999b56897116085777a0fea182fd93ee45730611e"}, + {file = "time_machine-2.16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0c766bea27a0600e36806d628ebc4b47178b12fcdfb6c24dc0a566a9c06bfe7f"}, + {file = "time_machine-2.16.0-cp312-cp312-win32.whl", hash = "sha256:6dae82ab647d107817e013db82223e20a9853fa88543fec853ae326382d03c2e"}, + {file = "time_machine-2.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:265462c77dc9576267c3c7f20707780a171a9fdbac93ac22e608c309efd68c33"}, + {file = "time_machine-2.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:ef768e14768eebe3bb1196c0dece8e14c1c6991605721214a0c3c68cf77eb216"}, + {file = "time_machine-2.16.0.tar.gz", hash = "sha256:4a99acc273d2f98add23a89b94d4dd9e14969c01214c8514bfa78e4e9364c7e2"}, +] + [[package]] name = "tomli" version = "2.1.0" @@ -2981,6 +3447,17 @@ files = [ {file = "tomli-2.1.0.tar.gz", hash = "sha256:3f646cae2aec94e17d04973e4249548320197cfabdf130015d023de4b74d8ab8"}, ] +[[package]] +name = "tomlkit" +version = "0.13.2" +requires_python = ">=3.8" +summary = "Style preserving TOML library" +groups = ["default"] +files = [ + {file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"}, + {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, +] + [[package]] name = "toposort" version = "1.10" @@ -3022,7 +3499,7 @@ files = [ [[package]] name = "typer" -version = "0.13.0" +version = "0.13.1" requires_python = ">=3.7" summary = "Typer, build great CLIs. Easy to code. Based on Python type hints." groups = ["default"] @@ -3033,8 +3510,19 @@ dependencies = [ "typing-extensions>=3.7.4.3", ] files = [ - {file = "typer-0.13.0-py3-none-any.whl", hash = "sha256:d85fe0b777b2517cc99c8055ed735452f2659cd45e451507c76f48ce5c1d00e2"}, - {file = "typer-0.13.0.tar.gz", hash = "sha256:f1c7198347939361eec90139ffa0fd8b3df3a2259d5852a0f7400e476d95985c"}, + {file = "typer-0.13.1-py3-none-any.whl", hash = "sha256:5b59580fd925e89463a29d363e0a43245ec02765bde9fb77d39e5d0f29dd7157"}, + {file = "typer-0.13.1.tar.gz", hash = "sha256:9d444cb96cc268ce6f8b94e13b4335084cef4c079998a9f4851a90229a3bd25c"}, +] + +[[package]] +name = "types-setuptools" +version = "75.5.0.20241121" +requires_python = ">=3.8" +summary = "Typing stubs for setuptools" +groups = ["default"] +files = [ + {file = "types_setuptools-75.5.0.20241121-py3-none-any.whl", hash = "sha256:8ea9489a8a9b98d4eb182d302117cbe1d5ae7c01a864e091b21a9107c2ba186c"}, + {file = "types_setuptools-75.5.0.20241121.tar.gz", hash = "sha256:47b261411cfca811681183f5371656ebc3d8e722b8ee857276fdb5402e5f6fb3"}, ] [[package]] @@ -3136,19 +3624,19 @@ files = [ [[package]] name = "uvloop" -version = "0.20.0" +version = "0.21.0" requires_python = ">=3.8.0" summary = "Fast implementation of asyncio event loop on top of libuv" groups = ["dev"] marker = "(sys_platform != \"cygwin\" and sys_platform != \"win32\") and platform_python_implementation != \"PyPy\"" files = [ - {file = "uvloop-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:4b75f2950ddb6feed85336412b9a0c310a2edbcf4cf931aa5cfe29034829676d"}, - {file = "uvloop-0.20.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:77fbc69c287596880ecec2d4c7a62346bef08b6209749bf6ce8c22bbaca0239e"}, - {file = "uvloop-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6462c95f48e2d8d4c993a2950cd3d31ab061864d1c226bbf0ee2f1a8f36674b9"}, - {file = "uvloop-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:649c33034979273fa71aa25d0fe120ad1777c551d8c4cd2c0c9851d88fcb13ab"}, - {file = "uvloop-0.20.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3a609780e942d43a275a617c0839d85f95c334bad29c4c0918252085113285b5"}, - {file = "uvloop-0.20.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aea15c78e0d9ad6555ed201344ae36db5c63d428818b4b2a42842b3870127c00"}, - {file = "uvloop-0.20.0.tar.gz", hash = "sha256:4603ca714a754fc8d9b197e325db25b2ea045385e8a3ad05d3463de725fdf469"}, + {file = "uvloop-0.21.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:359ec2c888397b9e592a889c4d72ba3d6befba8b2bb01743f72fffbde663b59c"}, + {file = "uvloop-0.21.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7089d2dc73179ce5ac255bdf37c236a9f914b264825fdaacaded6990a7fb4c2"}, + {file = "uvloop-0.21.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baa4dcdbd9ae0a372f2167a207cd98c9f9a1ea1188a8a526431eef2f8116cc8d"}, + {file = "uvloop-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86975dca1c773a2c9864f4c52c5a55631038e387b47eaf56210f873887b6c8dc"}, + {file = "uvloop-0.21.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:461d9ae6660fbbafedd07559c6a2e57cd553b34b0065b6550685f6653a98c1cb"}, + {file = "uvloop-0.21.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:183aef7c8730e54c9a3ee3227464daed66e37ba13040bb3f350bc2ddc040f22f"}, + {file = "uvloop-0.21.0.tar.gz", hash = "sha256:3bf12b0fda68447806a7ad847bfa591613177275d35b6724b1ee573faa3704e3"}, ] [[package]] @@ -3226,56 +3714,80 @@ files = [ [[package]] name = "websockets" -version = "13.1" -requires_python = ">=3.8" +version = "14.1" +requires_python = ">=3.9" summary = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" groups = ["dev"] files = [ - {file = "websockets-13.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9d75baf00138f80b48f1eac72ad1535aac0b6461265a0bcad391fc5aba875cfc"}, - {file = "websockets-13.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9b6f347deb3dcfbfde1c20baa21c2ac0751afaa73e64e5b693bb2b848efeaa49"}, - {file = "websockets-13.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de58647e3f9c42f13f90ac7e5f58900c80a39019848c5547bc691693098ae1bd"}, - {file = "websockets-13.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1b54689e38d1279a51d11e3467dd2f3a50f5f2e879012ce8f2d6943f00e83f0"}, - {file = "websockets-13.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf1781ef73c073e6b0f90af841aaf98501f975d306bbf6221683dd594ccc52b6"}, - {file = "websockets-13.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d23b88b9388ed85c6faf0e74d8dec4f4d3baf3ecf20a65a47b836d56260d4b9"}, - {file = "websockets-13.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3c78383585f47ccb0fcf186dcb8a43f5438bd7d8f47d69e0b56f71bf431a0a68"}, - {file = "websockets-13.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d6d300f8ec35c24025ceb9b9019ae9040c1ab2f01cddc2bcc0b518af31c75c14"}, - {file = "websockets-13.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a9dcaf8b0cc72a392760bb8755922c03e17a5a54e08cca58e8b74f6902b433cf"}, - {file = "websockets-13.1-cp312-cp312-win32.whl", hash = "sha256:2f85cf4f2a1ba8f602298a853cec8526c2ca42a9a4b947ec236eaedb8f2dc80c"}, - {file = "websockets-13.1-cp312-cp312-win_amd64.whl", hash = "sha256:38377f8b0cdeee97c552d20cf1865695fcd56aba155ad1b4ca8779a5b6ef4ac3"}, - {file = "websockets-13.1-py3-none-any.whl", hash = "sha256:a9a396a6ad26130cdae92ae10c36af09d9bfe6cafe69670fd3b6da9b07b4044f"}, - {file = "websockets-13.1.tar.gz", hash = "sha256:a3b3366087c1bc0a2795111edcadddb8b3b59509d5db5d7ea3fdd69f954a8878"}, + {file = "websockets-14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ed907449fe5e021933e46a3e65d651f641975a768d0649fee59f10c2985529ed"}, + {file = "websockets-14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:87e31011b5c14a33b29f17eb48932e63e1dcd3fa31d72209848652310d3d1f0d"}, + {file = "websockets-14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bc6ccf7d54c02ae47a48ddf9414c54d48af9c01076a2e1023e3b486b6e72c707"}, + {file = "websockets-14.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9777564c0a72a1d457f0848977a1cbe15cfa75fa2f67ce267441e465717dcf1a"}, + {file = "websockets-14.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a655bde548ca98f55b43711b0ceefd2a88a71af6350b0c168aa77562104f3f45"}, + {file = "websockets-14.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3dfff83ca578cada2d19e665e9c8368e1598d4e787422a460ec70e531dbdd58"}, + {file = "websockets-14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6a6c9bcf7cdc0fd41cc7b7944447982e8acfd9f0d560ea6d6845428ed0562058"}, + {file = "websockets-14.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4b6caec8576e760f2c7dd878ba817653144d5f369200b6ddf9771d64385b84d4"}, + {file = "websockets-14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eb6d38971c800ff02e4a6afd791bbe3b923a9a57ca9aeab7314c21c84bf9ff05"}, + {file = "websockets-14.1-cp312-cp312-win32.whl", hash = "sha256:1d045cbe1358d76b24d5e20e7b1878efe578d9897a25c24e6006eef788c0fdf0"}, + {file = "websockets-14.1-cp312-cp312-win_amd64.whl", hash = "sha256:90f4c7a069c733d95c308380aae314f2cb45bd8a904fb03eb36d1a4983a4993f"}, + {file = "websockets-14.1-py3-none-any.whl", hash = "sha256:4d4fc827a20abe6d544a119896f6b78ee13fe81cbfef416f3f2ddf09a03f0e2e"}, + {file = "websockets-14.1.tar.gz", hash = "sha256:398b10c77d471c0aab20a845e7a60076b6390bfdaac7a6d2edb0d2c59d75e8d8"}, +] + +[[package]] +name = "wheel" +version = "0.45.0" +requires_python = ">=3.8" +summary = "A built-package format for Python" +groups = ["default"] +files = [ + {file = "wheel-0.45.0-py3-none-any.whl", hash = "sha256:52f0baa5e6522155090a09c6bd95718cc46956d1b51d537ea5454249edb671c7"}, + {file = "wheel-0.45.0.tar.gz", hash = "sha256:a57353941a3183b3d5365346b567a260a0602a0f8a635926a7dede41b94c674a"}, +] + +[[package]] +name = "win-precise-time" +version = "1.4.2" +requires_python = ">=3.7" +summary = "" +groups = ["default"] +marker = "os_name == \"nt\"" +files = [ + {file = "win-precise-time-1.4.2.tar.gz", hash = "sha256:89274785cbc5f2997e01675206da3203835a442c60fd97798415c6b3c179c0b9"}, + {file = "win_precise_time-1.4.2-cp312-cp312-win32.whl", hash = "sha256:0210dcea88a520c91de1708ae4c881e3c0ddc956daa08b9eabf2b7c35f3109f5"}, + {file = "win_precise_time-1.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:85670f77cc8accd8f1e6d05073999f77561c23012a9ee988cbd44bb7ce655062"}, ] [[package]] name = "yarl" -version = "1.17.1" +version = "1.17.2" requires_python = ">=3.9" summary = "Yet another URL library" -groups = ["dev"] +groups = ["default", "dev"] dependencies = [ "idna>=2.0", "multidict>=4.0", "propcache>=0.2.0", ] files = [ - {file = "yarl-1.17.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:327828786da2006085a4d1feb2594de6f6d26f8af48b81eb1ae950c788d97f61"}, - {file = "yarl-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cc353841428d56b683a123a813e6a686e07026d6b1c5757970a877195f880c2d"}, - {file = "yarl-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c73df5b6e8fabe2ddb74876fb82d9dd44cbace0ca12e8861ce9155ad3c886139"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bdff5e0995522706c53078f531fb586f56de9c4c81c243865dd5c66c132c3b5"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:06157fb3c58f2736a5e47c8fcbe1afc8b5de6fb28b14d25574af9e62150fcaac"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1654ec814b18be1af2c857aa9000de7a601400bd4c9ca24629b18486c2e35463"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f6595c852ca544aaeeb32d357e62c9c780eac69dcd34e40cae7b55bc4fb1147"}, - {file = "yarl-1.17.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:459e81c2fb920b5f5df744262d1498ec2c8081acdcfe18181da44c50f51312f7"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e48cdb8226644e2fbd0bdb0a0f87906a3db07087f4de77a1b1b1ccfd9e93685"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d9b6b28a57feb51605d6ae5e61a9044a31742db557a3b851a74c13bc61de5172"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e594b22688d5747b06e957f1ef822060cb5cb35b493066e33ceac0cf882188b7"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5f236cb5999ccd23a0ab1bd219cfe0ee3e1c1b65aaf6dd3320e972f7ec3a39da"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a2a64e62c7a0edd07c1c917b0586655f3362d2c2d37d474db1a509efb96fea1c"}, - {file = "yarl-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d0eea830b591dbc68e030c86a9569826145df485b2b4554874b07fea1275a199"}, - {file = "yarl-1.17.1-cp312-cp312-win32.whl", hash = "sha256:46ddf6e0b975cd680eb83318aa1d321cb2bf8d288d50f1754526230fcf59ba96"}, - {file = "yarl-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:117ed8b3732528a1e41af3aa6d4e08483c2f0f2e3d3d7dca7cf538b3516d93df"}, - {file = "yarl-1.17.1-py3-none-any.whl", hash = "sha256:f1790a4b1e8e8e028c391175433b9c8122c39b46e1663228158e61e6f915bf06"}, - {file = "yarl-1.17.1.tar.gz", hash = "sha256:067a63fcfda82da6b198fa73079b1ca40b7c9b7994995b6ee38acda728b64d47"}, + {file = "yarl-1.17.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:dd90238d3a77a0e07d4d6ffdebc0c21a9787c5953a508a2231b5f191455f31e9"}, + {file = "yarl-1.17.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c74f0b0472ac40b04e6d28532f55cac8090e34c3e81f118d12843e6df14d0909"}, + {file = "yarl-1.17.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4d486ddcaca8c68455aa01cf53d28d413fb41a35afc9f6594a730c9779545876"}, + {file = "yarl-1.17.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25b7e93f5414b9a983e1a6c1820142c13e1782cc9ed354c25e933aebe97fcf2"}, + {file = "yarl-1.17.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3a0baff7827a632204060f48dca9e63fbd6a5a0b8790c1a2adfb25dc2c9c0d50"}, + {file = "yarl-1.17.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:460024cacfc3246cc4d9f47a7fc860e4fcea7d1dc651e1256510d8c3c9c7cde0"}, + {file = "yarl-1.17.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5870d620b23b956f72bafed6a0ba9a62edb5f2ef78a8849b7615bd9433384171"}, + {file = "yarl-1.17.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2941756754a10e799e5b87e2319bbec481ed0957421fba0e7b9fb1c11e40509f"}, + {file = "yarl-1.17.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9611b83810a74a46be88847e0ea616794c406dbcb4e25405e52bff8f4bee2d0a"}, + {file = "yarl-1.17.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cd7e35818d2328b679a13268d9ea505c85cd773572ebb7a0da7ccbca77b6a52e"}, + {file = "yarl-1.17.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6b981316fcd940f085f646b822c2ff2b8b813cbd61281acad229ea3cbaabeb6b"}, + {file = "yarl-1.17.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:688058e89f512fb7541cb85c2f149c292d3fa22f981d5a5453b40c5da49eb9e8"}, + {file = "yarl-1.17.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:56afb44a12b0864d17b597210d63a5b88915d680f6484d8d202ed68ade38673d"}, + {file = "yarl-1.17.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:17931dfbb84ae18b287279c1f92b76a3abcd9a49cd69b92e946035cff06bcd20"}, + {file = "yarl-1.17.2-cp312-cp312-win32.whl", hash = "sha256:ff8d95e06546c3a8c188f68040e9d0360feb67ba8498baf018918f669f7bc39b"}, + {file = "yarl-1.17.2-cp312-cp312-win_amd64.whl", hash = "sha256:4c840cc11163d3c01a9d8aad227683c48cd3e5be5a785921bcc2a8b4b758c4f3"}, + {file = "yarl-1.17.2-py3-none-any.whl", hash = "sha256:dd7abf4f717e33b7487121faf23560b3a50924f80e4bef62b22dab441ded8f3b"}, + {file = "yarl-1.17.2.tar.gz", hash = "sha256:753eaaa0c7195244c84b5cc159dc8204b7fd99f716f11198f999f2332a86b178"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index f33c7af18..44be485af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "dagster-k8s", "dagster-slack", "dagster-ssh", + "dagster-embedded-elt", "dbt-bigquery>=1.8,<1.9", "dbt-core>=1.8,<1.9", "beautifulsoup4>=4.12.2", @@ -24,6 +25,7 @@ dependencies = [ "gspread>=5.12.0", "ldap3>=2.9.1", "oracledb>=1.4.2", + "psycopg[binary,pool]>=3.2.3", "py-avro-schema>=3.4.1", "pycryptodome>=3.19.0", "pypdf>=5.0.0", diff --git a/requirements.txt b/requirements.txt index 21c5d6d43..73f92223b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,12 @@ # Please do not edit it manually. agate==1.9.1 +aiohappyeyeballs==2.4.3 +aiohttp==3.11.5 +aiosignal==1.3.1 alembic==1.14.0 annotated-types==0.7.0 +astunparse==1.6.3 attrs==24.2.0 avro==1.12.0 babel==2.16.0 @@ -26,6 +30,7 @@ dagster-airbyte==0.25.2 dagster-cloud==1.9.2 dagster-cloud-cli==1.9.2 dagster-dbt==0.25.2 +dagster-embedded-elt==0.25.2 dagster-fivetran==0.25.2 dagster-gcp==0.25.2 dagster-k8s==0.25.2 @@ -42,12 +47,17 @@ dbt-extractor==0.5.1 dbt-semantic-interfaces==0.5.1 deepdiff==7.0.1 defusedxml==0.7.1 +dlt==1.4.0 docstring-parser==0.16 durationpy==0.9 fastavro==1.9.7 filelock==3.16.1 -fsspec==2024.10.0; python_version >= "3.12" +frozenlist==1.5.0 +fsspec==2024.10.0 +gitdb==4.0.11 github3-py==4.0.1 +gitpython==3.1.43 +giturlparse==0.12.0 google-api-core[grpc]==2.23.0 google-api-python-client==2.153.0 google-auth==2.36.0 @@ -62,23 +72,27 @@ google-resumable-media==2.7.2 googleapis-common-protos[grpc]==1.66.0 greenlet==3.1.1; (platform_machine == "win32" or platform_machine == "WIN32" or platform_machine == "AMD64" or platform_machine == "amd64" or platform_machine == "x86_64" or platform_machine == "ppc64le" or platform_machine == "aarch64") and python_version < "3.13" grpc-google-iam-v1==0.13.1 -grpcio==1.67.1 +grpcio==1.68.0 grpcio-health-checking==1.62.3 grpcio-status==1.62.3 gspread==6.1.4 +hexbytes==1.2.1 httplib2==0.22.0 humanfriendly==10.0 +humanize==4.11.0 idna==3.10 importlib-metadata==6.11.0 isodate==0.6.1 jinja2==3.1.4 joblib==1.4.2 +jsonpath-ng==1.7.0 jsonschema==4.23.0 jsonschema-specifications==2024.10.1 kubernetes==31.0.0 ldap3==2.9.1 leather==0.4.0 logbook==1.5.3 +makefun==1.15.6 mako==1.3.6 markdown-it-py==3.0.0 markupsafe==3.0.2 @@ -88,6 +102,7 @@ memoization==0.4.0 minimal-snowplow-tracker==0.0.2 more-itertools==10.5.0 msgpack==1.1.0 +multidict==6.1.0 networkx==3.4.2 numpy==2.1.3 oauth2client==4.1.3 @@ -100,11 +115,19 @@ pandas==2.2.3 paramiko==3.5.0 parsedatetime==2.6 pathspec==0.12.1 +pathvalidate==3.2.1 +pendulum==3.0.0 pex==2.24.1 +pluggy==1.5.0 +ply==3.11 prompt-toolkit==3.0.36 +propcache==0.2.0 proto-plus==1.25.0 protobuf==4.25.5 psutil==6.1.0; platform_system == "Windows" +psycopg-binary==3.2.3; implementation_name != "pypy" +psycopg-pool==3.2.3 +psycopg[binary,pool]==3.2.3 py-avro-schema==3.8.2 pyarrow==18.0.0 pyasn1==0.6.1 @@ -114,7 +137,7 @@ pycryptodome==3.21.0 pydantic==2.9.2 pydantic-core==2.23.4 pygments==2.18.0 -pyjwt[crypto]==2.9.0 +pyjwt[crypto]==2.10.0 pynacl==1.5.0 pyparsing==3.2.0; python_version > "3.0" pypdf==5.1.0 @@ -130,19 +153,25 @@ questionary==2.0.1 referencing==0.35.1 requests==2.32.3 requests-oauthlib==2.0.0 +requirements-parser==0.11.0 rich==13.9.4 rpds-py==0.21.0 rsa==4.9 scikit-learn==1.5.2 scipy==1.14.1 +semver==3.0.2 setuptools==75.5.0 shellingham==1.5.4 +simplejson==3.19.3 six==1.16.0 -slack-sdk==3.33.3 +slack-sdk==3.33.4 +sling==1.2.22 +sling-linux-amd64==1.2.22 +smmap==5.0.1 soupsieve==2.6 sqlalchemy==2.0.36 -sqlglot[rs]==25.30.0 -sqlglotrs==0.2.13 +sqlglot[rs]==25.31.4 +sqlglotrs==0.2.14 sqlparse==0.5.2 sshtunnel==0.4.0 structlog==24.4.0 @@ -151,11 +180,14 @@ tabulate==0.9.0 tenacity==9.0.0 text-unidecode==1.3 threadpoolctl==3.5.0 +time-machine==2.16.0; implementation_name != "pypy" tomli==2.1.0 +tomlkit==0.13.2 toposort==1.10 tqdm==4.67.0 typeguard==4.4.1 -typer==0.13.0 +typer==0.13.1 +types-setuptools==75.5.0.20241121 typing-extensions==4.12.2 tzdata==2024.2 universal-pathlib==0.2.5; python_version >= "3.12" @@ -164,4 +196,7 @@ urllib3==2.2.3 watchdog==5.0.3 wcwidth==0.2.13 websocket-client==1.8.0 +wheel==0.45.0 +win-precise-time==1.4.2; os_name == "nt" +yarl==1.17.2 zipp==3.21.0 diff --git a/src/teamster/code_locations/kippcamden/deanslist/assets.py b/src/teamster/code_locations/kippcamden/deanslist/assets.py index 3a8139d1e..fb7a1898f 100644 --- a/src/teamster/code_locations/kippcamden/deanslist/assets.py +++ b/src/teamster/code_locations/kippcamden/deanslist/assets.py @@ -92,7 +92,7 @@ partitions_def=DEANSLIST_FISCAL_MULTI_PARTITIONS_DEF, op_tags={ "dagster-k8s/config": { - "container_config": {"resources": {"limits": {"memory": "4.5Gi"}}} + "container_config": {"resources": {"limits": {"memory": "5.0Gi"}}} } }, ) diff --git a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml index 803878425..a7194d153 100644 --- a/src/teamster/code_locations/kipptaf/dagster-cloud.yaml +++ b/src/teamster/code_locations/kipptaf/dagster-cloud.yaml @@ -41,6 +41,11 @@ locations: items: - key: service_account_gserviceaccount.json path: gcloud_service_account_json + - secret: + name: op-gcp-service-account-dlt + items: + - key: keyfile.json + path: gcloud_teamster_dlt_keyfile.json server_k8s_config: container_config: env: @@ -374,6 +379,31 @@ locations: secretKeyRef: name: op-slack-api key: credential + - name: ILLUMINATE_DB_DRIVERNAME + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: driver + - name: ILLUMINATE_DB_HOST + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: ip + - name: ILLUMINATE_DB_DATABASE + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: database + - name: ILLUMINATE_DB_USERNAME + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: username + - name: ILLUMINATE_DB_PASSWORD + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: password run_k8s_config: container_config: env: @@ -707,3 +737,28 @@ locations: secretKeyRef: name: op-slack-api key: credential + - name: ILLUMINATE_DB_DRIVERNAME + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: driver + - name: ILLUMINATE_DB_HOST + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: ip + - name: ILLUMINATE_DB_DATABASE + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: database + - name: ILLUMINATE_DB_USERNAME + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: username + - name: ILLUMINATE_DB_PASSWORD + valueFrom: + secretKeyRef: + name: op-illuminate-db + key: password diff --git a/src/teamster/code_locations/kipptaf/definitions.py b/src/teamster/code_locations/kipptaf/definitions.py index 32726e814..529d8a4d3 100644 --- a/src/teamster/code_locations/kipptaf/definitions.py +++ b/src/teamster/code_locations/kipptaf/definitions.py @@ -5,6 +5,7 @@ build_sensor_for_freshness_checks, load_assets_from_modules, ) +from dagster_embedded_elt.dlt import DagsterDltResource from dagster_k8s import k8s_job_executor from teamster.code_locations.kipptaf import ( @@ -19,6 +20,7 @@ deanslist, extracts, fivetran, + illuminate, ldap, overgrad, performance_management, @@ -54,6 +56,7 @@ amplify, extracts, deanslist, + illuminate, ldap, overgrad, performance_management, @@ -97,6 +100,7 @@ "db_bigquery": BIGQUERY_RESOURCE, "dbt_cli": get_dbt_cli_resource(DBT_PROJECT), "dds": resources.DIBELS_DATA_SYSTEM_RESOURCE, + "dlt": DagsterDltResource(), "fivetran": resources.FIVETRAN_RESOURCE, "gcs": GCS_RESOURCE, "google_directory": resources.GOOGLE_DIRECTORY_RESOURCE, diff --git a/src/teamster/code_locations/kipptaf/illuminate/__init__.py b/src/teamster/code_locations/kipptaf/illuminate/__init__.py new file mode 100644 index 000000000..25c0efc0a --- /dev/null +++ b/src/teamster/code_locations/kipptaf/illuminate/__init__.py @@ -0,0 +1,5 @@ +from teamster.code_locations.kipptaf.illuminate.assets import assets + +__all__ = [ + "assets", +] diff --git a/src/teamster/code_locations/kipptaf/illuminate/assets.py b/src/teamster/code_locations/kipptaf/illuminate/assets.py new file mode 100644 index 000000000..393ad62c0 --- /dev/null +++ b/src/teamster/code_locations/kipptaf/illuminate/assets.py @@ -0,0 +1,45 @@ +import json + +from dagster import AssetExecutionContext, EnvVar, _check +from dagster_embedded_elt.dlt import DagsterDltResource, dlt_assets +from dlt import pipeline +from dlt.sources.sql_database import sql_database +from sqlalchemy import URL, create_engine + + +@dlt_assets( + dlt_source=sql_database( + credentials=create_engine( + url=URL.create( + drivername=_check.not_none( + value=EnvVar("ILLUMINATE_DB_DRIVERNAME").get_value() + ), + host=EnvVar("ILLUMINATE_DB_HOST").get_value(), + database=EnvVar("ILLUMINATE_DB_DATABASE").get_value(), + username=EnvVar("ILLUMINATE_DB_USERNAME").get_value(), + password=EnvVar("ILLUMINATE_DB_PASSWORD").get_value(), + ) + ), + schema="dna_assessments", + table_names=["assessments", "agg_student_responses_standard"], + defer_table_reflect=True, + ), + dlt_pipeline=pipeline( + pipeline_name="illuminate", + destination="bigquery", + dataset_name="dlt_illuminate_dna_assessments", + progress="log", + ), +) +def illuminate_dna_assessments(context: AssetExecutionContext, dlt: DagsterDltResource): + yield from dlt.run( + context=context, + credentials=json.load( + fp=open(file="/etc/secret-volume/gcloud_teamster_dlt_keyfile.json") + ), + ) + + +assets = [ + illuminate_dna_assessments, +] diff --git a/src/teamster/code_locations/kipptaf/resources.py b/src/teamster/code_locations/kipptaf/resources.py index d660658bd..dd9315f4f 100644 --- a/src/teamster/code_locations/kipptaf/resources.py +++ b/src/teamster/code_locations/kipptaf/resources.py @@ -1,5 +1,6 @@ from dagster import EnvVar from dagster_airbyte import AirbyteCloudResource +from dagster_embedded_elt.dlt import DagsterDltResource from dagster_fivetran import FivetranResource from teamster.libraries.adp.workforce_manager.resources import ( @@ -50,6 +51,8 @@ username=EnvVar("AMPLIFY_DDS_USERNAME"), password=EnvVar("AMPLIFY_DDS_PASSWORD") ) +DLT_RESOURCE = DagsterDltResource() + FIVETRAN_RESOURCE = FivetranResource( api_key=EnvVar("FIVETRAN_API_KEY"), api_secret=EnvVar("FIVETRAN_API_SECRET"), diff --git a/src/teamster/libraries/dlt_sources/sql_database_pipeline.py b/src/teamster/libraries/dlt_sources/sql_database_pipeline.py new file mode 100644 index 000000000..a26f8cedf --- /dev/null +++ b/src/teamster/libraries/dlt_sources/sql_database_pipeline.py @@ -0,0 +1,364 @@ +import os +from typing import Any + +import dlt +import humanize +import sqlalchemy as sa +from dlt.common import pendulum +from dlt.sources.credentials import ConnectionStringCredentials +from dlt.sources.sql_database import Table, sql_database, sql_table + + +def load_select_tables_from_database() -> None: + """Use the sql_database source to reflect an entire database schema and load select tables from it. + + This example sources data from the public Rfam MySQL database. + """ + # Create a pipeline + pipeline = dlt.pipeline( + pipeline_name="rfam", destination="bigquery", dataset_name="rfam_data" + ) + + # Credentials for the sample database. + # Note: It is recommended to configure credentials in `.dlt/secrets.toml` under `sources.sql_database.credentials` + credentials = ConnectionStringCredentials( + "mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam" + ) + # To pass the credentials from `secrets.toml`, comment out the above credentials. + # And the credentials will be automatically read from `secrets.toml`. + + # Configure the source to load a few select tables incrementally + source_1 = sql_database(credentials).with_resources("family", "clan") + + # Add incremental config to the resources. "updated" is a timestamp column in these tables that gets used as a cursor + source_1.family.apply_hints(incremental=dlt.sources.incremental("updated")) + source_1.clan.apply_hints(incremental=dlt.sources.incremental("updated")) + + # Run the pipeline. The merge write disposition merges existing rows in the destination by primary key + info = pipeline.run(source_1, write_disposition="merge") + print(info) + + # Load some other tables with replace write disposition. This overwrites the existing tables in destination + source_2 = sql_database(credentials).with_resources("features", "author") + info = pipeline.run(source_2, write_disposition="replace") + print(info) + + # Load a table incrementally with append write disposition + # this is good when a table only has new rows inserted, but not updated + source_3 = sql_database(credentials).with_resources("genome") + source_3.genome.apply_hints(incremental=dlt.sources.incremental("created")) + + info = pipeline.run(source_3, write_disposition="append") + print(info) + + +def load_entire_database() -> None: + """Use the sql_database source to completely load all tables in a database""" + pipeline = dlt.pipeline( + pipeline_name="rfam", destination="bigquery", dataset_name="rfam_data" + ) + + # By default the sql_database source reflects all tables in the schema + # The database credentials are sourced from the `.dlt/secrets.toml` configuration + source = sql_database() + + # Run the pipeline. For a large db this may take a while + info = pipeline.run(source, write_disposition="replace") + print( + humanize.precisedelta( + pipeline.last_trace.finished_at - pipeline.last_trace.started_at + ) + ) + print(info) + + +def load_standalone_table_resource() -> None: + """Load a few known tables with the standalone sql_table resource, request full schema and deferred + table reflection""" + pipeline = dlt.pipeline( + pipeline_name="rfam_database", + destination="bigquery", + dataset_name="rfam_data", + full_refresh=True, + ) + + # Load a table incrementally starting at a given date + # Adding incremental via argument like this makes extraction more efficient + # as only rows newer than the start date are fetched from the table + # we also use `detect_precision_hints` to get detailed column schema + # and defer_table_reflect to reflect schema only during execution + family = sql_table( + credentials=ConnectionStringCredentials( + "mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam" + ), + table="family", + incremental=dlt.sources.incremental( + "updated", + ), + reflection_level="full_with_precision", + defer_table_reflect=True, + ) + # columns will be empty here due to defer_table_reflect set to True + print(family.compute_table_schema()) + + # Load all data from another table + genome = sql_table( + credentials="mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam", + table="genome", + reflection_level="full_with_precision", + defer_table_reflect=True, + ) + + # Run the resources together + info = pipeline.extract([family, genome], write_disposition="merge") + print(info) + # Show inferred columns + print(pipeline.default_schema.to_pretty_yaml()) + + +def select_columns() -> None: + """Uses table adapter callback to modify list of columns to be selected""" + pipeline = dlt.pipeline( + pipeline_name="rfam_database", + destination="bigquery", + dataset_name="rfam_data_cols", + full_refresh=True, + ) + + def table_adapter(table: Table) -> None: + print(table.name) + if table.name == "family": + # this is SqlAlchemy table. _columns are writable + # let's drop updated column + table._columns.remove(table.columns["updated"]) + + family = sql_table( + credentials="mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam", + table="family", + chunk_size=10, + reflection_level="full_with_precision", + table_adapter_callback=table_adapter, + ) + + # also we do not want the whole table, so we add limit to get just one chunk (10 records) + pipeline.run(family.add_limit(1)) + # only 10 rows + print(pipeline.last_trace.last_normalize_info) + # no "updated" column in "family" table + print(pipeline.default_schema.to_pretty_yaml()) + + +def select_with_end_value_and_row_order() -> None: + """Gets data from a table withing a specified range and sorts rows descending""" + pipeline = dlt.pipeline( + pipeline_name="rfam_database", + destination="bigquery", + dataset_name="rfam_data", + full_refresh=True, + ) + + # gets data from this range + start_date = pendulum.now().subtract(years=1) + end_date = pendulum.now() + + family = sql_table( + credentials="mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam", + table="family", + incremental=dlt.sources.incremental( # declares desc row order + "updated", initial_value=start_date, end_value=end_date, row_order="desc" + ), + chunk_size=10, + ) + # also we do not want the whole table, so we add limit to get just one chunk (10 records) + pipeline.run(family.add_limit(1)) + # only 10 rows + print(pipeline.last_trace.last_normalize_info) + + +def my_sql_via_pyarrow() -> None: + """Uses pyarrow backend to load tables from mysql""" + + # uncomment line below to get load_id into your data (slows pyarrow loading down) + # dlt.config["normalize.parquet_normalizer.add_dlt_load_id"] = True + + # Create a pipeline + pipeline = dlt.pipeline( + pipeline_name="rfam_cx", + destination="bigquery", + dataset_name="rfam_data_arrow_4", + ) + + def _double_as_decimal_adapter(table: sa.Table) -> None: + """Return double as double, not decimals, only works if you are using sqlalchemy 2.0""" + for column in table.columns.values(): + if hasattr(sa, "Double") and isinstance(column.type, sa.Double): + column.type.asdecimal = False + + sql_alchemy_source = sql_database( + "mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam?&binary_prefix=true", + backend="pyarrow", + table_adapter_callback=_double_as_decimal_adapter, + ).with_resources("family", "genome") + + info = pipeline.run(sql_alchemy_source) + print(info) + + +def create_unsw_flow() -> None: + """Uploads UNSW_Flow dataset to postgres via csv stream skipping dlt normalizer. + You need to download the dataset from https://github.com/rdpahalavan/nids-datasets + """ + from pyarrow.parquet import ParquetFile + + # from dlt.destinations import postgres + # use those config to get 3x speedup on parallelism + # [sources.data_writer] + # file_max_bytes=3000000 + # buffer_max_items=200000 + # [normalize] + # workers=3 + + data_iter = ParquetFile("UNSW-NB15/Network-Flows/UNSW_Flow.parquet").iter_batches( + batch_size=128 * 1024 + ) + + pipeline = dlt.pipeline( + pipeline_name="unsw_upload", + # destination=postgres("postgres://loader:loader@localhost:5432/dlt_data"), + destination="bigquery", + progress="log", + ) + pipeline.run( + data_iter, + dataset_name="speed_test", + table_name="unsw_flow_7", + loader_file_format="csv", + ) + + +def test_connectorx_speed() -> None: + """Uses unsw_flow dataset (~2mln rows, 25+ columns) to test connectorx speed""" + + # from dlt.destinations import filesystem + + unsw_table = sql_table( + "postgresql://loader:loader@localhost:5432/dlt_data", + "unsw_flow_7", + "speed_test", + # this is ignored by connectorx + chunk_size=100000, + backend="connectorx", + # keep source data types + reflection_level="full_with_precision", + # just to demonstrate how to setup a separate connection string for connectorx + backend_kwargs={"conn": "postgresql://loader:loader@localhost:5432/dlt_data"}, + ) + + pipeline = dlt.pipeline( + pipeline_name="unsw_download", + destination="bigquery", + # destination=filesystem(os.path.abspath("../_storage/unsw")), + progress="log", + full_refresh=True, + ) + + info = pipeline.run( + unsw_table, + dataset_name="speed_test", + table_name="unsw_flow", + loader_file_format="parquet", + ) + print(info) + + +def test_pandas_backend_verbatim_decimals() -> None: + pipeline = dlt.pipeline( + pipeline_name="rfam_cx", + destination="bigquery", + dataset_name="rfam_data_pandas_2", + ) + + def _double_as_decimal_adapter(table: sa.Table) -> None: + """Emits decimals instead of floats.""" + for column in table.columns.values(): + if isinstance(column.type, sa.Float): + column.type.asdecimal = True + + sql_alchemy_source = sql_database( + "mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam?&binary_prefix=true", + backend="pandas", + table_adapter_callback=_double_as_decimal_adapter, + chunk_size=100000, + # set coerce_float to False to represent them as string + backend_kwargs={"coerce_float": False, "dtype_backend": "numpy_nullable"}, + # preserve full typing info. this will parse + reflection_level="full_with_precision", + ).with_resources("family", "genome") + + info = pipeline.run(sql_alchemy_source) + print(info) + + +def use_type_adapter() -> None: + """Example use of type adapter to coerce unknown data types""" + pipeline = dlt.pipeline( + pipeline_name="dummy", + destination="bigquery", + dataset_name="dummy", + ) + + def type_adapter(sql_type: Any) -> Any: + if isinstance(sql_type, sa.ARRAY): + return sa.JSON() # Load arrays as JSON + return sql_type + + sql_alchemy_source = sql_database( + "postgresql://loader:loader@localhost:5432/dlt_data", + backend="pyarrow", + type_adapter_callback=type_adapter, + reflection_level="full_with_precision", + ).with_resources("table_with_array_column") + + info = pipeline.run(sql_alchemy_source) + print(info) + + +def specify_columns_to_load() -> None: + """Run the SQL database source with a subset of table columns loaded""" + pipeline = dlt.pipeline( + pipeline_name="dummy", + destination="bigquery", + dataset_name="dummy", + ) + + # Columns can be specified per table in env var (json array) or in `.dlt/config.toml` + os.environ["SOURCES__SQL_DATABASE__FAMILY__INCLUDED_COLUMNS"] = ( + '["rfam_acc", "description"]' + ) + + sql_alchemy_source = sql_database( + "mysql+pymysql://rfamro@mysql-rfam-public.ebi.ac.uk:4497/Rfam?&binary_prefix=true", + backend="pyarrow", + reflection_level="full_with_precision", + ).with_resources("family", "genome") + + info = pipeline.run(sql_alchemy_source) + print(info) + + +if __name__ == "__main__": + # Load selected tables with different settings + # load_select_tables_from_database() + + # load a table and select columns + # select_columns() + + # load_entire_database() + # select_with_end_value_and_row_order() + + # Load tables with the standalone table resource + load_standalone_table_resource() + + # Load all tables from the database. + # Warning: The sample database is very large + # load_entire_database()