diff --git a/examples/sparkle/__main__.py b/examples/sparkle/__main__.py index f67d2ff..2ff67c2 100644 --- a/examples/sparkle/__main__.py +++ b/examples/sparkle/__main__.py @@ -1,3 +1,4 @@ +import os from damavand.cloud.provider import AwsProvider from damavand.factories import SparkControllerFactory @@ -16,14 +17,15 @@ def main() -> None: spark_controller = spark_factory.new( name="my-spark", + applications=[ + Products(), + CustomerOrders(), + ], ) - spark_controller.applications = [ - Products(spark_controller.default_session()), - CustomerOrders(spark_controller.default_session()), - ] + app_name = os.getenv("APP_NAME", "default_app") # Get app name on runtime - spark_controller.run_application("products") + spark_controller.run_application(app_name) spark_controller.provision() diff --git a/examples/sparkle/applications/orders.py b/examples/sparkle/applications/orders.py index 3175714..d82514d 100644 --- a/examples/sparkle/applications/orders.py +++ b/examples/sparkle/applications/orders.py @@ -1,15 +1,14 @@ from sparkle.config import Config from sparkle.writer.iceberg_writer import IcebergWriter from sparkle.application import Sparkle +from sparkle.reader.kafka_reader import KafkaReader from pyspark.sql import DataFrame -from pyspark.sql import SparkSession class CustomerOrders(Sparkle): - def __init__(self, spark_session: SparkSession): + def __init__(self): super().__init__( - spark_session, config=Config( app_name="orders", app_id="orders-app", @@ -17,14 +16,8 @@ def __init__(self, spark_session: SparkSession): database_bucket="s3://test-bucket", checkpoints_bucket="s3://test-checkpoints", ), - writers=[ - IcebergWriter( - database_name="default", - database_path="s3://bucket-name/warehouse", - table_name="products", - spark_session=spark_session, - ) - ], + readers={"orders": KafkaReader}, + writers=[IcebergWriter], ) def process(self) -> DataFrame: diff --git a/examples/sparkle/applications/products.py b/examples/sparkle/applications/products.py index ddcae13..9362fd8 100644 --- a/examples/sparkle/applications/products.py +++ b/examples/sparkle/applications/products.py @@ -1,15 +1,15 @@ from sparkle.application import Sparkle from sparkle.config import Config from sparkle.writer.iceberg_writer import IcebergWriter +from sparkle.writer.kafka_writer import KafkaStreamPublisher +from sparkle.reader.table_reader import TableReader from pyspark.sql import DataFrame -from pyspark.sql import SparkSession class Products(Sparkle): - def __init__(self, spark_session: SparkSession): + def __init__(self): super().__init__( - spark_session, config=Config( app_name="products", app_id="products-app", @@ -17,13 +17,10 @@ def __init__(self, spark_session: SparkSession): database_bucket="s3://test-bucket", checkpoints_bucket="s3://test-checkpoints", ), + readers={"products": TableReader}, writers=[ - IcebergWriter( - database_name="default", - database_path="s3://bucket-name/warehouse", - table_name="products", - spark_session=spark_session, - ) + IcebergWriter, + KafkaStreamPublisher, ], ) diff --git a/pdm.lock b/pdm.lock index da8bd73..eac070a 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev", "llm"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:93255c38933f24e57c74fd658997ed3e4767545fbea3c6e0e25eb24ce7399aad" +content_hash = "sha256:8876bfee687350c2cca3535991e47503fc5a54baa28d6f0625216d56ef84630a" [[metadata.targets]] requires_python = ">=3.11.0" @@ -26,13 +26,13 @@ files = [ [[package]] name = "argcomplete" -version = "3.4.0" +version = "3.5.0" requires_python = ">=3.8" summary = "Bash tab completion for argparse" groups = ["dev"] files = [ - {file = "argcomplete-3.4.0-py3-none-any.whl", hash = "sha256:69a79e083a716173e5532e0fa3bef45f793f4e61096cf52b5a42c0211c8b8aa5"}, - {file = "argcomplete-3.4.0.tar.gz", hash = "sha256:c2abcdfe1be8ace47ba777d4fce319eb13bf8ad9dace8d085dcad6eded88057f"}, + {file = "argcomplete-3.5.0-py3-none-any.whl", hash = "sha256:d4bcf3ff544f51e16e54228a7ac7f486ed70ebf2ecfe49a63a91171c76bf029b"}, + {file = "argcomplete-3.5.0.tar.gz", hash = "sha256:4349400469dccfb7950bb60334a680c58d88699bff6159df61251878dc6bf74b"}, ] [[package]] @@ -72,25 +72,25 @@ files = [ [[package]] name = "boto3" -version = "1.34.156" +version = "1.35.31" requires_python = ">=3.8" summary = "The AWS SDK for Python" groups = ["default", "dev", "llm"] dependencies = [ - "botocore<1.35.0,>=1.34.156", + "botocore<1.36.0,>=1.35.31", "jmespath<2.0.0,>=0.7.1", "s3transfer<0.11.0,>=0.10.0", ] files = [ - {file = "boto3-1.34.156-py3-none-any.whl", hash = "sha256:cbbd453270b8ce94ef9da60dfbb6f9ceeb3eeee226b635aa9ec44b1def98cc96"}, - {file = "boto3-1.34.156.tar.gz", hash = "sha256:b33e9a8f8be80d3053b8418836a7c1900410b23a30c7cb040927d601a1082e68"}, + {file = "boto3-1.35.31-py3-none-any.whl", hash = "sha256:2e9af74d10d8af7610a8d8468d2914961f116912a024fce17351825260385a52"}, + {file = "boto3-1.35.31.tar.gz", hash = "sha256:8c593af260c4ea3eb6f079c09908f94494ca2222aa4e40a7ff490fab1cee8b39"}, ] [[package]] name = "boto3-stubs" -version = "1.34.156" +version = "1.35.31" requires_python = ">=3.8" -summary = "Type annotations for boto3 1.34.156 generated with mypy-boto3-builder 7.25.0" +summary = "Type annotations for boto3 1.35.31 generated with mypy-boto3-builder 8.1.2" groups = ["dev"] dependencies = [ "botocore-stubs", @@ -98,13 +98,13 @@ dependencies = [ "typing-extensions>=4.1.0; python_version < \"3.12\"", ] files = [ - {file = "boto3_stubs-1.34.156-py3-none-any.whl", hash = "sha256:69495bf3bc528617756467fa2ec9c7a19af6497b358836b7015f1cbf6c9534ec"}, - {file = "boto3_stubs-1.34.156.tar.gz", hash = "sha256:c34220a00b6bd00a3cd326bc5d7e3296b7d1dab1a15660df6f8fba4ae307ff39"}, + {file = "boto3_stubs-1.35.31-py3-none-any.whl", hash = "sha256:d440ad57493534074024c999045929c617007712d395c86208b95b48f716269f"}, + {file = "boto3_stubs-1.35.31.tar.gz", hash = "sha256:bc10fa6e0c0e46c2b7c664b2183bb524053eb7439d9ebf9ae66547b295b76675"}, ] [[package]] name = "botocore" -version = "1.34.156" +version = "1.35.31" requires_python = ">=3.8" summary = "Low-level, data-driven core of boto 3." groups = ["default", "dev", "llm"] @@ -115,14 +115,14 @@ dependencies = [ "urllib3<1.27,>=1.25.4; python_version < \"3.10\"", ] files = [ - {file = "botocore-1.34.156-py3-none-any.whl", hash = "sha256:c48f8c8996216dfdeeb0aa6d3c0f2c7ae25234766434a2ea3e57bdc08494bdda"}, - {file = "botocore-1.34.156.tar.gz", hash = "sha256:5d1478c41ab9681e660b3322432fe09c4055759c317984b7b8d3af9557ff769a"}, + {file = "botocore-1.35.31-py3-none-any.whl", hash = "sha256:4cee814875bc78656aef4011d3d6b2231e96f53ea3661ee428201afb579d5c31"}, + {file = "botocore-1.35.31.tar.gz", hash = "sha256:f7bfa910cf2cbcc8c2307c1cf7b93495d614c2d699883417893e0a337fe4eb63"}, ] [[package]] name = "botocore-stubs" -version = "1.34.156" -requires_python = "<4.0,>=3.8" +version = "1.35.31" +requires_python = ">=3.8" summary = "Type annotations and code completion for botocore" groups = ["dev"] dependencies = [ @@ -130,24 +130,24 @@ dependencies = [ "typing-extensions>=4.1.0; python_version < \"3.9\"", ] files = [ - {file = "botocore_stubs-1.34.156-py3-none-any.whl", hash = "sha256:c01d59fe531bf30e7a42b53ed031c77dcfcf4a767ffb0d8b1bfbf285246a5167"}, - {file = "botocore_stubs-1.34.156.tar.gz", hash = "sha256:bb5ad1687cd363035c3f000854542476040455629463d4e4db1ebf41048a4c06"}, + {file = "botocore_stubs-1.35.31-py3-none-any.whl", hash = "sha256:b7289c27b759dad40759c1421519cbd7ec65b79f7aa8be5dfc76004a61aef42b"}, + {file = "botocore_stubs-1.35.31.tar.gz", hash = "sha256:b26f79253d8f2460aa8f2d49ae933c3f352fe38be72ea033651c15cd47a822a9"}, ] [[package]] name = "certifi" -version = "2024.7.4" +version = "2024.8.30" requires_python = ">=3.6" summary = "Python package for providing Mozilla's CA Bundle." -groups = ["dev", "llm"] +groups = ["default", "dev", "llm"] files = [ - {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, - {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, + {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, + {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, ] [[package]] name = "cffi" -version = "1.17.0" +version = "1.17.1" requires_python = ">=3.8" summary = "Foreign Function Interface for Python calling C code." groups = ["dev"] @@ -156,41 +156,41 @@ dependencies = [ "pycparser", ] files = [ - {file = "cffi-1.17.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c5d97162c196ce54af6700949ddf9409e9833ef1003b4741c2b39ef46f1d9720"}, - {file = "cffi-1.17.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5ba5c243f4004c750836f81606a9fcb7841f8874ad8f3bf204ff5e56332b72b9"}, - {file = "cffi-1.17.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bb9333f58fc3a2296fb1d54576138d4cf5d496a2cc118422bd77835e6ae0b9cb"}, - {file = "cffi-1.17.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:435a22d00ec7d7ea533db494da8581b05977f9c37338c80bc86314bec2619424"}, - {file = "cffi-1.17.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d1df34588123fcc88c872f5acb6f74ae59e9d182a2707097f9e28275ec26a12d"}, - {file = "cffi-1.17.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df8bb0010fdd0a743b7542589223a2816bdde4d94bb5ad67884348fa2c1c67e8"}, - {file = "cffi-1.17.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8b5b9712783415695663bd463990e2f00c6750562e6ad1d28e072a611c5f2a6"}, - {file = "cffi-1.17.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ffef8fd58a36fb5f1196919638f73dd3ae0db1a878982b27a9a5a176ede4ba91"}, - {file = "cffi-1.17.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e67d26532bfd8b7f7c05d5a766d6f437b362c1bf203a3a5ce3593a645e870b8"}, - {file = "cffi-1.17.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:45f7cd36186db767d803b1473b3c659d57a23b5fa491ad83c6d40f2af58e4dbb"}, - {file = "cffi-1.17.0-cp311-cp311-win32.whl", hash = "sha256:a9015f5b8af1bb6837a3fcb0cdf3b874fe3385ff6274e8b7925d81ccaec3c5c9"}, - {file = "cffi-1.17.0-cp311-cp311-win_amd64.whl", hash = "sha256:b50aaac7d05c2c26dfd50c3321199f019ba76bb650e346a6ef3616306eed67b0"}, - {file = "cffi-1.17.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aec510255ce690d240f7cb23d7114f6b351c733a74c279a84def763660a2c3bc"}, - {file = "cffi-1.17.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2770bb0d5e3cc0e31e7318db06efcbcdb7b31bcb1a70086d3177692a02256f59"}, - {file = "cffi-1.17.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:db9a30ec064129d605d0f1aedc93e00894b9334ec74ba9c6bdd08147434b33eb"}, - {file = "cffi-1.17.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a47eef975d2b8b721775a0fa286f50eab535b9d56c70a6e62842134cf7841195"}, - {file = "cffi-1.17.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f3e0992f23bbb0be00a921eae5363329253c3b86287db27092461c887b791e5e"}, - {file = "cffi-1.17.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6107e445faf057c118d5050560695e46d272e5301feffda3c41849641222a828"}, - {file = "cffi-1.17.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb862356ee9391dc5a0b3cbc00f416b48c1b9a52d252d898e5b7696a5f9fe150"}, - {file = "cffi-1.17.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c1c13185b90bbd3f8b5963cd8ce7ad4ff441924c31e23c975cb150e27c2bf67a"}, - {file = "cffi-1.17.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17c6d6d3260c7f2d94f657e6872591fe8733872a86ed1345bda872cfc8c74885"}, - {file = "cffi-1.17.0-cp312-cp312-win32.whl", hash = "sha256:c3b8bd3133cd50f6b637bb4322822c94c5ce4bf0d724ed5ae70afce62187c492"}, - {file = "cffi-1.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:dca802c8db0720ce1c49cce1149ff7b06e91ba15fa84b1d59144fef1a1bc7ac2"}, - {file = "cffi-1.17.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6ce01337d23884b21c03869d2f68c5523d43174d4fc405490eb0091057943118"}, - {file = "cffi-1.17.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cab2eba3830bf4f6d91e2d6718e0e1c14a2f5ad1af68a89d24ace0c6b17cced7"}, - {file = "cffi-1.17.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:14b9cbc8f7ac98a739558eb86fabc283d4d564dafed50216e7f7ee62d0d25377"}, - {file = "cffi-1.17.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b00e7bcd71caa0282cbe3c90966f738e2db91e64092a877c3ff7f19a1628fdcb"}, - {file = "cffi-1.17.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:41f4915e09218744d8bae14759f983e466ab69b178de38066f7579892ff2a555"}, - {file = "cffi-1.17.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e4760a68cab57bfaa628938e9c2971137e05ce48e762a9cb53b76c9b569f1204"}, - {file = "cffi-1.17.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:011aff3524d578a9412c8b3cfaa50f2c0bd78e03eb7af7aa5e0df59b158efb2f"}, - {file = "cffi-1.17.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:a003ac9edc22d99ae1286b0875c460351f4e101f8c9d9d2576e78d7e048f64e0"}, - {file = "cffi-1.17.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ef9528915df81b8f4c7612b19b8628214c65c9b7f74db2e34a646a0a2a0da2d4"}, - {file = "cffi-1.17.0-cp313-cp313-win32.whl", hash = "sha256:70d2aa9fb00cf52034feac4b913181a6e10356019b18ef89bc7c12a283bf5f5a"}, - {file = "cffi-1.17.0-cp313-cp313-win_amd64.whl", hash = "sha256:b7b6ea9e36d32582cda3465f54c4b454f62f23cb083ebc7a94e2ca6ef011c3a7"}, - {file = "cffi-1.17.0.tar.gz", hash = "sha256:f3157624b7558b914cb039fd1af735e5e8049a87c817cc215109ad1c8779df76"}, + {file = "cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401"}, + {file = "cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6"}, + {file = "cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f"}, + {file = "cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b"}, + {file = "cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655"}, + {file = "cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0"}, + {file = "cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4"}, + {file = "cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99"}, + {file = "cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93"}, + {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3"}, + {file = "cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8"}, + {file = "cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65"}, + {file = "cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903"}, + {file = "cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e"}, + {file = "cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4"}, + {file = "cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd"}, + {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed"}, + {file = "cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9"}, + {file = "cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d"}, + {file = "cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a"}, + {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"}, ] [[package]] @@ -198,7 +198,7 @@ name = "charset-normalizer" version = "3.3.2" requires_python = ">=3.7.0" summary = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -groups = ["dev", "llm"] +groups = ["default", "dev", "llm"] files = [ {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, @@ -273,12 +273,12 @@ files = [ [[package]] name = "commitizen" -version = "3.28.0" +version = "3.29.1" requires_python = ">=3.8" summary = "Python commitizen client tool" groups = ["dev"] dependencies = [ - "argcomplete<3.5,>=1.12.1", + "argcomplete<3.6,>=1.12.1", "charset-normalizer<4,>=2.1.0", "colorama<0.5.0,>=0.4.1", "decli<0.7.0,>=0.6.0", @@ -292,8 +292,8 @@ dependencies = [ "typing-extensions<5.0.0,>=4.0.1; python_version < \"3.8\"", ] files = [ - {file = "commitizen-3.28.0-py3-none-any.whl", hash = "sha256:372dba10bb082be0e571cfa9ac0946f4333c3f4b2e242fbe32ae2f2afea143e4"}, - {file = "commitizen-3.28.0.tar.gz", hash = "sha256:de3a90b3246233260649e423963cd702d56a3b499ea02886a6412ebfb76f9462"}, + {file = "commitizen-3.29.1-py3-none-any.whl", hash = "sha256:83f6563fae6a6262238e4424c55db5743eaa9827d2044dc23719466e4e78a0ca"}, + {file = "commitizen-3.29.1.tar.gz", hash = "sha256:b9a56190f4f3b20c73600e5ba448c7b81e0e6f87be3092aec1db4de75bf0fa91"}, ] [[package]] @@ -403,7 +403,7 @@ files = [ [[package]] name = "cryptography" -version = "43.0.0" +version = "43.0.1" requires_python = ">=3.7" summary = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." groups = ["dev"] @@ -411,25 +411,44 @@ dependencies = [ "cffi>=1.12; platform_python_implementation != \"PyPy\"", ] files = [ - {file = "cryptography-43.0.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:64c3f16e2a4fc51c0d06af28441881f98c5d91009b8caaff40cf3548089e9c74"}, - {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3dcdedae5c7710b9f97ac6bba7e1052b95c7083c9d0e9df96e02a1932e777895"}, - {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d9a1eca329405219b605fac09ecfc09ac09e595d6def650a437523fcd08dd22"}, - {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ea9e57f8ea880eeea38ab5abf9fbe39f923544d7884228ec67d666abd60f5a47"}, - {file = "cryptography-43.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9a8d6802e0825767476f62aafed40532bd435e8a5f7d23bd8b4f5fd04cc80ecf"}, - {file = "cryptography-43.0.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:cc70b4b581f28d0a254d006f26949245e3657d40d8857066c2ae22a61222ef55"}, - {file = "cryptography-43.0.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4a997df8c1c2aae1e1e5ac49c2e4f610ad037fc5a3aadc7b64e39dea42249431"}, - {file = "cryptography-43.0.0-cp37-abi3-win32.whl", hash = "sha256:6e2b11c55d260d03a8cf29ac9b5e0608d35f08077d8c087be96287f43af3ccdc"}, - {file = "cryptography-43.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:31e44a986ceccec3d0498e16f3d27b2ee5fdf69ce2ab89b52eaad1d2f33d8778"}, - {file = "cryptography-43.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:7b3f5fe74a5ca32d4d0f302ffe6680fcc5c28f8ef0dc0ae8f40c0f3a1b4fca66"}, - {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac1955ce000cb29ab40def14fd1bbfa7af2017cca696ee696925615cafd0dce5"}, - {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:299d3da8e00b7e2b54bb02ef58d73cd5f55fb31f33ebbf33bd00d9aa6807df7e"}, - {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ee0c405832ade84d4de74b9029bedb7b31200600fa524d218fc29bfa371e97f5"}, - {file = "cryptography-43.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:cb013933d4c127349b3948aa8aaf2f12c0353ad0eccd715ca789c8a0f671646f"}, - {file = "cryptography-43.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fdcb265de28585de5b859ae13e3846a8e805268a823a12a4da2597f1f5afc9f0"}, - {file = "cryptography-43.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2905ccf93a8a2a416f3ec01b1a7911c3fe4073ef35640e7ee5296754e30b762b"}, - {file = "cryptography-43.0.0-cp39-abi3-win32.whl", hash = "sha256:47ca71115e545954e6c1d207dd13461ab81f4eccfcb1345eac874828b5e3eaaf"}, - {file = "cryptography-43.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:0663585d02f76929792470451a5ba64424acc3cd5227b03921dab0e2f27b1709"}, - {file = "cryptography-43.0.0.tar.gz", hash = "sha256:b88075ada2d51aa9f18283532c9f60e72170041bba88d7f37e49cbb10275299e"}, + {file = "cryptography-43.0.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277"}, + {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a"}, + {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042"}, + {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494"}, + {file = "cryptography-43.0.1-cp37-abi3-win32.whl", hash = "sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2"}, + {file = "cryptography-43.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d"}, + {file = "cryptography-43.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c"}, + {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1"}, + {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa"}, + {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4"}, + {file = "cryptography-43.0.1-cp39-abi3-win32.whl", hash = "sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47"}, + {file = "cryptography-43.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb"}, + {file = "cryptography-43.0.1.tar.gz", hash = "sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d"}, +] + +[[package]] +name = "debugpy" +version = "1.8.6" +requires_python = ">=3.8" +summary = "An implementation of the Debug Adapter Protocol for Python" +groups = ["default"] +files = [ + {file = "debugpy-1.8.6-cp311-cp311-macosx_14_0_universal2.whl", hash = "sha256:67479a94cf5fd2c2d88f9615e087fcb4fec169ec780464a3f2ba4a9a2bb79955"}, + {file = "debugpy-1.8.6-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fb8653f6cbf1dd0a305ac1aa66ec246002145074ea57933978346ea5afdf70b"}, + {file = "debugpy-1.8.6-cp311-cp311-win32.whl", hash = "sha256:cdaf0b9691879da2d13fa39b61c01887c34558d1ff6e5c30e2eb698f5384cd43"}, + {file = "debugpy-1.8.6-cp311-cp311-win_amd64.whl", hash = "sha256:43996632bee7435583952155c06881074b9a742a86cee74e701d87ca532fe833"}, + {file = "debugpy-1.8.6-cp312-cp312-macosx_14_0_universal2.whl", hash = "sha256:db891b141fc6ee4b5fc6d1cc8035ec329cabc64bdd2ae672b4550c87d4ecb128"}, + {file = "debugpy-1.8.6-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:567419081ff67da766c898ccf21e79f1adad0e321381b0dfc7a9c8f7a9347972"}, + {file = "debugpy-1.8.6-cp312-cp312-win32.whl", hash = "sha256:c9834dfd701a1f6bf0f7f0b8b1573970ae99ebbeee68314116e0ccc5c78eea3c"}, + {file = "debugpy-1.8.6-cp312-cp312-win_amd64.whl", hash = "sha256:e4ce0570aa4aca87137890d23b86faeadf184924ad892d20c54237bcaab75d8f"}, + {file = "debugpy-1.8.6-py2.py3-none-any.whl", hash = "sha256:b48892df4d810eff21d3ef37274f4c60d32cdcafc462ad5647239036b0f0649f"}, + {file = "debugpy-1.8.6.zip", hash = "sha256:c931a9371a86784cee25dec8d65bc2dc7a21f3f1552e3833d9ef8f919d22280a"}, ] [[package]] @@ -445,13 +464,13 @@ files = [ [[package]] name = "dill" -version = "0.3.8" +version = "0.3.9" requires_python = ">=3.8" summary = "serialize all of Python" groups = ["default", "llm"] files = [ - {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, - {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, + {file = "dill-0.3.9-py3-none-any.whl", hash = "sha256:468dff3b89520b474c0397703366b7b95eebe6303f108adf9b19da1f702be87a"}, + {file = "dill-0.3.9.tar.gz", hash = "sha256:81aa267dddf68cbfe8029c42ca9ec6a4ab3b22371d1c450abc54422577b4512c"}, ] [[package]] @@ -546,13 +565,13 @@ files = [ [[package]] name = "idna" -version = "3.7" -requires_python = ">=3.5" +version = "3.10" +requires_python = ">=3.6" summary = "Internationalized Domain Names in Applications (IDNA)" -groups = ["dev", "llm"] +groups = ["default", "dev", "llm"] files = [ - {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, - {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, + {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, + {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, ] [[package]] @@ -719,7 +738,7 @@ files = [ [[package]] name = "moto" -version = "5.0.12" +version = "5.0.16" requires_python = ">=3.8" summary = "" groups = ["dev"] @@ -735,28 +754,29 @@ dependencies = [ "xmltodict", ] files = [ - {file = "moto-5.0.12-py2.py3-none-any.whl", hash = "sha256:406095ba28da9e67e59bfc6b31b4a2627ca896d3e2c7eb82838fbe527e3c218a"}, - {file = "moto-5.0.12.tar.gz", hash = "sha256:10bd434bfda329639fe9529470c4c27938064c139984024e6a62513e50aff427"}, + {file = "moto-5.0.16-py2.py3-none-any.whl", hash = "sha256:4ce1f34830307f7b3d553d77a7ef26066ab3b70006203d4226b048c9d11a3be4"}, + {file = "moto-5.0.16.tar.gz", hash = "sha256:f4afb176a964cd7a70da9bc5e053d43109614ce3cab26044bcbb53610435dff4"}, ] [[package]] name = "multiprocess" -version = "0.70.16" +version = "0.70.17" requires_python = ">=3.8" summary = "better multiprocessing and multithreading in Python" groups = ["llm"] dependencies = [ - "dill>=0.3.8", + "dill>=0.3.9", ] files = [ - {file = "multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a"}, - {file = "multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e"}, - {file = "multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1"}, + {file = "multiprocess-0.70.17-py311-none-any.whl", hash = "sha256:2884701445d0177aec5bd5f6ee0df296773e4fb65b11903b94c613fb46cfb7d1"}, + {file = "multiprocess-0.70.17-py312-none-any.whl", hash = "sha256:2818af14c52446b9617d1b0755fa70ca2f77c28b25ed97bdaa2c69a22c47b46c"}, + {file = "multiprocess-0.70.17-py313-none-any.whl", hash = "sha256:20c28ca19079a6c879258103a6d60b94d4ffe2d9da07dda93fb1c8bc6243f522"}, + {file = "multiprocess-0.70.17.tar.gz", hash = "sha256:4ae2f11a3416809ebc9a48abfc8b14ecce0652a0944731a1493a3c1ba44ff57a"}, ] [[package]] name = "mypy" -version = "1.11.1" +version = "1.11.2" requires_python = ">=3.8" summary = "Optional static typing for Python" groups = ["dev"] @@ -766,18 +786,18 @@ dependencies = [ "typing-extensions>=4.6.0", ] files = [ - {file = "mypy-1.11.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7b6343d338390bb946d449677726edf60102a1c96079b4f002dedff375953fc5"}, - {file = "mypy-1.11.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e4fe9f4e5e521b458d8feb52547f4bade7ef8c93238dfb5bbc790d9ff2d770ca"}, - {file = "mypy-1.11.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:886c9dbecc87b9516eff294541bf7f3655722bf22bb898ee06985cd7269898de"}, - {file = "mypy-1.11.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fca4a60e1dd9fd0193ae0067eaeeb962f2d79e0d9f0f66223a0682f26ffcc809"}, - {file = "mypy-1.11.1-cp311-cp311-win_amd64.whl", hash = "sha256:0bd53faf56de9643336aeea1c925012837432b5faf1701ccca7fde70166ccf72"}, - {file = "mypy-1.11.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f39918a50f74dc5969807dcfaecafa804fa7f90c9d60506835036cc1bc891dc8"}, - {file = "mypy-1.11.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0bc71d1fb27a428139dd78621953effe0d208aed9857cb08d002280b0422003a"}, - {file = "mypy-1.11.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b868d3bcff720dd7217c383474008ddabaf048fad8d78ed948bb4b624870a417"}, - {file = "mypy-1.11.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a707ec1527ffcdd1c784d0924bf5cb15cd7f22683b919668a04d2b9c34549d2e"}, - {file = "mypy-1.11.1-cp312-cp312-win_amd64.whl", hash = "sha256:64f4a90e3ea07f590c5bcf9029035cf0efeae5ba8be511a8caada1a4893f5525"}, - {file = "mypy-1.11.1-py3-none-any.whl", hash = "sha256:0624bdb940255d2dd24e829d99a13cfeb72e4e9031f9492148f410ed30bcab54"}, - {file = "mypy-1.11.1.tar.gz", hash = "sha256:f404a0b069709f18bbdb702eb3dcfe51910602995de00bd39cea3050b5772d08"}, + {file = "mypy-1.11.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75746e06d5fa1e91bfd5432448d00d34593b52e7e91a187d981d08d1f33d4385"}, + {file = "mypy-1.11.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a976775ab2256aadc6add633d44f100a2517d2388906ec4f13231fafbb0eccca"}, + {file = "mypy-1.11.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd953f221ac1379050a8a646585a29574488974f79d8082cedef62744f0a0104"}, + {file = "mypy-1.11.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:57555a7715c0a34421013144a33d280e73c08df70f3a18a552938587ce9274f4"}, + {file = "mypy-1.11.2-cp311-cp311-win_amd64.whl", hash = "sha256:36383a4fcbad95f2657642a07ba22ff797de26277158f1cc7bd234821468b1b6"}, + {file = "mypy-1.11.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e8960dbbbf36906c5c0b7f4fbf2f0c7ffb20f4898e6a879fcf56a41a08b0d318"}, + {file = "mypy-1.11.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:06d26c277962f3fb50e13044674aa10553981ae514288cb7d0a738f495550b36"}, + {file = "mypy-1.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e7184632d89d677973a14d00ae4d03214c8bc301ceefcdaf5c474866814c987"}, + {file = "mypy-1.11.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3a66169b92452f72117e2da3a576087025449018afc2d8e9bfe5ffab865709ca"}, + {file = "mypy-1.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:969ea3ef09617aff826885a22ece0ddef69d95852cdad2f60c8bb06bf1f71f70"}, + {file = "mypy-1.11.2-py3-none-any.whl", hash = "sha256:b499bc07dbdcd3de92b0a8b29fdf592c111276f6a12fe29c30f6c417dd546d12"}, + {file = "mypy-1.11.2.tar.gz", hash = "sha256:7f9993ad3e0ffdc95c2a14b66dee63729f021968bff8ad911867579c65d13a79"}, ] [[package]] @@ -841,7 +861,7 @@ files = [ [[package]] name = "pandas" -version = "2.2.2" +version = "2.2.3" requires_python = ">=3.9" summary = "Powerful data structures for data analysis, time series, and statistics" groups = ["llm"] @@ -854,21 +874,34 @@ dependencies = [ "tzdata>=2022.7", ] files = [ - {file = "pandas-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:696039430f7a562b74fa45f540aca068ea85fa34c244d0deee539cb6d70aa288"}, - {file = "pandas-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8e90497254aacacbc4ea6ae5e7a8cd75629d6ad2b30025a4a8b09aa4faf55151"}, - {file = "pandas-2.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58b84b91b0b9f4bafac2a0ac55002280c094dfc6402402332c0913a59654ab2b"}, - {file = "pandas-2.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2123dc9ad6a814bcdea0f099885276b31b24f7edf40f6cdbc0912672e22eee"}, - {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2925720037f06e89af896c70bca73459d7e6a4be96f9de79e2d440bd499fe0db"}, - {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1"}, - {file = "pandas-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24"}, - {file = "pandas-2.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef"}, - {file = "pandas-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce"}, - {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad"}, - {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad"}, - {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76"}, - {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32"}, - {file = "pandas-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23"}, - {file = "pandas-2.2.2.tar.gz", hash = "sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54"}, + {file = "pandas-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039"}, + {file = "pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd"}, + {file = "pandas-2.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd8d0c3be0515c12fed0bdbae072551c8b54b7192c7b1fda0ba56059a0179698"}, + {file = "pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc"}, + {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:63cc132e40a2e084cf01adf0775b15ac515ba905d7dcca47e9a251819c575ef3"}, + {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32"}, + {file = "pandas-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5"}, + {file = "pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9"}, + {file = "pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4"}, + {file = "pandas-2.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3"}, + {file = "pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319"}, + {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8"}, + {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a"}, + {file = "pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13"}, + {file = "pandas-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f00d1345d84d8c86a63e476bb4955e46458b304b9575dcf71102b5c705320015"}, + {file = "pandas-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3508d914817e153ad359d7e069d752cdd736a247c322d932eb89e6bc84217f28"}, + {file = "pandas-2.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22a9d949bfc9a502d320aa04e5d02feab689d61da4e7764b62c30b991c42c5f0"}, + {file = "pandas-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3a255b2c19987fbbe62a9dfd6cff7ff2aa9ccab3fc75218fd4b7530f01efa24"}, + {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:800250ecdadb6d9c78eae4990da62743b857b470883fa27f652db8bdde7f6659"}, + {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6374c452ff3ec675a8f46fd9ab25c4ad0ba590b71cf0656f8b6daa5202bca3fb"}, + {file = "pandas-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:61c5ad4043f791b61dd4752191d9f07f0ae412515d59ba8f005832a532f8736d"}, + {file = "pandas-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3b71f27954685ee685317063bf13c7709a7ba74fc996b84fc6821c59b0f06468"}, + {file = "pandas-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:38cf8125c40dae9d5acc10fa66af8ea6fdf760b2714ee482ca691fc66e6fcb18"}, + {file = "pandas-2.2.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba96630bc17c875161df3818780af30e43be9b166ce51c9a18c1feae342906c2"}, + {file = "pandas-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db71525a1538b30142094edb9adc10be3f3e176748cd7acc2240c2f2e5aa3a4"}, + {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15c0e1e02e93116177d29ff83e8b1619c93ddc9c49083f237d4312337a61165d"}, + {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ad5b65698ab28ed8d7f18790a0dc58005c7629f227be9ecc1072aa74c0c1d43a"}, + {file = "pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667"}, ] [[package]] @@ -889,19 +922,19 @@ files = [ [[package]] name = "pathos" -version = "0.3.2" +version = "0.3.3" requires_python = ">=3.8" summary = "parallel graph management and execution in heterogeneous computing" groups = ["llm"] dependencies = [ - "dill>=0.3.8", - "multiprocess>=0.70.16", - "pox>=0.3.4", - "ppft>=1.7.6.8", + "dill>=0.3.9", + "multiprocess>=0.70.17", + "pox>=0.3.5", + "ppft>=1.7.6.9", ] files = [ - {file = "pathos-0.3.2-py3-none-any.whl", hash = "sha256:d669275e6eb4b3fbcd2846d7a6d1bba315fe23add0c614445ba1408d8b38bafe"}, - {file = "pathos-0.3.2.tar.gz", hash = "sha256:4f2a42bc1e10ccf0fe71961e7145fc1437018b6b21bd93b2446abc3983e49a7a"}, + {file = "pathos-0.3.3-py3-none-any.whl", hash = "sha256:e04616c6448608ad1f809360be22e3f2078d949a36a81e6991da6c2dd1f82513"}, + {file = "pathos-0.3.3.tar.gz", hash = "sha256:dcb2a5f321aa34ca541c1c1861011ea49df357bb908379c21dd5741f666e0a58"}, ] [[package]] @@ -954,24 +987,24 @@ files = [ [[package]] name = "pox" -version = "0.3.4" +version = "0.3.5" requires_python = ">=3.8" summary = "utilities for filesystem exploration and automated builds" groups = ["llm"] files = [ - {file = "pox-0.3.4-py3-none-any.whl", hash = "sha256:651b8ae8a7b341b7bfd267f67f63106daeb9805f1ac11f323d5280d2da93fdb6"}, - {file = "pox-0.3.4.tar.gz", hash = "sha256:16e6eca84f1bec3828210b06b052adf04cf2ab20c22fd6fbef5f78320c9a6fed"}, + {file = "pox-0.3.5-py3-none-any.whl", hash = "sha256:9e82bcc9e578b43e80a99cad80f0d8f44f4d424f0ee4ee8d4db27260a6aa365a"}, + {file = "pox-0.3.5.tar.gz", hash = "sha256:8120ee4c94e950e6e0483e050a4f0e56076e590ba0a9add19524c254bd23c2d1"}, ] [[package]] name = "ppft" -version = "1.7.6.8" +version = "1.7.6.9" requires_python = ">=3.8" summary = "distributed and parallel Python" groups = ["llm"] files = [ - {file = "ppft-1.7.6.8-py3-none-any.whl", hash = "sha256:de2dd4b1b080923dd9627fbdea52649fd741c752fce4f3cf37e26f785df23d9b"}, - {file = "ppft-1.7.6.8.tar.gz", hash = "sha256:76a429a7d7b74c4d743f6dba8351e58d62b6432ed65df9fe204790160dab996d"}, + {file = "ppft-1.7.6.9-py3-none-any.whl", hash = "sha256:dab36548db5ca3055067fbe6b1a17db5fee29f3c366c579a9a27cebb52ed96f0"}, + {file = "ppft-1.7.6.9.tar.gz", hash = "sha256:73161c67474ea9d81d04bcdad166d399cff3f084d5d2dc21ebdd46c075bbc265"}, ] [[package]] @@ -990,18 +1023,18 @@ files = [ [[package]] name = "protobuf" -version = "4.25.4" +version = "4.25.5" requires_python = ">=3.8" summary = "" groups = ["default", "llm"] files = [ - {file = "protobuf-4.25.4-cp310-abi3-win32.whl", hash = "sha256:db9fd45183e1a67722cafa5c1da3e85c6492a5383f127c86c4c4aa4845867dc4"}, - {file = "protobuf-4.25.4-cp310-abi3-win_amd64.whl", hash = "sha256:ba3d8504116a921af46499471c63a85260c1a5fc23333154a427a310e015d26d"}, - {file = "protobuf-4.25.4-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:eecd41bfc0e4b1bd3fa7909ed93dd14dd5567b98c941d6c1ad08fdcab3d6884b"}, - {file = "protobuf-4.25.4-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:4c8a70fdcb995dcf6c8966cfa3a29101916f7225e9afe3ced4395359955d3835"}, - {file = "protobuf-4.25.4-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:3319e073562e2515c6ddc643eb92ce20809f5d8f10fead3332f71c63be6a7040"}, - {file = "protobuf-4.25.4-py3-none-any.whl", hash = "sha256:bfbebc1c8e4793cfd58589acfb8a1026be0003e852b9da7db5a4285bde996978"}, - {file = "protobuf-4.25.4.tar.gz", hash = "sha256:0dc4a62cc4052a036ee2204d26fe4d835c62827c855c8a03f29fe6da146b380d"}, + {file = "protobuf-4.25.5-cp310-abi3-win32.whl", hash = "sha256:5e61fd921603f58d2f5acb2806a929b4675f8874ff5f330b7d6f7e2e784bbcd8"}, + {file = "protobuf-4.25.5-cp310-abi3-win_amd64.whl", hash = "sha256:4be0571adcbe712b282a330c6e89eae24281344429ae95c6d85e79e84780f5ea"}, + {file = "protobuf-4.25.5-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:b2fde3d805354df675ea4c7c6338c1aecd254dfc9925e88c6d31a2bcb97eb173"}, + {file = "protobuf-4.25.5-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:919ad92d9b0310070f8356c24b855c98df2b8bd207ebc1c0c6fcc9ab1e007f3d"}, + {file = "protobuf-4.25.5-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fe14e16c22be926d3abfcb500e60cab068baf10b542b8c858fa27e098123e331"}, + {file = "protobuf-4.25.5-py3-none-any.whl", hash = "sha256:0aebecb809cae990f8129ada5ca273d9d670b76d9bfc9b1809f0a9c02b7dbf41"}, + {file = "protobuf-4.25.5.tar.gz", hash = "sha256:7f8249476b4a9473645db7f8ab42b02fe1488cbe5fb72fddd445e0665afd8584"}, ] [[package]] @@ -1023,11 +1056,12 @@ files = [ [[package]] name = "pulumi" -version = "3.128.0" +version = "3.135.0" requires_python = ">=3.8" summary = "Pulumi's Python SDK" groups = ["default"] dependencies = [ + "debugpy~=1.8.5", "dill~=0.3", "grpcio~=1.60.1", "protobuf~=4.21", @@ -1036,12 +1070,12 @@ dependencies = [ "six~=1.12", ] files = [ - {file = "pulumi-3.128.0-py3-none-any.whl", hash = "sha256:5c7457974dddd2fb29602bd423a7a468bed3e87337b4d926cea95c394177c796"}, + {file = "pulumi-3.135.0-py3-none-any.whl", hash = "sha256:fadda283509e360afac92f8b569b1f14b15667984ad966a2a66c6595228f9042"}, ] [[package]] name = "pulumi-aws" -version = "6.48.0" +version = "6.54.1" requires_python = ">=3.8" summary = "A Pulumi package for creating and managing Amazon Web Services (AWS) cloud resources." groups = ["default"] @@ -1052,30 +1086,30 @@ dependencies = [ "typing-extensions>=4.11; python_version < \"3.11\"", ] files = [ - {file = "pulumi_aws-6.48.0-py3-none-any.whl", hash = "sha256:0e978190db8f3bb4152e63c1e7d818cb90b7011850fb3c97b521188ae6197785"}, - {file = "pulumi_aws-6.48.0.tar.gz", hash = "sha256:e0886ef29ee51a1f1072731c24c200746ddb4fcf8c6b8c30fd99499f7e16e890"}, + {file = "pulumi_aws-6.54.1-py3-none-any.whl", hash = "sha256:38157b7ff074c552d983f0bfa159a80a15731785d85ecddee934776eb39096d4"}, + {file = "pulumi_aws-6.54.1.tar.gz", hash = "sha256:6dd5fe2c9ce73c57297a93d2b2f52f8930c6584f3e67a597dc4e69257e9215d6"}, ] [[package]] name = "pulumi-azure-native" -version = "2.53.0" +version = "2.64.2" requires_python = ">=3.8" summary = "A native Pulumi package for creating and managing Azure resources." groups = ["default"] dependencies = [ "parver>=0.2.1", - "pulumi<4.0.0,>=3.35.0", + "pulumi<4.0.0,>=3.134.1", "semver>=2.8.1", "typing-extensions>=4.11; python_version < \"3.11\"", ] files = [ - {file = "pulumi_azure_native-2.53.0-py3-none-any.whl", hash = "sha256:036e230580bdf9f326e8455b7e99955390e6c67024611bc13d245b85258d393f"}, - {file = "pulumi_azure_native-2.53.0.tar.gz", hash = "sha256:b54775160afc25281e9694c638821e68d466dbada6436643c3c75450d0aeab89"}, + {file = "pulumi_azure_native-2.64.2-py3-none-any.whl", hash = "sha256:f6271e30e25c09080df3aef509607b362774d126016abfb50492c7587cfc47e8"}, + {file = "pulumi_azure_native-2.64.2.tar.gz", hash = "sha256:0adaf301dcdc06d32beb2baa86044d396f5ad69ef28789ae0143c134deb93128"}, ] [[package]] name = "pulumi-random" -version = "4.16.3" +version = "4.16.6" requires_python = ">=3.8" summary = "A Pulumi package to safely use randomness in Pulumi programs." groups = ["default"] @@ -1085,15 +1119,15 @@ dependencies = [ "semver>=2.8.1", ] files = [ - {file = "pulumi_random-4.16.3-py3-none-any.whl", hash = "sha256:5f4c260ec9dd0683d764194936f8513b5931a38a1a84d1fea91018883043b324"}, - {file = "pulumi_random-4.16.3.tar.gz", hash = "sha256:0eabfe233d4c59ab20b54c5651b61c488ca0d05cd1b14988646e884d2520a969"}, + {file = "pulumi_random-4.16.6-py3-none-any.whl", hash = "sha256:369e276eb4ad1b3b24bd95341f0449f035d0e642fb4edb33d378d94be1d2c5ae"}, + {file = "pulumi_random-4.16.6.tar.gz", hash = "sha256:9aff7552f03816d3ff217073cbf07f9692587b9c0a9373c882784e8974d0ac00"}, ] [[package]] name = "py4j" version = "0.10.9.5" summary = "Enables Python programs to dynamically access arbitrary Java objects" -groups = ["dev"] +groups = ["default", "dev"] files = [ {file = "py4j-0.10.9.5-py2.py3-none-any.whl", hash = "sha256:52d171a6a2b031d8a5d1de6efe451cf4f5baff1a2819aabc3741c8406539ba04"}, {file = "py4j-0.10.9.5.tar.gz", hash = "sha256:276a4a3c5a2154df1860ef3303a927460e02e97b047dc0a47c1c3fb8cce34db6"}, @@ -1209,17 +1243,17 @@ files = [ [[package]] name = "pyright" -version = "1.1.375" +version = "1.1.383" requires_python = ">=3.7" summary = "Command line wrapper for pyright" groups = ["dev"] dependencies = [ "nodeenv>=1.6.0", - "typing-extensions>=3.7; python_version < \"3.8\"", + "typing-extensions>=4.1", ] files = [ - {file = "pyright-1.1.375-py3-none-any.whl", hash = "sha256:4c5e27eddeaee8b41cc3120736a1dda6ae120edf8523bb2446b6073a52f286e3"}, - {file = "pyright-1.1.375.tar.gz", hash = "sha256:7765557b0d6782b2fadabff455da2014476404c9e9214f49977a4e49dec19a0f"}, + {file = "pyright-1.1.383-py3-none-any.whl", hash = "sha256:d864d1182a313f45aaf99e9bfc7d2668eeabc99b29a556b5344894fd73cb1959"}, + {file = "pyright-1.1.383.tar.gz", hash = "sha256:1df7f12407f3710c9c6df938d98ec53f70053e6c6bbf71ce7bcb038d42f10070"}, ] [[package]] @@ -1227,7 +1261,7 @@ name = "pyspark" version = "3.3.2" requires_python = ">=3.7" summary = "Apache Spark Python API" -groups = ["dev"] +groups = ["default", "dev"] dependencies = [ "py4j==0.10.9.5", ] @@ -1237,7 +1271,7 @@ files = [ [[package]] name = "pytest" -version = "8.3.2" +version = "8.3.3" requires_python = ">=3.8" summary = "pytest: simple powerful testing with Python" groups = ["dev"] @@ -1250,8 +1284,8 @@ dependencies = [ "tomli>=1; python_version < \"3.11\"", ] files = [ - {file = "pytest-8.3.2-py3-none-any.whl", hash = "sha256:4ba08f9ae7dcf84ded419494d229b48d0903ea6407b030eaec46df5e6a73bba5"}, - {file = "pytest-8.3.2.tar.gz", hash = "sha256:c132345d12ce551242c87269de812483f5bcc87cdbb4722e48487ba194f9fdce"}, + {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"}, + {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"}, ] [[package]] @@ -1405,7 +1439,7 @@ name = "requests" version = "2.32.3" requires_python = ">=3.8" summary = "Python HTTP for Humans." -groups = ["dev", "llm"] +groups = ["default", "dev", "llm"] dependencies = [ "certifi>=2017.4.17", "charset-normalizer<4,>=2", @@ -1422,7 +1456,7 @@ name = "responses" version = "0.25.3" requires_python = ">=3.8" summary = "A utility library for mocking out the `requests` Python library." -groups = ["dev"] +groups = ["default", "dev"] dependencies = [ "pyyaml", "requests<3.0,>=2.30.0", @@ -1435,18 +1469,18 @@ files = [ [[package]] name = "rich" -version = "13.7.1" -requires_python = ">=3.7.0" +version = "13.9.1" +requires_python = ">=3.8.0" summary = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" groups = ["default", "llm"] dependencies = [ "markdown-it-py>=2.2.0", "pygments<3.0.0,>=2.13.0", - "typing-extensions<5.0,>=4.0.0; python_version < \"3.9\"", + "typing-extensions<5.0,>=4.0.0; python_version < \"3.11\"", ] files = [ - {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"}, - {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"}, + {file = "rich-13.9.1-py3-none-any.whl", hash = "sha256:b340e739f30aa58921dc477b8adaa9ecdb7cecc217be01d93730ee1bc8aa83be"}, + {file = "rich-13.9.1.tar.gz", hash = "sha256:097cffdf85db1babe30cc7deba5ab3a29e1b9885047dab24c57e9a7f8a9c1466"}, ] [[package]] @@ -1514,7 +1548,7 @@ files = [ [[package]] name = "sagemaker" -version = "2.232.0" +version = "2.232.1" requires_python = ">=3.8" summary = "Open source library for training and deploying models on Amazon SageMaker." groups = ["llm"] @@ -1543,13 +1577,13 @@ dependencies = [ "urllib3<3.0.0,>=1.26.8", ] files = [ - {file = "sagemaker-2.232.0-py3-none-any.whl", hash = "sha256:0803d64acf887802f32dec59ce2f5fba591561b108cb4c4a9d63d5fb9f930698"}, - {file = "sagemaker-2.232.0.tar.gz", hash = "sha256:be622f4466d8a9e9f925138715a1b9e0d304f0f9e6eda0023266da096e4c47a1"}, + {file = "sagemaker-2.232.1-py3-none-any.whl", hash = "sha256:64b92639918613b8042ddbc13f34cfac65145d916ff0c0001b249f9f33012cb1"}, + {file = "sagemaker-2.232.1.tar.gz", hash = "sha256:e59e1ac79bc31235b8d5c766abee5c6b2fd526814d1dde62b98e7dca9654503c"}, ] [[package]] name = "sagemaker-core" -version = "1.0.5" +version = "1.0.9" requires_python = ">=3.8" summary = "An python package for sagemaker core functionalities" groups = ["llm"] @@ -1564,8 +1598,8 @@ dependencies = [ "rich<14.0.0,>=13.0.0", ] files = [ - {file = "sagemaker_core-1.0.5-py3-none-any.whl", hash = "sha256:999896411702c1703d4bc04f91e1bb6dcd30882092ec986a5c8024df35c00891"}, - {file = "sagemaker_core-1.0.5.tar.gz", hash = "sha256:855504bde44df1cc44368145bb5fd51d6748303bcc2e71b083bffe3d1bfa8964"}, + {file = "sagemaker_core-1.0.9-py3-none-any.whl", hash = "sha256:7a22c46cf93594f8d44e3523d4ba98407911f3530af68a8ffdde5082d3b26fa3"}, + {file = "sagemaker_core-1.0.9.tar.gz", hash = "sha256:664115faf797412553fb81b97a4777e78e51dfd4454c32edb2c8371bf203c535"}, ] [[package]] @@ -1616,13 +1650,18 @@ files = [ [[package]] name = "sparkle" -version = "0.0.0" -requires_python = ">=3.10.14" +version = "0.6.1" +requires_python = "<4.0,>=3.10.14" git = "https://github.com/DataChefHQ/sparkle.git" -ref = "v0.3.1" -revision = "48627c830977d136ad91e1ff6016e53c561bd95e" -summary = "" +ref = "v0.6.1" +revision = "7c59a62035142b51af8477f62829be46fa214b43" +summary = "✨ A meta framework for Apache Spark, helping data engineers to focus on solving business problems with highest quality!" groups = ["default"] +dependencies = [ + "pyspark==3.3.2", + "requests<3.0.0,>=2.32.3", + "responses<1.0.0,>=0.25.3", +] [[package]] name = "tblib" @@ -1648,13 +1687,13 @@ files = [ [[package]] name = "tomlkit" -version = "0.13.0" +version = "0.13.2" requires_python = ">=3.8" summary = "Style preserving TOML library" groups = ["dev"] files = [ - {file = "tomlkit-0.13.0-py3-none-any.whl", hash = "sha256:7075d3042d03b80f603482d69bf0c8f345c2b30e41699fd8883227f89972b264"}, - {file = "tomlkit-0.13.0.tar.gz", hash = "sha256:08ad192699734149f5b97b45f1f18dad7eb1b6d16bc72ad0c2335772650d7b72"}, + {file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"}, + {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, ] [[package]] @@ -1673,24 +1712,24 @@ files = [ [[package]] name = "types-awscrt" -version = "0.21.2" -requires_python = "<4.0,>=3.7" +version = "0.22.0" +requires_python = ">=3.8" summary = "Type annotations and code completion for awscrt" groups = ["dev"] files = [ - {file = "types_awscrt-0.21.2-py3-none-any.whl", hash = "sha256:0839fe12f0f914d8f7d63ed777c728cb4eccc2d5d79a26e377d12b0604e7bf0e"}, - {file = "types_awscrt-0.21.2.tar.gz", hash = "sha256:84a9f4f422ec525c314fdf54c23a1e73edfbcec968560943ca2d41cfae623b38"}, + {file = "types_awscrt-0.22.0-py3-none-any.whl", hash = "sha256:b2c196bbd3226bab42d80fae13c34548de9ddc195f5a366d79c15d18e5897aa9"}, + {file = "types_awscrt-0.22.0.tar.gz", hash = "sha256:67a660c90bad360c339f6a79310cc17094d12472042c7ca5a41450aaf5fc9a54"}, ] [[package]] name = "types-s3transfer" -version = "0.10.1" -requires_python = "<4.0,>=3.8" +version = "0.10.2" +requires_python = ">=3.8" summary = "Type annotations and code completion for s3transfer" groups = ["dev"] files = [ - {file = "types_s3transfer-0.10.1-py3-none-any.whl", hash = "sha256:49a7c81fa609ac1532f8de3756e64b58afcecad8767933310228002ec7adff74"}, - {file = "types_s3transfer-0.10.1.tar.gz", hash = "sha256:02154cce46528287ad76ad1a0153840e0492239a0887e8833466eccf84b98da0"}, + {file = "types_s3transfer-0.10.2-py3-none-any.whl", hash = "sha256:7a3fec8cd632e2b5efb665a355ef93c2a87fdd5a45b74a949f95a9e628a86356"}, + {file = "types_s3transfer-0.10.2.tar.gz", hash = "sha256:60167a3bfb5c536ec6cdb5818f7f9a28edca9dc3e0b5ff85ae374526fc5e576e"}, ] [[package]] @@ -1706,24 +1745,24 @@ files = [ [[package]] name = "tzdata" -version = "2024.1" +version = "2024.2" requires_python = ">=2" summary = "Provider of IANA time zone data" groups = ["llm"] files = [ - {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, - {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, + {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, + {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, ] [[package]] name = "urllib3" -version = "2.2.2" +version = "2.2.3" requires_python = ">=3.8" summary = "HTTP library with thread-safe connection pooling, file post, and more." groups = ["default", "dev", "llm"] files = [ - {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, - {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, + {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, + {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, ] [[package]] @@ -1741,7 +1780,7 @@ files = [ [[package]] name = "werkzeug" -version = "3.0.3" +version = "3.0.4" requires_python = ">=3.8" summary = "The comprehensive WSGI web application library." groups = ["default", "dev"] @@ -1749,8 +1788,8 @@ dependencies = [ "MarkupSafe>=2.1.1", ] files = [ - {file = "werkzeug-3.0.3-py3-none-any.whl", hash = "sha256:fc9645dc43e03e4d630d23143a04a7f947a9a3b5727cd535fdfe155a17cc48c8"}, - {file = "werkzeug-3.0.3.tar.gz", hash = "sha256:097e5bfda9f0aba8da6b8545146def481d06aa7d3266e7448e2cccf67dd8bd18"}, + {file = "werkzeug-3.0.4-py3-none-any.whl", hash = "sha256:02c9eb92b7d6c06f31a782811505d2157837cea66aaede3e217c7c27c039476c"}, + {file = "werkzeug-3.0.4.tar.gz", hash = "sha256:34f2371506b250df4d4f84bfe7b0921e4762525762bbd936614909fe25cd7306"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 3a41a4e..e7016b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "pulumi-aws>=6.47.0", "pulumi-azure-native>=2.51.0", "pulumi-random>=4.16.3", - "sparkle @ git+https://github.com/DataChefHQ/sparkle.git@v0.3.1", + "sparkle @ git+https://github.com/DataChefHQ/sparkle.git@v0.6.1", ] requires-python = ">=3.11.0" readme = "README.md" diff --git a/src/damavand/base/controllers/spark.py b/src/damavand/base/controllers/spark.py index d2d75f7..d591fe4 100644 --- a/src/damavand/base/controllers/spark.py +++ b/src/damavand/base/controllers/spark.py @@ -1,9 +1,5 @@ -import os import logging -from pyspark.conf import SparkConf -from pyspark.sql import SparkSession -from damavand.environment import Environment from damavand.base.controllers import ApplicationController from damavand.base.controllers.base_controller import runtime @@ -33,12 +29,6 @@ class SparkController(ApplicationController): Methods ------- - default_local_session() - Return the default local Spark session. - default_cloud_session() - Return the default cloud Spark session. - default_session() - Return the currently active Spark session. application_with_id(app_id) Return the Spark application with the given ID. run_application(app_id) @@ -48,86 +38,12 @@ class SparkController(ApplicationController): def __init__( self, name, + applications: list[Sparkle], tags: dict[str, str] = {}, **kwargs, ) -> None: ApplicationController.__init__(self, name, tags, **kwargs) - self.applications: list[Sparkle] = [] - - @property - def _spark_extensions(self) -> list[str]: - return [ - "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions", - ] - - @property - def _spark_packages(self) -> list[str]: - return [ - "org.apache.iceberg:iceberg-spark-runtime-3.2_2.12:1.3.1", - "org.apache.spark:spark-sql-kafka-0-10_2.12:3.3.0", - "org.apache.spark:spark-avro_2.12:3.3.0", - ] - - def default_local_session(self) -> SparkSession: - """Return the default local Spark session. - - Returns: - SparkSession: The Spark session. - """ - - ivy_settings_path = os.environ.get("IVY_SETTINGS_PATH", None) - LOCAL_CONFIG = { - "spark.sql.extensions": ",".join(self._spark_extensions), - "spark.jars.packages": ",".join(self._spark_packages), - "spark.sql.jsonGenerator.ignoreNullFields": False, - "spark.sql.session.timeZone": "UTC", - "spark.sql.catalog.spark_catalog": "org.apache.iceberg.spark.SparkSessionCatalog", - "spark.sql.catalog.spark_catalog.type": "hive", - "spark.sql.catalog.local": "org.apache.iceberg.spark.SparkCatalog", - "spark.sql.catalog.local.type": "hadoop", - "spark.sql.catalog.local.warehouse": "/tmp/warehouse", - "spark.sql.defaultCatalog": "local", - } - - spark_conf = SparkConf() - - for key, value in LOCAL_CONFIG.items(): - spark_conf.set(key, str(value)) - - spark_session = ( - # NOTE: Pyright does not work `@classproperty` decorator used in `SparkSession`. This however should be fixed in pyspark v4. - SparkSession.builder.master("local[*]") # type: ignore - .appName("LocalDataProductApp") - .config(conf=spark_conf) - ) - - if ivy_settings_path: - spark_session.config("spark.jars.ivySettings", ivy_settings_path) - - return spark_session.getOrCreate() - - def default_cloud_session(self) -> SparkSession: - """Return the default Spark session provided by the cloud spark machine. - - Returns: - SparkSession: The Spark session. - """ - - raise NotImplementedError - - def default_session(self) -> SparkSession: - """Return the currently active Spark session. If the environment is local, it - returns the local session. Otherwise, it returns the cloud session. - - Returns: - SparkSession: The Spark session. - """ - env = Environment.from_system_env() - match env: - case Environment.LOCAL: - return self.default_local_session() - case _: - return self.default_cloud_session() + self.applications: list[Sparkle] = applications def application_with_id(self, app_id: str) -> Sparkle: """Return the Spark application with the given ID. diff --git a/src/damavand/base/factory.py b/src/damavand/base/factory.py index 84156cb..1d5dd75 100644 --- a/src/damavand/base/factory.py +++ b/src/damavand/base/factory.py @@ -1,9 +1,9 @@ from dataclasses import field, dataclass from typing import Optional, Generic, TypeVar +from sparkle.application import Sparkle from damavand.base.controllers import ApplicationController from damavand.cloud.provider import AwsProvider, AzurermProvider, CloudProvider - from damavand.errors import UnsupportedProvider @@ -36,6 +36,7 @@ class ApplicationControllerFactory(Generic[ControllerType]): def new( self, name: str, + applications: list[Sparkle], id: Optional[str] = None, **kwargs, ) -> ControllerType: @@ -43,6 +44,7 @@ def new( case AwsProvider(): ctr = self._new_aws_controller( name=name, + applications=applications, region=self.provider.explicit_region, tags=self.tags, **kwargs, @@ -52,6 +54,7 @@ def new( case AzurermProvider(): ctr = self._new_azure_controller( name=name, + applications=applications, tags=self.tags, **kwargs, ) @@ -63,6 +66,7 @@ def new( def _new_aws_controller( self, name: str, + applications: list[Sparkle], region: str, tags: dict[str, str] = {}, **kwargs, @@ -72,6 +76,7 @@ def _new_aws_controller( def _new_azure_controller( self, name: str, + applications: list[Sparkle], tags: dict[str, str] = {}, **kwargs, ) -> ControllerType: diff --git a/src/damavand/cloud/aws/controllers/spark.py b/src/damavand/cloud/aws/controllers/spark.py index b1239b6..1f4713d 100644 --- a/src/damavand/cloud/aws/controllers/spark.py +++ b/src/damavand/cloud/aws/controllers/spark.py @@ -3,6 +3,7 @@ import boto3 from pulumi import Resource as PulumiResource +from sparkle.application import Sparkle from damavand.base.controllers import SparkController, buildtime from damavand.cloud.aws.resources import GlueComponent, GlueComponentArgs @@ -17,11 +18,12 @@ class AwsSparkController(SparkController): def __init__( self, name, + applications: list[Sparkle], region: str, tags: dict[str, str] = {}, **kwargs, ) -> None: - super().__init__(name, tags, **kwargs) + super().__init__(name, applications, tags, **kwargs) self._glue_client = boto3.client("glue", region_name=region) @buildtime diff --git a/src/damavand/cloud/azure/controllers/spark.py b/src/damavand/cloud/azure/controllers/spark.py index 1905727..c13435d 100644 --- a/src/damavand/cloud/azure/controllers/spark.py +++ b/src/damavand/cloud/azure/controllers/spark.py @@ -18,12 +18,13 @@ class AzureSparkController(SparkController): def __init__( self, name, + applications: list[Sparkle], region: str, tags: dict[str, str] = {}, **kwargs, ) -> None: - super().__init__(name, tags, **kwargs) - self.applications: list[Sparkle] + super().__init__(name, applications, tags, **kwargs) + self.applications = applications @buildtime def admin_username(self) -> str: diff --git a/src/damavand/factories.py b/src/damavand/factories.py index e8835f0..0367a2a 100644 --- a/src/damavand/factories.py +++ b/src/damavand/factories.py @@ -1,4 +1,4 @@ - +from sparkle.application import Sparkle from damavand.base.controllers.spark import SparkController from damavand.base.factory import ApplicationControllerFactory from damavand.cloud.aws.controllers import AwsSparkController @@ -9,6 +9,7 @@ class SparkControllerFactory(ApplicationControllerFactory[SparkController]): def _new_aws_controller( self, name: str, + applications: list[Sparkle], region: str, tags: dict[str, str] = {}, **kwargs, @@ -21,7 +22,11 @@ def _new_aws_controller( ) def _new_azure_controller( - self, name: str, tags: dict[str, str] = {}, **kwargs + self, + name: str, + applications: list[Sparkle], + tags: dict[str, str] = {}, + **kwargs, ) -> SparkController: return AzureSparkController( name=name, diff --git a/tests/base/test_spark.py b/tests/base/test_spark.py index 2e3455c..6fcb75f 100644 --- a/tests/base/test_spark.py +++ b/tests/base/test_spark.py @@ -1,6 +1,4 @@ import pytest -from unittest.mock import patch -from _pytest.monkeypatch import MonkeyPatch from damavand.base.controllers import SparkController @@ -9,33 +7,5 @@ def spark_controller(): return SparkController( name="test", + applications=[], ) - - -def test_default_session(monkeypatch: MonkeyPatch, spark_controller: SparkController): - monkeypatch.setenv("ENVIRONMENT", "local") - - with patch.object(spark_controller, "default_local_session") as mock_session: - spark_controller.default_session() - mock_session.assert_called_once() - - -@pytest.mark.parametrize( - "environment", - [ - "development", - "production", - "testing", - "acceptance", - ], -) -def test_default_session_with_environment( - monkeypatch: MonkeyPatch, - spark_controller: SparkController, - environment: str, -): - monkeypatch.setenv("ENVIRONMENT", environment) - - with patch.object(spark_controller, "default_cloud_session") as mock_session: - spark_controller.default_session() - mock_session.assert_called_once() diff --git a/tests/clouds/aws/controllers/test_aws_spark.py b/tests/clouds/aws/controllers/test_aws_spark.py index 5f1eda4..79912de 100644 --- a/tests/clouds/aws/controllers/test_aws_spark.py +++ b/tests/clouds/aws/controllers/test_aws_spark.py @@ -13,6 +13,7 @@ def controller(): ctr = AwsSparkController( "test-spark", + applications=[], region="us-east-1", )