fix(spark): lock package version, to ensure they match plugins.

DataChefHQ · Sep 11, 2024 · 28f9013 · 28f9013
1 parent 09eb0f5
commit 28f9013
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 14 deletions.
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -14,7 +14,10 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.10.14"
-pyspark = ">=3.3.2"
+# FIXME we need to lock this for now, given the plugins we install are
+# tightly coupled with the versions. Unless we require users to add
+# these plugins on runtime, we can't change this.
+pyspark = "3.3.2"
 
 [tool.poetry.group.dev.dependencies]
 commitizen = "^3.29.0"

diff --git a/src/sparkle/application/spark.py b/src/sparkle/application/spark.py
@@ -12,6 +12,7 @@
 _SPARK_EXTENSIONS = [
     "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
 ]
+
 _SPARK_PACKAGES = [
     "org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.3.1",
     "org.apache.spark:spark-sql-kafka-0-10_2.12:3.3.0",