Skip to content

Commit

Permalink
add pyspark and deltalake to requirements.txt
Browse files Browse the repository at this point in the history
  • Loading branch information
alsugiliazova committed Sep 27, 2024
1 parent f0dc626 commit 0264b7a
Show file tree
Hide file tree
Showing 3 changed files with 28 additions and 0 deletions.
2 changes: 2 additions & 0 deletions data_lakes/regression.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ def regression(
for node in nodes["clickhouse"]:
experimental_analyzer(node=cluster.node(node), with_analyzer=with_analyzer)

Feature(run=load("data_lakes.tests.check_healthy", "feature"))


if main():
regression()
24 changes: 24 additions & 0 deletions data_lakes/tests/check_healthy.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
from pyspark.sql import SparkSession
from delta import configure_spark_with_delta_pip

from testflows.core import *


@TestFeature
@Name("PySpark and Delta Lake are working")
def feature(self):
"""Check that PySpark and Delta Lake are working."""
builder = (
SparkSession.builder.appName("TestDeltaLake")
.config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
.config(
"spark.sql.catalog.spark_catalog",
"org.apache.spark.sql.delta.catalog.DeltaCatalog",
)
)

spark = configure_spark_with_delta_pip(builder).getOrCreate()

df = spark.createDataFrame([(1, "foo"), (2, "bar")], ["id", "value"])
df.write.format("delta").save("/tmp/delta-table")
df.show()
2 changes: 2 additions & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,5 @@ matplotlib
requests==2.31.0
bcrypt
clickhouse-driver==0.2.8
pyspark==3.5.3
delta-spark==3.2.1

0 comments on commit 0264b7a

Please sign in to comment.