-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add pyspark and deltalake to requirements.txt
- Loading branch information
1 parent
f0dc626
commit 0264b7a
Showing
3 changed files
with
28 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
from pyspark.sql import SparkSession | ||
from delta import configure_spark_with_delta_pip | ||
|
||
from testflows.core import * | ||
|
||
|
||
@TestFeature | ||
@Name("PySpark and Delta Lake are working") | ||
def feature(self): | ||
"""Check that PySpark and Delta Lake are working.""" | ||
builder = ( | ||
SparkSession.builder.appName("TestDeltaLake") | ||
.config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") | ||
.config( | ||
"spark.sql.catalog.spark_catalog", | ||
"org.apache.spark.sql.delta.catalog.DeltaCatalog", | ||
) | ||
) | ||
|
||
spark = configure_spark_with_delta_pip(builder).getOrCreate() | ||
|
||
df = spark.createDataFrame([(1, "foo"), (2, "bar")], ["id", "value"]) | ||
df.write.format("delta").save("/tmp/delta-table") | ||
df.show() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -16,3 +16,5 @@ matplotlib | |
requests==2.31.0 | ||
bcrypt | ||
clickhouse-driver==0.2.8 | ||
pyspark==3.5.3 | ||
delta-spark==3.2.1 |