spark-defaults.conf

# Retries
spark.port.maxRetries                  1
# For Iceberg Spark + Data "Lake" Catalog
spark.sql.extensions                   org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
spark.sql.catalog.lake                 org.apache.iceberg.spark.SparkCatalog
spark.sql.catalog.lake.type            hadoop
spark.sql.catalog.lake.warehouse       s3a://warehouse/lake/
spark.sql.catalog.lake.s3.endpoint     http://minio:9000
spark.sql.catalog.lake.io-impl         org.apache.iceberg.aws.s3.S3FileIO
spark.sql.defaultCatalog               lake
# For spark-submit Minio access to Python distribution files & Logging bucket
spark.hadoop.fs.s3a.path.style.access  true
spark.hadoop.fs.s3a.impl               org.apache.hadoop.fs.s3a.S3AFileSystem
spark.hadoop.fs.s3a.endpoint           http://minio:9000
# Memory Optimization
spark.driver.extraJavaOptions          -XX:+UseG1GC
spark.executor.extraJavaOptions        -XX:+UseG1GC
spark.serializer                       org.apache.spark.serializer.KryoSerializer
# Logging
spark.eventLog.enabled                 true
spark.eventLog.dir                     s3a://log/spark/spark-logs