luigi.cfg

[core]
# default-scheduler can be an IP central scheduler
default-scheduler-host:localhost
error-email:you@yourmail.com

[spark]
spark-submit:/path-to-spark-base/spark-1.6.1-bin-hadoop2.6/bin/spark-submit
jars:/path-to-spark-base/spark-1.6.1-bin-hadoop2.6/lib/postgresql-9.4.1208.jre6.jar
driver-library-path:/path-to-spark-base/spark-1.6.1-bin-hadoop2.6/lib/postgresql-9.4.1208.jre6.jar
driver-class-path:/path-to-spark-base/spark-1.6.1-bin-hadoop2.6/lib/postgresql-9.4.1208.jre6.jar

[pyspark]
jars:/path-to-spark-base/spark-1.6.1-bin-hadoop2.6/lib/postgresql-9.4.1208.jre6.jar
driver-library-path:/path-to-spark-base/spark-1.6.1-bin-hadoop2.6/lib/postgresql-9.4.1208.jre6.jar
driver-class-path:/path-to-spark-base/spark-1.6.1-bin-hadoop2.6/lib/postgresql-9.4.1208.jre6.jar

[scheduler]
# this was store the record of past task details, else it is discard either
# after the frequent luigi task refresh or when luigi daemon is shutdown
record_task_history = True
state_path = /home/user/luigi-state.pickle

[task_history]
# if we want to store the history in a DB
db_connection = sqlite:////home/user/luigi-task-history.db