template-demo-2.yaml

jobs:
- sparkJob:
    jarFileUris:
    - gs://dataproc-demo-bucket/dataprocJavaDemo-1.0-SNAPSHOT.jar
    mainClass: org.example.dataproc.InternationalLoansAppDataprocSmall
  stepId: ibrd-small-spark
- sparkJob:
    jarFileUris:
    - gs://dataproc-demo-bucket/dataprocJavaDemo-1.0-SNAPSHOT.jar
    mainClass: org.example.dataproc.InternationalLoansAppDataprocLarge
  stepId: ibrd-large-spark
- pysparkJob:
    args:
    - gs://dataproc-demo-bucket
    - ibrd-statement-of-loans-historical-data.csv
    - ibrd-summary-large-python
    mainPythonFileUri: gs://dataproc-demo-bucket/international_loans_dataproc.py
  stepId: ibrd-large-pyspark
placement:
  managedCluster:
    clusterName: three-node-cluster
    config:
      gceClusterConfig:
        zoneUri: us-east1-b
      masterConfig:
        diskConfig:
          bootDiskSizeGb: 500
        machineTypeUri: n1-standard-4
      softwareConfig:
        imageVersion: 1.3-deb9
      workerConfig:
        diskConfig:
          bootDiskSizeGb: 500
        machineTypeUri: n1-standard-4
        numInstances: 2