dataproc_pyspark.yaml

# Copyright 2018 Etsy Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.

# see: https://airflow.apache.org/code.html#airflow.contrib.operators.dataproc_operator.DataProcPySparkOperator
name: dataproc_pyspark
operator_class: DataProcPySparkOperator
operator_class_module: airflow.contrib.operators.dataproc_operator
schema_extends: base
parameters_jsonschema:
    properties:
        main:
            type: string
        arguments:
            type: array
            items:
                type: string
        archives:
            type: array
            items:
                type: string
        files:
            type: array
            items:
                type: string
        pyfiles:
            type: array
            items:
                type: string
        job_name:
            type: string
        cluster_name:
            type: string
        dataproc_pyspark_properties:
            type: object
            additionalProperties:
                type:
                    string
        dataproc_pyspark_jars:
            type: array
            items:
                type: string
        region:
            type: string
        job_error_states:
            type: array
            items:
                type: string
        gcp_conn_id:
            type: string
        delegate_to:
            type: string
    required:
        - main
        - cluster_name
        - region
    additionalProperties: false