dataproc_cluster_create.yaml

# Copyright 2018 Etsy Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.

name: dataproc_cluster_create
operator_class: DataprocClusterCreateOperator
operator_class_module: airflow.contrib.operators.dataproc_operator
schema_extends: base
parameters_jsonschema:
    properties:
        cluster_name:
            type: string
        project_id:
            type: string
        num_workers:
            type: integer
        storage_bucket:
            type: string
        init_actions_uris:
            type: array
            items:
                type: string
        init_action_timeout:
            type: string
        metadata:
            type: array
            items:
                type: string
        image_version:
            type: string
        custom_image:
            type: string
        properties:
            type: object
            additionalProperties:
                type: string
        master_machine_type:
            type: string
        master_disk_type:
            type: string
            enum: [ pd-standard, pd-ssd ]
        master_disk_size:
            type: integer
        worker_machine_type:
            type: string
        worker_disk_type:
            type: string
            enum: [ pd-standard, pd-ssd ]
        worker_disk_size:
            type: integer
        num_preemptible_workers:
            type: integer
        labels:
            type: object
            additionalProperties:
                type: string
        zone:
            type: string
        network_uri:
            type: string
        subnetwork_uri:
            type: string
        internal_ip_only:
            type: boolean
        tags:
            type: array
            items:
                type: string
        region:
            type: string
        gcp_conn_id:
            type: string
        delegate_to:
            type: string
        service_account:
            type: string
        service_account_scopes:
            type: array
            items:
                type: string
        idle_delete_ttl:
            type: integer
        auto_delete_time:
            type: string
        auto_delete_ttl:
            type: integer
    required:
        - project_id
        - cluster_name
        - region
        - num_workers
    additionalProperties: false
property_preprocessors:
    - type: ensure_rendered_string_pattern
      apply_to_properties:
      - cluster_name
      properties:
          pattern: "[a-z][a-z0-9-]{,54}(?<!-)"