diff --git a/README.md b/README.md index 02ea9cf..9c01f44 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ OAuth flow for installed applications. | gcs_bucket | string | optional | nil | See [GCS Bucket](#gcs-bucket) | | auto_create_gcs_bucket | boolean | optional | false | See [GCS Bucket](#gcs-bucket) | | progress_log_interval | float | optional | nil (Disabled) | Progress log interval. The progress log is disabled by nil (default). NOTE: This option may be removed in a future because a filter plugin can achieve the same goal | +| description | string | optional | nil | description of table | Client or request options @@ -329,6 +330,7 @@ Column options are used to aid guessing BigQuery schema, or to define conversion - json: `STRING`, `RECORD` (default: `STRING`) - **mode**: BigQuery mode such as `NULLABLE`, `REQUIRED`, and `REPEATED` (string, default: `NULLABLE`) - **fields**: Describes the nested schema fields if the type property is set to RECORD. Please note that this is **required** for `RECORD` column. + - **description**: description (string, default is `None`). - **timestamp_format**: timestamp format to convert into/from `timestamp` (string, default is `default_timestamp_format`) - **timezone**: timezone to convert into/from `timestamp`, `date` (string, default is `default_timezone`). - **default_timestamp_format**: default timestamp format for column_options (string, default is "%Y-%m-%d %H:%M:%S.%6N") diff --git a/lib/embulk/output/bigquery.rb b/lib/embulk/output/bigquery.rb index dd9b92d..10cbd00 100644 --- a/lib/embulk/output/bigquery.rb +++ b/lib/embulk/output/bigquery.rb @@ -63,6 +63,8 @@ def self.configure(config, schema, task_count) 'payload_column' => config.param('payload_column', :string, :default => nil), 'payload_column_index' => config.param('payload_column_index', :integer, :default => nil), + 'description' => config.param('description', :string, :default => nil), + 'open_timeout_sec' => config.param('open_timeout_sec', :integer, :default => nil), 'timeout_sec' => config.param('timeout_sec', :integer, :default => nil), # google-api-ruby-client < v0.11.0 'send_timeout_sec' => config.param('send_timeout_sec', :integer, :default => nil), # google-api-ruby-client >= v0.11.0 diff --git a/lib/embulk/output/bigquery/bigquery_client.rb b/lib/embulk/output/bigquery/bigquery_client.rb index 4e0750b..2fcdb5e 100644 --- a/lib/embulk/output/bigquery/bigquery_client.rb +++ b/lib/embulk/output/bigquery/bigquery_client.rb @@ -420,6 +420,7 @@ def create_table_if_not_exists(table, dataset: nil, options: nil) table_reference: { table_id: table, }, + description: @task['description'], schema: { fields: fields, } diff --git a/lib/embulk/output/bigquery/helper.rb b/lib/embulk/output/bigquery/helper.rb index f0b41b3..37cac69 100644 --- a/lib/embulk/output/bigquery/helper.rb +++ b/lib/embulk/output/bigquery/helper.rb @@ -46,10 +46,11 @@ def self.fields_from_embulk_schema(task, schema) embulk_type = column[:type] column_option = column_options_map[column_name] || {} {}.tap do |field| - field[:name] = column_name - field[:type] = (column_option['type'] || bq_type_from_embulk_type(embulk_type)).upcase - field[:mode] = column_option['mode'] if column_option['mode'] - field[:fields] = deep_symbolize_keys(column_option['fields']) if column_option['fields'] + field[:name] = column_name + field[:type] = (column_option['type'] || bq_type_from_embulk_type(embulk_type)).upcase + field[:mode] = column_option['mode'] if column_option['mode'] + field[:fields] = deep_symbolize_keys(column_option['fields']) if column_option['fields'] + field[:description] = column_option['description'] if column_option['description'] end end end diff --git a/test/test_helper.rb b/test/test_helper.rb index 11f30eb..17cb1c7 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -68,7 +68,7 @@ def test_fields_from_embulk_schema_with_column_options ]) task = { 'column_options' => [ - {'name' => 'boolean', 'type' => 'STRING', 'mode' => 'REQUIRED'}, + {'name' => 'boolean', 'type' => 'STRING', 'mode' => 'REQUIRED', 'description' => 'hoge'}, {'name' => 'long', 'type' => 'STRING'}, {'name' => 'double', 'type' => 'STRING'}, {'name' => 'string', 'type' => 'INTEGER'}, @@ -81,7 +81,7 @@ def test_fields_from_embulk_schema_with_column_options ], } expected = [ - {name: 'boolean', type: 'STRING', mode: 'REQUIRED'}, + {name: 'boolean', type: 'STRING', mode: 'REQUIRED', description: 'hoge'}, {name: 'long', type: 'STRING'}, {name: 'double', type: 'STRING'}, {name: 'string', type: 'INTEGER'},