From 15d1a2c9625bddad599b0b503359d6e67f5dce73 Mon Sep 17 00:00:00 2001 From: Katsuhiko YOSHIDA Date: Sun, 4 Apr 2021 01:10:25 +0900 Subject: [PATCH 1/6] Add description of tables support --- lib/embulk/output/bigquery/bigquery_client.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/embulk/output/bigquery/bigquery_client.rb b/lib/embulk/output/bigquery/bigquery_client.rb index 13753d2..e956291 100644 --- a/lib/embulk/output/bigquery/bigquery_client.rb +++ b/lib/embulk/output/bigquery/bigquery_client.rb @@ -419,6 +419,7 @@ def create_table_if_not_exists(table, dataset: nil, options: nil) table_reference: { table_id: table, }, + description: @task['description'], schema: { fields: fields, } From 06a64767363dae525a8e1661494bda9d0b883729 Mon Sep 17 00:00:00 2001 From: Katsuhiko YOSHIDA Date: Sun, 4 Apr 2021 01:13:37 +0900 Subject: [PATCH 2/6] Add description of columns support --- lib/embulk/output/bigquery/helper.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/embulk/output/bigquery/helper.rb b/lib/embulk/output/bigquery/helper.rb index f0b41b3..37cac69 100644 --- a/lib/embulk/output/bigquery/helper.rb +++ b/lib/embulk/output/bigquery/helper.rb @@ -46,10 +46,11 @@ def self.fields_from_embulk_schema(task, schema) embulk_type = column[:type] column_option = column_options_map[column_name] || {} {}.tap do |field| - field[:name] = column_name - field[:type] = (column_option['type'] || bq_type_from_embulk_type(embulk_type)).upcase - field[:mode] = column_option['mode'] if column_option['mode'] - field[:fields] = deep_symbolize_keys(column_option['fields']) if column_option['fields'] + field[:name] = column_name + field[:type] = (column_option['type'] || bq_type_from_embulk_type(embulk_type)).upcase + field[:mode] = column_option['mode'] if column_option['mode'] + field[:fields] = deep_symbolize_keys(column_option['fields']) if column_option['fields'] + field[:description] = column_option['description'] if column_option['description'] end end end From bd2a74814750ddc1d5d32606bf7f31b0cdbe3d36 Mon Sep 17 00:00:00 2001 From: Katsuhiko YOSHIDA Date: Sun, 4 Apr 2021 01:29:53 +0900 Subject: [PATCH 3/6] Update document --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b36847b..c592845 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ OAuth flow for installed applications. | gcs_bucket | string | optional | nil | See [GCS Bucket](#gcs-bucket) | | auto_create_gcs_bucket | boolean | optional | false | See [GCS Bucket](#gcs-bucket) | | progress_log_interval | float | optional | nil (Disabled) | Progress log interval. The progress log is disabled by nil (default). NOTE: This option may be removed in a future because a filter plugin can achieve the same goal | +| description | string | optional | nil | description of table | Client or request options @@ -316,6 +317,7 @@ Column options are used to aid guessing BigQuery schema, or to define conversion - json: `STRING`, `RECORD` (default: `STRING`) - **mode**: BigQuery mode such as `NULLABLE`, `REQUIRED`, and `REPEATED` (string, default: `NULLABLE`) - **fields**: Describes the nested schema fields if the type property is set to RECORD. Please note that this is **required** for `RECORD` column. + - **description**: description (string, default is `None`). - **timestamp_format**: timestamp format to convert into/from `timestamp` (string, default is `default_timestamp_format`) - **timezone**: timezone to convert into/from `timestamp`, `date` (string, default is `default_timezone`). - **default_timestamp_format**: default timestamp format for column_options (string, default is "%Y-%m-%d %H:%M:%S.%6N") From 1ebfcace4da06726b6386e1372aca6e1465207fc Mon Sep 17 00:00:00 2001 From: fagai Date: Tue, 18 Jan 2022 22:30:08 +0900 Subject: [PATCH 4/6] fix test --- test/test_transaction.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_transaction.rb b/test/test_transaction.rb index 575d0ca..196715a 100644 --- a/test/test_transaction.rb +++ b/test/test_transaction.rb @@ -109,7 +109,7 @@ def test_replace task = Bigquery.configure(config, schema, processor_count) any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'], { :options => { "expiration_time" => nil } }) mock(obj).create_table_if_not_exists(config['table']) mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_TRUNCATE') mock(obj).delete_table(config['temp_table']) @@ -122,7 +122,7 @@ def test_replace_with_partitioning task = Bigquery.configure(config, schema, processor_count) any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'], { :options => { "expiration_time" => nil } }) mock(obj).create_table_if_not_exists(config['table']) mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_TRUNCATE') mock(obj).delete_table(config['temp_table']) @@ -138,7 +138,7 @@ def test_replace_backup any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) mock(obj).get_dataset(config['dataset_old']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'], { :options => { "expiration_time" => nil } }) mock(obj).create_table_if_not_exists(config['table']) mock(obj).create_table_if_not_exists(config['table_old'], dataset: config['dataset_old']) @@ -158,7 +158,7 @@ def test_replace_backup_auto_create_dataset mock(obj).create_dataset(config['dataset']) mock(obj).create_dataset(config['dataset_old'], reference: config['dataset']) mock(obj).create_table_if_not_exists(config['table']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'], { :options => { "expiration_time" => nil } }) mock(obj).create_table_if_not_exists(config['table_old'], dataset: config['dataset_old']) mock(obj).get_table_or_partition(config['table']) @@ -176,7 +176,7 @@ def test_replace_backup_with_partitioning any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) mock(obj).get_dataset(config['dataset_old']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'], { :options => { "expiration_time" => nil } }) mock(obj).create_table_if_not_exists(config['table']) mock(obj).create_table_if_not_exists(config['table_old'], dataset: config['dataset_old']) @@ -196,7 +196,7 @@ def test_append task = Bigquery.configure(config, schema, processor_count) any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'], { :options => { "expiration_time" => nil } }) mock(obj).create_table_if_not_exists(config['table']) mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_APPEND') mock(obj).delete_table(config['temp_table']) @@ -209,7 +209,7 @@ def test_append_with_partitioning task = Bigquery.configure(config, schema, processor_count) any_instance_of(BigqueryClient) do |obj| mock(obj).get_dataset(config['dataset']) - mock(obj).create_table_if_not_exists(config['temp_table']) + mock(obj).create_table_if_not_exists(config['temp_table'], { :options => { "expiration_time" => nil } }) mock(obj).create_table_if_not_exists(config['table']) mock(obj).copy(config['temp_table'], config['table'], write_disposition: 'WRITE_APPEND') mock(obj).delete_table(config['temp_table']) From 72372d1f1d4c7ebfae2471275cc43bbf97375891 Mon Sep 17 00:00:00 2001 From: fagai Date: Tue, 18 Jan 2022 22:30:30 +0900 Subject: [PATCH 5/6] add description test --- test/test_helper.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_helper.rb b/test/test_helper.rb index 11f30eb..17cb1c7 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -68,7 +68,7 @@ def test_fields_from_embulk_schema_with_column_options ]) task = { 'column_options' => [ - {'name' => 'boolean', 'type' => 'STRING', 'mode' => 'REQUIRED'}, + {'name' => 'boolean', 'type' => 'STRING', 'mode' => 'REQUIRED', 'description' => 'hoge'}, {'name' => 'long', 'type' => 'STRING'}, {'name' => 'double', 'type' => 'STRING'}, {'name' => 'string', 'type' => 'INTEGER'}, @@ -81,7 +81,7 @@ def test_fields_from_embulk_schema_with_column_options ], } expected = [ - {name: 'boolean', type: 'STRING', mode: 'REQUIRED'}, + {name: 'boolean', type: 'STRING', mode: 'REQUIRED', description: 'hoge'}, {name: 'long', type: 'STRING'}, {name: 'double', type: 'STRING'}, {name: 'string', type: 'INTEGER'}, From dd311f54a0a977448345e8f15f22f7040f1a900c Mon Sep 17 00:00:00 2001 From: fagai Date: Sat, 2 Mar 2024 00:31:23 +0900 Subject: [PATCH 6/6] add config 'description' --- lib/embulk/output/bigquery.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/embulk/output/bigquery.rb b/lib/embulk/output/bigquery.rb index dd9b92d..10cbd00 100644 --- a/lib/embulk/output/bigquery.rb +++ b/lib/embulk/output/bigquery.rb @@ -63,6 +63,8 @@ def self.configure(config, schema, task_count) 'payload_column' => config.param('payload_column', :string, :default => nil), 'payload_column_index' => config.param('payload_column_index', :integer, :default => nil), + 'description' => config.param('description', :string, :default => nil), + 'open_timeout_sec' => config.param('open_timeout_sec', :integer, :default => nil), 'timeout_sec' => config.param('timeout_sec', :integer, :default => nil), # google-api-ruby-client < v0.11.0 'send_timeout_sec' => config.param('send_timeout_sec', :integer, :default => nil), # google-api-ruby-client >= v0.11.0