From 011315caa6a31cd2f7244c4b83d1082ec0510b2c Mon Sep 17 00:00:00 2001 From: Sebastien Haentjens Date: Fri, 12 Apr 2024 16:49:23 +0200 Subject: [PATCH] Fix: Resolve load failures for large files by running Snowflake `PUT` queries sequentially (#188) --- airbyte/_processors/sql/snowflake.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte/_processors/sql/snowflake.py b/airbyte/_processors/sql/snowflake.py index b4ce4298..2fb70f8f 100644 --- a/airbyte/_processors/sql/snowflake.py +++ b/airbyte/_processors/sql/snowflake.py @@ -69,10 +69,10 @@ def _write_files_to_new_table( def path_str(path: Path) -> str: return str(path.absolute()).replace("\\", "\\\\") - put_files_statements = "\n".join( - [f"PUT 'file://{path_str(file_path)}' {internal_sf_stage_name};" for file_path in files] - ) - self._execute_sql(put_files_statements) + for file_path in files: + query = f"PUT 'file://{path_str(file_path)}' {internal_sf_stage_name};" + self._execute_sql(query) + columns_list = [ self._quote_identifier(c) for c in list(self._get_sql_column_definitions(stream_name).keys())