Skip to content

Commit

Permalink
Fix: Resolve load failures for large files by running Snowflake PUT
Browse files Browse the repository at this point in the history
… queries sequentially (#188)
  • Loading branch information
SebastienN15 authored Apr 12, 2024
1 parent 6c9dab8 commit 011315c
Showing 1 changed file with 4 additions and 4 deletions.
8 changes: 4 additions & 4 deletions airbyte/_processors/sql/snowflake.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,10 @@ def _write_files_to_new_table(
def path_str(path: Path) -> str:
return str(path.absolute()).replace("\\", "\\\\")

put_files_statements = "\n".join(
[f"PUT 'file://{path_str(file_path)}' {internal_sf_stage_name};" for file_path in files]
)
self._execute_sql(put_files_statements)
for file_path in files:
query = f"PUT 'file://{path_str(file_path)}' {internal_sf_stage_name};"
self._execute_sql(query)

columns_list = [
self._quote_identifier(c)
for c in list(self._get_sql_column_definitions(stream_name).keys())
Expand Down

0 comments on commit 011315c

Please sign in to comment.