Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(data-warehouse): Add incremental syncs to snowflake: #23928

Merged
merged 3 commits into from
Jul 24, 2024
Merged

feat(data-warehouse): Add incremental syncs to snowflake: #23928

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
35634c5
Add incremental syncs to snowflake:
Gilbert09 Jul 23, 2024
d4fb8f5
Merge branch 'master' into tom/snowflake-incremental
Gilbert09 Jul 23, 2024
753cf12
Merge branch 'master' into tom/snowflake-incremental
Gilbert09 Jul 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions posthog/temporal/data_imports/pipelines/sql_database/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ def snowflake_source(
table_names: list[str],
role: Optional[str] = None,
incremental_field: Optional[str] = None,
incremental_field_type: Optional[str] = None,
incremental_field_type: Optional[IncrementalFieldType] = None,
) -> DltSource:
account_id = quote(account_id)
user = quote(user)
Expand All @@ -88,10 +88,17 @@ def snowflake_source(
warehouse = quote(warehouse)
role = quote(role) if role else None

if incremental_field is not None and incremental_field_type is not None:
incremental: dlt.sources.incremental | None = dlt.sources.incremental(
cursor_path=incremental_field, initial_value=incremental_type_to_initial_value(incremental_field_type)
)
else:
incremental = None

credentials = ConnectionStringCredentials(
f"snowflake://{user}:{password}@{account_id}/{database}/{schema}?warehouse={warehouse}{f'&role={role}' if role else ''}"
)
db_source = sql_database(credentials, schema=schema, table_names=table_names)
db_source = sql_database(credentials, schema=schema, table_names=table_names, incremental=incremental)

return db_source

Expand Down
4 changes: 4 additions & 0 deletions posthog/temporal/data_imports/workflow_activities/import_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,10 @@ async def import_data_activity(inputs: ImportDataActivityInputs):
warehouse=warehouse,
role=role,
table_names=endpoints,
incremental_field=schema.sync_type_config.get("incremental_field") if schema.is_incremental else None,
incremental_field_type=schema.sync_type_config.get("incremental_field_type")
if schema.is_incremental
else None,
)

return await _run(
Expand Down
Loading