Skip to content

Commit

Permalink
fix: Pass use_json_type input to insert bq activity (#19650)
Browse files Browse the repository at this point in the history
* fix: Pass use_json_type input to insert bq activity

* fix: Extend tests with fixture
  • Loading branch information
tomasfarias authored Jan 8, 2024
1 parent 290baab commit 9efa662
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 5 deletions.
1 change: 1 addition & 0 deletions posthog/temporal/batch_exports/bigquery_batch_export.py
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,7 @@ async def run(self, inputs: BigQueryBatchExportInputs):
data_interval_end=data_interval_end.isoformat(),
exclude_events=inputs.exclude_events,
include_events=inputs.include_events,
use_json_type=inputs.use_json_type,
)

await execute_batch_export_insert_activity(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,8 +146,17 @@ def bigquery_dataset(bigquery_config, bigquery_client) -> typing.Generator[bigqu
# bigquery_client.delete_dataset(dataset_id, delete_contents=True, not_found_ok=True)


@pytest.fixture
def use_json_type(request) -> bool:
"""A parametrizable fixture to configure the bool use_json_type setting."""
try:
return request.param
except AttributeError:
return False


@pytest.mark.parametrize("exclude_events", [None, ["test-exclude"]], indirect=True)
@pytest.mark.parametrize("use_json_type", [False, True])
@pytest.mark.parametrize("use_json_type", [False, True], indirect=True)
async def test_insert_into_bigquery_activity_inserts_data_into_bigquery_table(
clickhouse_client,
activity_environment,
Expand Down Expand Up @@ -248,7 +257,7 @@ def table_id(ateam, interval):

@pytest_asyncio.fixture
async def bigquery_batch_export(
ateam, table_id, bigquery_config, interval, exclude_events, temporal_client, bigquery_dataset
ateam, table_id, bigquery_config, interval, exclude_events, use_json_type, temporal_client, bigquery_dataset
):
destination_data = {
"type": "BigQuery",
Expand All @@ -257,6 +266,7 @@ async def bigquery_batch_export(
"table_id": table_id,
"dataset_id": bigquery_dataset.dataset_id,
"exclude_events": exclude_events,
"use_json_type": use_json_type,
},
}

Expand All @@ -280,7 +290,7 @@ async def bigquery_batch_export(

@pytest.mark.parametrize("interval", ["hour", "day"])
@pytest.mark.parametrize("exclude_events", [None, ["test-exclude"]], indirect=True)
@pytest.mark.parametrize("use_json_type", [False, True])
@pytest.mark.parametrize("use_json_type", [False, True], indirect=True)
async def test_bigquery_export_workflow(
clickhouse_client,
bigquery_client,
Expand All @@ -289,7 +299,6 @@ async def test_bigquery_export_workflow(
exclude_events,
ateam,
table_id,
use_json_type,
):
"""Test BigQuery Export Workflow end-to-end.
Expand Down Expand Up @@ -331,7 +340,6 @@ async def test_bigquery_export_workflow(
batch_export_id=str(bigquery_batch_export.id),
data_interval_end=data_interval_end.isoformat(),
interval=interval,
use_json_type=use_json_type,
**bigquery_batch_export.destination.config,
)

Expand Down

0 comments on commit 9efa662

Please sign in to comment.