From 9efa662f6dbf5ae90ea2edec0a069a1a80cd0039 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Mon, 8 Jan 2024 15:54:32 +0100 Subject: [PATCH] fix: Pass use_json_type input to insert bq activity (#19650) * fix: Pass use_json_type input to insert bq activity * fix: Extend tests with fixture --- .../batch_exports/bigquery_batch_export.py | 1 + .../test_bigquery_batch_export_workflow.py | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/posthog/temporal/batch_exports/bigquery_batch_export.py b/posthog/temporal/batch_exports/bigquery_batch_export.py index ca11b60be5e25..c9488739d5cef 100644 --- a/posthog/temporal/batch_exports/bigquery_batch_export.py +++ b/posthog/temporal/batch_exports/bigquery_batch_export.py @@ -308,6 +308,7 @@ async def run(self, inputs: BigQueryBatchExportInputs): data_interval_end=data_interval_end.isoformat(), exclude_events=inputs.exclude_events, include_events=inputs.include_events, + use_json_type=inputs.use_json_type, ) await execute_batch_export_insert_activity( diff --git a/posthog/temporal/tests/batch_exports/test_bigquery_batch_export_workflow.py b/posthog/temporal/tests/batch_exports/test_bigquery_batch_export_workflow.py index 719cd37e08e40..7a2872795a789 100644 --- a/posthog/temporal/tests/batch_exports/test_bigquery_batch_export_workflow.py +++ b/posthog/temporal/tests/batch_exports/test_bigquery_batch_export_workflow.py @@ -146,8 +146,17 @@ def bigquery_dataset(bigquery_config, bigquery_client) -> typing.Generator[bigqu # bigquery_client.delete_dataset(dataset_id, delete_contents=True, not_found_ok=True) +@pytest.fixture +def use_json_type(request) -> bool: + """A parametrizable fixture to configure the bool use_json_type setting.""" + try: + return request.param + except AttributeError: + return False + + @pytest.mark.parametrize("exclude_events", [None, ["test-exclude"]], indirect=True) -@pytest.mark.parametrize("use_json_type", [False, True]) +@pytest.mark.parametrize("use_json_type", [False, True], indirect=True) async def test_insert_into_bigquery_activity_inserts_data_into_bigquery_table( clickhouse_client, activity_environment, @@ -248,7 +257,7 @@ def table_id(ateam, interval): @pytest_asyncio.fixture async def bigquery_batch_export( - ateam, table_id, bigquery_config, interval, exclude_events, temporal_client, bigquery_dataset + ateam, table_id, bigquery_config, interval, exclude_events, use_json_type, temporal_client, bigquery_dataset ): destination_data = { "type": "BigQuery", @@ -257,6 +266,7 @@ async def bigquery_batch_export( "table_id": table_id, "dataset_id": bigquery_dataset.dataset_id, "exclude_events": exclude_events, + "use_json_type": use_json_type, }, } @@ -280,7 +290,7 @@ async def bigquery_batch_export( @pytest.mark.parametrize("interval", ["hour", "day"]) @pytest.mark.parametrize("exclude_events", [None, ["test-exclude"]], indirect=True) -@pytest.mark.parametrize("use_json_type", [False, True]) +@pytest.mark.parametrize("use_json_type", [False, True], indirect=True) async def test_bigquery_export_workflow( clickhouse_client, bigquery_client, @@ -289,7 +299,6 @@ async def test_bigquery_export_workflow( exclude_events, ateam, table_id, - use_json_type, ): """Test BigQuery Export Workflow end-to-end. @@ -331,7 +340,6 @@ async def test_bigquery_export_workflow( batch_export_id=str(bigquery_batch_export.id), data_interval_end=data_interval_end.isoformat(), interval=interval, - use_json_type=use_json_type, **bigquery_batch_export.destination.config, )