diff --git a/frontend/__snapshots__/exporter-exporter--trends-line-insight-detailed--dark.png b/frontend/__snapshots__/exporter-exporter--trends-line-insight-detailed--dark.png index e3e93b3dd9678..b54d4facb0bfe 100644 Binary files a/frontend/__snapshots__/exporter-exporter--trends-line-insight-detailed--dark.png and b/frontend/__snapshots__/exporter-exporter--trends-line-insight-detailed--dark.png differ diff --git a/frontend/__snapshots__/exporter-exporter--trends-line-insight-detailed--light.png b/frontend/__snapshots__/exporter-exporter--trends-line-insight-detailed--light.png index d4377362d9270..332ea24ce3084 100644 Binary files a/frontend/__snapshots__/exporter-exporter--trends-line-insight-detailed--light.png and b/frontend/__snapshots__/exporter-exporter--trends-line-insight-detailed--light.png differ diff --git a/frontend/src/layout/navigation/EnvironmentSwitcher.tsx b/frontend/src/layout/navigation/EnvironmentSwitcher.tsx index 2432cb72c63f7..83c85092ff3ab 100644 --- a/frontend/src/layout/navigation/EnvironmentSwitcher.tsx +++ b/frontend/src/layout/navigation/EnvironmentSwitcher.tsx @@ -157,6 +157,17 @@ function determineProjectSwitchUrl(pathname: string, newTeamId: number): string // and after switching is on a different page than before. let route = removeProjectIdIfPresent(pathname) route = removeFlagIdIfPresent(route) + + // List of routes that should redirect to project home + // instead of keeping the current path. + const redirectToHomeRoutes = ['/products', '/onboarding'] + + const shouldRedirectToHome = redirectToHomeRoutes.some((redirectRoute) => route.includes(redirectRoute)) + + if (shouldRedirectToHome) { + return urls.project(newTeamId) // Go to project home + } + return urls.project(newTeamId, route) } diff --git a/mypy-baseline.txt b/mypy-baseline.txt index 6f5de8a4ccf38..57a7e095f6298 100644 --- a/mypy-baseline.txt +++ b/mypy-baseline.txt @@ -808,7 +808,6 @@ posthog/temporal/tests/external_data/test_external_data_job.py:0: error: Invalid posthog/temporal/tests/external_data/test_external_data_job.py:0: error: Invalid index type "str" for "dict[Type, Sequence[str]]"; expected type "Type" [index] posthog/temporal/tests/external_data/test_external_data_job.py:0: error: Invalid index type "str" for "dict[Type, Sequence[str]]"; expected type "Type" [index] posthog/temporal/tests/external_data/test_external_data_job.py:0: error: Invalid index type "str" for "dict[Type, Sequence[str]]"; expected type "Type" [index] -posthog/temporal/tests/external_data/test_external_data_job.py:0: error: Invalid index type "str" for "dict[Type, Sequence[str]]"; expected type "Type" [index] posthog/api/test/batch_exports/conftest.py:0: error: Signature of "run" incompatible with supertype "Worker" [override] posthog/api/test/batch_exports/conftest.py:0: note: Superclass: posthog/api/test/batch_exports/conftest.py:0: note: def run(self) -> Coroutine[Any, Any, None] diff --git a/posthog/api/test/__snapshots__/test_api_docs.ambr b/posthog/api/test/__snapshots__/test_api_docs.ambr index 6ef31c6530176..a5f9b394809ae 100644 --- a/posthog/api/test/__snapshots__/test_api_docs.ambr +++ b/posthog/api/test/__snapshots__/test_api_docs.ambr @@ -97,8 +97,8 @@ '/home/runner/work/posthog/posthog/posthog/api/survey.py: Warning [SurveyViewSet > SurveySerializer]: unable to resolve type hint for function "get_conditions". Consider using a type hint or @extend_schema_field. Defaulting to string.', '/home/runner/work/posthog/posthog/posthog/api/web_experiment.py: Warning [WebExperimentViewSet]: could not derive type of path parameter "project_id" because model "posthog.models.web_experiment.WebExperiment" contained no such field. Consider annotating parameter with @extend_schema. Defaulting to "string".', 'Warning: encountered multiple names for the same choice set (HrefMatchingEnum). This may be unwanted even though the generated schema is technically correct. Add an entry to ENUM_NAME_OVERRIDES to fix the naming.', - 'Warning: enum naming encountered a non-optimally resolvable collision for fields named "kind". The same name has been used for multiple choice sets in multiple components. The collision was resolved with "Kind069Enum". add an entry to ENUM_NAME_OVERRIDES to fix the naming.', 'Warning: enum naming encountered a non-optimally resolvable collision for fields named "kind". The same name has been used for multiple choice sets in multiple components. The collision was resolved with "KindCfaEnum". add an entry to ENUM_NAME_OVERRIDES to fix the naming.', + 'Warning: enum naming encountered a non-optimally resolvable collision for fields named "kind". The same name has been used for multiple choice sets in multiple components. The collision was resolved with "Kind069Enum". add an entry to ENUM_NAME_OVERRIDES to fix the naming.', 'Warning: enum naming encountered a non-optimally resolvable collision for fields named "type". The same name has been used for multiple choice sets in multiple components. The collision was resolved with "TypeF73Enum". add an entry to ENUM_NAME_OVERRIDES to fix the naming.', 'Warning: encountered multiple names for the same choice set (EffectivePrivilegeLevelEnum). This may be unwanted even though the generated schema is technically correct. Add an entry to ENUM_NAME_OVERRIDES to fix the naming.', 'Warning: encountered multiple names for the same choice set (MembershipLevelEnum). This may be unwanted even though the generated schema is technically correct. Add an entry to ENUM_NAME_OVERRIDES to fix the naming.', diff --git a/posthog/hogql/database/schema/persons.py b/posthog/hogql/database/schema/persons.py index 0b0747593b7a7..04e8eaa47dd5e 100644 --- a/posthog/hogql/database/schema/persons.py +++ b/posthog/hogql/database/schema/persons.py @@ -1,10 +1,12 @@ from typing import cast, Optional, Self +from posthog.hogql.parser import parse_select import posthoganalytics from posthog.hogql.ast import SelectQuery, And, CompareOperation, CompareOperationOp, Field, JoinExpr from posthog.hogql.base import Expr from posthog.hogql.constants import HogQLQuerySettings from posthog.hogql.context import HogQLContext +from posthog.hogql import ast from posthog.hogql.database.argmax import argmax_select from posthog.hogql.database.models import ( BooleanDatabaseField, @@ -56,10 +58,46 @@ def select_from_persons_table( version = PersonsArgMaxVersion.V2 break - if version == PersonsArgMaxVersion.V2: - from posthog.hogql import ast - from posthog.hogql.parser import parse_select + and_conditions = [] + + if filter is not None: + and_conditions.append(filter) + # For now, only do this optimization for directly querying the persons table (without joins or as part of a subquery) to avoid knock-on effects to insight queries + if ( + node.select_from + and node.select_from.type + and hasattr(node.select_from.type, "table") + and node.select_from.type.table + and isinstance(node.select_from.type.table, PersonsTable) + ): + extractor = WhereClauseExtractor(context) + extractor.add_local_tables(join_or_table) + where = extractor.get_inner_where(node) + if where: + select = argmax_select( + table_name="raw_persons", + select_fields=join_or_table.fields_accessed, + group_fields=["id"], + argmax_field="version", + deleted_field="is_deleted", + timestamp_field_to_clamp="created_at", + ) + inner_select = cast( + ast.SelectQuery, + parse_select( + """ + SELECT id FROM raw_persons as where_optimization + """ + ), + ) + inner_select.where = where + select.where = ast.CompareOperation( + left=ast.Field(chain=["id"]), right=inner_select, op=ast.CompareOperationOp.In + ) + return select + + if version == PersonsArgMaxVersion.V2: select = cast( ast.SelectQuery, parse_select( diff --git a/posthog/hogql/database/schema/test/__snapshots__/test_persons.ambr b/posthog/hogql/database/schema/test/__snapshots__/test_persons.ambr new file mode 100644 index 0000000000000..90d7cb108ef8d --- /dev/null +++ b/posthog/hogql/database/schema/test/__snapshots__/test_persons.ambr @@ -0,0 +1,58 @@ +# serializer version: 1 +# name: TestPersonOptimization.test_joins_are_left_alone_for_now + ''' + SELECT events.uuid AS uuid + FROM events + INNER JOIN + (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS events__pdi___person_id, + argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, + person_distinct_id2.distinct_id AS distinct_id + FROM person_distinct_id2 + WHERE equals(person_distinct_id2.team_id, 2) + GROUP BY person_distinct_id2.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) + INNER JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS `properties___$some_prop` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS events__pdi__person ON equals(events__pdi.events__pdi___person_id, events__pdi__person.id) + WHERE and(equals(events.team_id, 2), ifNull(equals(events__pdi__person.`properties___$some_prop`, 'something'), 0)) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestPersonOptimization.test_simple_filter + ''' + SELECT persons.id AS id, + persons.properties AS properties + FROM + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$some_prop'), ''), 'null'), '^"|"$', ''), person.version) AS `properties___$some_prop`, + argMax(person.properties, person.version) AS properties, + person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT where_optimization.id AS id + FROM person AS where_optimization + WHERE and(equals(where_optimization.team_id, 2), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(where_optimization.properties, '$some_prop'), ''), 'null'), '^"|"$', ''), 'something'), 0))))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0))) AS persons + WHERE ifNull(equals(persons.`properties___$some_prop`, 'something'), 0) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- diff --git a/posthog/hogql/database/schema/test/test_persons.py b/posthog/hogql/database/schema/test/test_persons.py new file mode 100644 index 0000000000000..e259df691d06e --- /dev/null +++ b/posthog/hogql/database/schema/test/test_persons.py @@ -0,0 +1,128 @@ +from posthog.hogql.parser import parse_select +from posthog.schema import ( + PersonsOnEventsMode, + InsightActorsQuery, + TrendsQuery, + ActorsQuery, + EventsNode, + InsightDateRange, +) +from posthog.hogql_queries.actors_query_runner import ActorsQueryRunner +from posthog.hogql.modifiers import create_default_modifiers_for_team +from posthog.hogql.query import execute_hogql_query +from posthog.test.base import ( + APIBaseTest, + ClickhouseTestMixin, + _create_person, + _create_event, + snapshot_clickhouse_queries, +) +from posthog.models.person.util import create_person +from datetime import datetime + +from unittest.mock import patch, Mock + + +@patch("posthoganalytics.feature_enabled", new=Mock(return_value=True)) # for persons-inner-where-optimization +class TestPersonOptimization(ClickhouseTestMixin, APIBaseTest): + """ + Mostly tests for the optimization of pre-filtering before aggregating. See https://github.com/PostHog/posthog/pull/25604 + """ + + def setUp(self): + super().setUp() + self.first_person = _create_person( + team_id=self.team.pk, + distinct_ids=["1"], + properties={"$some_prop": "something", "$another_prop": "something1"}, + created_at=datetime(2024, 1, 1, 12), + ) + self.second_person = _create_person( + team_id=self.team.pk, + properties={"$some_prop": "ifwematcholdversionsthiswillmatch", "$another_prop": "something2"}, + distinct_ids=["2"], + version=1, + created_at=datetime(2024, 1, 1, 13), + ) + # update second_person with the correct prop + create_person( + team_id=self.team.pk, + uuid=str(self.second_person.uuid), + properties={"$some_prop": "something", "$another_prop": "something2"}, + created_at=datetime(2024, 1, 1, 13), + version=2, + ) + self.third_person = _create_person( + team_id=self.team.pk, + distinct_ids=["3"], + properties={"$some_prop": "not something", "$another_prop": "something3"}, + created_at=datetime(2024, 1, 1, 14), + ) + # deleted + self.deleted_person = _create_person( + team_id=self.team.pk, + properties={"$some_prop": "ifwematcholdversionsthiswillmatch", "$another_prop": "something2"}, + distinct_ids=["deleted"], + created_at=datetime(2024, 1, 1, 13), + version=1, + ) + create_person(team_id=self.team.pk, uuid=str(self.deleted_person.uuid), version=2, is_deleted=True) + _create_event(event="$pageview", distinct_id="1", team=self.team) + _create_event(event="$pageview", distinct_id="2", team=self.team) + _create_event(event="$pageview", distinct_id="3", team=self.team) + self.modifiers = create_default_modifiers_for_team(self.team) + self.modifiers.personsOnEventsMode = PersonsOnEventsMode.DISABLED + # self.modifiers.optimizeJoinedFilters = True + # self.modifiers.personsArgMaxVersion = PersonsArgMaxVersion.V1 + + @snapshot_clickhouse_queries + def test_simple_filter(self): + response = execute_hogql_query( + parse_select("select id, properties from persons where properties.$some_prop = 'something'"), + self.team, + modifiers=self.modifiers, + ) + assert len(response.results) == 2 + assert response.clickhouse + self.assertIn("where_optimization", response.clickhouse) + self.assertNotIn("in(tuple(person.id, person.version)", response.clickhouse) + + @snapshot_clickhouse_queries + def test_joins_are_left_alone_for_now(self): + response = execute_hogql_query( + parse_select("select uuid from events where person.properties.$some_prop = 'something'"), + self.team, + modifiers=self.modifiers, + ) + assert len(response.results) == 2 + assert response.clickhouse + self.assertIn("in(tuple(person.id, person.version)", response.clickhouse) + self.assertNotIn("where_optimization", response.clickhouse) + + def test_person_modal_not_optimized_yet(self): + source_query = TrendsQuery( + series=[EventsNode(event="$pageview")], + dateRange=InsightDateRange(date_from="2024-01-01", date_to="2024-01-07"), + # breakdownFilter=BreakdownFilter(breakdown="$", breakdown_type=BreakdownType.PERSON), + ) + insight_actors_query = InsightActorsQuery( + source=source_query, + day="2024-01-01", + modifiers=self.modifiers, + ) + actors_query = ActorsQuery( + source=insight_actors_query, + offset=0, + select=[ + "actor", + "created_at", + "event_count", + # "matched_recordings", + ], + orderBy=["event_count DESC"], + modifiers=self.modifiers, + ) + query_runner = ActorsQueryRunner(query=actors_query, team=self.team) + response = execute_hogql_query(query_runner.to_query(), self.team, modifiers=self.modifiers) + assert response.clickhouse + self.assertNotIn("where_optimization", response.clickhouse) diff --git a/posthog/hogql/test/__snapshots__/test_query.ambr b/posthog/hogql/test/__snapshots__/test_query.ambr index 96bfad37a5a50..d41d6417cdbd4 100644 --- a/posthog/hogql/test/__snapshots__/test_query.ambr +++ b/posthog/hogql/test/__snapshots__/test_query.ambr @@ -459,16 +459,15 @@ SELECT DISTINCT persons.properties___sneaky_mail AS sneaky_mail FROM ( - SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', '') AS properties___random_uuid + SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', ''), person.version) AS properties___sneaky_mail, argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', ''), person.version) AS properties___random_uuid, person.id AS id FROM person - WHERE and(equals(person.team_id, 420), ifNull(in(tuple(person.id, person.version), ( - SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 420) + WHERE and(equals(person.team_id, 420), in(id, ( + SELECT where_optimization.id AS id + FROM person AS where_optimization + WHERE and(equals(where_optimization.team_id, 420), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(where_optimization.properties, %(hogql_val_2)s), ''), 'null'), '^"|"$', ''), %(hogql_val_3)s), 0))))) GROUP BY person.id - HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, %(hogql_val_2)s), person.version), plus(now64(6, %(hogql_val_3)s), toIntervalDay(1))), 0)))), 0)) - SETTINGS optimize_aggregation_in_order=1) AS persons - WHERE ifNull(equals(persons.properties___random_uuid, %(hogql_val_4)s), 0) + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, %(hogql_val_4)s), person.version), plus(now64(6, %(hogql_val_5)s), toIntervalDay(1))), 0))) AS persons + WHERE ifNull(equals(persons.properties___random_uuid, %(hogql_val_6)s), 0) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0 diff --git a/posthog/temporal/data_imports/external_data_job.py b/posthog/temporal/data_imports/external_data_job.py index 1f44942a422b7..01367cb710812 100644 --- a/posthog/temporal/data_imports/external_data_job.py +++ b/posthog/temporal/data_imports/external_data_job.py @@ -2,6 +2,7 @@ import datetime as dt import json +import posthoganalytics from temporalio import activity, exceptions, workflow from temporalio.common import RetryPolicy @@ -21,6 +22,7 @@ create_external_data_job_model_activity, ) from posthog.temporal.data_imports.workflow_activities.import_data import ImportDataActivityInputs, import_data_activity +from posthog.utils import get_machine_id from posthog.warehouse.data_load.service import ( a_delete_external_data_schedule, a_external_data_workflow_exists, @@ -37,16 +39,27 @@ ExternalDataJob, get_active_schemas_for_source_id, ExternalDataSource, + get_external_data_source, ) from posthog.temporal.common.logger import bind_temporal_worker_logger from posthog.warehouse.models.external_data_schema import aupdate_should_sync -Non_Retryable_Schema_Errors = [ - "NoSuchTableError", - "401 Client Error: Unauthorized for url: https://api.stripe.com", - "403 Client Error: Forbidden for url: https://api.stripe.com", -] +Non_Retryable_Schema_Errors: dict[ExternalDataSource.Type, list[str]] = { + ExternalDataSource.Type.STRIPE: [ + "401 Client Error: Unauthorized for url: https://api.stripe.com", + "403 Client Error: Forbidden for url: https://api.stripe.com", + ], + ExternalDataSource.Type.POSTGRES: [ + "NoSuchTableError", + "is not permitted to log in", + "Tenant or user not found connection to server", + "FATAL: Tenant or user not found", + "error received from server in SCRAM exchange: Wrong password", + "could not translate host name", + ], + ExternalDataSource.Type.ZENDESK: ["404 Client Error: Not Found for url", "403 Client Error: Forbidden for url"], +} @dataclasses.dataclass @@ -54,6 +67,7 @@ class UpdateExternalDataJobStatusInputs: team_id: int job_id: str | None schema_id: str + source_id: str status: str internal_error: str | None latest_error: str | None @@ -78,10 +92,26 @@ async def update_external_data_job_model(inputs: UpdateExternalDataJobStatusInpu f"External data job failed for external data schema {inputs.schema_id} with error: {inputs.internal_error}" ) - has_non_retryable_error = any(error in inputs.internal_error for error in Non_Retryable_Schema_Errors) - if has_non_retryable_error: - logger.info("Schema has a non-retryable error - turning off syncing") - await aupdate_should_sync(schema_id=inputs.schema_id, team_id=inputs.team_id, should_sync=False) + source: ExternalDataSource = await get_external_data_source(inputs.source_id) + non_retryable_errors = Non_Retryable_Schema_Errors.get(ExternalDataSource.Type(source.source_type)) + + if non_retryable_errors is not None: + has_non_retryable_error = any(error in inputs.internal_error for error in non_retryable_errors) + if has_non_retryable_error: + logger.info("Schema has a non-retryable error - turning off syncing") + posthoganalytics.capture( + get_machine_id(), + "schema non-retryable error", + { + "schemaId": inputs.schema_id, + "sourceId": inputs.source_id, + "sourceType": source.source_type, + "jobId": inputs.job_id, + "teamId": inputs.team_id, + "error": inputs.internal_error, + }, + ) + await aupdate_should_sync(schema_id=inputs.schema_id, team_id=inputs.team_id, should_sync=False) await aupdate_external_job_status( job_id=job_id, @@ -166,6 +196,7 @@ async def run(self, inputs: ExternalDataWorkflowInputs): internal_error=None, team_id=inputs.team_id, schema_id=str(inputs.external_data_schema_id), + source_id=str(inputs.external_data_source_id), ) try: diff --git a/posthog/temporal/tests/data_imports/test_end_to_end.py b/posthog/temporal/tests/data_imports/test_end_to_end.py index cb92492b29c4c..98d81cc153bbe 100644 --- a/posthog/temporal/tests/data_imports/test_end_to_end.py +++ b/posthog/temporal/tests/data_imports/test_end_to_end.py @@ -6,6 +6,7 @@ from asgiref.sync import sync_to_async from django.conf import settings from django.test import override_settings +import posthoganalytics import pytest import pytest_asyncio import psycopg @@ -905,3 +906,56 @@ def get_jobs(): with pytest.raises(Exception): await sync_to_async(execute_hogql_query)("SELECT * FROM stripe_customer", team) + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.asyncio +async def test_non_retryable_error(team, stripe_customer): + source = await sync_to_async(ExternalDataSource.objects.create)( + source_id=uuid.uuid4(), + connection_id=uuid.uuid4(), + destination_id=uuid.uuid4(), + team=team, + status="running", + source_type="Stripe", + job_inputs={"stripe_secret_key": "test-key", "stripe_account_id": "acct_id"}, + ) + + schema = await sync_to_async(ExternalDataSchema.objects.create)( + name="Customer", + team_id=team.pk, + source_id=source.pk, + sync_type=ExternalDataSchema.SyncType.FULL_REFRESH, + sync_type_config={}, + ) + + workflow_id = str(uuid.uuid4()) + inputs = ExternalDataWorkflowInputs( + team_id=team.id, + external_data_source_id=source.pk, + external_data_schema_id=schema.id, + ) + + with ( + mock.patch( + "posthog.temporal.data_imports.workflow_activities.check_billing_limits.list_limited_team_attributes", + ) as mock_list_limited_team_attributes, + mock.patch.object(posthoganalytics, "capture") as capture_mock, + ): + mock_list_limited_team_attributes.side_effect = Exception( + "401 Client Error: Unauthorized for url: https://api.stripe.com" + ) + + with pytest.raises(Exception): + await _execute_run(workflow_id, inputs, stripe_customer["data"]) + + capture_mock.assert_called_once() + + job: ExternalDataJob = await sync_to_async(ExternalDataJob.objects.get)(team_id=team.id, schema_id=schema.pk) + await sync_to_async(schema.refresh_from_db)() + + assert job.status == ExternalDataJob.Status.FAILED + assert schema.should_sync is False + + with pytest.raises(Exception): + await sync_to_async(execute_hogql_query)("SELECT * FROM stripe_customer", team) diff --git a/posthog/temporal/tests/external_data/test_external_data_job.py b/posthog/temporal/tests/external_data/test_external_data_job.py index d5839cd4ce0ef..1b5b3d692d5ed 100644 --- a/posthog/temporal/tests/external_data/test_external_data_job.py +++ b/posthog/temporal/tests/external_data/test_external_data_job.py @@ -252,6 +252,7 @@ async def test_update_external_job_activity(activity_environment, team, **kwargs latest_error=None, internal_error=None, schema_id=str(schema.pk), + source_id=str(new_source.pk), team_id=team.id, ) @@ -296,6 +297,7 @@ async def test_update_external_job_activity_with_retryable_error(activity_enviro latest_error=None, internal_error="Some other retryable error", schema_id=str(schema.pk), + source_id=str(new_source.pk), team_id=team.id, ) @@ -317,11 +319,11 @@ async def test_update_external_job_activity_with_non_retryable_error(activity_en destination_id=uuid.uuid4(), team=team, status="running", - source_type="Stripe", + source_type="Postgres", ) schema = await sync_to_async(ExternalDataSchema.objects.create)( - name=PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING[new_source.source_type][0], + name="test_123", team_id=team.id, source_id=new_source.pk, should_sync=True, @@ -341,6 +343,7 @@ async def test_update_external_job_activity_with_non_retryable_error(activity_en latest_error=None, internal_error="NoSuchTableError: TableA", schema_id=str(schema.pk), + source_id=str(new_source.pk), team_id=team.id, ) with mock.patch("posthog.warehouse.models.external_data_schema.external_data_workflow_exists", return_value=False):