From af6b23c88639af00f2e82f3894204c7c40331115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Thu, 16 Nov 2023 14:33:12 +0100 Subject: [PATCH 1/2] fix: Compatibility with Redshift (#18677) --- .../workflows/redshift_batch_export.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/posthog/temporal/workflows/redshift_batch_export.py b/posthog/temporal/workflows/redshift_batch_export.py index a853c91e79e1f..7b008d4f5b674 100644 --- a/posthog/temporal/workflows/redshift_batch_export.py +++ b/posthog/temporal/workflows/redshift_batch_export.py @@ -3,6 +3,7 @@ import datetime as dt import itertools import json +import os import typing from dataclasses import dataclass @@ -32,6 +33,31 @@ ) +@contextlib.asynccontextmanager +async def redshift_connection(inputs) -> typing.AsyncIterator[psycopg.AsyncConnection]: + """Manage a Redshift connection. + + This just yields a Postgres connection but we adjust a couple of things required for + psycopg to work with Redshift: + 1. Set PGCLIENTENCODING to utf-8 as Redshift reports back UNICODE. + 2. Set prepare_threshold to None on the connection as psycopg attempts to run DEALLOCATE ALL otherwise + which is not supported on Redshift. + """ + old_value = os.environ.get("PGCLIENTENCODING", None) + os.environ["PGCLIENTENCODING"] = "utf-8" + + try: + async with postgres_connection(inputs) as connection: + connection.prepare_threshold = None + yield connection + + finally: + if old_value is None: + del os.environ["PGCLIENTENCODING"] + else: + os.environ["PGCLIENTENCODING"] = old_value + + async def insert_records_to_redshift( records: collections.abc.Iterator[dict[str, typing.Any]], redshift_connection: psycopg.AsyncConnection, @@ -186,7 +212,7 @@ async def insert_into_redshift_activity(inputs: RedshiftInsertInputs): ) properties_type = "VARCHAR(65535)" if inputs.properties_data_type == "varchar" else "SUPER" - async with postgres_connection(inputs) as connection: + async with redshift_connection(inputs) as connection: await create_table_in_postgres( connection, schema=inputs.schema, From e83de60f8961226f9ad8253720363b7bd51b44cd Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 16 Nov 2023 15:23:31 +0100 Subject: [PATCH 2/2] feat(insigts): remove aggregation_group_type_index for lifecycle and stickiness (#18678) --- .../nodes/InsightQuery/utils/queryNodeToFilter.ts | 10 +++++++++- frontend/src/queries/schema.json | 8 -------- frontend/src/queries/schema.ts | 4 ++-- .../src/scenes/insights/filters/AggregationSelect.tsx | 6 ++++-- .../src/scenes/retention/retentionLineGraphLogic.ts | 7 ++++++- frontend/src/scenes/retention/retentionModalLogic.ts | 6 +++++- .../legacy_compatibility/filter_to_query.py | 2 +- posthog/schema.py | 2 -- 8 files changed, 27 insertions(+), 18 deletions(-) diff --git a/frontend/src/queries/nodes/InsightQuery/utils/queryNodeToFilter.ts b/frontend/src/queries/nodes/InsightQuery/utils/queryNodeToFilter.ts index 53432d3c1009f..03d904476be9a 100644 --- a/frontend/src/queries/nodes/InsightQuery/utils/queryNodeToFilter.ts +++ b/frontend/src/queries/nodes/InsightQuery/utils/queryNodeToFilter.ts @@ -86,7 +86,6 @@ export const queryNodeToFilter = (query: InsightQueryNode): Partial date_from: query.dateRange?.date_from, entity_type: 'events', sampling_factor: query.samplingFactor, - aggregation_group_type_index: query.aggregation_group_type_index, }) if (!isRetentionQuery(query) && !isPathsQuery(query)) { @@ -107,6 +106,15 @@ export const queryNodeToFilter = (query: InsightQueryNode): Partial Object.assign(filters, objectClean>>(query.breakdown)) } + if (!isLifecycleQuery(query) && !isStickinessQuery(query)) { + Object.assign( + filters, + objectClean({ + aggregation_group_type_index: query.aggregation_group_type_index, + }) + ) + } + if (isTrendsQuery(query) || isStickinessQuery(query) || isLifecycleQuery(query) || isFunnelsQuery(query)) { filters.interval = query.interval } diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index 62825d2ac3f9c..2e127952f7d27 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -1805,10 +1805,6 @@ "LifecycleQuery": { "additionalProperties": false, "properties": { - "aggregation_group_type_index": { - "description": "Groups aggregation", - "type": "number" - }, "dateRange": { "$ref": "#/definitions/DateRange", "description": "Date range for the query" @@ -2761,10 +2757,6 @@ "StickinessQuery": { "additionalProperties": false, "properties": { - "aggregation_group_type_index": { - "description": "Groups aggregation", - "type": "number" - }, "dateRange": { "$ref": "#/definitions/DateRange", "description": "Date range for the query" diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index 7b51deead7f6a..c0216197ba6f9 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -502,7 +502,7 @@ export type StickinessFilter = Omit< StickinessFilterType & { hidden_legend_indexes?: number[] }, keyof FilterType | 'hidden_legend_keys' | 'stickiness_days' | 'shown_as' > -export interface StickinessQuery extends InsightsQueryBase { +export interface StickinessQuery extends Omit { kind: NodeKind.StickinessQuery /** Granularity of the response. Can be one of `hour`, `day`, `week` or `month` */ interval?: IntervalType @@ -531,7 +531,7 @@ export interface LifecycleQueryResponse extends QueryResponse { results: Record[] } -export interface LifecycleQuery extends InsightsQueryBase { +export interface LifecycleQuery extends Omit { kind: NodeKind.LifecycleQuery /** Granularity of the response. Can be one of `hour`, `day`, `week` or `month` */ interval?: IntervalType diff --git a/frontend/src/scenes/insights/filters/AggregationSelect.tsx b/frontend/src/scenes/insights/filters/AggregationSelect.tsx index 766860da847a8..97104d7f22315 100644 --- a/frontend/src/scenes/insights/filters/AggregationSelect.tsx +++ b/frontend/src/scenes/insights/filters/AggregationSelect.tsx @@ -4,7 +4,7 @@ import { LemonSelect, LemonSelectSection } from '@posthog/lemon-ui' import { groupsAccessLogic } from 'lib/introductions/groupsAccessLogic' import { GroupIntroductionFooter } from 'scenes/groups/GroupsIntroduction' import { InsightLogicProps } from '~/types' -import { isFunnelsQuery, isInsightQueryNode } from '~/queries/utils' +import { isFunnelsQuery, isInsightQueryNode, isLifecycleQuery, isStickinessQuery } from '~/queries/utils' import { insightVizDataLogic } from 'scenes/insights/insightVizDataLogic' import { FunnelsQuery } from '~/queries/schema' import { HogQLEditor } from 'lib/components/HogQLEditor/HogQLEditor' @@ -51,7 +51,9 @@ export function AggregationSelect({ } const value = getHogQLValue( - querySource.aggregation_group_type_index, + isLifecycleQuery(querySource) || isStickinessQuery(querySource) + ? undefined + : querySource.aggregation_group_type_index, isFunnelsQuery(querySource) ? querySource.funnelsFilter?.funnel_aggregate_by_hogql : undefined ) const onChange = (value: string): void => { diff --git a/frontend/src/scenes/retention/retentionLineGraphLogic.ts b/frontend/src/scenes/retention/retentionLineGraphLogic.ts index 127ff04440385..656307fb41d38 100644 --- a/frontend/src/scenes/retention/retentionLineGraphLogic.ts +++ b/frontend/src/scenes/retention/retentionLineGraphLogic.ts @@ -9,6 +9,7 @@ import { insightVizDataLogic } from 'scenes/insights/insightVizDataLogic' import { retentionLogic } from './retentionLogic' import type { retentionLineGraphLogicType } from './retentionLineGraphLogicType' +import { isLifecycleQuery, isStickinessQuery } from '~/queries/utils' const DEFAULT_RETENTION_LOGIC_KEY = 'default_retention_key' @@ -117,7 +118,11 @@ export const retentionLineGraphLogic = kea([ aggregationGroupTypeIndex: [ (s) => [s.querySource], (querySource) => { - return querySource?.aggregation_group_type_index ?? 'people' + return ( + (isLifecycleQuery(querySource) || isStickinessQuery(querySource) + ? null + : querySource?.aggregation_group_type_index) ?? 'people' + ) }, ], }), diff --git a/frontend/src/scenes/retention/retentionModalLogic.ts b/frontend/src/scenes/retention/retentionModalLogic.ts index ebd464a94f4b0..5d5c4043a8112 100644 --- a/frontend/src/scenes/retention/retentionModalLogic.ts +++ b/frontend/src/scenes/retention/retentionModalLogic.ts @@ -7,6 +7,7 @@ import { retentionPeopleLogic } from './retentionPeopleLogic' import { insightVizDataLogic } from 'scenes/insights/insightVizDataLogic' import type { retentionModalLogicType } from './retentionModalLogicType' +import { isLifecycleQuery, isStickinessQuery } from '~/queries/utils' const DEFAULT_RETENTION_LOGIC_KEY = 'default_retention_key' @@ -35,7 +36,10 @@ export const retentionModalLogic = kea([ aggregationTargetLabel: [ (s) => [s.querySource, s.aggregationLabel], (querySource, aggregationLabel): Noun => { - const { aggregation_group_type_index } = querySource || {} + const aggregation_group_type_index = + isLifecycleQuery(querySource) || isStickinessQuery(querySource) + ? undefined + : querySource?.aggregation_group_type_index return aggregationLabel(aggregation_group_type_index) }, ], diff --git a/posthog/hogql_queries/legacy_compatibility/filter_to_query.py b/posthog/hogql_queries/legacy_compatibility/filter_to_query.py index ce490cadfc834..1fb210226f619 100644 --- a/posthog/hogql_queries/legacy_compatibility/filter_to_query.py +++ b/posthog/hogql_queries/legacy_compatibility/filter_to_query.py @@ -298,7 +298,7 @@ def _breakdown_filter(_filter: Dict): def _group_aggregation_filter(filter: Dict): - if _insight_type(filter) == "STICKINESS": + if _insight_type(filter) == "STICKINESS" or _insight_type(filter) == "LIFECYCLE": return {} return {"aggregation_group_type_index": filter.get("aggregation_group_type_index")} diff --git a/posthog/schema.py b/posthog/schema.py index fbba06675133a..c70fc0e400d3e 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -1447,7 +1447,6 @@ class StickinessQuery(BaseModel): model_config = ConfigDict( extra="forbid", ) - aggregation_group_type_index: Optional[float] = Field(default=None, description="Groups aggregation") dateRange: Optional[DateRange] = Field(default=None, description="Date range for the query") filterTestAccounts: Optional[bool] = Field( default=None, description="Exclude internal and test users by applying the respective filters" @@ -1645,7 +1644,6 @@ class LifecycleQuery(BaseModel): model_config = ConfigDict( extra="forbid", ) - aggregation_group_type_index: Optional[float] = Field(default=None, description="Groups aggregation") dateRange: Optional[DateRange] = Field(default=None, description="Date range for the query") filterTestAccounts: Optional[bool] = Field( default=None, description="Exclude internal and test users by applying the respective filters"