Skip to content

Commit

Permalink
chore(hogql): Add retention queries in HogQL (#18831)
Browse files Browse the repository at this point in the history 
* Set up retention query runnner
* Use and adapt existing code
* Streamline things by using HogQL AST
* Add feature flag for insights retention to frontend
  • Loading branch information
@webjunkie
webjunkie authored Dec 6, 2023
1 parent 19b418d commit 65e5ba7
Show file tree
Hide file tree
Showing 20 changed files with 2,942 additions and 40 deletions.
1 change: 1 addition & 0 deletions frontend/src/lib/constants.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ export const FEATURE_FLAGS = {
APPS_AND_EXPORTS_UI: 'apps-and-exports-ui', // owner: @benjackwhite
SESSION_REPLAY_CORS_PROXY: 'session-replay-cors-proxy', // owner: #team-replay
HOGQL_INSIGHTS_LIFECYCLE: 'hogql-insights-lifecycle', // owner: @mariusandra
HOGQL_INSIGHTS_RETENTION: 'hogql-insights-retention', // owner: @webjunkie
HOGQL_INSIGHTS_TRENDS: 'hogql-insights-trends', // owner: @Gilbert09
HOGQL_INSIGHT_LIVE_COMPARE: 'hogql-insight-live-compare', // owner: @mariusandra
BI_VIZ: 'bi_viz', // owner: @Gilbert09
Expand Down
4 changes: 4 additions & 0 deletions frontend/src/queries/nodes/DataNode/dataNodeLogic.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,10 @@ export const dataNodeLogic = kea<dataNodeLogicType>([
(s) => [s.featureFlags],
(featureFlags) => !!featureFlags[FEATURE_FLAGS.HOGQL_INSIGHTS_LIFECYCLE],
],
hogQLInsightsRetentionFlagEnabled: [
(s) => [s.featureFlags],
(featureFlags) => !!featureFlags[FEATURE_FLAGS.HOGQL_INSIGHTS_RETENTION],
],
hogQLInsightsTrendsFlagEnabled: [
(s) => [s.featureFlags],
(featureFlags) => !!featureFlags[FEATURE_FLAGS.HOGQL_INSIGHTS_TRENDS],
Expand Down
14 changes: 7 additions & 7 deletions frontend/src/queries/nodes/InsightQuery/utils/filtersToQueryNode.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ describe('filtersToQueryNode', () => {
const query: InsightQueryNode = {
kind: NodeKind.RetentionQuery,
filterTestAccounts: true,
}
} as InsightQueryNode
expect(result).toEqual(query)
})

Expand Down Expand Up @@ -212,7 +212,7 @@ describe('filtersToQueryNode', () => {
},
],
},
}
} as InsightQueryNode
expect(result).toEqual(query)
})

Expand All @@ -231,7 +231,7 @@ describe('filtersToQueryNode', () => {
date_to: '2021-12-08',
date_from: '2021-12-08',
},
}
} as InsightQueryNode
expect(result).toEqual(query)
})
})
Expand Down Expand Up @@ -401,8 +401,8 @@ describe('filtersToQueryNode', () => {
retention_type: 'retention_first_time',
retention_reference: 'total',
total_intervals: 2,
returning_entity: [{ a: 1 }],
target_entity: [{ b: 1 }],
returning_entity: { id: '1' },
target_entity: { id: '1' },
period: RetentionPeriod.Day,
}

Expand All @@ -414,8 +414,8 @@ describe('filtersToQueryNode', () => {
retention_type: 'retention_first_time',
retention_reference: 'total',
total_intervals: 2,
returning_entity: [{ a: 1 }],
target_entity: [{ b: 1 }],
returning_entity: { id: '1' },
target_entity: { id: '1' },
period: RetentionPeriod.Day,
},
}
Expand Down
5 changes: 5 additions & 0 deletions frontend/src/queries/query.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import {
isLifecycleQuery,
isPersonsNode,
isPersonsQuery,
isRetentionQuery,
isTimeToSeeDataQuery,
isTimeToSeeDataSessionsNode,
isTimeToSeeDataSessionsQuery,
Expand Down Expand Up @@ -148,6 +149,9 @@ export async function query<N extends DataNode = DataNode>(
const hogQLInsightsLifecycleFlagEnabled = Boolean(
featureFlagLogic.findMounted()?.values.featureFlags?.[FEATURE_FLAGS.HOGQL_INSIGHTS_LIFECYCLE]
)
const hogQLInsightsRetentionFlagEnabled = Boolean(
featureFlagLogic.findMounted()?.values.featureFlags?.[FEATURE_FLAGS.HOGQL_INSIGHTS_RETENTION]
)
const hogQLInsightsTrendsFlagEnabled = Boolean(
featureFlagLogic.findMounted()?.values.featureFlags?.[FEATURE_FLAGS.HOGQL_INSIGHTS_TRENDS]
)
Expand Down Expand Up @@ -193,6 +197,7 @@ export async function query<N extends DataNode = DataNode>(
} else if (isInsightQueryNode(queryNode)) {
if (
(hogQLInsightsLifecycleFlagEnabled && isLifecycleQuery(queryNode)) ||
(hogQLInsightsRetentionFlagEnabled && isRetentionQuery(queryNode)) ||
(hogQLInsightsTrendsFlagEnabled && isTrendsQuery(queryNode))
) {
if (hogQLInsightsLiveCompareEnabled) {
Expand Down
100 changes: 96 additions & 4 deletions frontend/src/queries/schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2536,6 +2536,34 @@
"required": ["key", "operator", "type", "value"],
"type": "object"
},
"RetentionEntity": {
"additionalProperties": false,
"properties": {
"custom_name": {
"type": "string"
},
"id": {
"type": ["string", "number"]
},
"kind": {
"enum": ["ActionsNode", "EventsNode"],
"type": "string"
},
"name": {
"type": "string"
},
"order": {
"type": "number"
},
"type": {
"$ref": "#/definitions/EntityType"
},
"uuid": {
"type": "string"
}
},
"type": "object"
},
"RetentionFilter": {
"additionalProperties": false,
"description": "`RetentionFilterType` minus everything inherited from `FilterType`",
Expand All @@ -2551,13 +2579,13 @@
"$ref": "#/definitions/RetentionType"
},
"returning_entity": {
"type": "object"
"$ref": "#/definitions/RetentionEntity"
},
"target_entity": {
"type": "object"
"$ref": "#/definitions/RetentionEntity"
},
"total_intervals": {
"type": "number"
"type": "integer"
}
},
"type": "object"
Expand Down Expand Up @@ -2599,6 +2627,9 @@
],
"description": "Property filters for all series"
},
"response": {
"$ref": "#/definitions/RetentionQueryResponse"
},
"retentionFilter": {
"$ref": "#/definitions/RetentionFilter",
"description": "Properties specific to the retention insight"
Expand All @@ -2608,13 +2639,74 @@
"type": ["number", "null"]
}
},
"required": ["kind"],
"required": ["kind", "retentionFilter"],
"type": "object"
},
"RetentionQueryResponse": {
"additionalProperties": false,
"properties": {
"hogql": {
"type": "string"
},
"is_cached": {
"type": "boolean"
},
"last_refresh": {
"type": "string"
},
"next_allowed_client_refresh": {
"type": "string"
},
"results": {
"items": {
"$ref": "#/definitions/RetentionResult"
},
"type": "array"
},
"timings": {
"items": {
"$ref": "#/definitions/QueryTiming"
},
"type": "array"
}
},
"required": ["results"],
"type": "object"
},
"RetentionResult": {
"additionalProperties": false,
"properties": {
"date": {
"format": "date-time",
"type": "string"
},
"label": {
"type": "string"
},
"values": {
"items": {
"$ref": "#/definitions/RetentionValue"
},
"type": "array"
}
},
"required": ["values", "label", "date"],
"type": "object"
},
"RetentionType": {
"enum": ["retention_recurring", "retention_first_time"],
"type": "string"
},
"RetentionValue": {
"additionalProperties": false,
"properties": {
"count": {
"type": "integer"
}
},
"required": ["count"],
"type": "object"
},
"SavedInsightNode": {
"additionalProperties": false,
"properties": {
Expand Down
19 changes: 18 additions & 1 deletion frontend/src/queries/schema.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -512,10 +512,27 @@ export interface FunnelsQuery extends InsightsQueryBase {

/** `RetentionFilterType` minus everything inherited from `FilterType` */
export type RetentionFilter = Omit<RetentionFilterType, keyof FilterType>

export interface RetentionValue {
/** @asType integer */
count: number
}

export interface RetentionResult {
values: RetentionValue[]
label: string
/** @format date-time */
date: string
}

export interface RetentionQueryResponse extends QueryResponse {
results: RetentionResult[]
}
export interface RetentionQuery extends InsightsQueryBase {
kind: NodeKind.RetentionQuery
response?: RetentionQueryResponse
/** Properties specific to the retention insight */
retentionFilter?: RetentionFilter
retentionFilter: RetentionFilter
}

/** `PathsFilterType` minus everything inherited from `FilterType` and persons modal related params */
Expand Down
4 changes: 2 additions & 2 deletions frontend/src/scenes/insights/InsightNav/insightNavLogic.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,9 @@ export interface CommonInsightFilter
Partial<LifecycleFilter> {}

export interface QueryPropertyCache
extends Omit<Partial<TrendsQuery>, 'kind'>,
extends Omit<Partial<TrendsQuery>, 'kind' | 'response'>,
Omit<Partial<FunnelsQuery>, 'kind'>,
Omit<Partial<RetentionQuery>, 'kind'>,
Omit<Partial<RetentionQuery>, 'kind' | 'response'>,
Omit<Partial<PathsQuery>, 'kind'>,
Omit<Partial<StickinessQuery>, 'kind'>,
Omit<Partial<LifecycleQuery>, 'kind'> {
Expand Down
16 changes: 8 additions & 8 deletions frontend/src/scenes/insights/summarizeInsight.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,12 +308,12 @@ describe('summarizing insights', () => {
target_entity: {
id: '$autocapture',
name: '$autocapture',
type: 'event',
type: 'events',
},
returning_entity: {
id: '$autocapture',
name: '$autocapture',
type: 'event',
type: 'events',
},
retention_type: RETENTION_FIRST_TIME,
} as RetentionFilterType,
Expand All @@ -333,12 +333,12 @@ describe('summarizing insights', () => {
target_entity: {
id: 'purchase',
name: 'purchase',
type: 'event',
type: 'events',
},
returning_entity: {
id: '$pageview',
name: '$pageview',
type: 'event',
type: 'events',
},
retention_type: RETENTION_RECURRING,
aggregation_group_type_index: 0,
Expand Down Expand Up @@ -731,12 +731,12 @@ describe('summarizing insights', () => {
target_entity: {
id: '$autocapture',
name: '$autocapture',
type: 'event',
type: 'events',
},
returning_entity: {
id: '$autocapture',
name: '$autocapture',
type: 'event',
type: 'events',
},
retention_type: RETENTION_FIRST_TIME,
},
Expand All @@ -760,12 +760,12 @@ describe('summarizing insights', () => {
target_entity: {
id: 'purchase',
name: 'purchase',
type: 'event',
type: 'events',
},
returning_entity: {
id: '$pageview',
name: '$pageview',
type: 'event',
type: 'events',
},
retention_type: RETENTION_RECURRING,
},
Expand Down
18 changes: 16 additions & 2 deletions frontend/src/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import type {
InsightVizNode,
Node,
} from './queries/schema'
import { NodeKind } from './queries/schema'

export type Optional<T, K extends string | number | symbol> = Omit<T, K> & { [K in keyof T]?: T[K] }

Expand Down Expand Up @@ -1857,12 +1858,25 @@ export interface PathsFilterType extends FilterType {
path_end_key?: string // Paths People End Key
path_dropoff_key?: string // Paths People Dropoff Key
}

export interface RetentionEntity {
id?: string | number // TODO: Fix weird typing issues
kind?: NodeKind.ActionsNode | NodeKind.EventsNode
name?: string
type?: EntityType
// @asType integer
order?: number
uuid?: string
custom_name?: string
}

export interface RetentionFilterType extends FilterType {
retention_type?: RetentionType
retention_reference?: 'total' | 'previous' // retention wrt cohort size or previous period
/** @asType integer */
total_intervals?: number // retention total intervals
returning_entity?: Record<string, any>
target_entity?: Record<string, any>
returning_entity?: RetentionEntity
target_entity?: RetentionEntity
period?: RetentionPeriod
}
export interface LifecycleFilterType extends FilterType {
Expand Down
1 change: 1 addition & 0 deletions posthog/api/services/query.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

QUERY_WITH_RUNNER = [
"LifecycleQuery",
"RetentionQuery",
"TrendsQuery",
"WebOverviewQuery",
"WebTopSourcesQuery",
Expand Down
Loading

0 comments on commit 65e5ba7

Please sign in to comment.