feat(data-warehouse): integrating data warehouse with trends insight

.github/workflows/ci-backend-update-test-timing.yml

@@ -11,6 +11,10 @@ env:
     CLICKHOUSE_SECURE: 'False'
     CLICKHOUSE_VERIFY: 'False'
     TEST: 1
+    OBJECT_STORAGE_ENABLED: 'True'
+    OBJECT_STORAGE_ENDPOINT: 'http://localhost:19000'
+    OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
+    OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'
 
 jobs:
     django:

docker-compose.dev.yml

@@ -48,6 +48,8 @@ services:
             - ./docker/clickhouse/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
             - ./docker/clickhouse/config.xml:/etc/clickhouse-server/config.xml
             - ./docker/clickhouse/users-dev.xml:/etc/clickhouse-server/users.xml
+        extra_hosts:
+            - 'host.docker.internal:host-gateway'
 
     zookeeper:
         extends:

frontend/src/lib/components/PropertyFilters/utils.ts

@@ -99,6 +99,7 @@ export const PROPERTY_FILTER_TYPE_TO_TAXONOMIC_FILTER_GROUP_TYPE: Omit<
     [PropertyFilterType.Session]: TaxonomicFilterGroupType.Sessions,
     [PropertyFilterType.HogQL]: TaxonomicFilterGroupType.HogQLExpression,
     [PropertyFilterType.Group]: TaxonomicFilterGroupType.GroupsPrefix,
+    [PropertyFilterType.DataWarehouse]: TaxonomicFilterGroupType.DataWarehouse,
 }
 
 export function formatPropertyLabel(

frontend/src/lib/components/TaxonomicFilter/types.ts

@@ -79,6 +79,7 @@ export enum TaxonomicFilterGroupType {
     Actions = 'actions',
     Cohorts = 'cohorts',
     CohortsWithAllUsers = 'cohorts_with_all',
+    DataWarehouse = 'data_warehouse',
     Elements = 'elements',
     Events = 'events',
     EventProperties = 'event_properties',

frontend/src/queries/nodes/InsightQuery/utils/queryNodeToFilter.ts

@@ -4,6 +4,7 @@ import { isFunnelsFilter, isLifecycleFilter, isStickinessFilter, isTrendsFilter
 import {
     ActionsNode,
     BreakdownFilter,
+    DataWarehouseNode,
     EventsNode,
     FunnelsFilterLegacy,
     InsightNodeKind,
@@ -17,6 +18,7 @@ import {
 } from '~/queries/schema'
 import {
     isActionsNode,
+    isDataWarehouseNode,
     isEventsNode,
     isFunnelsQuery,
     isLifecycleQuery,
@@ -27,12 +29,24 @@ import {
 } from '~/queries/utils'
 import { ActionFilter, EntityTypes, FilterType, InsightType } from '~/types'
 
-type FilterTypeActionsAndEvents = { events?: ActionFilter[]; actions?: ActionFilter[]; new_entity?: ActionFilter[] }
+type FilterTypeActionsAndEvents = {
+    events?: ActionFilter[]
+    actions?: ActionFilter[]
+    data_warehouse?: ActionFilter[]
+    new_entity?: ActionFilter[]
+}
 
-export const seriesNodeToFilter = (node: EventsNode | ActionsNode, index?: number): ActionFilter => {
+export const seriesNodeToFilter = (
+    node: EventsNode | ActionsNode | DataWarehouseNode,
+    index?: number
+): ActionFilter => {
     const entity: ActionFilter = objectClean({
-        type: isActionsNode(node) ? EntityTypes.ACTIONS : EntityTypes.EVENTS,
-        id: (!isActionsNode(node) ? node.event : node.id) || null,
+        type: isDataWarehouseNode(node)
+            ? EntityTypes.DATA_WAREHOUSE
+            : isActionsNode(node)
+            ? EntityTypes.ACTIONS
+            : EntityTypes.EVENTS,
+        id: isDataWarehouseNode(node) ? node.table_name : (!isActionsNode(node) ? node.event : node.id) || null,
         order: index,
         name: node.name,
         custom_name: node.custom_name,
@@ -47,23 +61,26 @@ export const seriesNodeToFilter = (node: EventsNode | ActionsNode, index?: numbe
 }
 
 export const seriesToActionsAndEvents = (
-    series: (EventsNode | ActionsNode)[]
+    series: (EventsNode | ActionsNode | DataWarehouseNode)[]
 ): Required<FilterTypeActionsAndEvents> => {
     const actions: ActionFilter[] = []
     const events: ActionFilter[] = []
+    const data_warehouse: ActionFilter[] = []
     const new_entity: ActionFilter[] = []
     series.forEach((node, index) => {
         const entity = seriesNodeToFilter(node, index)
         if (isEventsNode(node)) {
             events.push(entity)
         } else if (isActionsNode(node)) {
             actions.push(entity)
+        } else if (isDataWarehouseNode(node)) {
+            data_warehouse.push(entity)
         } else {
             new_entity.push(entity)
         }
     })
 
-    return { actions, events, new_entity }
+    return { actions, events, data_warehouse, new_entity }
 }
 
 export const hiddenLegendItemsToKeys = (

frontend/src/queries/nodes/InsightViz/utils.ts

@@ -1,7 +1,14 @@
 import equal from 'fast-deep-equal'
 import { getEventNamesForAction, isEmptyObject } from 'lib/utils'
 
-import { ActionsNode, BreakdownFilter, EventsNode, InsightQueryNode, TrendsQuery } from '~/queries/schema'
+import {
+    ActionsNode,
+    BreakdownFilter,
+    DataWarehouseNode,
+    EventsNode,
+    InsightQueryNode,
+    TrendsQuery,
+} from '~/queries/schema'
 import {
     isInsightQueryWithBreakdown,
     isInsightQueryWithSeries,
@@ -59,7 +66,7 @@ export const getFormula = (query: InsightQueryNode): string | undefined => {
     }
 }
 
-export const getSeries = (query: InsightQueryNode): (EventsNode | ActionsNode)[] | undefined => {
+export const getSeries = (query: InsightQueryNode): (EventsNode | ActionsNode | DataWarehouseNode)[] | undefined => {
     if (isInsightQueryWithSeries(query)) {
         return query.series
     } else {

frontend/src/queries/schema.json

@@ -234,6 +234,9 @@
                 },
                 {
                     "$ref": "#/definitions/ActionsNode"
+                },
+                {
+                    "$ref": "#/definitions/DataWarehouseNode"
                 }
             ]
         },
@@ -268,6 +271,9 @@
                 },
                 {
                     "$ref": "#/definitions/EmptyPropertyFilter"
+                },
+                {
+                    "$ref": "#/definitions/DataWarehousePropertyFilter"
                 }
             ]
         },
@@ -473,7 +479,7 @@
             ]
         },
         "BreakdownType": {
-            "enum": ["cohort", "person", "event", "group", "session", "hogql"],
+            "enum": ["cohort", "person", "event", "group", "session", "hogql", "data_warehouse"],
             "type": "string"
         },
         "BreakdownValueInt": {
@@ -755,6 +761,105 @@
             "required": ["kind", "source"],
             "type": "object"
         },
+        "DataWarehouseNode": {
+            "additionalProperties": false,
+            "properties": {
+                "custom_name": {
+                    "type": "string"
+                },
+                "fixedProperties": {
+                    "description": "Fixed properties in the query, can't be edited in the interface (e.g. scoping down by person)",
+                    "items": {
+                        "$ref": "#/definitions/AnyPropertyFilter"
+                    },
+                    "type": "array"
+                },
+                "id": {
+                    "type": "string"
+                },
+                "id_field": {
+                    "type": "string"
+                },
+                "kind": {
+                    "const": "DataWarehouseNode",
+                    "type": "string"
+                },
+                "math": {
+                    "anyOf": [
+                        {
+                            "$ref": "#/definitions/BaseMathType"
+                        },
+                        {
+                            "$ref": "#/definitions/PropertyMathType"
+                        },
+                        {
+                            "$ref": "#/definitions/CountPerActorMathType"
+                        },
+                        {
+                            "$ref": "#/definitions/GroupMathType"
+                        },
+                        {
+                            "$ref": "#/definitions/HogQLMathType"
+                        }
+                    ]
+                },
+                "math_group_type_index": {
+                    "enum": [0, 1, 2, 3, 4],
+                    "type": "number"
+                },
+                "math_hogql": {
+                    "type": "string"
+                },
+                "math_property": {
+                    "type": "string"
+                },
+                "name": {
+                    "type": "string"
+                },
+                "properties": {
+                    "description": "Properties configurable in the interface",
+                    "items": {
+                        "$ref": "#/definitions/AnyPropertyFilter"
+                    },
+                    "type": "array"
+                },
+                "response": {
+                    "description": "Cached query response",
+                    "type": "object"
+                },
+                "table_name": {
+                    "type": "string"
+                },
+                "timestamp_field": {
+                    "type": "string"
+                }
+            },
+            "required": ["id", "id_field", "kind", "table_name", "timestamp_field"],
+            "type": "object"
+        },
+        "DataWarehousePropertyFilter": {
+            "additionalProperties": false,
+            "properties": {
+                "key": {
+                    "type": "string"
+                },
+                "label": {
+                    "type": "string"
+                },
+                "operator": {
+                    "$ref": "#/definitions/PropertyOperator"
+                },
+                "type": {
+                    "const": "data_warehouse",
+                    "type": "string"
+                },
+                "value": {
+                    "$ref": "#/definitions/PropertyFilterValue"
+                }
+            },
+            "required": ["key", "operator", "type"],
+            "type": "object"
+        },
         "DatabaseSchemaQuery": {
             "additionalProperties": false,
             "properties": {
@@ -955,7 +1060,7 @@
             "type": "object"
         },
         "EntityType": {
-            "enum": ["actions", "events", "new_entity"],
+            "enum": ["actions", "events", "data_warehouse", "new_entity"],
             "type": "string"
         },
         "EventPropertyFilter": {
@@ -2852,6 +2957,7 @@
             "enum": [
                 "EventsNode",
                 "ActionsNode",
+                "DataWarehouseNode",
                 "EventsQuery",
                 "PersonsNode",
                 "HogQLQuery",
@@ -3193,7 +3299,8 @@
                 "cohort",
                 "recording",
                 "group",
-                "hogql"
+                "hogql",
+                "data_warehouse"
             ],
             "type": "string"
         },
@@ -3997,6 +4104,9 @@
                 {
                     "$ref": "#/definitions/PersonsNode"
                 },
+                {
+                    "$ref": "#/definitions/DataWarehouseNode"
+                },
                 {
                     "$ref": "#/definitions/TimeToSeeDataSessionsQuery"
                 },

frontend/src/queries/schema.ts

@@ -46,6 +46,7 @@ export enum NodeKind {
     // Data nodes
     EventsNode = 'EventsNode',
     ActionsNode = 'ActionsNode',
+    DataWarehouseNode = 'DataWarehouseNode',
     EventsQuery = 'EventsQuery',
     PersonsNode = 'PersonsNode',
     HogQLQuery = 'HogQLQuery',
@@ -110,6 +111,7 @@ export type QuerySchema =
     | EventsNode // never queried directly
     | ActionsNode // old actions API endpoint
     | PersonsNode // old persons API endpoint
+    | DataWarehouseNode
     | TimeToSeeDataSessionsQuery // old API
     | EventsQuery
     | ActorsQuery
@@ -364,12 +366,20 @@ export interface EventsNode extends EntityNode {
     }
 }
 
+export interface DataWarehouseNode extends EntityNode {
+    id: string
+    kind: NodeKind.DataWarehouseNode
+    id_field: string
+    table_name: string
+    timestamp_field: string
+}
+
 export interface ActionsNode extends EntityNode {
     kind: NodeKind.ActionsNode
     id: integer
 }
 
-export type AnyEntityNode = EventsNode | ActionsNode
+export type AnyEntityNode = EventsNode | ActionsNode | DataWarehouseNode
 
 export interface QueryTiming {
     /** Key. Shortened to 'k' to save on data. */

frontend/src/queries/utils.ts

@@ -8,6 +8,7 @@ import {
     DatabaseSchemaQuery,
     DataTableNode,
     DataVisualizationNode,
+    DataWarehouseNode,
     DateRange,
     EventsNode,
     EventsQuery,
@@ -88,6 +89,10 @@ export function isActionsNode(node?: Node | null): node is ActionsNode {
     return node?.kind === NodeKind.ActionsNode
 }
 
+export function isDataWarehouseNode(node?: Node | null): node is DataWarehouseNode {
+    return node?.kind === NodeKind.DataWarehouseNode
+}
+
 export function isPersonsNode(node?: Node | null): node is PersonsNode {
     return node?.kind === NodeKind.PersonsNode
 }

frontend/src/scenes/insights/InsightNav/insightNavLogic.tsx

@@ -12,6 +12,7 @@ import { insightMap } from '~/queries/nodes/InsightQuery/utils/queryNodeToFilter
 import { getDisplay, getShowPercentStackView, getShowValueOnSeries } from '~/queries/nodes/InsightViz/utils'
 import {
     ActionsNode,
+    DataWarehouseNode,
     EventsNode,
     FunnelsFilter,
     FunnelsQuery,
@@ -70,9 +71,9 @@ export interface QueryPropertyCache
 }
 
 const cleanSeriesEntityMath = (
-    entity: EventsNode | ActionsNode,
+    entity: EventsNode | ActionsNode | DataWarehouseNode,
     mathAvailability: MathAvailability
-): EventsNode | ActionsNode => {
+): EventsNode | ActionsNode | DataWarehouseNode => {
     const { math, math_property, math_group_type_index, math_hogql, ...baseEntity } = entity
 
     // TODO: This should be improved to keep a math that differs from the default.
@@ -91,9 +92,9 @@ const cleanSeriesEntityMath = (
 }
 
 const cleanSeriesMath = (
-    series: (EventsNode | ActionsNode)[],
+    series: (EventsNode | ActionsNode | DataWarehouseNode)[],
     mathAvailability: MathAvailability
-): (EventsNode | ActionsNode)[] => {
+): (EventsNode | ActionsNode | DataWarehouseNode)[] => {
     return series.map((entity) => cleanSeriesEntityMath(entity, mathAvailability))
 }