From ed01d5b9f5c98869030c9ab70105aaf0516a71dc Mon Sep 17 00:00:00 2001 From: Robbie Coomber Date: Fri, 29 Sep 2023 14:08:23 +0100 Subject: [PATCH] feat(web-analytics): Change query runner query type --- .../Cards/InsightCard/InsightCard.tsx | 2 +- frontend/src/queries/Query/Query.tsx | 5 +- .../src/queries/QueryEditor/QueryEditor.tsx | 3 +- .../src/queries/nodes/DataTable/DataTable.tsx | 11 +- .../queries/nodes/DataTable/dataTableLogic.ts | 2 +- .../queries/nodes/DataTable/renderColumn.tsx | 3 +- .../nodes/DataTable/renderColumnMeta.tsx | 3 +- .../nodes/InsightViz/InsightContainer.tsx | 4 +- .../queries/nodes/InsightViz/InsightViz.tsx | 3 +- .../nodes/SavedInsight/SavedInsight.tsx | 3 +- frontend/src/queries/schema.json | 502 +++++++++++++++++- frontend/src/queries/schema.ts | 45 +- frontend/src/queries/types.ts | 20 + frontend/src/types.ts | 10 +- package.json | 2 +- posthog/api/insight.py | 2 +- posthog/api/query.py | 2 +- posthog/hogql_queries/query_runner.py | 15 +- posthog/schema.py | 311 ++++++++++- 19 files changed, 873 insertions(+), 75 deletions(-) create mode 100644 frontend/src/queries/types.ts diff --git a/frontend/src/lib/components/Cards/InsightCard/InsightCard.tsx b/frontend/src/lib/components/Cards/InsightCard/InsightCard.tsx index b7c0dd1aecae49..f71e8f73f6e5c5 100644 --- a/frontend/src/lib/components/Cards/InsightCard/InsightCard.tsx +++ b/frontend/src/lib/components/Cards/InsightCard/InsightCard.tsx @@ -45,13 +45,13 @@ import { import { Resizeable } from 'lib/components/Cards/CardMeta' import { Query } from '~/queries/Query/Query' import { QueriesUnsupportedHere } from 'lib/components/Cards/InsightCard/QueriesUnsupportedHere' -import { QueryContext } from '~/queries/schema' import { InsightMeta } from './InsightMeta' import { dataNodeLogic, DataNodeLogicProps } from '~/queries/nodes/DataNode/dataNodeLogic' import { filtersToQueryNode } from '~/queries/nodes/InsightQuery/utils/filtersToQueryNode' import { insightVizDataNodeKey } from '~/queries/nodes/InsightViz/InsightViz' import { getCachedResults } from '~/queries/nodes/InsightViz/utils' import { funnelDataLogic } from 'scenes/funnels/funnelDataLogic' +import { QueryContext } from '~/queries/types' type DisplayedType = ChartDisplayType | 'RetentionContainer' | 'FunnelContainer' | 'PathsContainer' diff --git a/frontend/src/queries/Query/Query.tsx b/frontend/src/queries/Query/Query.tsx index c77e88e5cdb61a..4c3cf5c3c2338f 100644 --- a/frontend/src/queries/Query/Query.tsx +++ b/frontend/src/queries/Query/Query.tsx @@ -1,20 +1,21 @@ import { isDataNode, isDataTableNode, - isSavedInsightNode, isInsightVizNode, + isSavedInsightNode, isTimeToSeeDataSessionsNode, } from '../utils' import { DataTable } from '~/queries/nodes/DataTable/DataTable' import { DataNode } from '~/queries/nodes/DataNode/DataNode' import { InsightViz } from '~/queries/nodes/InsightViz/InsightViz' -import { AnyResponseType, Node, QueryContext, QuerySchema } from '~/queries/schema' +import { AnyResponseType, Node, QuerySchema } from '~/queries/schema' import { ErrorBoundary } from '~/layout/ErrorBoundary' import { useEffect, useState } from 'react' import { TimeToSeeData } from '../nodes/TimeToSeeData/TimeToSeeData' import { QueryEditor } from '~/queries/QueryEditor/QueryEditor' import { LemonDivider } from 'lib/lemon-ui/LemonDivider' import { SavedInsight } from '../nodes/SavedInsight/SavedInsight' +import { QueryContext } from '~/queries/types' export interface QueryProps { /** An optional key to identify the query */ diff --git a/frontend/src/queries/QueryEditor/QueryEditor.tsx b/frontend/src/queries/QueryEditor/QueryEditor.tsx index 93413869db173e..d959e779b3fd5a 100644 --- a/frontend/src/queries/QueryEditor/QueryEditor.tsx +++ b/frontend/src/queries/QueryEditor/QueryEditor.tsx @@ -6,8 +6,9 @@ import { LemonButton } from 'lib/lemon-ui/LemonButton' import { queryEditorLogic } from '~/queries/QueryEditor/queryEditorLogic' import { AutoSizer } from 'react-virtualized/dist/es/AutoSizer' import clsx from 'clsx' -import { QueryContext } from '~/queries/schema' import { CodeEditor } from 'lib/components/CodeEditors' +import { QueryContext } from '~/queries/types' + export interface QueryEditorProps { query: string setQuery?: (query: string) => void diff --git a/frontend/src/queries/nodes/DataTable/DataTable.tsx b/frontend/src/queries/nodes/DataTable/DataTable.tsx index 435922fa671ba5..15b3b0a8202482 100644 --- a/frontend/src/queries/nodes/DataTable/DataTable.tsx +++ b/frontend/src/queries/nodes/DataTable/DataTable.tsx @@ -1,13 +1,5 @@ import './DataTable.scss' -import { - AnyResponseType, - DataTableNode, - EventsNode, - EventsQuery, - HogQLQuery, - PersonsNode, - QueryContext, -} from '~/queries/schema' +import { AnyResponseType, DataTableNode, EventsNode, EventsQuery, HogQLQuery, PersonsNode } from '~/queries/schema' import { useCallback, useState } from 'react' import { BindLogic, useValues } from 'kea' import { dataNodeLogic, DataNodeLogicProps } from '~/queries/nodes/DataNode/dataNodeLogic' @@ -43,6 +35,7 @@ import { EventType } from '~/types' import { SavedQueries } from '~/queries/nodes/DataTable/SavedQueries' import { HogQLQueryEditor } from '~/queries/nodes/HogQLQuery/HogQLQueryEditor' import { QueryFeature } from '~/queries/nodes/DataTable/queryFeatures' +import { QueryContext } from '~/queries/types' interface DataTableProps { uniqueKey?: string | number diff --git a/frontend/src/queries/nodes/DataTable/dataTableLogic.ts b/frontend/src/queries/nodes/DataTable/dataTableLogic.ts index f3ae0423c563e4..ee81ac338c01a5 100644 --- a/frontend/src/queries/nodes/DataTable/dataTableLogic.ts +++ b/frontend/src/queries/nodes/DataTable/dataTableLogic.ts @@ -6,7 +6,6 @@ import { EventsQuery, HogQLExpression, NodeKind, - QueryContext, TimeToSeeDataSessionsQuery, } from '~/queries/schema' import { getColumnsForQuery, removeExpressionComment } from './utils' @@ -18,6 +17,7 @@ import { dataNodeLogic } from '~/queries/nodes/DataNode/dataNodeLogic' import { dayjs } from 'lib/dayjs' import equal from 'fast-deep-equal' import { getQueryFeatures, QueryFeature } from '~/queries/nodes/DataTable/queryFeatures' +import { QueryContext } from '~/queries/types' export interface DataTableLogicProps { vizKey: string diff --git a/frontend/src/queries/nodes/DataTable/renderColumn.tsx b/frontend/src/queries/nodes/DataTable/renderColumn.tsx index 39f6dd6a6194bf..f5680fb4d92496 100644 --- a/frontend/src/queries/nodes/DataTable/renderColumn.tsx +++ b/frontend/src/queries/nodes/DataTable/renderColumn.tsx @@ -6,7 +6,7 @@ import { TZLabel } from 'lib/components/TZLabel' import { Property } from 'lib/components/Property' import { urls } from 'scenes/urls' import { PersonDisplay } from 'scenes/persons/PersonDisplay' -import { DataTableNode, EventsQueryPersonColumn, HasPropertiesNode, QueryContext } from '~/queries/schema' +import { DataTableNode, EventsQueryPersonColumn, HasPropertiesNode } from '~/queries/schema' import { isEventsQuery, isHogQLQuery, isPersonsNode, isTimeToSeeDataSessionsQuery, trimQuotes } from '~/queries/utils' import { combineUrl, router } from 'kea-router' import { CopyToClipboardInline } from 'lib/components/CopyToClipboard' @@ -17,6 +17,7 @@ import { Spinner } from 'lib/lemon-ui/Spinner/Spinner' import { LemonTag } from 'lib/lemon-ui/LemonTag/LemonTag' import { TableCellSparkline } from 'lib/lemon-ui/LemonTable/TableCellSparkline' import { Tooltip } from 'lib/lemon-ui/Tooltip' +import { QueryContext } from '~/queries/types' export function renderColumn( key: string, diff --git a/frontend/src/queries/nodes/DataTable/renderColumnMeta.tsx b/frontend/src/queries/nodes/DataTable/renderColumnMeta.tsx index 7f8e546277a7c5..917e70b654a2c3 100644 --- a/frontend/src/queries/nodes/DataTable/renderColumnMeta.tsx +++ b/frontend/src/queries/nodes/DataTable/renderColumnMeta.tsx @@ -1,9 +1,10 @@ import { PropertyFilterType } from '~/types' import { PropertyKeyInfo } from 'lib/components/PropertyKeyInfo' -import { QueryContext, DataTableNode } from '~/queries/schema' +import { DataTableNode } from '~/queries/schema' import { isEventsQuery, isHogQLQuery, trimQuotes } from '~/queries/utils' import { extractExpressionComment } from '~/queries/nodes/DataTable/utils' import { SortingIndicator } from 'lib/lemon-ui/LemonTable/sorting' +import { QueryContext } from '~/queries/types' export interface ColumnMeta { title?: JSX.Element | string diff --git a/frontend/src/queries/nodes/InsightViz/InsightContainer.tsx b/frontend/src/queries/nodes/InsightViz/InsightContainer.tsx index b9ed3ab50983bd..87b3a37f389866 100644 --- a/frontend/src/queries/nodes/InsightViz/InsightContainer.tsx +++ b/frontend/src/queries/nodes/InsightViz/InsightContainer.tsx @@ -6,8 +6,7 @@ import { funnelDataLogic } from 'scenes/funnels/funnelDataLogic' import { insightDataLogic } from 'scenes/insights/insightDataLogic' import { insightNavLogic } from 'scenes/insights/InsightNav/insightNavLogic' -import { QueryContext } from '~/queries/schema' -import { ChartDisplayType, FunnelVizType, ExporterFormat, InsightType, ItemMode } from '~/types' +import { ChartDisplayType, ExporterFormat, FunnelVizType, InsightType, ItemMode } from '~/types' import { Tooltip } from 'lib/lemon-ui/Tooltip' import { Animation } from 'lib/components/Animation/Animation' import { AnimationType } from 'lib/animations/animations' @@ -34,6 +33,7 @@ import { FunnelStepsTable } from 'scenes/insights/views/Funnels/FunnelStepsTable import { insightVizDataLogic } from 'scenes/insights/insightVizDataLogic' import { FunnelCorrelation } from 'scenes/insights/views/Funnels/FunnelCorrelation' import { InsightResultMetadata } from './InsightResultMetadata' +import { QueryContext } from '~/queries/types' const VIEW_MAP = { [`${InsightType.TRENDS}`]: , diff --git a/frontend/src/queries/nodes/InsightViz/InsightViz.tsx b/frontend/src/queries/nodes/InsightViz/InsightViz.tsx index 85f4d0f0e76304..083d09532bd715 100644 --- a/frontend/src/queries/nodes/InsightViz/InsightViz.tsx +++ b/frontend/src/queries/nodes/InsightViz/InsightViz.tsx @@ -6,7 +6,7 @@ import { insightSceneLogic } from 'scenes/insights/insightSceneLogic' import { isFunnelsQuery } from '~/queries/utils' import { dataNodeLogic, DataNodeLogicProps } from '../DataNode/dataNodeLogic' -import { InsightVizNode, QueryContext } from '../../schema' +import { InsightVizNode } from '../../schema' import { InsightContainer } from './InsightContainer' import { EditorFilters } from './EditorFilters' @@ -17,6 +17,7 @@ import { useState } from 'react' import './Insight.scss' import { insightVizDataLogic } from 'scenes/insights/insightVizDataLogic' +import { QueryContext } from '~/queries/types' /** The key for the dataNodeLogic mounted by an InsightViz for insight of insightProps */ export const insightVizDataNodeKey = (insightProps: InsightLogicProps): string => { diff --git a/frontend/src/queries/nodes/SavedInsight/SavedInsight.tsx b/frontend/src/queries/nodes/SavedInsight/SavedInsight.tsx index 1b14be082c320c..0fc0d2736ae4b9 100644 --- a/frontend/src/queries/nodes/SavedInsight/SavedInsight.tsx +++ b/frontend/src/queries/nodes/SavedInsight/SavedInsight.tsx @@ -2,11 +2,12 @@ import { useValues } from 'kea' import { insightLogic } from 'scenes/insights/insightLogic' import { Query } from '~/queries/Query/Query' -import { SavedInsightNode, QueryContext } from '~/queries/schema' +import { SavedInsightNode } from '~/queries/schema' import { InsightLogicProps } from '~/types' import { Animation } from 'lib/components/Animation/Animation' import { AnimationType } from 'lib/animations/animations' import { insightDataLogic } from 'scenes/insights/insightDataLogic' +import { QueryContext } from '~/queries/types' interface InsightProps { query: SavedInsightNode diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index 7edaeb0039cf1e..9bf741a29a3740 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -1,5 +1,4 @@ { - "$ref": "#/definitions/QuerySchema", "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { "ActionsNode": { @@ -132,6 +131,46 @@ } ] }, + "AnyResponseType": { + "anyOf": [ + { + "type": "object" + }, + { + "$ref": "#/definitions/HogQLQueryResponse" + }, + { + "$ref": "#/definitions/HogQLMetadataResponse" + }, + { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "next": { + "type": "string" + }, + "results": { + "items": { + "$ref": "#/definitions/EventType" + }, + "type": "array" + } + }, + "required": ["results"], + "type": "object" + }, + { + "not": {} + } + ], + "description": "Return a limited set of data" + }, + { + "$ref": "#/definitions/EventsQueryResponse" + } + ] + }, "BaseMathType": { "enum": ["total", "dau", "weekly_active", "monthly_active", "unique_session"], "type": "string" @@ -272,6 +311,20 @@ ], "type": "string" }, + "DataNode": { + "additionalProperties": false, + "properties": { + "kind": { + "$ref": "#/definitions/NodeKind" + }, + "response": { + "description": "Cached query response", + "type": "object" + } + }, + "required": ["kind"], + "type": "object" + }, "DataTableNode": { "additionalProperties": false, "properties": { @@ -530,6 +583,69 @@ "additionalProperties": false, "type": "object" }, + "EntityNode": { + "additionalProperties": false, + "properties": { + "custom_name": { + "type": "string" + }, + "fixedProperties": { + "description": "Fixed properties in the query, can't be edited in the interface (e.g. scoping down by person)", + "items": { + "$ref": "#/definitions/AnyPropertyFilter" + }, + "type": "array" + }, + "kind": { + "$ref": "#/definitions/NodeKind" + }, + "math": { + "anyOf": [ + { + "$ref": "#/definitions/BaseMathType" + }, + { + "$ref": "#/definitions/PropertyMathType" + }, + { + "$ref": "#/definitions/CountPerActorMathType" + }, + { + "$ref": "#/definitions/GroupMathType" + }, + { + "$ref": "#/definitions/HogQLMathType" + } + ] + }, + "math_group_type_index": { + "enum": [0, 1, 2, 3, 4], + "type": "number" + }, + "math_hogql": { + "type": "string" + }, + "math_property": { + "type": "string" + }, + "name": { + "type": "string" + }, + "properties": { + "description": "Properties configurable in the interface", + "items": { + "$ref": "#/definitions/AnyPropertyFilter" + }, + "type": "array" + }, + "response": { + "description": "Cached query response", + "type": "object" + } + }, + "required": ["kind"], + "type": "object" + }, "EntityType": { "enum": ["actions", "events", "new_entity"], "type": "string" @@ -781,6 +897,34 @@ "required": ["kind", "select"], "type": "object" }, + "EventsQueryPersonColumn": { + "additionalProperties": false, + "properties": { + "created_at": { + "type": "string" + }, + "distinct_id": { + "type": "string" + }, + "properties": { + "additionalProperties": false, + "properties": { + "email": { + "type": "string" + }, + "name": { + "type": "string" + } + }, + "type": "object" + }, + "uuid": { + "type": "string" + } + }, + "required": ["uuid", "created_at", "properties", "distinct_id"], + "type": "object" + }, "EventsQueryResponse": { "additionalProperties": false, "properties": { @@ -842,6 +986,105 @@ "enum": ["AND", "OR"], "type": "string" }, + "FilterType": { + "additionalProperties": false, + "properties": { + "actions": { + "items": { + "type": "object" + }, + "type": "array" + }, + "aggregation_group_type_index": { + "type": "number" + }, + "breakdown": { + "$ref": "#/definitions/BreakdownKeyType" + }, + "breakdown_group_type_index": { + "type": ["number", "null"] + }, + "breakdown_normalize_url": { + "type": "boolean" + }, + "breakdown_type": { + "anyOf": [ + { + "$ref": "#/definitions/BreakdownType" + }, + { + "type": "null" + } + ] + }, + "breakdowns": { + "items": { + "$ref": "#/definitions/Breakdown" + }, + "type": "array" + }, + "date_from": { + "type": ["string", "null"] + }, + "date_to": { + "type": ["string", "null"] + }, + "entity_id": { + "type": ["string", "number"] + }, + "entity_math": { + "type": "string" + }, + "entity_type": { + "$ref": "#/definitions/EntityType" + }, + "events": { + "items": { + "type": "object" + }, + "type": "array" + }, + "explicit_date": { + "description": "Whether the `date_from` and `date_to` should be used verbatim. Disables rounding to the start and end of period. Strings are cast to bools, e.g. \"true\" -> true.", + "type": ["boolean", "string", "null"] + }, + "filter_test_accounts": { + "type": "boolean" + }, + "from_dashboard": { + "type": ["boolean", "number"] + }, + "insight": { + "$ref": "#/definitions/InsightType" + }, + "interval": { + "$ref": "#/definitions/IntervalType" + }, + "new_entity": { + "items": { + "type": "object" + }, + "type": "array" + }, + "properties": { + "anyOf": [ + { + "items": { + "$ref": "#/definitions/AnyPropertyFilter" + }, + "type": "array" + }, + { + "$ref": "#/definitions/PropertyGroupFilter" + } + ] + }, + "sampling_factor": { + "type": ["number", "null"] + } + }, + "type": "object" + }, "FunnelConversionWindowTimeUnit": { "enum": ["second", "minute", "hour", "day", "week", "month"], "type": "string" @@ -1044,6 +1287,19 @@ "required": ["key", "operator", "type"], "type": "object" }, + "HasPropertiesNode": { + "anyOf": [ + { + "$ref": "#/definitions/EventsNode" + }, + { + "$ref": "#/definitions/EventsQuery" + }, + { + "$ref": "#/definitions/PersonsNode" + } + ] + }, "HogQLExpression": { "type": "string" }, @@ -1221,6 +1477,50 @@ }, "type": "object" }, + "InsightFilter": { + "anyOf": [ + { + "$ref": "#/definitions/TrendsFilter" + }, + { + "$ref": "#/definitions/FunnelsFilter" + }, + { + "$ref": "#/definitions/RetentionFilter" + }, + { + "$ref": "#/definitions/PathsFilter" + }, + { + "$ref": "#/definitions/StickinessFilter" + }, + { + "$ref": "#/definitions/LifecycleFilter" + } + ] + }, + "InsightFilterProperty": { + "enum": [ + "trendsFilter", + "funnelsFilter", + "retentionFilter", + "pathsFilter", + "stickinessFilter", + "lifecycleFilter" + ], + "type": "string" + }, + "InsightNodeKind": { + "enum": [ + "TrendsQuery", + "FunnelsQuery", + "RetentionQuery", + "PathsQuery", + "StickinessQuery", + "LifecycleQuery" + ], + "type": "string" + }, "InsightQueryNode": { "anyOf": [ { @@ -1246,6 +1546,10 @@ "InsightShortId": { "type": "string" }, + "InsightType": { + "enum": ["TRENDS", "STICKINESS", "LIFECYCLE", "FUNNELS", "RETENTION", "PATHS", "JSON", "SQL"], + "type": "string" + }, "InsightVizNode": { "additionalProperties": false, "properties": { @@ -1289,6 +1593,47 @@ "required": ["kind", "source"], "type": "object" }, + "InsightsQueryBase": { + "additionalProperties": false, + "description": "Base class for insight query nodes. Should not be used directly.", + "properties": { + "aggregation_group_type_index": { + "description": "Groups aggregation", + "type": "number" + }, + "dateRange": { + "$ref": "#/definitions/DateRange", + "description": "Date range for the query" + }, + "filterTestAccounts": { + "description": "Exclude internal and test users by applying the respective filters", + "type": "boolean" + }, + "kind": { + "$ref": "#/definitions/NodeKind" + }, + "properties": { + "anyOf": [ + { + "items": { + "$ref": "#/definitions/AnyPropertyFilter" + }, + "type": "array" + }, + { + "$ref": "#/definitions/PropertyGroupFilter" + } + ], + "description": "Property filters for all series" + }, + "samplingFactor": { + "description": "Sampling rate", + "type": ["number", "null"] + } + }, + "required": ["kind"], + "type": "object" + }, "IntervalType": { "enum": ["hour", "day", "week", "month"], "type": "string" @@ -1410,6 +1755,52 @@ "enum": ["new", "resurrecting", "returning", "dormant"], "type": "string" }, + "NamedParameters": { + "additionalProperties": false, + "properties": { + "source": { + "$ref": "#/definitions/FilterType" + } + }, + "type": "object" + }, + "Node": { + "additionalProperties": false, + "description": "Node base class, everything else inherits from here", + "properties": { + "kind": { + "$ref": "#/definitions/NodeKind" + } + }, + "required": ["kind"], + "type": "object" + }, + "NodeKind": { + "description": "PostHog Query Schema definition.\n\nThis file acts as the source of truth for:\n\n- frontend/src/queries/schema.json - generated from typescript via \"pnpm run generate:schema:json\"\n\n- posthog/schema.py - generated from json the above json via \"pnpm run generate:schema:python\"", + "enum": [ + "EventsNode", + "ActionsNode", + "EventsQuery", + "PersonsNode", + "HogQLQuery", + "HogQLMetadata", + "DataTableNode", + "SavedInsightNode", + "InsightVizNode", + "TrendsQuery", + "FunnelsQuery", + "RetentionQuery", + "PathsQuery", + "StickinessQuery", + "LifecycleQuery", + "TimeToSeeDataSessionsQuery", + "TimeToSeeDataQuery", + "TimeToSeeDataSessionsJSONNode", + "TimeToSeeDataSessionsWaterfallNode", + "DatabaseSchemaQuery" + ], + "type": "string" + }, "PathCleaningFilter": { "additionalProperties": false, "properties": { @@ -1604,6 +1995,21 @@ "required": ["kind"], "type": "object" }, + "PropertyFilterType": { + "enum": [ + "meta", + "event", + "person", + "element", + "feature", + "session", + "cohort", + "recording", + "group", + "hogql" + ], + "type": "string" + }, "PropertyFilterValue": { "anyOf": [ { @@ -1691,6 +2097,29 @@ ], "type": "string" }, + "QueryResponse": { + "additionalProperties": false, + "properties": { + "is_cached": { + "type": "boolean" + }, + "last_refresh": { + "type": "string" + }, + "next_allowed_client_refresh": { + "type": "string" + }, + "result": {}, + "timings": { + "items": { + "$ref": "#/definitions/QueryTiming" + }, + "type": "array" + } + }, + "required": ["result"], + "type": "object" + }, "QuerySchema": { "anyOf": [ { @@ -1729,7 +2158,8 @@ { "$ref": "#/definitions/DatabaseSchemaQuery" } - ] + ], + "description": "Query Schema" }, "QueryTiming": { "additionalProperties": false, @@ -2087,6 +2517,60 @@ "required": ["kind", "series"], "type": "object" }, + "TimeToSeeDataJSONNode": { + "additionalProperties": false, + "properties": { + "kind": { + "const": "TimeToSeeDataSessionsJSONNode", + "type": "string" + }, + "source": { + "$ref": "#/definitions/TimeToSeeDataQuery" + } + }, + "required": ["kind", "source"], + "type": "object" + }, + "TimeToSeeDataNode": { + "anyOf": [ + { + "$ref": "#/definitions/TimeToSeeDataJSONNode" + }, + { + "$ref": "#/definitions/TimeToSeeDataWaterfallNode" + } + ] + }, + "TimeToSeeDataQuery": { + "additionalProperties": false, + "properties": { + "kind": { + "const": "TimeToSeeDataQuery", + "type": "string" + }, + "response": { + "description": "Cached query response", + "type": "object" + }, + "sessionEnd": { + "type": "string" + }, + "sessionId": { + "description": "Project to filter on. Defaults to current session", + "type": "string" + }, + "sessionStart": { + "description": "Session start time. Defaults to current time - 2 hours", + "type": "string" + }, + "teamId": { + "description": "Project to filter on. Defaults to current project", + "type": "number" + } + }, + "required": ["kind"], + "type": "object" + }, "TimeToSeeDataSessionsQuery": { "additionalProperties": false, "properties": { @@ -2123,6 +2607,20 @@ "required": ["results"], "type": "object" }, + "TimeToSeeDataWaterfallNode": { + "additionalProperties": false, + "properties": { + "kind": { + "const": "TimeToSeeDataSessionsWaterfallNode", + "type": "string" + }, + "source": { + "$ref": "#/definitions/TimeToSeeDataQuery" + } + }, + "required": ["kind", "source"], + "type": "object" + }, "TrendsFilter": { "additionalProperties": false, "description": "`TrendsFilterType` minus everything inherited from `FilterType` and `hidden_legend_keys` replaced by `hidden_legend_indexes`", diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index 54c296a3521a0b..c8d128a51854da 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -1,26 +1,25 @@ import { AnyPropertyFilter, + BaseMathType, Breakdown, BreakdownKeyType, BreakdownType, - PropertyGroupFilter, - EventType, - IntervalType, - BaseMathType, - PropertyMathType, CountPerActorMathType, - GroupMathType, + EventType, FilterType, - TrendsFilterType, FunnelsFilterType, - RetentionFilterType, - PathsFilterType, - StickinessFilterType, - LifecycleFilterType, - LifecycleToggle, + GroupMathType, HogQLMathType, - InsightLogicProps, InsightShortId, + IntervalType, + LifecycleFilterType, + LifecycleToggle, + PathsFilterType, + PropertyGroupFilter, + PropertyMathType, + RetentionFilterType, + StickinessFilterType, + TrendsFilterType, } from '~/types' /** @@ -76,6 +75,7 @@ export type AnyDataNode = | HogQLMetadata | TimeToSeeDataSessionsQuery +/** Query Schema */ export type QuerySchema = // Data nodes (see utils.ts) | AnyDataNode @@ -587,22 +587,3 @@ export interface BreakdownFilter { breakdown_group_type_index?: number | null breakdown_histogram_bin_count?: number // trends breakdown histogram bin count } - -/** Pass custom metadata to queries. Used for e.g. custom columns in the DataTable. */ -export interface QueryContext { - /** Column templates for the DataTable */ - columns?: Record - /** used to override the value in the query */ - showOpenEditorButton?: boolean - showQueryEditor?: boolean - /* Adds help and examples to the query editor component */ - showQueryHelp?: boolean - insightProps?: InsightLogicProps - emptyStateHeading?: string - emptyStateDetail?: string -} - -interface QueryContextColumn { - title?: string - render?: (props: { record: any }) => JSX.Element -} diff --git a/frontend/src/queries/types.ts b/frontend/src/queries/types.ts new file mode 100644 index 00000000000000..b126874dc39a18 --- /dev/null +++ b/frontend/src/queries/types.ts @@ -0,0 +1,20 @@ +import { InsightLogicProps } from '~/types' + +/** Pass custom metadata to queries. Used for e.g. custom columns in the DataTable. */ +export interface QueryContext { + /** Column templates for the DataTable */ + columns?: Record + /** used to override the value in the query */ + showOpenEditorButton?: boolean + showQueryEditor?: boolean + /* Adds help and examples to the query editor component */ + showQueryHelp?: boolean + insightProps?: InsightLogicProps + emptyStateHeading?: string + emptyStateDetail?: string +} + +interface QueryContextColumn { + title?: string + render?: (props: { record: any }) => JSX.Element +} diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 8a038927dc90e3..f2e8988d9d3db1 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -26,16 +26,10 @@ import { BehavioralFilterKey, BehavioralFilterType } from 'scenes/cohorts/Cohort import { LogicWrapper } from 'kea' import { AggregationAxisFormat } from 'scenes/insights/aggregationAxisFormat' import { Layout } from 'react-grid-layout' -import { - DatabaseSchemaQueryResponseField, - HogQLQuery, - InsightQueryNode, - InsightVizNode, - Node, - QueryContext, -} from './queries/schema' +import { DatabaseSchemaQueryResponseField, HogQLQuery, InsightQueryNode, InsightVizNode, Node } from './queries/schema' import { JSONContent } from 'scenes/notebooks/Notebook/utils' import { DashboardCompatibleScenes } from 'lib/components/SceneDashboardChoice/sceneDashboardChoiceModalLogic' +import { QueryContext } from '~/queries/types' export type Optional = Omit & { [K in keyof T]?: T[K] } diff --git a/package.json b/package.json index c1a3772e89de9d..cdf03e5234d898 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "build": "pnpm copy-scripts && pnpm build:esbuild", "build:esbuild": "node frontend/build.mjs", "schema:build": "pnpm run schema:build:json && pnpm run schema:build:python", - "schema:build:json": "ts-json-schema-generator -f tsconfig.json --path 'frontend/src/*.ts' --type 'QuerySchema' --no-type-check > frontend/src/queries/schema.json && prettier --write frontend/src/queries/schema.json", + "schema:build:json": "ts-json-schema-generator -f tsconfig.json --path 'frontend/src/queries/schema.ts' --no-type-check > frontend/src/queries/schema.json && prettier --write frontend/src/queries/schema.json", "schema:build:python": "datamodel-codegen --collapse-root-models --disable-timestamp --use-one-literal-as-default --use-default-kwarg --use-subclass-enum --input frontend/src/queries/schema.json --input-file-type jsonschema --output posthog/schema.py --output-model-type pydantic_v2.BaseModel && black posthog/schema.py", "grammar:build": "cd posthog/hogql/grammar && antlr -Dlanguage=Python3 HogQLLexer.g4 && antlr -visitor -no-listener -Dlanguage=Python3 HogQLParser.g4", "packages:build": "pnpm packages:build:apps-common && pnpm packages:build:lemon-ui", diff --git a/posthog/api/insight.py b/posthog/api/insight.py index 8fc0dfa2827358..7eb4010cdbc0f4 100644 --- a/posthog/api/insight.py +++ b/posthog/api/insight.py @@ -138,7 +138,7 @@ def parse(self, stream, media_type=None, parser_context=None): try: query = data.get("query", None) if query: - schema.Model.model_validate(query) + schema.QuerySchema.model_validate(query) except Exception as error: raise ParseError(detail=str(error)) else: diff --git a/posthog/api/query.py b/posthog/api/query.py index 628a55da744eea..9264e2e75ec854 100644 --- a/posthog/api/query.py +++ b/posthog/api/query.py @@ -52,7 +52,7 @@ class QuerySchemaParser(JSONParser): @staticmethod def validate_query(data) -> Dict: try: - schema.Model.model_validate(data) + schema.QuerySchema.model_validate(data) # currently we have to return data not the parsed Model # because pydantic doesn't know to discriminate on 'kind' # if we can get this correctly typed we can return the parsed model diff --git a/posthog/hogql_queries/query_runner.py b/posthog/hogql_queries/query_runner.py index 5dbd4850e599d7..331e8bad27a6b9 100644 --- a/posthog/hogql_queries/query_runner.py +++ b/posthog/hogql_queries/query_runner.py @@ -2,9 +2,9 @@ from datetime import datetime from typing import Any, Generic, List, Optional, Type, Dict, TypeVar -from prometheus_client import Counter -from django.core.cache import cache from django.conf import settings +from django.core.cache import cache +from prometheus_client import Counter from pydantic import BaseModel, ConfigDict from posthog.clickhouse.query_tagging import tag_queries @@ -14,8 +14,7 @@ from posthog.hogql.timings import HogQLTimings from posthog.metrics import LABEL_TEAM_ID from posthog.models import Team -from posthog.schema import QueryTiming -from posthog.types import InsightQueryNode +from posthog.schema import QueryTiming, QuerySchema from posthog.utils import generate_cache_key, get_safe_cache QUERY_CACHE_WRITE_COUNTER = Counter( @@ -51,16 +50,16 @@ class CachedQueryResponse(QueryResponse): class QueryRunner(ABC): - query: InsightQueryNode - query_type: Type[InsightQueryNode] + query: QuerySchema + query_type: Type[QuerySchema] team: Team timings: HogQLTimings - def __init__(self, query: InsightQueryNode | Dict[str, Any], team: Team, timings: Optional[HogQLTimings] = None): + def __init__(self, query: QuerySchema | Dict[str, Any], team: Team, timings: Optional[HogQLTimings] = None): self.team = team self.timings = timings or HogQLTimings() if isinstance(query, self.query_type): - self.query = query # type: ignore + self.query = query else: self.query = self.query_type.model_validate(query) diff --git a/posthog/schema.py b/posthog/schema.py index 207fb07d6e62c1..388a040c76a8ea 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -10,6 +10,10 @@ from typing_extensions import Literal +class Model(RootModel): + root: Any + + class MathGroupTypeIndex(float, Enum): number_0 = 0 number_1 = 1 @@ -168,6 +172,24 @@ class Response(BaseModel): results: List[EventType] +class Properties(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + email: Optional[str] = None + name: Optional[str] = None + + +class EventsQueryPersonColumn(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + created_at: str + distinct_id: str + properties: Properties + uuid: str + + class FilterLogicalOperator(str, Enum): AND = "AND" OR = "OR" @@ -228,6 +250,35 @@ class HogQLNotice(BaseModel): start: Optional[float] = None +class InsightFilterProperty(str, Enum): + trendsFilter = "trendsFilter" + funnelsFilter = "funnelsFilter" + retentionFilter = "retentionFilter" + pathsFilter = "pathsFilter" + stickinessFilter = "stickinessFilter" + lifecycleFilter = "lifecycleFilter" + + +class InsightNodeKind(str, Enum): + TrendsQuery = "TrendsQuery" + FunnelsQuery = "FunnelsQuery" + RetentionQuery = "RetentionQuery" + PathsQuery = "PathsQuery" + StickinessQuery = "StickinessQuery" + LifecycleQuery = "LifecycleQuery" + + +class InsightType(str, Enum): + TRENDS = "TRENDS" + STICKINESS = "STICKINESS" + LIFECYCLE = "LIFECYCLE" + FUNNELS = "FUNNELS" + RETENTION = "RETENTION" + PATHS = "PATHS" + JSON = "JSON" + SQL = "SQL" + + class IntervalType(str, Enum): hour = "hour" day = "day" @@ -242,6 +293,29 @@ class LifecycleToggle(str, Enum): dormant = "dormant" +class NodeKind(str, Enum): + EventsNode = "EventsNode" + ActionsNode = "ActionsNode" + EventsQuery = "EventsQuery" + PersonsNode = "PersonsNode" + HogQLQuery = "HogQLQuery" + HogQLMetadata = "HogQLMetadata" + DataTableNode = "DataTableNode" + SavedInsightNode = "SavedInsightNode" + InsightVizNode = "InsightVizNode" + TrendsQuery = "TrendsQuery" + FunnelsQuery = "FunnelsQuery" + RetentionQuery = "RetentionQuery" + PathsQuery = "PathsQuery" + StickinessQuery = "StickinessQuery" + LifecycleQuery = "LifecycleQuery" + TimeToSeeDataSessionsQuery = "TimeToSeeDataSessionsQuery" + TimeToSeeDataQuery = "TimeToSeeDataQuery" + TimeToSeeDataSessionsJSONNode = "TimeToSeeDataSessionsJSONNode" + TimeToSeeDataSessionsWaterfallNode = "TimeToSeeDataSessionsWaterfallNode" + DatabaseSchemaQuery = "DatabaseSchemaQuery" + + class PathCleaningFilter(BaseModel): model_config = ConfigDict( extra="forbid", @@ -278,6 +352,19 @@ class PathsFilter(BaseModel): step_limit: Optional[float] = None +class PropertyFilterType(str, Enum): + meta = "meta" + event = "event" + person = "person" + element = "element" + feature = "feature" + session = "session" + cohort = "cohort" + recording = "recording" + group = "group" + hogql = "hogql" + + class PropertyMathType(str, Enum): avg = "avg" sum = "sum" @@ -431,6 +518,20 @@ class StickinessFilter(BaseModel): shown_as: Optional[ShownAsValue] = None +class TimeToSeeDataQuery(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + kind: Literal["TimeToSeeDataQuery"] = "TimeToSeeDataQuery" + response: Optional[Dict[str, Any]] = Field(default=None, description="Cached query response") + sessionEnd: Optional[str] = None + sessionId: Optional[str] = Field(default=None, description="Project to filter on. Defaults to current session") + sessionStart: Optional[str] = Field( + default=None, description="Session start time. Defaults to current time - 2 hours" + ) + teamId: Optional[float] = Field(default=None, description="Project to filter on. Defaults to current project") + + class TimeToSeeDataSessionsQueryResponse(BaseModel): model_config = ConfigDict( extra="forbid", @@ -438,6 +539,14 @@ class TimeToSeeDataSessionsQueryResponse(BaseModel): results: List[Dict[str, Any]] +class TimeToSeeDataWaterfallNode(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + kind: Literal["TimeToSeeDataSessionsWaterfallNode"] = "TimeToSeeDataSessionsWaterfallNode" + source: TimeToSeeDataQuery + + class TrendsFilter(BaseModel): model_config = ConfigDict( extra="forbid", @@ -468,6 +577,14 @@ class TrendsQueryResponse(BaseModel): timings: Optional[List[QueryTiming]] = None +class AnyResponseTypeItem(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + next: Optional[str] = None + results: List[EventType] + + class Breakdown(BaseModel): model_config = ConfigDict( extra="forbid", @@ -489,6 +606,14 @@ class BreakdownFilter(BaseModel): breakdowns: Optional[List[Breakdown]] = None +class DataNode(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + kind: NodeKind + response: Optional[Dict[str, Any]] = Field(default=None, description="Cached query response") + + class ElementPropertyFilter(BaseModel): model_config = ConfigDict( extra="forbid", @@ -621,6 +746,13 @@ class LifecycleQueryResponse(BaseModel): timings: Optional[List[QueryTiming]] = None +class Node(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + kind: NodeKind + + class PersonPropertyFilter(BaseModel): model_config = ConfigDict( extra="forbid", @@ -632,6 +764,17 @@ class PersonPropertyFilter(BaseModel): value: Optional[Union[str, float, List[Union[str, float]]]] = None +class QueryResponse(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + is_cached: Optional[bool] = None + last_refresh: Optional[str] = None + next_allowed_client_refresh: Optional[str] = None + result: Any + timings: Optional[List[QueryTiming]] = None + + class RetentionFilter(BaseModel): model_config = ConfigDict( extra="forbid", @@ -644,6 +787,18 @@ class RetentionFilter(BaseModel): total_intervals: Optional[float] = None +class TimeToSeeDataJSONNode(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + kind: Literal["TimeToSeeDataSessionsJSONNode"] = "TimeToSeeDataSessionsJSONNode" + source: TimeToSeeDataQuery + + +class TimeToSeeDataNode(RootModel): + root: Union[TimeToSeeDataJSONNode, TimeToSeeDataWaterfallNode] + + class TimeToSeeDataSessionsQuery(BaseModel): model_config = ConfigDict( extra="forbid", @@ -654,6 +809,12 @@ class TimeToSeeDataSessionsQuery(BaseModel): teamId: Optional[float] = Field(default=None, description="Project to filter on. Defaults to current project") +class AnyResponseType(RootModel): + root: Union[ + Dict[str, Any], HogQLQueryResponse, HogQLMetadataResponse, Union[AnyResponseTypeItem, Any], EventsQueryResponse + ] + + class DatabaseSchemaQuery(BaseModel): model_config = ConfigDict( extra="forbid", @@ -664,6 +825,57 @@ class DatabaseSchemaQuery(BaseModel): ) +class EntityNode(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + custom_name: Optional[str] = None + fixedProperties: Optional[ + List[ + Union[ + EventPropertyFilter, + PersonPropertyFilter, + ElementPropertyFilter, + SessionPropertyFilter, + CohortPropertyFilter, + RecordingDurationFilter, + GroupPropertyFilter, + FeaturePropertyFilter, + HogQLPropertyFilter, + EmptyPropertyFilter, + ] + ] + ] = Field( + default=None, + description="Fixed properties in the query, can't be edited in the interface (e.g. scoping down by person)", + ) + kind: NodeKind + math: Optional[ + Union[BaseMathType, PropertyMathType, CountPerActorMathType, Literal["unique_group"], Literal["hogql"]] + ] = None + math_group_type_index: Optional[MathGroupTypeIndex] = None + math_hogql: Optional[str] = None + math_property: Optional[str] = None + name: Optional[str] = None + properties: Optional[ + List[ + Union[ + EventPropertyFilter, + PersonPropertyFilter, + ElementPropertyFilter, + SessionPropertyFilter, + CohortPropertyFilter, + RecordingDurationFilter, + GroupPropertyFilter, + FeaturePropertyFilter, + HogQLPropertyFilter, + EmptyPropertyFilter, + ] + ] + ] = Field(default=None, description="Properties configurable in the interface") + response: Optional[Dict[str, Any]] = Field(default=None, description="Cached query response") + + class EventsNode(BaseModel): model_config = ConfigDict( extra="forbid", @@ -815,6 +1027,10 @@ class HogQLQuery(BaseModel): response: Optional[HogQLQueryResponse] = Field(default=None, description="Cached query response") +class InsightFilter(RootModel): + root: Union[TrendsFilter, FunnelsFilter, RetentionFilter, PathsFilter, StickinessFilter, LifecycleFilter] + + class PersonsNode(BaseModel): model_config = ConfigDict( extra="forbid", @@ -987,6 +1203,10 @@ class DataTableNode(BaseModel): ) +class HasPropertiesNode(RootModel): + root: Union[EventsNode, EventsQuery, PersonsNode] + + class PropertyGroupFilter(BaseModel): model_config = ConfigDict( extra="forbid", @@ -1108,6 +1328,54 @@ class TrendsQuery(BaseModel): trendsFilter: Optional[TrendsFilter] = Field(default=None, description="Properties specific to the trends insight") +class FilterType(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + actions: Optional[List[Dict[str, Any]]] = None + aggregation_group_type_index: Optional[float] = None + breakdown: Optional[Union[str, float, List[Union[str, float]]]] = None + breakdown_group_type_index: Optional[float] = None + breakdown_normalize_url: Optional[bool] = None + breakdown_type: Optional[BreakdownType] = None + breakdowns: Optional[List[Breakdown]] = None + date_from: Optional[str] = None + date_to: Optional[str] = None + entity_id: Optional[Union[str, float]] = None + entity_math: Optional[str] = None + entity_type: Optional[EntityType] = None + events: Optional[List[Dict[str, Any]]] = None + explicit_date: Optional[Union[bool, str]] = Field( + default=None, + description='Whether the `date_from` and `date_to` should be used verbatim. Disables rounding to the start and end of period. Strings are cast to bools, e.g. "true" -> true.', + ) + filter_test_accounts: Optional[bool] = None + from_dashboard: Optional[Union[bool, float]] = None + insight: Optional[InsightType] = None + interval: Optional[IntervalType] = None + new_entity: Optional[List[Dict[str, Any]]] = None + properties: Optional[ + Union[ + List[ + Union[ + EventPropertyFilter, + PersonPropertyFilter, + ElementPropertyFilter, + SessionPropertyFilter, + CohortPropertyFilter, + RecordingDurationFilter, + GroupPropertyFilter, + FeaturePropertyFilter, + HogQLPropertyFilter, + EmptyPropertyFilter, + ] + ], + PropertyGroupFilter, + ] + ] = None + sampling_factor: Optional[float] = None + + class FunnelsQuery(BaseModel): model_config = ConfigDict( extra="forbid", @@ -1148,6 +1416,38 @@ class FunnelsQuery(BaseModel): series: List[Union[EventsNode, ActionsNode]] = Field(..., description="Events and actions to include") +class InsightsQueryBase(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + aggregation_group_type_index: Optional[float] = Field(default=None, description="Groups aggregation") + dateRange: Optional[DateRange] = Field(default=None, description="Date range for the query") + filterTestAccounts: Optional[bool] = Field( + default=None, description="Exclude internal and test users by applying the respective filters" + ) + kind: NodeKind + properties: Optional[ + Union[ + List[ + Union[ + EventPropertyFilter, + PersonPropertyFilter, + ElementPropertyFilter, + SessionPropertyFilter, + CohortPropertyFilter, + RecordingDurationFilter, + GroupPropertyFilter, + FeaturePropertyFilter, + HogQLPropertyFilter, + EmptyPropertyFilter, + ] + ], + PropertyGroupFilter, + ] + ] = Field(default=None, description="Property filters for all series") + samplingFactor: Optional[float] = Field(default=None, description="Sampling rate") + + class LifecycleQuery(BaseModel): model_config = ConfigDict( extra="forbid", @@ -1188,6 +1488,13 @@ class LifecycleQuery(BaseModel): series: List[Union[EventsNode, ActionsNode]] = Field(..., description="Events and actions to include") +class NamedParametersTypeofDateRangeForFilter(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + source: Optional[FilterType] = None + + class PathsQuery(BaseModel): model_config = ConfigDict( extra="forbid", @@ -1240,7 +1547,7 @@ class InsightVizNode(BaseModel): source: Union[TrendsQuery, FunnelsQuery, RetentionQuery, PathsQuery, StickinessQuery, LifecycleQuery] -class Model(RootModel): +class QuerySchema(RootModel): root: Union[ DataTableNode, SavedInsightNode, @@ -1254,7 +1561,7 @@ class Model(RootModel): TimeToSeeDataSessionsQuery, DatabaseSchemaQuery, Union[EventsNode, EventsQuery, ActionsNode, PersonsNode, HogQLQuery, HogQLMetadata, TimeToSeeDataSessionsQuery], - ] + ] = Field(..., description="Query Schema") PropertyGroupFilterValue.model_rebuild()