From 1456301b8d743ac03d96364151a234505f89cfcf Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 22 Feb 2023 10:25:01 +0100 Subject: [PATCH] feat(hogql): HogQLQuery node and data table support (#14265) --- .../lib/lemon-ui/LemonTable/LemonTable.tsx | 2 +- frontend/src/queries/examples.ts | 22 ++++++ .../src/queries/nodes/DataNode/DataNode.tsx | 4 +- .../queries/nodes/DataNode/dataNodeLogic.ts | 15 +++- .../src/queries/nodes/DataTable/DataTable.tsx | 37 +++++++--- .../queries/nodes/DataTable/dataTableLogic.ts | 12 ++-- .../nodes/HogQLQuery/HogQLQueryEditor.tsx | 55 ++++++++++++++ .../nodes/HogQLQuery/hogQLQueryEditorLogic.ts | 45 ++++++++++++ frontend/src/queries/query.ts | 5 ++ frontend/src/queries/schema.json | 55 ++++++++++++++ frontend/src/queries/schema.ts | 21 +++++- frontend/src/queries/utils.ts | 7 +- frontend/src/scenes/query/QueryScene.tsx | 25 +++++-- .../scenes/saved-insights/SavedInsights.tsx | 6 ++ package.json | 1 + pnpm-lock.yaml | 45 +++++++++++- posthog/api/query.py | 72 +++++++++++++------ .../api/test/__snapshots__/test_query.ambr | 24 +++++++ posthog/api/test/test_query.py | 47 +++++++++++- posthog/hogql/printer.py | 4 +- posthog/hogql/query.py | 8 ++- .../test/__snapshots__/test_formula.ambr | 8 +-- posthog/schema.py | 26 ++++++- webpack.config.js | 2 +- 24 files changed, 486 insertions(+), 62 deletions(-) create mode 100644 frontend/src/queries/nodes/HogQLQuery/HogQLQueryEditor.tsx create mode 100644 frontend/src/queries/nodes/HogQLQuery/hogQLQueryEditorLogic.ts diff --git a/frontend/src/lib/lemon-ui/LemonTable/LemonTable.tsx b/frontend/src/lib/lemon-ui/LemonTable/LemonTable.tsx index aeffd4a92400b..f800eb7f840ad 100644 --- a/frontend/src/lib/lemon-ui/LemonTable/LemonTable.tsx +++ b/frontend/src/lib/lemon-ui/LemonTable/LemonTable.tsx @@ -143,7 +143,7 @@ export function LemonTable>({ ) const columnGroups = ( - 'children' in rawColumns[0] + rawColumns.length > 0 && 'children' in rawColumns[0] ? rawColumns : [ { diff --git a/frontend/src/queries/examples.ts b/frontend/src/queries/examples.ts index 87ca2b5d229c8..c53e9ea62a590 100644 --- a/frontend/src/queries/examples.ts +++ b/frontend/src/queries/examples.ts @@ -5,6 +5,7 @@ import { EventsNode, EventsQuery, FunnelsQuery, + HogQLQuery, LegacyQuery, LifecycleQuery, Node, @@ -290,6 +291,25 @@ const TimeToSeeDataWaterfall: TimeToSeeDataWaterfallNode = { }, } +const HogQL: HogQLQuery = { + kind: NodeKind.HogQLQuery, + query: + ' select event,\n' + + ' properties.$geoip_country_name as `Country Name`,\n' + + ' count() as `Event count`\n' + + ' from events\n' + + ' group by event,\n' + + ' properties.$geoip_country_name\n' + + ' order by count() desc\n' + + ' limit 100', +} + +const HogQLTable: DataTableNode = { + kind: NodeKind.DataTableNode, + full: true, + source: HogQL, +} + export const examples: Record = { Events, EventsTable, @@ -310,6 +330,8 @@ export const examples: Record = { TimeToSeeDataSessions, TimeToSeeDataWaterfall, TimeToSeeDataJSON, + HogQL, + HogQLTable, } export const stringifiedExamples: Record = Object.fromEntries( diff --git a/frontend/src/queries/nodes/DataNode/DataNode.tsx b/frontend/src/queries/nodes/DataNode/DataNode.tsx index 992bc84126299..465651fa1963d 100644 --- a/frontend/src/queries/nodes/DataNode/DataNode.tsx +++ b/frontend/src/queries/nodes/DataNode/DataNode.tsx @@ -18,7 +18,7 @@ let uniqueNode = 0 export function DataNode(props: DataNodeProps): JSX.Element { const [key] = useState(() => `DataNode.${uniqueNode++}`) const logic = dataNodeLogic({ ...props, key }) - const { response, responseLoading } = useValues(logic) + const { response, responseLoading, responseErrorObject } = useValues(logic) return (
@@ -36,7 +36,7 @@ export function DataNode(props: DataNodeProps): JSX.Element { theme="vs-light" className="border" language={'json'} - value={JSON.stringify(response, null, 2)} + value={JSON.stringify(response ?? responseErrorObject, null, 2)} height={Math.max(height, 300)} /> )} diff --git a/frontend/src/queries/nodes/DataNode/dataNodeLogic.ts b/frontend/src/queries/nodes/DataNode/dataNodeLogic.ts index 44d0a467c599e..9ea069e8800c4 100644 --- a/frontend/src/queries/nodes/DataNode/dataNodeLogic.ts +++ b/frontend/src/queries/nodes/DataNode/dataNodeLogic.ts @@ -166,11 +166,24 @@ export const dataNodeLogic = kea([ // Clear the response if a failure to avoid showing inconsistencies in the UI loadDataFailure: () => null, }, + responseErrorObject: [ + null as Record | null, + { + loadData: () => null, + loadDataFailure: (_, { errorObject }) => errorObject, + loadDataSuccess: () => null, + }, + ], responseError: [ null as string | null, { loadData: () => null, - loadDataFailure: () => 'Error loading data', + loadDataFailure: (_, { error, errorObject }) => { + if (errorObject && 'error' in errorObject) { + return errorObject.error + } + return error ?? 'Error loading data' + }, loadDataSuccess: () => null, }, ], diff --git a/frontend/src/queries/nodes/DataTable/DataTable.tsx b/frontend/src/queries/nodes/DataTable/DataTable.tsx index 3c9acf2be5fbc..298fffde66a7b 100644 --- a/frontend/src/queries/nodes/DataTable/DataTable.tsx +++ b/frontend/src/queries/nodes/DataTable/DataTable.tsx @@ -1,5 +1,5 @@ import './DataTable.scss' -import { DataTableNode, EventsNode, EventsQuery, Node, PersonsNode, QueryContext } from '~/queries/schema' +import { DataTableNode, EventsNode, EventsQuery, HogQLQuery, Node, PersonsNode, QueryContext } from '~/queries/schema' import { useCallback, useState } from 'react' import { BindLogic, useValues } from 'kea' import { dataNodeLogic, DataNodeLogicProps } from '~/queries/nodes/DataNode/dataNodeLogic' @@ -21,7 +21,7 @@ import { EventBufferNotice } from 'scenes/events/EventBufferNotice' import clsx from 'clsx' import { SessionPlayerModal } from 'scenes/session-recordings/player/modal/SessionPlayerModal' import { InlineEditorButton } from '~/queries/nodes/Node/InlineEditorButton' -import { isEventsQuery, isHogQlAggregation, isPersonsNode, taxonomicFilterToHogQl } from '~/queries/utils' +import { isEventsQuery, isHogQlAggregation, isHogQLQuery, isPersonsNode, taxonomicFilterToHogQl } from '~/queries/utils' import { PersonPropertyFilters } from '~/queries/nodes/PersonsNode/PersonPropertyFilters' import { PersonsSearch } from '~/queries/nodes/PersonsNode/PersonsSearch' import { PersonDeleteModal } from 'scenes/persons/PersonDeleteModal' @@ -34,6 +34,7 @@ import { extractExpressionComment, removeExpressionComment } from '~/queries/nod import { InsightEmptyState, InsightErrorState } from 'scenes/insights/EmptyStates' import { EventType } from '~/types' import { SavedQueries } from '~/queries/nodes/DataTable/SavedQueries' +import { HogQLQueryEditor } from '~/queries/nodes/HogQLQuery/HogQLQueryEditor' interface DataTableProps { query: DataTableNode @@ -79,6 +80,7 @@ export function DataTable({ query, setQuery, context }: DataTableProps): JSX.Ele showSearch, showEventFilter, showPropertyFilter, + showHogQLEditor, showReload, showExport, showElapsedTime, @@ -89,8 +91,9 @@ export function DataTable({ query, setQuery, context }: DataTableProps): JSX.Ele } = queryWithDefaults const actionsColumnShown = showActions && isEventsQuery(query.source) && columnsInResponse?.includes('*') + const columnsInLemonTable = isHogQLQuery(query.source) ? columnsInResponse ?? columnsInQuery : columnsInQuery const lemonColumns: LemonTableColumn[] = [ - ...columnsInQuery.map((key, index) => ({ + ...columnsInLemonTable.map((key, index) => ({ dataIndex: key as any, ...renderColumnMeta(key, query, context), render: function RenderDataTableColumn(_: any, { result, label }: DataTableRow) { @@ -98,13 +101,13 @@ export function DataTable({ query, setQuery, context }: DataTableProps): JSX.Ele if (index === (expandable ? 1 : 0)) { return { children: label, - props: { colSpan: columnsInQuery.length + (actionsColumnShown ? 1 : 0) }, + props: { colSpan: columnsInLemonTable.length + (actionsColumnShown ? 1 : 0) }, } } else { return { props: { colSpan: 0 } } } } else if (result) { - if (isEventsQuery(query.source)) { + if (isEventsQuery(query.source) || isHogQLQuery(query.source)) { return renderColumn(key, result[index], result, query, setQuery, context) } return renderColumn(key, result[key], result, query, setQuery, context) @@ -292,7 +295,7 @@ export function DataTable({ query, setQuery, context }: DataTableProps): JSX.Ele ].filter((column) => !query.hiddenColumns?.includes(column.dataIndex) && column.dataIndex !== '*') const setQuerySource = useCallback( - (source: EventsNode | EventsQuery | PersonsNode) => setQuery?.({ ...query, source }), + (source: EventsNode | EventsQuery | PersonsNode | HogQLQuery) => setQuery?.({ ...query, source }), [setQuery] ) @@ -337,7 +340,10 @@ export function DataTable({ query, setQuery, context }: DataTableProps): JSX.Ele return ( -
+
+ {showHogQLEditor && isHogQLQuery(query.source) ? ( + + ) : null} {showFirstRow && (
{firstRowLeft} @@ -398,7 +404,22 @@ export function DataTable({ query, setQuery, context }: DataTableProps): JSX.Ele }} sorting={null} useURLForSorting={false} - emptyState={responseError ? : } + emptyState={ + responseError ? ( + isHogQLQuery(query.source) ? ( + + ) : ( + + ) + ) : ( + + ) + } expandable={ expandable && isEventsQuery(query.source) && columnsInResponse?.includes('*') ? { diff --git a/frontend/src/queries/nodes/DataTable/dataTableLogic.ts b/frontend/src/queries/nodes/DataTable/dataTableLogic.ts index f9193f38c18c2..e4bf1edbf9c3d 100644 --- a/frontend/src/queries/nodes/DataTable/dataTableLogic.ts +++ b/frontend/src/queries/nodes/DataTable/dataTableLogic.ts @@ -59,12 +59,7 @@ export const dataTableLogic = kea([ columnsInResponse: [ (s) => [s.response], (response: AnyDataNode['response']): string[] | null => - response && - 'columns' in response && - Array.isArray(response.columns) && - !response.columns.find((c) => typeof c !== 'string') - ? (response?.columns as string[]) - : null, + response && 'columns' in response && Array.isArray(response.columns) ? response?.columns : null, ], dataTableRows: [ (s) => [s.sourceKind, s.orderBy, s.response, s.columnsInQuery, s.columnsInResponse], @@ -143,10 +138,13 @@ export const dataTableLogic = kea([ showDateRange: query.showDateRange ?? showIfFull, showExport: query.showExport ?? showIfFull, showReload: query.showReload ?? showIfFull, - showElapsedTime: query.showElapsedTime ?? (flagQueryRunningTimeEnabled ? showIfFull : false), + showElapsedTime: + query.showElapsedTime ?? + (flagQueryRunningTimeEnabled || source.kind === NodeKind.HogQLQuery ? showIfFull : false), showColumnConfigurator: query.showColumnConfigurator ?? showIfFull, showSavedQueries: query.showSavedQueries ?? false, showEventsBufferWarning: query.showEventsBufferWarning ?? showIfFull, + showHogQLEditor: query.showHogQLEditor ?? showIfFull, allowSorting: query.allowSorting ?? true, }), } diff --git a/frontend/src/queries/nodes/HogQLQuery/HogQLQueryEditor.tsx b/frontend/src/queries/nodes/HogQLQuery/HogQLQueryEditor.tsx new file mode 100644 index 0000000000000..0d38c3b4beeab --- /dev/null +++ b/frontend/src/queries/nodes/HogQLQuery/HogQLQueryEditor.tsx @@ -0,0 +1,55 @@ +import { useActions, useValues } from 'kea' +import { HogQLQuery } from '~/queries/schema' +import { useState } from 'react' +import { hogQLQueryEditorLogic } from './hogQLQueryEditorLogic' +import MonacoEditor from '@monaco-editor/react' +import { AutoSizer } from 'react-virtualized/dist/es/AutoSizer' +import { LemonButton } from 'lib/lemon-ui/LemonButton' + +export interface HogQLQueryEditorProps { + query: HogQLQuery + setQuery?: (query: HogQLQuery) => void +} + +let uniqueNode = 0 +export function HogQLQueryEditor(props: HogQLQueryEditorProps): JSX.Element { + const [key] = useState(() => uniqueNode++) + const hogQLQueryEditorLogicProps = { query: props.query, setQuery: props.setQuery, key } + const { queryInput } = useValues(hogQLQueryEditorLogic(hogQLQueryEditorLogicProps)) + const { setQueryInput, saveQuery } = useActions(hogQLQueryEditorLogic(hogQLQueryEditorLogicProps)) + + return ( +
+
+ + {({ height }) => ( + setQueryInput(v ?? '')} + height={height} + options={{ + minimap: { + enabled: false, + }, + wordWrap: 'on', + }} + /> + )} + +
+ + {!props.setQuery ? 'No permission to update' : 'Update'} + +
+ ) +} diff --git a/frontend/src/queries/nodes/HogQLQuery/hogQLQueryEditorLogic.ts b/frontend/src/queries/nodes/HogQLQuery/hogQLQueryEditorLogic.ts new file mode 100644 index 0000000000000..566c2dfaa51fb --- /dev/null +++ b/frontend/src/queries/nodes/HogQLQuery/hogQLQueryEditorLogic.ts @@ -0,0 +1,45 @@ +import { actions, kea, key, listeners, path, props, propsChanged, reducers } from 'kea' +import { format } from 'sql-formatter' +import { HogQLQuery } from '~/queries/schema' + +import type { hogQLQueryEditorLogicType } from './hogQLQueryEditorLogicType' + +function formatSQL(sql: string): string { + return format(sql, { + language: 'mysql', + tabWidth: 2, + keywordCase: 'preserve', + linesBetweenQueries: 2, + indentStyle: 'tabularRight', + }) +} +export interface HogQLQueryEditorLogicProps { + key: number + query: HogQLQuery + setQuery?: (query: HogQLQuery) => void +} + +export const hogQLQueryEditorLogic = kea([ + path(['queries', 'nodes', 'HogQLQuery', 'hogQLQueryEditorLogic']), + props({} as HogQLQueryEditorLogicProps), + key((props) => props.key), + propsChanged(({ actions, props }, oldProps) => { + if (props.query.query !== oldProps.query.query) { + actions.setQueryInput(formatSQL(props.query.query)) + } + }), + actions({ + saveQuery: true, + setQueryInput: (queryInput: string) => ({ queryInput }), + }), + reducers(({ props }) => ({ + queryInput: [formatSQL(props.query.query), { setQueryInput: (_, { queryInput }) => queryInput }], + })), + listeners(({ actions, props, values }) => ({ + saveQuery: () => { + const formattedQuery = formatSQL(values.queryInput) + actions.setQueryInput(formattedQuery) + props.setQuery?.({ ...props.query, query: formattedQuery }) + }, + })), +]) diff --git a/frontend/src/queries/query.ts b/frontend/src/queries/query.ts index c416d072b0c1d..196a056da2ea8 100644 --- a/frontend/src/queries/query.ts +++ b/frontend/src/queries/query.ts @@ -9,6 +9,7 @@ import { isRecentPerformancePageViewNode, isDataTableNode, isTimeToSeeDataSessionsNode, + isHogQLQuery, isInsightVizNode, } from './utils' import api, { ApiMethodOptions } from 'lib/api' @@ -47,6 +48,8 @@ export function queryExportContext( after: now().subtract(EVENTS_DAYS_FIRST_FETCH, 'day').toISOString(), }, } + } else if (isHogQLQuery(query)) { + return { path: api.queryURL(), method: 'POST', body: query } } else if (isPersonsNode(query)) { return { path: getPersonsEndpoint(query) } } else if (isInsightQueryNode(query)) { @@ -118,6 +121,8 @@ export async function query( } } return await api.query({ after: now().subtract(1, 'year').toISOString(), ...query }, methodOptions) + } else if (isHogQLQuery(query)) { + return api.query(query, methodOptions) } else if (isPersonsNode(query)) { return await api.get(getPersonsEndpoint(query), methodOptions) } else if (isInsightQueryNode(query)) { diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index 58773a21b9657..b3316100b9325 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -80,6 +80,9 @@ }, { "$ref": "#/definitions/PersonsNode" + }, + { + "$ref": "#/definitions/HogQLQuery" } ] }, @@ -1302,6 +1305,10 @@ "description": "Show the export button", "type": "boolean" }, + "showHogQLEditor": { + "description": "Include a HogQL query editor above HogQL tables", + "type": "boolean" + }, "showPropertyFilter": { "description": "Include a property filter above the table", "type": "boolean" @@ -1331,6 +1338,9 @@ }, { "$ref": "#/definitions/RecentPerformancePageViewNode" + }, + { + "$ref": "#/definitions/HogQLQuery" } ], "description": "Source of the events" @@ -2002,6 +2012,51 @@ "required": ["key", "type"], "type": "object" }, + "HogQLQuery": { + "additionalProperties": false, + "properties": { + "kind": { + "const": "HogQLQuery", + "type": "string" + }, + "query": { + "type": "string" + }, + "response": { + "$ref": "#/definitions/HogQLQueryResponse", + "description": "Cached query response" + } + }, + "required": ["kind", "query"], + "type": "object" + }, + "HogQLQueryResponse": { + "additionalProperties": false, + "properties": { + "clickhouse": { + "type": "string" + }, + "columns": { + "items": {}, + "type": "array" + }, + "hogql": { + "type": "string" + }, + "query": { + "type": "string" + }, + "results": { + "items": {}, + "type": "array" + }, + "types": { + "items": {}, + "type": "array" + } + }, + "type": "object" + }, "InsightQueryNode": { "anyOf": [ { diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index 03f0134ca698c..7ca8fafb769ec 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -40,6 +40,7 @@ export enum NodeKind { NewEntityNode = 'NewEntityNode', EventsQuery = 'EventsQuery', PersonsNode = 'PersonsNode', + HogQLQuery = 'HogQLQuery', // Interface nodes DataTableNode = 'DataTableNode', @@ -64,7 +65,7 @@ export enum NodeKind { RecentPerformancePageViewNode = 'RecentPerformancePageViewNode', } -export type AnyDataNode = EventsNode | EventsQuery | ActionsNode | PersonsNode +export type AnyDataNode = EventsNode | EventsQuery | ActionsNode | PersonsNode | HogQLQuery export type QuerySchema = // Data nodes (see utils.ts) @@ -101,6 +102,20 @@ export interface DataNode extends Node { response?: Record } +export interface HogQLQueryResponse { + query?: string + hogql?: string + clickhouse?: string + results?: any[] + types?: any[] + columns?: any[] +} + +export interface HogQLQuery extends DataNode { + kind: NodeKind.HogQLQuery + query: string + response?: HogQLQueryResponse +} export interface EntityNode extends DataNode { name?: string custom_name?: string @@ -201,7 +216,7 @@ export type HasPropertiesNode = EventsNode | EventsQuery | PersonsNode export interface DataTableNode extends Node { kind: NodeKind.DataTableNode /** Source of the events */ - source: EventsNode | EventsQuery | PersonsNode | RecentPerformancePageViewNode + source: EventsNode | EventsQuery | PersonsNode | RecentPerformancePageViewNode | HogQLQuery /** Columns shown in the table, unless the `source` provides them. */ columns?: HogQLExpression[] /** Columns that aren't shown in the table, even if in columns or returned data */ @@ -214,6 +229,8 @@ export interface DataTableNode extends Node { showSearch?: boolean /** Include a property filter above the table */ showPropertyFilter?: boolean + /** Include a HogQL query editor above HogQL tables */ + showHogQLEditor?: boolean /** Show the kebab menu at the end of the row */ showActions?: boolean /** Show date range selector */ diff --git a/frontend/src/queries/utils.ts b/frontend/src/queries/utils.ts index a99d7e71387fd..cdf36b268e261 100644 --- a/frontend/src/queries/utils.ts +++ b/frontend/src/queries/utils.ts @@ -4,6 +4,7 @@ import { DateRange, EventsNode, EventsQuery, + HogQLQuery, TrendsQuery, FunnelsQuery, RetentionQuery, @@ -27,7 +28,7 @@ import { import { TaxonomicFilterGroupType, TaxonomicFilterValue } from 'lib/components/TaxonomicFilter/types' export function isDataNode(node?: Node): node is EventsQuery | PersonsNode | TimeToSeeDataSessionsQuery { - return isEventsQuery(node) || isPersonsNode(node) || isTimeToSeeDataSessionsQuery(node) + return isEventsQuery(node) || isPersonsNode(node) || isTimeToSeeDataSessionsQuery(node) || isHogQLQuery(node) } export function isEventsNode(node?: Node): node is EventsNode { @@ -58,6 +59,10 @@ export function isLegacyQuery(node?: Node): node is LegacyQuery { return node?.kind === NodeKind.LegacyQuery } +export function isHogQLQuery(node?: Node): node is HogQLQuery { + return node?.kind === NodeKind.HogQLQuery +} + /* * Insight Queries */ diff --git a/frontend/src/scenes/query/QueryScene.tsx b/frontend/src/scenes/query/QueryScene.tsx index 6c0337dd51fb5..fb44c06db3813 100644 --- a/frontend/src/scenes/query/QueryScene.tsx +++ b/frontend/src/scenes/query/QueryScene.tsx @@ -14,6 +14,19 @@ export function QueryScene(): JSX.Element { const { query } = useValues(querySceneLogic) const { setQuery } = useActions(querySceneLogic) + let showEditor = true + try { + const parsed = JSON.parse(query) + if ( + parsed && + parsed.kind == 'DataTableNode' && + parsed.source.kind == 'HogQLQuery' && + (parsed.full || parsed.showHogQLEditor) + ) { + showEditor = false + } + } catch (e) {} + return (
@@ -35,10 +48,14 @@ export function QueryScene(): JSX.Element { ))}
- -
- -
+ {showEditor ? ( + <> + +
+ +
+ + ) : null} setQuery(JSON.stringify(query, null, 2))} />
diff --git a/frontend/src/scenes/saved-insights/SavedInsights.tsx b/frontend/src/scenes/saved-insights/SavedInsights.tsx index 0e04c893fc2d5..504743a42098d 100644 --- a/frontend/src/scenes/saved-insights/SavedInsights.tsx +++ b/frontend/src/scenes/saved-insights/SavedInsights.tsx @@ -218,6 +218,12 @@ export const QUERY_TYPES_METADATA: Record = { icon: IconCoffee, inMenu: true, }, + [NodeKind.HogQLQuery]: { + name: 'HogQL', + description: 'Direct HogQL query', + icon: IconCoffee, + inMenu: true, + }, } export const INSIGHT_TYPE_OPTIONS: LemonSelectOptions = [ diff --git a/package.json b/package.json index e3bff0a82450d..b00760aa53c84 100644 --- a/package.json +++ b/package.json @@ -139,6 +139,7 @@ "resize-observer-polyfill": "^1.5.1", "rrweb": "^1.1.3", "sass": "^1.26.2", + "sql-formatter": "^12.1.2", "use-debounce": "^6.0.1", "use-resize-observer": "^8.0.0", "wildcard-match": "^5.1.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d85f5ab447c79..eb6359bcf0769 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -173,6 +173,7 @@ specifiers: rrweb: ^1.1.3 sass: ^1.26.2 sass-loader: ^10.0.1 + sql-formatter: ^12.1.2 storybook-addon-pseudo-states: ^1.15.1 style-loader: ^2.0.0 sucrase: ^3.29.0 @@ -269,6 +270,7 @@ dependencies: resize-observer-polyfill: 1.5.1 rrweb: 1.1.3 sass: 1.56.0 + sql-formatter: 12.1.2 use-debounce: 6.0.1_react@16.14.0 use-resize-observer: 8.0.0_wcqkhtmu7mswc6yz4uyexck3ty wildcard-match: 5.1.2 @@ -5643,6 +5645,10 @@ packages: sprintf-js: 1.0.3 dev: true + /argparse/2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + dev: false + /aria-hidden/1.2.1_edij4neeagymnxmr7qklvezyj4: resolution: {integrity: sha512-PN344VAf9j1EAi+jyVHOJ8XidQdPVssGco39eNcsGdM4wcsILtxrKLkbuiMfLWYROK1FjRQasMWCBttrhjnr6A==} engines: {node: '>=10'} @@ -8190,6 +8196,10 @@ packages: path-type: 4.0.0 dev: true + /discontinuous-range/1.0.0: + resolution: {integrity: sha512-c68LpLbO+7kP/b1Hr1qs8/BJ09F5khZGTxqxZuhzxpmwJKOgRFHJWIb9/KmqnqHhLdO55aOxFH/EGBvUQbL/RQ==} + dev: false + /doctrine/2.1.0: resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==} engines: {node: '>=0.10.0'} @@ -13283,6 +13293,10 @@ packages: color-name: 1.1.4 dev: true + /moo/0.5.2: + resolution: {integrity: sha512-iSAJLHYKnX41mKcJKjqvnAN9sf0LMDTXDEvFv+ffuRR9a1MIuXLjMNL6EsnDHSkKLTWNqQQ5uo61P4EbU4NU+Q==} + dev: false + /move-concurrently/1.0.1: resolution: {integrity: sha512-hdrFxZOycD/g6A6SoI2bB5NA/5NEqD0569+S47WZhPvm46sD50ZHdYaFmnua5lndde9rCHGjmfK7Z8BuCt/PcQ==} dependencies: @@ -13415,6 +13429,16 @@ packages: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} dev: true + /nearley/2.20.1: + resolution: {integrity: sha512-+Mc8UaAebFzgV+KpI5n7DasuuQCHA89dmwm7JXw3TV43ukfNQ9DnBH3Mdb2g/I4Fdxc26pwimBWvjIw0UAILSQ==} + hasBin: true + dependencies: + commander: 2.20.3 + moo: 0.5.2 + railroad-diagrams: 1.0.0 + randexp: 0.4.6 + dev: false + /needle/3.2.0: resolution: {integrity: sha512-oUvzXnyLiVyVGoianLijF9O/RecZUf7TkBfimjGrLM4eQhXyeJwM6GeAWccwfQ9aa4gMCZKqhAOuLaMIcQxajQ==} engines: {node: '>= 4.4.x'} @@ -14898,10 +14922,22 @@ packages: engines: {node: '>=10'} dev: false + /railroad-diagrams/1.0.0: + resolution: {integrity: sha512-cz93DjNeLY0idrCNOH6PviZGRN9GJhsdm9hpn1YCS879fj4W+x5IFJhhkRZcwVgMmFF7R82UA/7Oh+R8lLZg6A==} + dev: false + /ramda/0.28.0: resolution: {integrity: sha512-9QnLuG/kPVgWvMQ4aODhsBUFKOUmnbUnsSXACv+NCQZcHbeb+v8Lodp8OVxtRULN1/xOyYLLaL6npE6dMq5QTA==} dev: true + /randexp/0.4.6: + resolution: {integrity: sha512-80WNmd9DA0tmZrw9qQa62GPPWfuXJknrmVmLcxvq4uZBdYqb1wYoKTmnlGUchvVWe0XiLupYkBoXVOxz3C8DYQ==} + engines: {node: '>=0.12'} + dependencies: + discontinuous-range: 1.0.0 + ret: 0.1.15 + dev: false + /randombytes/2.1.0: resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==} dependencies: @@ -16239,7 +16275,6 @@ packages: /ret/0.1.15: resolution: {integrity: sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==} engines: {node: '>=0.12'} - dev: true /reusify/1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} @@ -16843,6 +16878,14 @@ packages: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} dev: true + /sql-formatter/12.1.2: + resolution: {integrity: sha512-SoFn+9ZflUt8+HYZ/PaifXt1RptcDUn8HXqsWmfXdPV3WeHPgT0qOSJXxHU24d7NOVt9X40MLqf263fNk79XqA==} + hasBin: true + dependencies: + argparse: 2.0.1 + nearley: 2.20.1 + dev: false + /sshpk/1.17.0: resolution: {integrity: sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ==} engines: {node: '>=0.10.0'} diff --git a/posthog/api/query.py b/posthog/api/query.py index 95909caafd620..535fdd80e3d77 100644 --- a/posthog/api/query.py +++ b/posthog/api/query.py @@ -1,9 +1,11 @@ import json -from typing import Dict +from typing import Dict, cast +import posthoganalytics from django.http import HttpResponse, JsonResponse from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import OpenApiParameter +from pydantic import BaseModel from rest_framework import viewsets from rest_framework.exceptions import ValidationError from rest_framework.permissions import IsAuthenticated @@ -11,11 +13,13 @@ from posthog.api.documentation import extend_schema from posthog.api.routing import StructuredViewSetMixin -from posthog.models import Team +from posthog.cloud_utils import is_cloud +from posthog.hogql.query import execute_hogql_query +from posthog.models import Team, User from posthog.models.event.query_event_list import run_events_query from posthog.permissions import ProjectMembershipNecessaryPermissions, TeamMemberAccessPermission from posthog.rate_limit import ClickHouseBurstRateThrottle, ClickHouseSustainedRateThrottle -from posthog.schema import EventsQuery +from posthog.schema import EventsQuery, HogQLQuery class QueryViewSet(StructuredViewSetMixin, viewsets.ViewSet): @@ -33,13 +37,29 @@ class QueryViewSet(StructuredViewSetMixin, viewsets.ViewSet): ) def list(self, request: Request, **kw) -> HttpResponse: query_json = self._query_json_from_request(request) - query_result = process_query(self.team, query_json) - return JsonResponse(query_result) + return self._process_query(self.team, query_json) def post(self, request, *args, **kwargs): query_json = request.data - query_result = process_query(self.team, query_json) - return JsonResponse(query_result) + return self._process_query(self.team, query_json) + + def _process_query(self, team: Team, query_json: Dict) -> JsonResponse: + try: + query_kind = query_json.get("kind") + if query_kind == "EventsQuery": + events_query = EventsQuery.parse_obj(query_json) + response = run_events_query(query=events_query, team=team) + return self._response_to_json_response(response) + elif query_kind == "HogQLQuery": + if not self._is_hogql_enabled(): + return JsonResponse({"error": "HogQL is not enabled for this organization"}, status=400) + hogql_query = HogQLQuery.parse_obj(query_json) + response = execute_hogql_query(query=hogql_query.query, team=team) + return self._response_to_json_response(response) + else: + raise ValidationError("Unsupported query kind: %s" % query_kind) + except Exception as e: + return JsonResponse({"error": str(e)}, status=400) def _query_json_from_request(self, request): if request.method == "POST": @@ -65,21 +85,27 @@ def parsing_error(ex): raise ValidationError("Invalid JSON: %s" % (str(error_main))) return query + def _response_to_json_response(self, response: BaseModel) -> JsonResponse: + dict = {} + for key in response.__fields__.keys(): + dict[key] = getattr(response, key) + return JsonResponse(dict) + + def _is_hogql_enabled(self) -> bool: + # enabled for all self-hosted + if not is_cloud(): + return True -def process_query(team: Team, query_json: Dict) -> Dict: - query_kind = query_json.get("kind") - if query_kind == "EventsQuery": - query = EventsQuery.parse_obj(query_json) - query_result = run_events_query( - team=team, - query=query, + # on PostHog Cloud, use the feature flag + user: User = cast(User, self.request.user) + return posthoganalytics.feature_enabled( + "hogql-queries", + str(user.distinct_id), + person_properties={"email": user.email}, + groups={"organization": str(self.organization_id)}, + group_properties={ + "organization": {"id": str(self.organization_id), "created_at": self.organization.created_at} + }, + only_evaluate_locally=True, + send_feature_flag_events=False, ) - # :KLUDGE: Calling `query_result.dict()` without the following deconstruction fails with a cryptic error - return { - "columns": query_result.columns, - "types": query_result.types, - "results": query_result.results, - "hasMore": query_result.hasMore, - } - else: - raise ValidationError("Unsupported query kind: %s" % query_kind) diff --git a/posthog/api/test/__snapshots__/test_query.ambr b/posthog/api/test/__snapshots__/test_query.ambr index 9b049fa990e6c..158a059e51a88 100644 --- a/posthog/api/test/__snapshots__/test_query.ambr +++ b/posthog/api/test/__snapshots__/test_query.ambr @@ -92,6 +92,30 @@ LIMIT 101 ' --- +# name: TestQuery.test_full_hogql_query + ' + /* user_id:0 request:_snapshot_ */ + SELECT event, + distinct_id, + replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '') + FROM events + WHERE equals(team_id, 68) + ORDER BY timestamp ASC + LIMIT 1000 + ' +--- +# name: TestQuery.test_full_hogql_query_materialized + ' + /* user_id:0 request:_snapshot_ */ + SELECT event, + distinct_id, + mat_key + FROM events + WHERE equals(team_id, 69) + ORDER BY timestamp ASC + LIMIT 1000 + ' +--- # name: TestQuery.test_hogql_property_filter ' /* user_id:0 request:_snapshot_ */ diff --git a/posthog/api/test/test_query.py b/posthog/api/test/test_query.py index 4c65e8ee5eec6..4b164743f5580 100644 --- a/posthog/api/test/test_query.py +++ b/posthog/api/test/test_query.py @@ -1,7 +1,15 @@ from freezegun import freeze_time from rest_framework import status -from posthog.schema import EventPropertyFilter, EventsQuery, HogQLPropertyFilter, PersonPropertyFilter, PropertyOperator +from posthog.schema import ( + EventPropertyFilter, + EventsQuery, + HogQLPropertyFilter, + HogQLQuery, + HogQLQueryResponse, + PersonPropertyFilter, + PropertyOperator, +) from posthog.test.base import ( APIBaseTest, ClickhouseTestMixin, @@ -256,3 +264,40 @@ def test_property_filter_aggregations(self): query.where = ["count() > 1"] response = self.client.post(f"/api/projects/{self.team.id}/query/", query.dict()).json() self.assertEqual(len(response["results"]), 1) + + @also_test_with_materialized_columns(event_properties=["key"]) + @snapshot_clickhouse_queries + def test_full_hogql_query(self): + with freeze_time("2020-01-10 12:00:00"): + _create_person( + properties={"email": "tom@posthog.com"}, + distinct_ids=["2", "some-random-uid"], + team=self.team, + immediate=True, + ) + _create_event(team=self.team, event="sign up", distinct_id="2", properties={"key": "test_val1"}) + with freeze_time("2020-01-10 12:11:00"): + _create_event(team=self.team, event="sign out", distinct_id="2", properties={"key": "test_val2"}) + with freeze_time("2020-01-10 12:12:00"): + _create_event(team=self.team, event="sign out", distinct_id="3", properties={"key": "test_val2"}) + with freeze_time("2020-01-10 12:13:00"): + _create_event( + team=self.team, event="sign out", distinct_id="4", properties={"key": "test_val3", "path": "a/b/c"} + ) + flush_persons_and_events() + + with freeze_time("2020-01-10 12:14:00"): + query = HogQLQuery(query="select event, distinct_id, properties.key from events order by timestamp") + api_response = self.client.post(f"/api/projects/{self.team.id}/query/", query.dict()).json() + query.response = HogQLQueryResponse.parse_obj(api_response) + + self.assertEqual(query.response.results and len(query.response.results), 4) + self.assertEqual( + query.response.results, + [ + ["sign up", "2", "test_val1"], + ["sign out", "2", "test_val2"], + ["sign out", "3", "test_val2"], + ["sign out", "4", "test_val3"], + ], + ) diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 5c77705250bf8..20930c183a06b 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -8,7 +8,7 @@ from posthog.hogql.database import Table, database from posthog.hogql.print_string import print_clickhouse_identifier, print_hogql_identifier from posthog.hogql.resolver import ResolverException, lookup_field_by_name, resolve_symbols -from posthog.hogql.visitor import Visitor, clone_expr +from posthog.hogql.visitor import Visitor from posthog.models.property import PropertyName, TableColumn @@ -35,8 +35,6 @@ def print_ast( """Print an AST into a string. Does not modify the node.""" symbol = stack[-1].symbol if stack else None - # make a clean copy of the object - node = clone_expr(node) # resolve symbols resolve_symbols(node, symbol) diff --git a/posthog/hogql/query.py b/posthog/hogql/query.py index e1979d05706d3..45251b69a610b 100644 --- a/posthog/hogql/query.py +++ b/posthog/hogql/query.py @@ -56,7 +56,13 @@ def execute_hogql_query( query_type=query_type, workload=workload, ) - print_columns = [print_ast(col, HogQLContext(), "hogql") for col in select_query.select] + print_columns = [] + for node in select_query.select: + if isinstance(node, ast.Alias): + print_columns.append(node.alias) + else: + print_columns.append(print_ast(node=node, context=hogql_context, dialect="hogql", stack=[select_query])) + return HogQLQueryResponse( query=query, hogql=hogql, diff --git a/posthog/queries/trends/test/__snapshots__/test_formula.ambr b/posthog/queries/trends/test/__snapshots__/test_formula.ambr index ca045a9c8901e..945f56de6aad8 100644 --- a/posthog/queries/trends/test/__snapshots__/test_formula.ambr +++ b/posthog/queries/trends/test/__snapshots__/test_formula.ambr @@ -153,7 +153,7 @@ CROSS JOIN (SELECT breakdown_value FROM - (SELECT [36, 0] as breakdown_value) ARRAY + (SELECT [37, 0] as breakdown_value) ARRAY JOIN breakdown_value) as sec ORDER BY breakdown_value, day_start @@ -163,7 +163,7 @@ FROM events e INNER JOIN (SELECT distinct_id, - 36 as value + 37 as value FROM (SELECT distinct_id, argMax(person_id, version) as person_id @@ -222,7 +222,7 @@ CROSS JOIN (SELECT breakdown_value FROM - (SELECT [36, 0] as breakdown_value) ARRAY + (SELECT [37, 0] as breakdown_value) ARRAY JOIN breakdown_value) as sec ORDER BY breakdown_value, day_start @@ -232,7 +232,7 @@ FROM events e INNER JOIN (SELECT distinct_id, - 36 as value + 37 as value FROM (SELECT distinct_id, argMax(person_id, version) as person_id diff --git a/posthog/schema.py b/posthog/schema.py index cea0aadae6af7..336ec2418a72f 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -240,6 +240,18 @@ class FunnelCorrelationPersonConverted1(str, Enum): false = "false" +class HogQLQueryResponse(BaseModel): + class Config: + extra = Extra.forbid + + clickhouse: Optional[str] = None + columns: Optional[List] = None + hogql: Optional[str] = None + query: Optional[str] = None + results: Optional[List] = None + types: Optional[List] = None + + class InsightType(str, Enum): TRENDS = "TRENDS" STICKINESS = "STICKINESS" @@ -538,6 +550,15 @@ class Config: value: Optional[Union[str, float, List[Union[str, float]]]] = None +class HogQLQuery(BaseModel): + class Config: + extra = Extra.forbid + + kind: str = Field("HogQLQuery", const=True) + query: str + response: Optional[HogQLQueryResponse] = Field(None, description="Cached query response") + + class LifecycleFilter(BaseModel): class Config: extra = Extra.forbid @@ -827,11 +848,12 @@ class Config: None, description="Show warning about live events being buffered max 60 sec (default: false)" ) showExport: Optional[bool] = Field(None, description="Show the export button") + showHogQLEditor: Optional[bool] = Field(None, description="Include a HogQL query editor above HogQL tables") showPropertyFilter: Optional[bool] = Field(None, description="Include a property filter above the table") showReload: Optional[bool] = Field(None, description="Show a reload button") showSavedQueries: Optional[bool] = Field(None, description="Shows a list of saved queries") showSearch: Optional[bool] = Field(None, description="Include a free text search field (PersonsNode only)") - source: Union[EventsNode, EventsQuery, PersonsNode, RecentPerformancePageViewNode] = Field( + source: Union[EventsNode, EventsQuery, PersonsNode, RecentPerformancePageViewNode, HogQLQuery] = Field( ..., description="Source of the events" ) @@ -1501,7 +1523,7 @@ class Model(BaseModel): LifecycleQuery, RecentPerformancePageViewNode, TimeToSeeDataSessionsQuery, - Union[EventsNode, EventsQuery, ActionsNode, PersonsNode], + Union[EventsNode, EventsQuery, ActionsNode, PersonsNode, HogQLQuery], ] diff --git a/webpack.config.js b/webpack.config.js index 33228a392c92f..dc958a503b5c2 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -65,7 +65,7 @@ function createEntry(entry) { rules: [ { test: /\.[jt]sx?$/, - exclude: /(node_modules)/, + exclude: /node_modules(?!(\/\.pnpm|)(\/sql-formatter))/, use: { loader: 'babel-loader', },