From e0bbd342ed89d70c8690cd83f1688ccfdda18c2c Mon Sep 17 00:00:00 2001 From: Robbie Coomber Date: Fri, 13 Oct 2023 14:22:26 +0100 Subject: [PATCH 01/13] Add web analytics tabs --- .../src/scenes/web-analytics/WebDashboard.tsx | 34 +++ .../scenes/web-analytics/webAnalyticsLogic.ts | 193 ++++++++++++++++-- frontend/src/styles/utilities.scss | 4 + 3 files changed, 210 insertions(+), 21 deletions(-) diff --git a/frontend/src/scenes/web-analytics/WebDashboard.tsx b/frontend/src/scenes/web-analytics/WebDashboard.tsx index b081446350627..da7fd3a9e6202 100644 --- a/frontend/src/scenes/web-analytics/WebDashboard.tsx +++ b/frontend/src/scenes/web-analytics/WebDashboard.tsx @@ -96,6 +96,40 @@ export const WebAnalyticsDashboard = (): JSX.Element => { ) + } else if ('tabs' in tile) { + const { tabs, activeTabId, layout, setTabId } = tile + const tab = tabs.find((t) => t.id === activeTabId) + if (!tab) { + return null + } + const { query, title } = tab + return ( +
+
+ {

{title}

} +
+ {/* TODO switch to a select if more than 3 */} + {tabs.map(({ id, linkText }) => ( + setTabId(id)} + > + {linkText} + + ))} +
+
+ +
+ ) } else { return null } diff --git a/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts b/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts index 8df1288047610..729292892c68b 100644 --- a/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts +++ b/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts @@ -20,15 +20,35 @@ interface QueryTile extends BaseTile { } interface TabsTile extends BaseTile { + activeTabId: string + setTabId: (id: string) => void tabs: { + id: string title: string linkText: string query: QuerySchema - } + }[] } export type WebDashboardTile = QueryTile | TabsTile +export enum SourceTab { + REFERRING_DOMAIN = 'REFERRING_DOMAIN', + UTM_SOURCE = 'UTM_SOURCE', + UTM_CAMPAIGN = 'UTM_CAMPAIGN', +} + +export enum DeviceTab { + BROWSER = 'BROWSER', + OS = 'OS', + DEVICE_TYPE = 'DEVICE_TYPE', +} + +export enum PathTab { + PATH = 'PATH', + INITIAL_PATH = 'INITIAL_PATH', +} + export const initialWebAnalyticsFilter = [] as WebAnalyticsPropertyFilters export const webAnalyticsLogic = kea([ @@ -37,6 +57,15 @@ export const webAnalyticsLogic = kea([ actions({ setWebAnalyticsFilters: (webAnalyticsFilters: WebAnalyticsPropertyFilters) => ({ webAnalyticsFilters }), togglePropertyFilter: (key: string, value: string) => ({ key, value }), + setSourceTab: (tab: string) => ({ + tab, + }), + setDeviceTab: (tab: string) => ({ + tab, + }), + setPathTab: (tab: string) => ({ + tab, + }), }), reducers({ webAnalyticsFilters: [ @@ -86,11 +115,29 @@ export const webAnalyticsLogic = kea([ }, }, ], + sourceTab: [ + SourceTab.REFERRING_DOMAIN as string, + { + setSourceTab: (_, { tab }) => tab, + }, + ], + deviceTab: [ + DeviceTab.BROWSER as string, + { + setDeviceTab: (_, { tab }) => tab, + }, + ], + pathTab: [ + PathTab.PATH as string, + { + setPathTab: (_, { tab }) => tab, + }, + ], }), - selectors({ + selectors(({ actions }) => ({ tiles: [ - (s) => [s.webAnalyticsFilters], - (webAnalyticsFilters): WebDashboardTile[] => [ + (s) => [s.webAnalyticsFilters, s.pathTab, s.deviceTab, s.sourceTab], + (webAnalyticsFilters, pathTab, deviceTab, sourceTab): WebDashboardTile[] => [ { layout: { colSpan: 12, @@ -105,32 +152,136 @@ export const webAnalyticsLogic = kea([ }, }, { - title: 'Pages', layout: { colSpan: 6, }, - query: { - full: true, - kind: NodeKind.DataTableNode, - source: { - kind: NodeKind.WebTopPagesQuery, - properties: webAnalyticsFilters, + activeTabId: pathTab, + setTabId: actions.setPathTab, + tabs: [ + { + id: PathTab.PATH, + title: 'Top Paths', + linkText: 'Path', + query: { + full: true, + kind: NodeKind.DataTableNode, + source: { + kind: NodeKind.WebTopPagesQuery, + properties: webAnalyticsFilters, + }, + }, }, - }, + { + id: PathTab.INITIAL_PATH, + title: 'Top Initial Paths', + linkText: 'Initial Path', + query: { + full: true, + kind: NodeKind.DataTableNode, + source: { + kind: NodeKind.WebTopPagesQuery, // TODO + properties: webAnalyticsFilters, + }, + }, + }, + ], }, { - title: 'Traffic Sources', layout: { colSpan: 6, }, - query: { - full: true, - kind: NodeKind.DataTableNode, - source: { - kind: NodeKind.WebTopSourcesQuery, - properties: webAnalyticsFilters, + activeTabId: sourceTab, + setTabId: actions.setSourceTab, + tabs: [ + { + id: SourceTab.REFERRING_DOMAIN, + title: 'Top Referrers', + linkText: 'Referrer', + query: { + full: true, + kind: NodeKind.DataTableNode, + source: { + kind: NodeKind.WebTopSourcesQuery, // TODO + properties: webAnalyticsFilters, + }, + }, + }, + { + id: SourceTab.UTM_SOURCE, + title: 'Top Sources', + linkText: 'UTM Source', + query: { + full: true, + kind: NodeKind.DataTableNode, + source: { + kind: NodeKind.WebTopSourcesQuery, + properties: webAnalyticsFilters, + }, + }, }, + { + id: SourceTab.UTM_CAMPAIGN, + title: 'Top Campaigns', + linkText: 'UTM Campaign', + query: { + full: true, + kind: NodeKind.DataTableNode, + source: { + kind: NodeKind.WebTopSourcesQuery, // TODO + properties: webAnalyticsFilters, + }, + }, + }, + ], + }, + { + layout: { + colSpan: 6, }, + activeTabId: deviceTab, + setTabId: actions.setDeviceTab, + + tabs: [ + { + id: DeviceTab.BROWSER, + title: 'Top Browsers', + linkText: 'Browser', + query: { + full: true, + kind: NodeKind.DataTableNode, + source: { + kind: NodeKind.WebTopSourcesQuery, // TODO + properties: webAnalyticsFilters, + }, + }, + }, + { + id: DeviceTab.OS, + title: 'Top OSs', + linkText: 'OS', + query: { + full: true, + kind: NodeKind.DataTableNode, + source: { + kind: NodeKind.WebTopSourcesQuery, // TODO + properties: webAnalyticsFilters, + }, + }, + }, + { + id: DeviceTab.DEVICE_TYPE, + title: 'Top Device Types', + linkText: 'Device Type', + query: { + full: true, + kind: NodeKind.DataTableNode, + source: { + kind: NodeKind.WebTopSourcesQuery, // TODO + properties: webAnalyticsFilters, + }, + }, + }, + ], }, { title: 'Unique users', @@ -164,7 +315,7 @@ export const webAnalyticsLogic = kea([ }, }, { - title: 'User locations', + title: 'World Map (Unique Users)', layout: { colSpan: 6, }, @@ -196,7 +347,7 @@ export const webAnalyticsLogic = kea([ }, ], ], - }), + })), sharedListeners(() => ({})), listeners(() => ({})), ]) diff --git a/frontend/src/styles/utilities.scss b/frontend/src/styles/utilities.scss index 05cff33f5d4d3..b441a308aeaeb 100644 --- a/frontend/src/styles/utilities.scss +++ b/frontend/src/styles/utilities.scss @@ -406,6 +406,10 @@ $decorations: underline, overline, line-through, no-underline; } } +.text-inherit { + color: inherit; +} + @each $name, $hex in $colors { .hover\:text-#{$name}:hover { color: $hex; From 27b2af60b5ec17df97d7393009b3cc0c42632d26 Mon Sep 17 00:00:00 2001 From: Robbie Coomber Date: Sat, 14 Oct 2023 08:29:39 +0100 Subject: [PATCH 02/13] Add stats table query runner and use it for pathname --- frontend/src/lib/utils.tsx | 48 +++++++++ .../queries/nodes/DataTable/queryFeatures.ts | 4 +- .../queries/nodes/DataTable/renderColumn.tsx | 2 +- .../nodes/DataTable/renderColumnMeta.tsx | 8 +- frontend/src/queries/schema.json | 70 +++++++++++++ frontend/src/queries/schema.ts | 34 ++++++- frontend/src/queries/utils.ts | 5 + .../src/scenes/web-analytics/WebDashboard.tsx | 81 ++++++++++----- .../scenes/web-analytics/webAnalyticsLogic.ts | 18 +--- posthog/api/query.py | 1 + posthog/hogql_queries/query_runner.py | 4 + posthog/hogql_queries/web_analytics/ctes.py | 44 +++++++++ .../web_analytics/stats_table.py | 98 +++++++++++++++++++ .../web_analytics_query_runner.py | 2 + posthog/schema.py | 27 +++++ 15 files changed, 402 insertions(+), 44 deletions(-) create mode 100644 posthog/hogql_queries/web_analytics/stats_table.py diff --git a/frontend/src/lib/utils.tsx b/frontend/src/lib/utils.tsx index 1df911788009a..03147c82c8156 100644 --- a/frontend/src/lib/utils.tsx +++ b/frontend/src/lib/utils.tsx @@ -1603,6 +1603,54 @@ export function isNotNil(arg: T): arg is Exclude { return arg !== null && arg !== undefined } +/** An error signaling that a value of type `never` in TypeScript was used unexpectedly at runtime. + * + * Useful for type-narrowing, will give a compile-time error if the type of x is not `never`. + * See the example below where it catches a missing branch at compile-time. + * + * @example + * + * enum MyEnum { + * a, + * b, + * } + * + * function handleEnum(x: MyEnum) { + * switch (x) { + * case MyEnum.a: + * return + * // missing branch + * default: + * throw new UnexpectedNeverError(x) // TS2345: Argument of type MyEnum is not assignable to parameter of type never + * } + * } + * + * function handleEnum(x: MyEnum) { + * switch (x) { + * case MyEnum.a: + * return + * case MyEnum.b: + * return + * default: + * throw new UnexpectedNeverError(x) // no type error + * } + * } + * + */ +export class UnexpectedNeverError extends Error { + constructor(x: never, message?: string) { + message = message ?? 'Unexpected never: ' + String(x) + super(message) + + // restore prototype chain, which is broken by Error + // see https://stackoverflow.com/questions/41102060/typescript-extending-error-class + const actualProto = new.target.prototype + if (Object.setPrototypeOf) { + Object.setPrototypeOf(this, actualProto) + } + } +} + export function calculateDays(timeValue: number, timeUnit: TimeUnitType): number { if (timeUnit === TimeUnitType.Year) { return timeValue * 365 diff --git a/frontend/src/queries/nodes/DataTable/queryFeatures.ts b/frontend/src/queries/nodes/DataTable/queryFeatures.ts index 4813def3438bd..03d327390dff1 100644 --- a/frontend/src/queries/nodes/DataTable/queryFeatures.ts +++ b/frontend/src/queries/nodes/DataTable/queryFeatures.ts @@ -4,6 +4,7 @@ import { isPersonsNode, isPersonsQuery, isWebOverviewStatsQuery, + isWebStatsTableQuery, isWebTopClicksQuery, isWebTopPagesQuery, isWebTopSourcesQuery, @@ -59,7 +60,8 @@ export function getQueryFeatures(query: Node): Set { isWebOverviewStatsQuery(query) || isWebTopSourcesQuery(query) || isWebTopPagesQuery(query) || - isWebTopClicksQuery(query) + isWebTopClicksQuery(query) || + isWebStatsTableQuery(query) ) { features.add(QueryFeature.columnsInResponse) features.add(QueryFeature.resultIsArrayOfArrays) diff --git a/frontend/src/queries/nodes/DataTable/renderColumn.tsx b/frontend/src/queries/nodes/DataTable/renderColumn.tsx index 422fc0934da03..7077620b18abe 100644 --- a/frontend/src/queries/nodes/DataTable/renderColumn.tsx +++ b/frontend/src/queries/nodes/DataTable/renderColumn.tsx @@ -229,7 +229,7 @@ export function renderColumn( } else if (key.startsWith('context.columns.')) { const columnName = trimQuotes(key.substring(16)) // 16 = "context.columns.".length const Component = context?.columns?.[columnName]?.render - return Component ? : '' + return Component ? : '' } else if (key === 'id' && (isPersonsNode(query.source) || isPersonsQuery(query.source))) { return ( } else if (key.startsWith('context.columns.')) { const column = trimQuotes(key.substring(16)) - title = context?.columns?.[column]?.title ?? column.replace('_', ' ') + const queryContextColumn = context?.columns?.[column] + const Component = queryContextColumn?.renderTitle + title = Component ? ( + + ) : ( + queryContextColumn?.title ?? column.replace('_', ' ') + ) } else if (key === 'person.$delete') { title = '' width = 0 diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index 0647d0debf9d8..091496fd7c42f 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -102,6 +102,9 @@ { "$ref": "#/definitions/WebOverviewStatsQuery" }, + { + "$ref": "#/definitions/WebStatsTableQuery" + }, { "$ref": "#/definitions/WebTopSourcesQuery" }, @@ -411,6 +414,9 @@ { "$ref": "#/definitions/WebOverviewStatsQuery" }, + { + "$ref": "#/definitions/WebStatsTableQuery" + }, { "$ref": "#/definitions/WebTopSourcesQuery" }, @@ -2474,6 +2480,70 @@ "required": ["results"], "type": "object" }, + "WebStatsBreakdown": { + "const": "Page", + "type": "string" + }, + "WebStatsTableQuery": { + "additionalProperties": false, + "properties": { + "breakdownBy": { + "$ref": "#/definitions/WebStatsBreakdown" + }, + "dateRange": { + "$ref": "#/definitions/DateRange" + }, + "kind": { + "const": "WebStatsTableQuery", + "type": "string" + }, + "properties": { + "$ref": "#/definitions/WebAnalyticsPropertyFilters" + }, + "response": { + "$ref": "#/definitions/WebStatsTableQueryResponse" + } + }, + "required": ["kind", "properties", "breakdownBy"], + "type": "object" + }, + "WebStatsTableQueryResponse": { + "additionalProperties": false, + "properties": { + "columns": { + "items": {}, + "type": "array" + }, + "hogql": { + "type": "string" + }, + "is_cached": { + "type": "boolean" + }, + "last_refresh": { + "type": "string" + }, + "next_allowed_client_refresh": { + "type": "string" + }, + "results": { + "items": {}, + "type": "array" + }, + "timings": { + "items": { + "$ref": "#/definitions/QueryTiming" + }, + "type": "array" + }, + "types": { + "items": {}, + "type": "array" + } + }, + "required": ["results"], + "type": "object" + }, "WebTopClicksQuery": { "additionalProperties": false, "properties": { diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index be055c28a899c..a2a7bcf19ea4d 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -65,6 +65,7 @@ export enum NodeKind { WebTopSourcesQuery = 'WebTopSourcesQuery', WebTopPagesQuery = 'WebTopPagesQuery', WebTopClicksQuery = 'WebTopClicksQuery', + WebStatsTableQuery = 'WebStatsTableQuery', // Time to see data TimeToSeeDataSessionsQuery = 'TimeToSeeDataSessionsQuery', @@ -86,6 +87,7 @@ export type AnyDataNode = | HogQLQuery | HogQLMetadata | WebOverviewStatsQuery + | WebStatsTableQuery | WebTopSourcesQuery | WebTopClicksQuery | WebTopPagesQuery @@ -314,6 +316,7 @@ export interface DataTableNode extends Node, DataTableNodeViewProps { | HogQLQuery | TimeToSeeDataSessionsQuery | WebOverviewStatsQuery + | WebStatsTableQuery | WebTopSourcesQuery | WebTopClicksQuery | WebTopPagesQuery @@ -594,6 +597,24 @@ export interface WebTopPagesQueryResponse extends QueryResponse { columns?: unknown[] } +export enum WebStatsBreakdown { + Page = 'Page', + // InitialPage = 'InitialPage', + // ExitPage = 'ExitPage' +} +export interface WebStatsTableQuery extends WebAnalyticsQueryBase { + kind: NodeKind.WebStatsTableQuery + properties: WebAnalyticsPropertyFilters + breakdownBy: WebStatsBreakdown + response?: WebStatsTableQueryResponse +} +export interface WebStatsTableQueryResponse extends QueryResponse { + results: unknown[] + types?: unknown[] + columns?: unknown[] + hogql?: string +} + export type InsightQueryNode = | TrendsQuery | FunnelsQuery @@ -713,9 +734,20 @@ export interface QueryContext { emptyStateDetail?: string } -export type QueryContextColumnComponent = ComponentType<{ record: any; columnName: string; value: any }> +export type QueryContextColumnTitleComponent = ComponentType<{ + columnName: string + query: DataTableNode +}> + +export type QueryContextColumnComponent = ComponentType<{ + record: any + columnName: string + value: any + query: DataTableNode +}> interface QueryContextColumn { title?: string + renderTitle?: QueryContextColumnTitleComponent render?: QueryContextColumnComponent } diff --git a/frontend/src/queries/utils.ts b/frontend/src/queries/utils.ts index fa673e53085a7..442b540e2b25b 100644 --- a/frontend/src/queries/utils.ts +++ b/frontend/src/queries/utils.ts @@ -32,6 +32,7 @@ import { WebOverviewStatsQuery, PersonsQuery, HogQLMetadata, + WebStatsTableQuery, } from '~/queries/schema' import { TaxonomicFilterGroupType, TaxonomicFilterValue } from 'lib/components/TaxonomicFilter/types' import { dayjs } from 'lib/dayjs' @@ -117,6 +118,10 @@ export function isWebOverviewStatsQuery(node?: Node | null): node is WebOverview return node?.kind === NodeKind.WebOverviewStatsQuery } +export function isWebStatsTableQuery(node?: Node | null): node is WebStatsTableQuery { + return node?.kind === NodeKind.WebStatsTableQuery +} + export function isWebTopSourcesQuery(node?: Node | null): node is WebTopSourcesQuery { return node?.kind === NodeKind.WebTopSourcesQuery } diff --git a/frontend/src/scenes/web-analytics/WebDashboard.tsx b/frontend/src/scenes/web-analytics/WebDashboard.tsx index da7fd3a9e6202..a473cd9e72f6d 100644 --- a/frontend/src/scenes/web-analytics/WebDashboard.tsx +++ b/frontend/src/scenes/web-analytics/WebDashboard.tsx @@ -4,8 +4,15 @@ import { webAnalyticsLogic } from 'scenes/web-analytics/webAnalyticsLogic' import { PropertyFilters } from 'lib/components/PropertyFilters/PropertyFilters' import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types' import { isEventPropertyFilter } from 'lib/components/PropertyFilters/utils' -import { QueryContext, QueryContextColumnComponent } from '~/queries/schema' +import { + NodeKind, + QueryContext, + QueryContextColumnComponent, + QueryContextColumnTitleComponent, + WebStatsBreakdown, +} from '~/queries/schema' import { useCallback } from 'react' +import { UnexpectedNeverError } from 'lib/utils' const PercentageCell: QueryContextColumnComponent = ({ value }) => { if (typeof value === 'number') { @@ -27,16 +34,35 @@ const NumericCell: QueryContextColumnComponent = ({ value }) => { ) } -const ClickablePropertyCell: QueryContextColumnComponent = (props) => { - const { columnName, value } = props +const BreakdownValueTitle: QueryContextColumnTitleComponent = (props) => { + const { query } = props + const { source } = query + if (source.kind !== NodeKind.WebStatsTableQuery) { + return null + } + const { breakdownBy } = source + switch (breakdownBy) { + case WebStatsBreakdown.Page: + return <>Path + default: + throw new UnexpectedNeverError(breakdownBy) + } +} +const BreakdownValueCell: QueryContextColumnComponent = (props) => { + const { value, query } = props const { togglePropertyFilter } = useActions(webAnalyticsLogic) + const { source } = query + if (source.kind !== NodeKind.WebStatsTableQuery) { + return null + } + const { breakdownBy } = source let propertyName: string - switch (columnName) { - case 'pathname': + switch (breakdownBy) { + case WebStatsBreakdown.Page: propertyName = '$pathname' break default: - return null + throw new UnexpectedNeverError(breakdownBy) } const onClick = useCallback(() => { @@ -48,14 +74,14 @@ const ClickablePropertyCell: QueryContextColumnComponent = (props) => { const queryContext: QueryContext = { columns: { + breakdown_value: { + renderTitle: BreakdownValueTitle, + render: BreakdownValueCell, + }, bounce_rate: { title: 'Bounce Rate', render: PercentageCell, }, - pathname: { - title: 'Path', - render: ClickablePropertyCell, - }, views: { title: 'Views', render: NumericCell, @@ -112,22 +138,27 @@ export const WebAnalyticsDashboard = (): JSX.Element => { >
{

{title}

} -
- {/* TODO switch to a select if more than 3 */} - {tabs.map(({ id, linkText }) => ( - setTabId(id)} - > - {linkText} - - ))} -
+ {tabs.length > 1 && ( +
+ {/* TODO switch to a select if more than 3 */} + {tabs.map(({ id, linkText }) => ( + setTabId(id)} + > + {linkText} + + ))} +
+ )}
- + {/* Setting key forces the component to be recreated when the tab changes */} + ) } else { diff --git a/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts b/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts index 729292892c68b..5fe0ef07b72c2 100644 --- a/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts +++ b/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts @@ -1,7 +1,7 @@ import { actions, connect, kea, listeners, path, reducers, selectors, sharedListeners } from 'kea' import type { webAnalyticsLogicType } from './webAnalyticsLogicType' -import { NodeKind, QuerySchema, WebAnalyticsPropertyFilters } from '~/queries/schema' +import { NodeKind, QuerySchema, WebAnalyticsPropertyFilters, WebStatsBreakdown } from '~/queries/schema' import { BaseMathType, ChartDisplayType, EventPropertyFilter, PropertyFilterType, PropertyOperator } from '~/types' import { isNotNil } from 'lib/utils' @@ -166,21 +166,9 @@ export const webAnalyticsLogic = kea([ full: true, kind: NodeKind.DataTableNode, source: { - kind: NodeKind.WebTopPagesQuery, - properties: webAnalyticsFilters, - }, - }, - }, - { - id: PathTab.INITIAL_PATH, - title: 'Top Initial Paths', - linkText: 'Initial Path', - query: { - full: true, - kind: NodeKind.DataTableNode, - source: { - kind: NodeKind.WebTopPagesQuery, // TODO + kind: NodeKind.WebStatsTableQuery, properties: webAnalyticsFilters, + breakdownBy: WebStatsBreakdown.Page, }, }, }, diff --git a/posthog/api/query.py b/posthog/api/query.py index c93594dbb463b..41b4375436f8c 100644 --- a/posthog/api/query.py +++ b/posthog/api/query.py @@ -45,6 +45,7 @@ "WebTopSourcesQuery", "WebTopClicksQuery", "WebTopPagesQuery", + "WebStatsTableQuery", ] QUERY_WITH_RUNNER_NO_CACHE = [ "EventsQuery", diff --git a/posthog/hogql_queries/query_runner.py b/posthog/hogql_queries/query_runner.py index 4677774a65615..59bce744fd26b 100644 --- a/posthog/hogql_queries/query_runner.py +++ b/posthog/hogql_queries/query_runner.py @@ -122,6 +122,10 @@ def get_query_runner( from .web_analytics.top_pages import WebTopPagesQueryRunner return WebTopPagesQueryRunner(query=query, team=team, timings=timings) + if kind == "WebStatsTableQuery": + from .web_analytics.stats_table import WebStatsTableQueryRunner + + return WebStatsTableQueryRunner(query=query, team=team, timings=timings) raise ValueError(f"Can't get a runner for an unknown query kind: {kind}") diff --git a/posthog/hogql_queries/web_analytics/ctes.py b/posthog/hogql_queries/web_analytics/ctes.py index 0e20fcdc3c7dc..4981c2389f6b5 100644 --- a/posthog/hogql_queries/web_analytics/ctes.py +++ b/posthog/hogql_queries/web_analytics/ctes.py @@ -73,3 +73,47 @@ AND ({pathname_scroll_where}) GROUP BY $pathname """ + +COUNTS_CTE = """ +SELECT + {breakdown_by} AS breakdown_value, + count() as total_pageviews, + uniq(events.person_id) as unique_visitors +FROM + events +WHERE + (event = '$pageview') + AND ({counts_where}) + GROUP BY breakdown_value +""" + +BOUNCE_RATE_CTE = """ +SELECT + breakdown_value, + avg(session.is_bounce) as bounce_rate +FROM ( + SELECT + events.properties.`$session_id` AS session_id, + min(events.timestamp) AS min_timestamp, + max(events.timestamp) AS max_timestamp, + dateDiff('second', min_timestamp, max_timestamp) AS duration_s, + countIf(events.event == '$pageview') AS num_pageviews, + countIf(events.event == '$autocapture') AS num_autocaptures, + {breakdown_by} AS breakdown_value, + + -- definition of a GA4 bounce from here https://support.google.com/analytics/answer/12195621?hl=en + (num_autocaptures == 0 AND num_pageviews <= 1 AND duration_s < 10) AS is_bounce + FROM + events + WHERE + session_id IS NOT NULL + AND (events.event == '$pageview' OR events.event == '$autocapture' OR events.event == '$pageleave') + AND ({session_where}) + GROUP BY + events.properties.`$session_id` + HAVING + ({session_having}) +) AS session +GROUP BY + breakdown_value + """ diff --git a/posthog/hogql_queries/web_analytics/stats_table.py b/posthog/hogql_queries/web_analytics/stats_table.py new file mode 100644 index 0000000000000..a92665c2c6684 --- /dev/null +++ b/posthog/hogql_queries/web_analytics/stats_table.py @@ -0,0 +1,98 @@ +from posthog.hogql import ast +from posthog.hogql.parser import parse_select, parse_expr +from posthog.hogql.query import execute_hogql_query +from posthog.hogql_queries.web_analytics.ctes import ( + COUNTS_CTE, + BOUNCE_RATE_CTE, +) +from posthog.hogql_queries.web_analytics.web_analytics_query_runner import WebAnalyticsQueryRunner +from posthog.schema import ( + WebStatsTableQuery, + WebStatsBreakdown, + WebStatsTableQueryResponse, +) + + +class WebStatsTableQueryRunner(WebAnalyticsQueryRunner): + query: WebStatsTableQuery + query_type = WebStatsTableQuery + + def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery: + with self.timings.measure("bounce_rate_query"): + bounce_rate_query = parse_select( + BOUNCE_RATE_CTE, + timings=self.timings, + placeholders={ + "session_where": self.session_where(), + "session_having": self.session_having(), + "breakdown_by": self.bounce_breakdown(), + }, + ) + with self.timings.measure("counts_query"): + counts_query = parse_select( + COUNTS_CTE, + timings=self.timings, + placeholders={ + "counts_where": self.events_where(), + "breakdown_by": self.counts_breakdown(), + }, + ) + with self.timings.measure("top_pages_query"): + top_sources_query = parse_select( + """ +SELECT + counts.breakdown_value as "context.columns.breakdown_value", + counts.total_pageviews as "context.columns.views", + counts.unique_visitors as "context.columns.visitors", + bounce_rate.bounce_rate as "context.columns.bounce_rate" +FROM + {counts_query} AS counts +LEFT OUTER JOIN + {bounce_rate_query} AS bounce_rate +ON + counts.breakdown_value = bounce_rate.breakdown_value +ORDER BY + "context.columns.views" DESC +LIMIT 10 + """, + timings=self.timings, + placeholders={ + "counts_query": counts_query, + "bounce_rate_query": bounce_rate_query, + }, + ) + return top_sources_query + + def calculate(self): + response = execute_hogql_query( + query_type="top_sources_query", + query=self.to_query(), + team=self.team, + timings=self.timings, + ) + + return WebStatsTableQueryResponse( + columns=response.columns, + results=response.results, + timings=response.timings, + types=response.types, + hogql=response.hogql, + ) + + def counts_breakdown(self): + match self.query.breakdownBy: + case WebStatsBreakdown.Page: + return parse_expr("properties.$pathname") + case WebStatsBreakdown.InitialPage: + return parse_expr("events.properties.$set_once.$initial_pathname") + case _: + raise NotImplementedError("Breakdown not implemented") + + def bounce_breakdown(self): + match self.query.breakdownBy: + case WebStatsBreakdown.Page: + return parse_expr("any(events.properties.$set_once.$initial_pathname)") + case WebStatsBreakdown.InitialPage: + return parse_expr("any(events.properties.$set_once.$initial_pathname)") + case _: + raise NotImplementedError("Breakdown not implemented") diff --git a/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py b/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py index 9458692bb44c7..3c9c6d2b34e2a 100644 --- a/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py +++ b/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py @@ -14,6 +14,7 @@ WebTopClicksQuery, WebTopPagesQuery, WebOverviewStatsQuery, + WebStatsTableQuery, ) WebQueryNode = Union[ @@ -21,6 +22,7 @@ WebTopSourcesQuery, WebTopClicksQuery, WebTopPagesQuery, + WebStatsTableQuery, ] diff --git a/posthog/schema.py b/posthog/schema.py index 56b6eb2dbc1cf..a5902a8b55cc6 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -495,6 +495,20 @@ class WebOverviewStatsQueryResponse(BaseModel): types: Optional[List] = None +class WebStatsTableQueryResponse(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + columns: Optional[List] = None + hogql: Optional[str] = None + is_cached: Optional[bool] = None + last_refresh: Optional[str] = None + next_allowed_client_refresh: Optional[str] = None + results: List + timings: Optional[List[QueryTiming]] = None + types: Optional[List] = None + + class WebTopClicksQueryResponse(BaseModel): model_config = ConfigDict( extra="forbid", @@ -747,6 +761,17 @@ class WebOverviewStatsQuery(BaseModel): response: Optional[WebOverviewStatsQueryResponse] = None +class WebStatsTableQuery(BaseModel): + model_config = ConfigDict( + extra="forbid", + ) + breakdownBy: Literal["Page"] = "Page" + dateRange: Optional[DateRange] = None + kind: Literal["WebStatsTableQuery"] = "WebStatsTableQuery" + properties: List[EventPropertyFilter] + response: Optional[WebStatsTableQueryResponse] = None + + class WebTopClicksQuery(BaseModel): model_config = ConfigDict( extra="forbid", @@ -1167,6 +1192,7 @@ class DataTableNode(BaseModel): HogQLQuery, TimeToSeeDataSessionsQuery, WebOverviewStatsQuery, + WebStatsTableQuery, WebTopSourcesQuery, WebTopClicksQuery, WebTopPagesQuery, @@ -1449,6 +1475,7 @@ class Model(RootModel): HogQLQuery, HogQLMetadata, WebOverviewStatsQuery, + WebStatsTableQuery, WebTopSourcesQuery, WebTopClicksQuery, WebTopPagesQuery, From 1c272a2953c0a66373c04400bfb0eb66301d2ac2 Mon Sep 17 00:00:00 2001 From: Robbie Coomber Date: Sat, 14 Oct 2023 09:47:18 +0100 Subject: [PATCH 03/13] Add stats query runner, replacing page and source --- frontend/src/queries/schema.json | 2 +- frontend/src/queries/schema.ts | 5 +- .../src/scenes/web-analytics/WebDashboard.tsx | 20 +++++ .../scenes/web-analytics/webAnalyticsLogic.ts | 23 +++++- posthog/hogql/property.py | 2 +- .../web_analytics/stats_table.py | 12 +++ .../hogql_queries/web_analytics/top_pages.py | 81 ------------------- .../web_analytics/top_sources.py | 66 --------------- posthog/schema.py | 10 ++- 9 files changed, 67 insertions(+), 154 deletions(-) delete mode 100644 posthog/hogql_queries/web_analytics/top_pages.py delete mode 100644 posthog/hogql_queries/web_analytics/top_sources.py diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index 091496fd7c42f..a858f2fc13307 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -2481,7 +2481,7 @@ "type": "object" }, "WebStatsBreakdown": { - "const": "Page", + "enum": ["Page", "InitialPage", "InitialReferringDomain", "InitialUTMSource", "InitialUTMCampaign"], "type": "string" }, "WebStatsTableQuery": { diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index a2a7bcf19ea4d..631a56da30f13 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -599,8 +599,11 @@ export interface WebTopPagesQueryResponse extends QueryResponse { export enum WebStatsBreakdown { Page = 'Page', - // InitialPage = 'InitialPage', + InitialPage = 'InitialPage', // ExitPage = 'ExitPage' + InitialReferringDomain = 'InitialReferringDomain', + InitialUTMSource = 'InitialUTMSource', + InitialUTMCampaign = 'InitialUTMCampaign', } export interface WebStatsTableQuery extends WebAnalyticsQueryBase { kind: NodeKind.WebStatsTableQuery diff --git a/frontend/src/scenes/web-analytics/WebDashboard.tsx b/frontend/src/scenes/web-analytics/WebDashboard.tsx index a473cd9e72f6d..ec6404003a8ac 100644 --- a/frontend/src/scenes/web-analytics/WebDashboard.tsx +++ b/frontend/src/scenes/web-analytics/WebDashboard.tsx @@ -44,6 +44,14 @@ const BreakdownValueTitle: QueryContextColumnTitleComponent = (props) => { switch (breakdownBy) { case WebStatsBreakdown.Page: return <>Path + case WebStatsBreakdown.InitialPage: + return <>Initial Path + case WebStatsBreakdown.InitialReferringDomain: + return <>Referring Domain + case WebStatsBreakdown.InitialUTMSource: + return <>UTM Source + case WebStatsBreakdown.InitialUTMCampaign: + return <>UTM Campaign default: throw new UnexpectedNeverError(breakdownBy) } @@ -61,6 +69,18 @@ const BreakdownValueCell: QueryContextColumnComponent = (props) => { case WebStatsBreakdown.Page: propertyName = '$pathname' break + case WebStatsBreakdown.InitialPage: + propertyName = '$set_once.$initial_pathname' + break + case WebStatsBreakdown.InitialReferringDomain: + propertyName = '$set_once.$initial_referrer' + break + case WebStatsBreakdown.InitialUTMSource: + propertyName = '$set_once.$initial_utm_source' + break + case WebStatsBreakdown.InitialUTMCampaign: + propertyName = '$set_once.$initial_utm_campaign' + break default: throw new UnexpectedNeverError(breakdownBy) } diff --git a/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts b/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts index 5fe0ef07b72c2..a71590d19a9b2 100644 --- a/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts +++ b/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts @@ -172,6 +172,20 @@ export const webAnalyticsLogic = kea([ }, }, }, + { + id: PathTab.INITIAL_PATH, + title: 'Top Entry Paths', + linkText: 'Entry Path', + query: { + full: true, + kind: NodeKind.DataTableNode, + source: { + kind: NodeKind.WebStatsTableQuery, + properties: webAnalyticsFilters, + breakdownBy: WebStatsBreakdown.InitialPage, + }, + }, + }, ], }, { @@ -189,8 +203,9 @@ export const webAnalyticsLogic = kea([ full: true, kind: NodeKind.DataTableNode, source: { - kind: NodeKind.WebTopSourcesQuery, // TODO + kind: NodeKind.WebStatsTableQuery, properties: webAnalyticsFilters, + breakdownBy: WebStatsBreakdown.InitialReferringDomain, }, }, }, @@ -202,8 +217,9 @@ export const webAnalyticsLogic = kea([ full: true, kind: NodeKind.DataTableNode, source: { - kind: NodeKind.WebTopSourcesQuery, + kind: NodeKind.WebStatsTableQuery, properties: webAnalyticsFilters, + breakdownBy: WebStatsBreakdown.InitialUTMSource, }, }, }, @@ -215,8 +231,9 @@ export const webAnalyticsLogic = kea([ full: true, kind: NodeKind.DataTableNode, source: { - kind: NodeKind.WebTopSourcesQuery, // TODO + kind: NodeKind.WebStatsTableQuery, properties: webAnalyticsFilters, + breakdownBy: WebStatsBreakdown.InitialUTMCampaign, }, }, }, diff --git a/posthog/hogql/property.py b/posthog/hogql/property.py index 293005bce9822..9cb8025d6446d 100644 --- a/posthog/hogql/property.py +++ b/posthog/hogql/property.py @@ -129,7 +129,7 @@ def property_to_expr( return ast.Or(exprs=exprs) chain = ["person", "properties"] if property.type == "person" and scope != "person" else ["properties"] - field = ast.Field(chain=chain + [property.key]) + field = ast.Field(chain=chain + property.key.split(".")) if operator == PropertyOperator.is_set: return ast.CompareOperation(op=ast.CompareOperationOp.NotEq, left=field, right=ast.Constant(value=None)) diff --git a/posthog/hogql_queries/web_analytics/stats_table.py b/posthog/hogql_queries/web_analytics/stats_table.py index a92665c2c6684..273308b6ff653 100644 --- a/posthog/hogql_queries/web_analytics/stats_table.py +++ b/posthog/hogql_queries/web_analytics/stats_table.py @@ -85,6 +85,12 @@ def counts_breakdown(self): return parse_expr("properties.$pathname") case WebStatsBreakdown.InitialPage: return parse_expr("events.properties.$set_once.$initial_pathname") + case WebStatsBreakdown.InitialReferringDomain: + return parse_expr("events.properties.$set_once.$initial_referring_domain") + case WebStatsBreakdown.InitialUTMSource: + return parse_expr("events.properties.$set_once.$initial_utm_source") + case WebStatsBreakdown.InitialUTMCampaign: + return parse_expr("events.properties.$set_once.$initial_utm_campaign") case _: raise NotImplementedError("Breakdown not implemented") @@ -94,5 +100,11 @@ def bounce_breakdown(self): return parse_expr("any(events.properties.$set_once.$initial_pathname)") case WebStatsBreakdown.InitialPage: return parse_expr("any(events.properties.$set_once.$initial_pathname)") + case WebStatsBreakdown.InitialReferringDomain: + return parse_expr("any(events.properties.$set_once.$initial_referring_domain)") + case WebStatsBreakdown.InitialUTMSource: + return parse_expr("any(events.properties.$set_once.$initial_utm_source)") + case WebStatsBreakdown.InitialUTMCampaign: + return parse_expr("any(events.properties.$set_once.$initial_utm_campaign)") case _: raise NotImplementedError("Breakdown not implemented") diff --git a/posthog/hogql_queries/web_analytics/top_pages.py b/posthog/hogql_queries/web_analytics/top_pages.py deleted file mode 100644 index d219495d3aa24..0000000000000 --- a/posthog/hogql_queries/web_analytics/top_pages.py +++ /dev/null @@ -1,81 +0,0 @@ -from posthog.hogql import ast -from posthog.hogql.parser import parse_select -from posthog.hogql.query import execute_hogql_query -from posthog.hogql_queries.web_analytics.ctes import SESSION_CTE, PATHNAME_CTE, PATHNAME_SCROLL_CTE -from posthog.hogql_queries.web_analytics.web_analytics_query_runner import WebAnalyticsQueryRunner -from posthog.schema import WebTopPagesQuery, WebTopPagesQueryResponse - - -class WebTopPagesQueryRunner(WebAnalyticsQueryRunner): - query: WebTopPagesQuery - query_type = WebTopPagesQuery - - def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery: - with self.timings.measure("session_query"): - session_query = parse_select( - SESSION_CTE, - timings=self.timings, - placeholders={"session_where": self.session_where(), "session_having": self.session_having()}, - ) - with self.timings.measure("pathname_query"): - pathname_query = parse_select( - PATHNAME_CTE, timings=self.timings, placeholders={"pathname_where": self.events_where()} - ) - with self.timings.measure("pathname_scroll_query"): - pathname_scroll_query = parse_select( - PATHNAME_SCROLL_CTE, - timings=self.timings, - placeholders={"pathname_scroll_where": self.events_where()}, - ) - with self.timings.measure("top_pages_query"): - top_sources_query = parse_select( - """ -SELECT - pathname.$pathname as "context.columns.pathname", - pathname.total_pageviews as "context.columns.views", - pathname.unique_visitors as "context.columns.visitors", - bounce_rate.bounce_rate as "context.columns.bounce_rate", - scroll_data.scroll_gt80_percentage as scroll_gt80_percentage, - scroll_data.average_scroll_percentage as average_scroll_percentage -FROM - {pathname_query} AS pathname -LEFT OUTER JOIN - ( - SELECT - session.$initial_pathname, - avg(session.is_bounce) as bounce_rate - FROM - {session_query} AS session - GROUP BY - session.$initial_pathname - ) AS bounce_rate -ON - pathname.$pathname = bounce_rate.$initial_pathname -LEFT OUTER JOIN - {pathname_scroll_query} AS scroll_data -ON - pathname.$pathname = scroll_data.$pathname -ORDER BY - "context.columns.views" DESC -LIMIT 10 - """, - timings=self.timings, - placeholders={ - "pathname_query": pathname_query, - "session_query": session_query, - "pathname_scroll_query": pathname_scroll_query, - }, - ) - return top_sources_query - - def calculate(self): - response = execute_hogql_query( - query_type="top_sources_query", - query=self.to_query(), - team=self.team, - timings=self.timings, - ) - - return WebTopPagesQueryResponse( - columns=response.columns, results=response.results, timings=response.timings, types=response.types - ) diff --git a/posthog/hogql_queries/web_analytics/top_sources.py b/posthog/hogql_queries/web_analytics/top_sources.py deleted file mode 100644 index 1cc65c8ee9cec..0000000000000 --- a/posthog/hogql_queries/web_analytics/top_sources.py +++ /dev/null @@ -1,66 +0,0 @@ -from posthog.hogql import ast -from posthog.hogql.parser import parse_select -from posthog.hogql.query import execute_hogql_query -from posthog.hogql_queries.web_analytics.ctes import SESSION_CTE, SOURCE_CTE -from posthog.hogql_queries.web_analytics.web_analytics_query_runner import WebAnalyticsQueryRunner -from posthog.schema import WebTopSourcesQuery, WebTopSourcesQueryResponse - - -class WebTopSourcesQueryRunner(WebAnalyticsQueryRunner): - query: WebTopSourcesQuery - query_type = WebTopSourcesQuery - - def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery: - with self.timings.measure("session_query"): - session_query = parse_select( - SESSION_CTE, - timings=self.timings, - placeholders={"session_where": self.session_where(), "session_having": self.session_having()}, - ) - with self.timings.measure("sources_query"): - source_query = parse_select( - SOURCE_CTE, - timings=self.timings, - placeholders={"source_where": self.events_where()}, - ) - with self.timings.measure("top_sources_query"): - top_sources_query = parse_select( - """ -SELECT - source_query.$initial_utm_source as "Initial UTM Source", - source_query.total_pageviews as "context.columns.views", - source_query.unique_visitors as "context.columns.visitors", - bounce_rate.bounce_rate AS "context.columns.bounce_rate" -FROM - {source_query} AS source_query -LEFT JOIN ( - SELECT - session.$initial_utm_source, - avg(session.is_bounce) as bounce_rate - FROM - {session_query} AS session - GROUP BY - session.$initial_utm_source - ) AS bounce_rate -ON source_query.$initial_utm_source = bounce_rate.$initial_utm_source -WHERE - "Initial UTM Source" IS NOT NULL -ORDER BY "context.columns.views" DESC -LIMIT 10 - """, - timings=self.timings, - placeholders={"session_query": session_query, "source_query": source_query}, - ) - return top_sources_query - - def calculate(self): - response = execute_hogql_query( - query_type="top_sources_query", - query=self.to_query(), - team=self.team, - timings=self.timings, - ) - - return WebTopSourcesQueryResponse( - columns=response.columns, results=response.results, timings=response.timings, types=response.types - ) diff --git a/posthog/schema.py b/posthog/schema.py index a5902a8b55cc6..2505654fc694c 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -495,6 +495,14 @@ class WebOverviewStatsQueryResponse(BaseModel): types: Optional[List] = None +class WebStatsBreakdown(str, Enum): + Page = "Page" + InitialPage = "InitialPage" + InitialReferringDomain = "InitialReferringDomain" + InitialUTMSource = "InitialUTMSource" + InitialUTMCampaign = "InitialUTMCampaign" + + class WebStatsTableQueryResponse(BaseModel): model_config = ConfigDict( extra="forbid", @@ -765,7 +773,7 @@ class WebStatsTableQuery(BaseModel): model_config = ConfigDict( extra="forbid", ) - breakdownBy: Literal["Page"] = "Page" + breakdownBy: WebStatsBreakdown dateRange: Optional[DateRange] = None kind: Literal["WebStatsTableQuery"] = "WebStatsTableQuery" properties: List[EventPropertyFilter] From 415116a98c228cd6ca4d0c8a76168f9f1433411a Mon Sep 17 00:00:00 2001 From: Robbie Coomber Date: Sat, 14 Oct 2023 09:55:14 +0100 Subject: [PATCH 04/13] Remove classes for now-unused queries --- frontend/src/queries/schema.json | 126 ------------------ frontend/src/queries/schema.ts | 27 ---- posthog/hogql_queries/query_runner.py | 12 -- .../web_analytics_query_runner.py | 4 - posthog/schema.py | 52 -------- 5 files changed, 221 deletions(-) diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index a858f2fc13307..18148a584d882 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -105,14 +105,8 @@ { "$ref": "#/definitions/WebStatsTableQuery" }, - { - "$ref": "#/definitions/WebTopSourcesQuery" - }, { "$ref": "#/definitions/WebTopClicksQuery" - }, - { - "$ref": "#/definitions/WebTopPagesQuery" } ] }, @@ -417,14 +411,8 @@ { "$ref": "#/definitions/WebStatsTableQuery" }, - { - "$ref": "#/definitions/WebTopSourcesQuery" - }, { "$ref": "#/definitions/WebTopClicksQuery" - }, - { - "$ref": "#/definitions/WebTopPagesQuery" } ], "description": "Source of the events" @@ -2600,120 +2588,6 @@ }, "required": ["results"], "type": "object" - }, - "WebTopPagesQuery": { - "additionalProperties": false, - "properties": { - "dateRange": { - "$ref": "#/definitions/DateRange" - }, - "kind": { - "const": "WebTopPagesQuery", - "type": "string" - }, - "properties": { - "$ref": "#/definitions/WebAnalyticsPropertyFilters" - }, - "response": { - "$ref": "#/definitions/WebTopPagesQueryResponse" - } - }, - "required": ["kind", "properties"], - "type": "object" - }, - "WebTopPagesQueryResponse": { - "additionalProperties": false, - "properties": { - "columns": { - "items": {}, - "type": "array" - }, - "hogql": { - "type": "string" - }, - "is_cached": { - "type": "boolean" - }, - "last_refresh": { - "type": "string" - }, - "next_allowed_client_refresh": { - "type": "string" - }, - "results": { - "items": {}, - "type": "array" - }, - "timings": { - "items": { - "$ref": "#/definitions/QueryTiming" - }, - "type": "array" - }, - "types": { - "items": {}, - "type": "array" - } - }, - "required": ["results"], - "type": "object" - }, - "WebTopSourcesQuery": { - "additionalProperties": false, - "properties": { - "dateRange": { - "$ref": "#/definitions/DateRange" - }, - "kind": { - "const": "WebTopSourcesQuery", - "type": "string" - }, - "properties": { - "$ref": "#/definitions/WebAnalyticsPropertyFilters" - }, - "response": { - "$ref": "#/definitions/WebTopSourcesQueryResponse" - } - }, - "required": ["kind", "properties"], - "type": "object" - }, - "WebTopSourcesQueryResponse": { - "additionalProperties": false, - "properties": { - "columns": { - "items": {}, - "type": "array" - }, - "hogql": { - "type": "string" - }, - "is_cached": { - "type": "boolean" - }, - "last_refresh": { - "type": "string" - }, - "next_allowed_client_refresh": { - "type": "string" - }, - "results": { - "items": {}, - "type": "array" - }, - "timings": { - "items": { - "$ref": "#/definitions/QueryTiming" - }, - "type": "array" - }, - "types": { - "items": {}, - "type": "array" - } - }, - "required": ["results"], - "type": "object" } } } diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index 631a56da30f13..1bf1f0e3c14b1 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -62,8 +62,6 @@ export enum NodeKind { // Web analytics queries WebOverviewStatsQuery = 'WebOverviewStatsQuery', - WebTopSourcesQuery = 'WebTopSourcesQuery', - WebTopPagesQuery = 'WebTopPagesQuery', WebTopClicksQuery = 'WebTopClicksQuery', WebStatsTableQuery = 'WebStatsTableQuery', @@ -88,9 +86,7 @@ export type AnyDataNode = | HogQLMetadata | WebOverviewStatsQuery | WebStatsTableQuery - | WebTopSourcesQuery | WebTopClicksQuery - | WebTopPagesQuery export type QuerySchema = // Data nodes (see utils.ts) @@ -317,9 +313,7 @@ export interface DataTableNode extends Node, DataTableNodeViewProps { | TimeToSeeDataSessionsQuery | WebOverviewStatsQuery | WebStatsTableQuery - | WebTopSourcesQuery | WebTopClicksQuery - | WebTopPagesQuery /** Columns shown in the table, unless the `source` provides them. */ columns?: HogQLExpression[] @@ -564,16 +558,6 @@ export interface WebOverviewStatsQueryResponse extends QueryResponse { types?: unknown[] columns?: unknown[] } -export interface WebTopSourcesQuery extends WebAnalyticsQueryBase { - kind: NodeKind.WebTopSourcesQuery - properties: WebAnalyticsPropertyFilters - response?: WebTopSourcesQueryResponse -} -export interface WebTopSourcesQueryResponse extends QueryResponse { - results: unknown[] - types?: unknown[] - columns?: unknown[] -} export interface WebTopClicksQuery extends WebAnalyticsQueryBase { kind: NodeKind.WebTopClicksQuery @@ -586,17 +570,6 @@ export interface WebTopClicksQueryResponse extends QueryResponse { columns?: unknown[] } -export interface WebTopPagesQuery extends WebAnalyticsQueryBase { - kind: NodeKind.WebTopPagesQuery - properties: WebAnalyticsPropertyFilters - response?: WebTopPagesQueryResponse -} -export interface WebTopPagesQueryResponse extends QueryResponse { - results: unknown[] - types?: unknown[] - columns?: unknown[] -} - export enum WebStatsBreakdown { Page = 'Page', InitialPage = 'InitialPage', diff --git a/posthog/hogql_queries/query_runner.py b/posthog/hogql_queries/query_runner.py index 59bce744fd26b..9b22b4ad5094d 100644 --- a/posthog/hogql_queries/query_runner.py +++ b/posthog/hogql_queries/query_runner.py @@ -19,9 +19,7 @@ QueryTiming, TrendsQuery, LifecycleQuery, - WebTopSourcesQuery, WebTopClicksQuery, - WebTopPagesQuery, WebOverviewStatsQuery, PersonsQuery, EventsQuery, @@ -70,9 +68,7 @@ class CachedQueryResponse(QueryResponse): EventsQuery, PersonsQuery, WebOverviewStatsQuery, - WebTopSourcesQuery, WebTopClicksQuery, - WebTopPagesQuery, ] @@ -110,18 +106,10 @@ def get_query_runner( from .web_analytics.overview_stats import WebOverviewStatsQueryRunner return WebOverviewStatsQueryRunner(query=query, team=team, timings=timings) - if kind == "WebTopSourcesQuery": - from .web_analytics.top_sources import WebTopSourcesQueryRunner - - return WebTopSourcesQueryRunner(query=query, team=team, timings=timings) if kind == "WebTopClicksQuery": from .web_analytics.top_clicks import WebTopClicksQueryRunner return WebTopClicksQueryRunner(query=query, team=team, timings=timings) - if kind == "WebTopPagesQuery": - from .web_analytics.top_pages import WebTopPagesQueryRunner - - return WebTopPagesQueryRunner(query=query, team=team, timings=timings) if kind == "WebStatsTableQuery": from .web_analytics.stats_table import WebStatsTableQueryRunner diff --git a/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py b/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py index 3c9c6d2b34e2a..01e778f1b7c67 100644 --- a/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py +++ b/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py @@ -10,18 +10,14 @@ from posthog.models.filters.mixins.utils import cached_property from posthog.schema import ( EventPropertyFilter, - WebTopSourcesQuery, WebTopClicksQuery, - WebTopPagesQuery, WebOverviewStatsQuery, WebStatsTableQuery, ) WebQueryNode = Union[ WebOverviewStatsQuery, - WebTopSourcesQuery, WebTopClicksQuery, - WebTopPagesQuery, WebStatsTableQuery, ] diff --git a/posthog/schema.py b/posthog/schema.py index 2505654fc694c..f9d404c4387c3 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -531,34 +531,6 @@ class WebTopClicksQueryResponse(BaseModel): types: Optional[List] = None -class WebTopPagesQueryResponse(BaseModel): - model_config = ConfigDict( - extra="forbid", - ) - columns: Optional[List] = None - hogql: Optional[str] = None - is_cached: Optional[bool] = None - last_refresh: Optional[str] = None - next_allowed_client_refresh: Optional[str] = None - results: List - timings: Optional[List[QueryTiming]] = None - types: Optional[List] = None - - -class WebTopSourcesQueryResponse(BaseModel): - model_config = ConfigDict( - extra="forbid", - ) - columns: Optional[List] = None - hogql: Optional[str] = None - is_cached: Optional[bool] = None - last_refresh: Optional[str] = None - next_allowed_client_refresh: Optional[str] = None - results: List - timings: Optional[List[QueryTiming]] = None - types: Optional[List] = None - - class Breakdown(BaseModel): model_config = ConfigDict( extra="forbid", @@ -790,26 +762,6 @@ class WebTopClicksQuery(BaseModel): response: Optional[WebTopClicksQueryResponse] = None -class WebTopPagesQuery(BaseModel): - model_config = ConfigDict( - extra="forbid", - ) - dateRange: Optional[DateRange] = None - kind: Literal["WebTopPagesQuery"] = "WebTopPagesQuery" - properties: List[EventPropertyFilter] - response: Optional[WebTopPagesQueryResponse] = None - - -class WebTopSourcesQuery(BaseModel): - model_config = ConfigDict( - extra="forbid", - ) - dateRange: Optional[DateRange] = None - kind: Literal["WebTopSourcesQuery"] = "WebTopSourcesQuery" - properties: List[EventPropertyFilter] - response: Optional[WebTopSourcesQueryResponse] = None - - class DatabaseSchemaQuery(BaseModel): model_config = ConfigDict( extra="forbid", @@ -1201,9 +1153,7 @@ class DataTableNode(BaseModel): TimeToSeeDataSessionsQuery, WebOverviewStatsQuery, WebStatsTableQuery, - WebTopSourcesQuery, WebTopClicksQuery, - WebTopPagesQuery, ] = Field(..., description="Source of the events") @@ -1484,9 +1434,7 @@ class Model(RootModel): HogQLMetadata, WebOverviewStatsQuery, WebStatsTableQuery, - WebTopSourcesQuery, WebTopClicksQuery, - WebTopPagesQuery, ], ] From 1ce34374384ad6660e6be74f0362d2a68bbc745b Mon Sep 17 00:00:00 2001 From: Robbie Coomber Date: Sat, 14 Oct 2023 10:09:41 +0100 Subject: [PATCH 05/13] Add remaining breakdowns --- frontend/src/queries/schema.json | 11 ++++++++++- frontend/src/queries/schema.ts | 3 +++ .../src/scenes/web-analytics/WebDashboard.tsx | 15 +++++++++++++++ .../src/scenes/web-analytics/webAnalyticsLogic.ts | 10 ++++++---- .../hogql_queries/web_analytics/stats_table.py | 12 ++++++++++++ posthog/schema.py | 3 +++ 6 files changed, 49 insertions(+), 5 deletions(-) diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index 18148a584d882..b3001deae55df 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -2469,7 +2469,16 @@ "type": "object" }, "WebStatsBreakdown": { - "enum": ["Page", "InitialPage", "InitialReferringDomain", "InitialUTMSource", "InitialUTMCampaign"], + "enum": [ + "Page", + "InitialPage", + "InitialReferringDomain", + "InitialUTMSource", + "InitialUTMCampaign", + "Browser", + "OS", + "DeviceType" + ], "type": "string" }, "WebStatsTableQuery": { diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index 1bf1f0e3c14b1..657f8f3fa8f4a 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -577,6 +577,9 @@ export enum WebStatsBreakdown { InitialReferringDomain = 'InitialReferringDomain', InitialUTMSource = 'InitialUTMSource', InitialUTMCampaign = 'InitialUTMCampaign', + Browser = 'Browser', + OS = 'OS', + DeviceType = 'DeviceType', } export interface WebStatsTableQuery extends WebAnalyticsQueryBase { kind: NodeKind.WebStatsTableQuery diff --git a/frontend/src/scenes/web-analytics/WebDashboard.tsx b/frontend/src/scenes/web-analytics/WebDashboard.tsx index ec6404003a8ac..8f1f08ccd1e79 100644 --- a/frontend/src/scenes/web-analytics/WebDashboard.tsx +++ b/frontend/src/scenes/web-analytics/WebDashboard.tsx @@ -52,6 +52,12 @@ const BreakdownValueTitle: QueryContextColumnTitleComponent = (props) => { return <>UTM Source case WebStatsBreakdown.InitialUTMCampaign: return <>UTM Campaign + case WebStatsBreakdown.Browser: + return <>Browser + case WebStatsBreakdown.OS: + return <>OS + case WebStatsBreakdown.DeviceType: + return <>Device Type default: throw new UnexpectedNeverError(breakdownBy) } @@ -81,6 +87,15 @@ const BreakdownValueCell: QueryContextColumnComponent = (props) => { case WebStatsBreakdown.InitialUTMCampaign: propertyName = '$set_once.$initial_utm_campaign' break + case WebStatsBreakdown.Browser: + propertyName = '$browser' + break + case WebStatsBreakdown.OS: + propertyName = '$os' + break + case WebStatsBreakdown.DeviceType: + propertyName = '$device_type' + break default: throw new UnexpectedNeverError(breakdownBy) } diff --git a/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts b/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts index a71590d19a9b2..e1bfca3595c88 100644 --- a/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts +++ b/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts @@ -245,7 +245,6 @@ export const webAnalyticsLogic = kea([ }, activeTabId: deviceTab, setTabId: actions.setDeviceTab, - tabs: [ { id: DeviceTab.BROWSER, @@ -255,8 +254,9 @@ export const webAnalyticsLogic = kea([ full: true, kind: NodeKind.DataTableNode, source: { - kind: NodeKind.WebTopSourcesQuery, // TODO + kind: NodeKind.WebStatsTableQuery, properties: webAnalyticsFilters, + breakdownBy: WebStatsBreakdown.Browser, }, }, }, @@ -268,8 +268,9 @@ export const webAnalyticsLogic = kea([ full: true, kind: NodeKind.DataTableNode, source: { - kind: NodeKind.WebTopSourcesQuery, // TODO + kind: NodeKind.WebStatsTableQuery, properties: webAnalyticsFilters, + breakdownBy: WebStatsBreakdown.OS, }, }, }, @@ -281,8 +282,9 @@ export const webAnalyticsLogic = kea([ full: true, kind: NodeKind.DataTableNode, source: { - kind: NodeKind.WebTopSourcesQuery, // TODO + kind: NodeKind.WebStatsTableQuery, properties: webAnalyticsFilters, + breakdownBy: WebStatsBreakdown.DeviceType, }, }, }, diff --git a/posthog/hogql_queries/web_analytics/stats_table.py b/posthog/hogql_queries/web_analytics/stats_table.py index 273308b6ff653..6af53cf1c9ddd 100644 --- a/posthog/hogql_queries/web_analytics/stats_table.py +++ b/posthog/hogql_queries/web_analytics/stats_table.py @@ -91,6 +91,12 @@ def counts_breakdown(self): return parse_expr("events.properties.$set_once.$initial_utm_source") case WebStatsBreakdown.InitialUTMCampaign: return parse_expr("events.properties.$set_once.$initial_utm_campaign") + case WebStatsBreakdown.Browser: + return parse_expr("events.properties.$browser") + case WebStatsBreakdown.OS: + return parse_expr("events.properties.$os") + case WebStatsBreakdown.DeviceType: + return parse_expr("events.properties.$device_type") case _: raise NotImplementedError("Breakdown not implemented") @@ -106,5 +112,11 @@ def bounce_breakdown(self): return parse_expr("any(events.properties.$set_once.$initial_utm_source)") case WebStatsBreakdown.InitialUTMCampaign: return parse_expr("any(events.properties.$set_once.$initial_utm_campaign)") + case WebStatsBreakdown.Browser: + return parse_expr("any(events.properties.$browser)") + case WebStatsBreakdown.OS: + return parse_expr("any(events.properties.$os)") + case WebStatsBreakdown.DeviceType: + return parse_expr("any(events.properties.$device_type)") case _: raise NotImplementedError("Breakdown not implemented") diff --git a/posthog/schema.py b/posthog/schema.py index f9d404c4387c3..2c07e1109209d 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -501,6 +501,9 @@ class WebStatsBreakdown(str, Enum): InitialReferringDomain = "InitialReferringDomain" InitialUTMSource = "InitialUTMSource" InitialUTMCampaign = "InitialUTMCampaign" + Browser = "Browser" + OS = "OS" + DeviceType = "DeviceType" class WebStatsTableQueryResponse(BaseModel): From 5e691888211404e3513cd9288d4eae4ad4cf07c7 Mon Sep 17 00:00:00 2001 From: Robbie Coomber Date: Sat, 14 Oct 2023 10:57:38 +0100 Subject: [PATCH 06/13] Prefer unknown to any --- frontend/src/queries/schema.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index 657f8f3fa8f4a..87225904e1f5b 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -719,10 +719,10 @@ export type QueryContextColumnTitleComponent = ComponentType<{ }> export type QueryContextColumnComponent = ComponentType<{ - record: any columnName: string - value: any query: DataTableNode + record: unknown + value: unknown }> interface QueryContextColumn { From 5bb0e73df151675cce76909ce983a2552843f78a Mon Sep 17 00:00:00 2001 From: Robbie Coomber Date: Sat, 14 Oct 2023 11:12:03 +0100 Subject: [PATCH 07/13] Remove unused ctes --- posthog/hogql_queries/web_analytics/ctes.py | 55 --------------------- 1 file changed, 55 deletions(-) diff --git a/posthog/hogql_queries/web_analytics/ctes.py b/posthog/hogql_queries/web_analytics/ctes.py index 4981c2389f6b5..3328092fb3493 100644 --- a/posthog/hogql_queries/web_analytics/ctes.py +++ b/posthog/hogql_queries/web_analytics/ctes.py @@ -2,61 +2,6 @@ # while these queries are under development they are left as CTEs so that they can be iterated # on without needing database migrations -SESSION_CTE = """ -SELECT - events.properties.`$session_id` AS session_id, - min(events.timestamp) AS min_timestamp, - max(events.timestamp) AS max_timestamp, - dateDiff('second', min_timestamp, max_timestamp) AS duration_s, - - any(events.properties.$initial_referring_domain) AS $initial_referring_domain, - any(events.properties.$set_once.$initial_pathname) AS $initial_pathname, - any(events.properties.$set_once.$initial_utm_source) AS $initial_utm_source, - - countIf(events.event == '$pageview') AS num_pageviews, - countIf(events.event == '$autocapture') AS num_autocaptures, - -- in v1 we'd also want to count whether there were any conversion events - - any(events.person_id) as person_id, - -- definition of a GA4 bounce from here https://support.google.com/analytics/answer/12195621?hl=en - (num_autocaptures == 0 AND num_pageviews <= 1 AND duration_s < 10) AS is_bounce -FROM - events -WHERE - session_id IS NOT NULL - AND ({session_where}) -GROUP BY - events.properties.`$session_id` -HAVING - ({session_having}) - """ - -SOURCE_CTE = """ -SELECT - events.properties.$set_once.$initial_utm_source AS $initial_utm_source, - count() as total_pageviews, - uniq(events.person_id) as unique_visitors -FROM - events -WHERE - (event = '$pageview') - AND ({source_where}) - GROUP BY $initial_utm_source -""" - -PATHNAME_CTE = """ -SELECT - events.properties.`$pathname` AS $pathname, - count() as total_pageviews, - uniq(events.person_id) as unique_visitors -FROM - events -WHERE - (event = '$pageview') - AND ({pathname_where}) - GROUP BY $pathname -""" - PATHNAME_SCROLL_CTE = """ SELECT events.properties.`$prev_pageview_pathname` AS $pathname, From 06ced1a9d6a26d2f9f1660f6f71a26eb25133fc9 Mon Sep 17 00:00:00 2001 From: Robbie Coomber Date: Mon, 16 Oct 2023 09:11:00 +0100 Subject: [PATCH 08/13] Fix typing issues --- .../queries/nodes/DataTable/queryFeatures.ts | 10 +----- frontend/src/queries/utils.ts | 36 +++++++------------ .../scenes/saved-insights/SavedInsights.tsx | 12 ++----- .../src/scenes/web-analytics/WebDashboard.tsx | 3 ++ 4 files changed, 20 insertions(+), 41 deletions(-) diff --git a/frontend/src/queries/nodes/DataTable/queryFeatures.ts b/frontend/src/queries/nodes/DataTable/queryFeatures.ts index 03d327390dff1..b1cedfa1386ee 100644 --- a/frontend/src/queries/nodes/DataTable/queryFeatures.ts +++ b/frontend/src/queries/nodes/DataTable/queryFeatures.ts @@ -6,8 +6,6 @@ import { isWebOverviewStatsQuery, isWebStatsTableQuery, isWebTopClicksQuery, - isWebTopPagesQuery, - isWebTopSourcesQuery, } from '~/queries/utils' import { Node } from '~/queries/schema' @@ -56,13 +54,7 @@ export function getQueryFeatures(query: Node): Set { } } - if ( - isWebOverviewStatsQuery(query) || - isWebTopSourcesQuery(query) || - isWebTopPagesQuery(query) || - isWebTopClicksQuery(query) || - isWebStatsTableQuery(query) - ) { + if (isWebOverviewStatsQuery(query) || isWebTopClicksQuery(query) || isWebStatsTableQuery(query)) { features.add(QueryFeature.columnsInResponse) features.add(QueryFeature.resultIsArrayOfArrays) } diff --git a/frontend/src/queries/utils.ts b/frontend/src/queries/utils.ts index 442b540e2b25b..5d4b01a08db99 100644 --- a/frontend/src/queries/utils.ts +++ b/frontend/src/queries/utils.ts @@ -1,38 +1,36 @@ import { ActionsNode, + DatabaseSchemaQuery, DataTableNode, DateRange, EventsNode, EventsQuery, - HogQLQuery, - TrendsQuery, FunnelsQuery, - RetentionQuery, - PathsQuery, - StickinessQuery, - LifecycleQuery, + HogQLMetadata, + HogQLQuery, InsightFilter, InsightFilterProperty, + InsightNodeKind, InsightQueryNode, InsightVizNode, + LifecycleQuery, Node, NodeKind, + PathsQuery, PersonsNode, + PersonsQuery, + RetentionQuery, + SavedInsightNode, + StickinessQuery, + TimeToSeeDataJSONNode, TimeToSeeDataNode, TimeToSeeDataQuery, TimeToSeeDataSessionsQuery, - InsightNodeKind, TimeToSeeDataWaterfallNode, - TimeToSeeDataJSONNode, - DatabaseSchemaQuery, - SavedInsightNode, - WebTopSourcesQuery, - WebTopClicksQuery, - WebTopPagesQuery, + TrendsQuery, WebOverviewStatsQuery, - PersonsQuery, - HogQLMetadata, WebStatsTableQuery, + WebTopClicksQuery, } from '~/queries/schema' import { TaxonomicFilterGroupType, TaxonomicFilterValue } from 'lib/components/TaxonomicFilter/types' import { dayjs } from 'lib/dayjs' @@ -122,18 +120,10 @@ export function isWebStatsTableQuery(node?: Node | null): node is WebStatsTableQ return node?.kind === NodeKind.WebStatsTableQuery } -export function isWebTopSourcesQuery(node?: Node | null): node is WebTopSourcesQuery { - return node?.kind === NodeKind.WebTopSourcesQuery -} - export function isWebTopClicksQuery(node?: Node | null): node is WebTopClicksQuery { return node?.kind === NodeKind.WebTopClicksQuery } -export function isWebTopPagesQuery(node?: Node | null): node is WebTopPagesQuery { - return node?.kind === NodeKind.WebTopPagesQuery -} - export function containsHogQLQuery(node?: Node | null): boolean { if (!node) { return false diff --git a/frontend/src/scenes/saved-insights/SavedInsights.tsx b/frontend/src/scenes/saved-insights/SavedInsights.tsx index c868d7906b410..88fd684e6d4c2 100644 --- a/frontend/src/scenes/saved-insights/SavedInsights.tsx +++ b/frontend/src/scenes/saved-insights/SavedInsights.tsx @@ -254,15 +254,9 @@ export const QUERY_TYPES_METADATA: Record = { icon: InsightsTrendsIcon, inMenu: true, }, - [NodeKind.WebTopSourcesQuery]: { - name: 'Top Sources', - description: 'View top sources for a website', - icon: InsightsTrendsIcon, - inMenu: true, - }, - [NodeKind.WebTopPagesQuery]: { - name: 'Top Pages', - description: 'View top pages for a website', + [NodeKind.WebStatsTableQuery]: { + name: 'Web Table', + description: 'A table of results from web analytics, with a breakdown', icon: InsightsTrendsIcon, inMenu: true, }, diff --git a/frontend/src/scenes/web-analytics/WebDashboard.tsx b/frontend/src/scenes/web-analytics/WebDashboard.tsx index 8f1f08ccd1e79..55536b3988fb9 100644 --- a/frontend/src/scenes/web-analytics/WebDashboard.tsx +++ b/frontend/src/scenes/web-analytics/WebDashboard.tsx @@ -69,6 +69,9 @@ const BreakdownValueCell: QueryContextColumnComponent = (props) => { if (source.kind !== NodeKind.WebStatsTableQuery) { return null } + if (typeof value !== 'string') { + return null + } const { breakdownBy } = source let propertyName: string switch (breakdownBy) { From 295bdca8db912eb029966929ac226a7333cfb299 Mon Sep 17 00:00:00 2001 From: Robbie Coomber Date: Mon, 16 Oct 2023 10:19:52 +0100 Subject: [PATCH 09/13] Fix python typing --- posthog/hogql_queries/query_runner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/posthog/hogql_queries/query_runner.py b/posthog/hogql_queries/query_runner.py index 9b22b4ad5094d..49f7ca369fdc5 100644 --- a/posthog/hogql_queries/query_runner.py +++ b/posthog/hogql_queries/query_runner.py @@ -23,6 +23,7 @@ WebOverviewStatsQuery, PersonsQuery, EventsQuery, + WebStatsTableQuery, ) from posthog.utils import generate_cache_key, get_safe_cache @@ -69,6 +70,7 @@ class CachedQueryResponse(QueryResponse): PersonsQuery, WebOverviewStatsQuery, WebTopClicksQuery, + WebStatsTableQuery, ] From d1bc162e4be523d9d61ffe4443a29fd8e2af3baa Mon Sep 17 00:00:00 2001 From: Robbie Coomber Date: Mon, 16 Oct 2023 11:59:50 +0100 Subject: [PATCH 10/13] Switch to hogql filters for set_once properties --- frontend/src/queries/schema.json | 9 +- frontend/src/queries/schema.ts | 29 ++--- .../src/scenes/web-analytics/WebDashboard.tsx | 8 +- .../scenes/web-analytics/webAnalyticsLogic.ts | 106 +++++++++++++----- posthog/hogql/property.py | 2 +- .../web_analytics/stats_table.py | 30 ++--- posthog/schema.py | 6 +- 7 files changed, 124 insertions(+), 66 deletions(-) diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index b3001deae55df..0574a8b44ceb0 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -2407,7 +2407,14 @@ }, "WebAnalyticsPropertyFilters": { "items": { - "$ref": "#/definitions/EventPropertyFilter" + "anyOf": [ + { + "$ref": "#/definitions/EventPropertyFilter" + }, + { + "$ref": "#/definitions/HogQLPropertyFilter" + } + ] }, "type": "array" }, diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index 87225904e1f5b..f2b718d62407b 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -1,27 +1,28 @@ import { AnyPropertyFilter, + BaseMathType, Breakdown, BreakdownKeyType, BreakdownType, - PropertyGroupFilter, - EventType, - IntervalType, - BaseMathType, - PropertyMathType, CountPerActorMathType, - GroupMathType, + EventPropertyFilter, + EventType, FilterType, - TrendsFilterType, FunnelsFilterType, - RetentionFilterType, - PathsFilterType, - StickinessFilterType, - LifecycleFilterType, - LifecycleToggle, + GroupMathType, HogQLMathType, + HogQLPropertyFilter, InsightLogicProps, InsightShortId, - EventPropertyFilter, + IntervalType, + LifecycleFilterType, + LifecycleToggle, + PathsFilterType, + PropertyGroupFilter, + PropertyMathType, + RetentionFilterType, + StickinessFilterType, + TrendsFilterType, } from '~/types' import { ComponentType } from 'react' @@ -541,7 +542,7 @@ export interface PersonsQuery extends DataNode { response?: PersonsQueryResponse } -export type WebAnalyticsPropertyFilters = EventPropertyFilter[] +export type WebAnalyticsPropertyFilters = (EventPropertyFilter | HogQLPropertyFilter)[] export interface WebAnalyticsQueryBase { dateRange?: DateRange diff --git a/frontend/src/scenes/web-analytics/WebDashboard.tsx b/frontend/src/scenes/web-analytics/WebDashboard.tsx index 55536b3988fb9..9b5d8a4c14c1b 100644 --- a/frontend/src/scenes/web-analytics/WebDashboard.tsx +++ b/frontend/src/scenes/web-analytics/WebDashboard.tsx @@ -79,16 +79,16 @@ const BreakdownValueCell: QueryContextColumnComponent = (props) => { propertyName = '$pathname' break case WebStatsBreakdown.InitialPage: - propertyName = '$set_once.$initial_pathname' + propertyName = '$initial_pathname' break case WebStatsBreakdown.InitialReferringDomain: - propertyName = '$set_once.$initial_referrer' + propertyName = '$initial_referrer' break case WebStatsBreakdown.InitialUTMSource: - propertyName = '$set_once.$initial_utm_source' + propertyName = '$initial_utm_source' break case WebStatsBreakdown.InitialUTMCampaign: - propertyName = '$set_once.$initial_utm_campaign' + propertyName = '$initial_utm_campaign' break case WebStatsBreakdown.Browser: propertyName = '$browser' diff --git a/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts b/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts index e1bfca3595c88..96ec0094fb489 100644 --- a/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts +++ b/frontend/src/scenes/web-analytics/webAnalyticsLogic.ts @@ -2,7 +2,14 @@ import { actions, connect, kea, listeners, path, reducers, selectors, sharedList import type { webAnalyticsLogicType } from './webAnalyticsLogicType' import { NodeKind, QuerySchema, WebAnalyticsPropertyFilters, WebStatsBreakdown } from '~/queries/schema' -import { BaseMathType, ChartDisplayType, EventPropertyFilter, PropertyFilterType, PropertyOperator } from '~/types' +import { + BaseMathType, + ChartDisplayType, + EventPropertyFilter, + HogQLPropertyFilter, + PropertyFilterType, + PropertyOperator, +} from '~/types' import { isNotNil } from 'lib/utils' interface Layout { @@ -51,6 +58,11 @@ export enum PathTab { export const initialWebAnalyticsFilter = [] as WebAnalyticsPropertyFilters +const setOncePropertyNames = ['$initial_pathname', '$initial_referrer', '$initial_utm_source', '$initial_utm_campaign'] +const hogqlForSetOnceProperty = (key: string, value: string): string => `properties.$set_once.${key} = '${value}'` +const isHogqlForSetOnceProperty = (key: string, p: HogQLPropertyFilter): boolean => + setOncePropertyNames.includes(key) && p.key.startsWith(`properties.$set_once.${key} = `) + export const webAnalyticsLogic = kea([ path(['scenes', 'webAnalytics', 'webAnalyticsSceneLogic']), connect({}), @@ -73,43 +85,81 @@ export const webAnalyticsLogic = kea([ { setWebAnalyticsFilters: (_, { webAnalyticsFilters }) => webAnalyticsFilters, togglePropertyFilter: (oldPropertyFilters, { key, value }) => { - if (oldPropertyFilters.some((f) => f.key === key && f.operator === PropertyOperator.Exact)) { + if ( + oldPropertyFilters.some( + (f) => + (f.type === PropertyFilterType.Event && + f.key === key && + f.operator === PropertyOperator.Exact) || + (f.type === PropertyFilterType.HogQL && isHogqlForSetOnceProperty(key, f)) + ) + ) { return oldPropertyFilters .map((f) => { - if ( - f.type !== PropertyFilterType.Event || - f.key !== key || - f.operator !== PropertyOperator.Exact - ) { - return f - } - const oldValue = (Array.isArray(f.value) ? f.value : [f.value]).filter(isNotNil) - let newValue: (string | number)[] - if (oldValue.includes(value)) { - // If there are multiple values for this filter, reduce that to just the one being clicked - if (oldValue.length > 1) { - newValue = [value] - } else { + if (setOncePropertyNames.includes(key)) { + if (f.type !== PropertyFilterType.HogQL) { + return f + } + if (!isHogqlForSetOnceProperty(key, f)) { + return f + } + // With the hogql properties, we don't even attempt to handle arrays, to avoiding + // needing a parser on the front end. Instead the logic is much simpler + const hogql = hogqlForSetOnceProperty(key, value) + if (f.key === hogql) { return null + } else { + return { + type: PropertyFilterType.HogQL, + key, + value: hogql, + } as const } } else { - newValue = [...oldValue, value] + if ( + f.key !== key || + f.type !== PropertyFilterType.Event || + f.operator !== PropertyOperator.Exact + ) { + return f + } + const oldValue = (Array.isArray(f.value) ? f.value : [f.value]).filter(isNotNil) + let newValue: (string | number)[] + if (oldValue.includes(value)) { + // If there are multiple values for this filter, reduce that to just the one being clicked + if (oldValue.length > 1) { + newValue = [value] + } else { + return null + } + } else { + newValue = [...oldValue, value] + } + return { + type: PropertyFilterType.Event, + key, + operator: PropertyOperator.Exact, + value: newValue, + } as const } - return { - type: PropertyFilterType.Event, - key, - operator: PropertyOperator.Exact, - value: newValue, - } as const }) .filter(isNotNil) } else { - const newFilter: EventPropertyFilter = { - type: PropertyFilterType.Event, - key, - value, - operator: PropertyOperator.Exact, + let newFilter: EventPropertyFilter | HogQLPropertyFilter + if (setOncePropertyNames.includes(key)) { + newFilter = { + type: PropertyFilterType.HogQL, + key: hogqlForSetOnceProperty(key, value), + } + } else { + newFilter = { + type: PropertyFilterType.Event, + key, + value, + operator: PropertyOperator.Exact, + } } + return [...oldPropertyFilters, newFilter] } }, diff --git a/posthog/hogql/property.py b/posthog/hogql/property.py index 9cb8025d6446d..293005bce9822 100644 --- a/posthog/hogql/property.py +++ b/posthog/hogql/property.py @@ -129,7 +129,7 @@ def property_to_expr( return ast.Or(exprs=exprs) chain = ["person", "properties"] if property.type == "person" and scope != "person" else ["properties"] - field = ast.Field(chain=chain + property.key.split(".")) + field = ast.Field(chain=chain + [property.key]) if operator == PropertyOperator.is_set: return ast.CompareOperation(op=ast.CompareOperationOp.NotEq, left=field, right=ast.Constant(value=None)) diff --git a/posthog/hogql_queries/web_analytics/stats_table.py b/posthog/hogql_queries/web_analytics/stats_table.py index 6af53cf1c9ddd..e70cf4a115651 100644 --- a/posthog/hogql_queries/web_analytics/stats_table.py +++ b/posthog/hogql_queries/web_analytics/stats_table.py @@ -84,39 +84,39 @@ def counts_breakdown(self): case WebStatsBreakdown.Page: return parse_expr("properties.$pathname") case WebStatsBreakdown.InitialPage: - return parse_expr("events.properties.$set_once.$initial_pathname") + return parse_expr("properties.$set_once.$initial_pathname") case WebStatsBreakdown.InitialReferringDomain: - return parse_expr("events.properties.$set_once.$initial_referring_domain") + return parse_expr("properties.$set_once.$initial_referring_domain") case WebStatsBreakdown.InitialUTMSource: - return parse_expr("events.properties.$set_once.$initial_utm_source") + return parse_expr("properties.$set_once.$initial_utm_source") case WebStatsBreakdown.InitialUTMCampaign: - return parse_expr("events.properties.$set_once.$initial_utm_campaign") + return parse_expr("properties.$set_once.$initial_utm_campaign") case WebStatsBreakdown.Browser: - return parse_expr("events.properties.$browser") + return parse_expr("properties.$browser") case WebStatsBreakdown.OS: - return parse_expr("events.properties.$os") + return parse_expr("properties.$os") case WebStatsBreakdown.DeviceType: - return parse_expr("events.properties.$device_type") + return parse_expr("properties.$device_type") case _: raise NotImplementedError("Breakdown not implemented") def bounce_breakdown(self): match self.query.breakdownBy: case WebStatsBreakdown.Page: - return parse_expr("any(events.properties.$set_once.$initial_pathname)") + return parse_expr("any(properties.$set_once.$initial_pathname)") case WebStatsBreakdown.InitialPage: - return parse_expr("any(events.properties.$set_once.$initial_pathname)") + return parse_expr("any(properties.$set_once.$initial_pathname)") case WebStatsBreakdown.InitialReferringDomain: - return parse_expr("any(events.properties.$set_once.$initial_referring_domain)") + return parse_expr("any(properties.$set_once.$initial_referring_domain)") case WebStatsBreakdown.InitialUTMSource: - return parse_expr("any(events.properties.$set_once.$initial_utm_source)") + return parse_expr("any(properties.$set_once.$initial_utm_source)") case WebStatsBreakdown.InitialUTMCampaign: - return parse_expr("any(events.properties.$set_once.$initial_utm_campaign)") + return parse_expr("any(properties.$set_once.$initial_utm_campaign)") case WebStatsBreakdown.Browser: - return parse_expr("any(events.properties.$browser)") + return parse_expr("any(properties.$browser)") case WebStatsBreakdown.OS: - return parse_expr("any(events.properties.$os)") + return parse_expr("any(properties.$os)") case WebStatsBreakdown.DeviceType: - return parse_expr("any(events.properties.$device_type)") + return parse_expr("any(properties.$device_type)") case _: raise NotImplementedError("Breakdown not implemented") diff --git a/posthog/schema.py b/posthog/schema.py index 2c07e1109209d..70e104f14a67c 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -740,7 +740,7 @@ class WebOverviewStatsQuery(BaseModel): ) dateRange: Optional[DateRange] = None kind: Literal["WebOverviewStatsQuery"] = "WebOverviewStatsQuery" - properties: List[EventPropertyFilter] + properties: List[Union[EventPropertyFilter, HogQLPropertyFilter]] response: Optional[WebOverviewStatsQueryResponse] = None @@ -751,7 +751,7 @@ class WebStatsTableQuery(BaseModel): breakdownBy: WebStatsBreakdown dateRange: Optional[DateRange] = None kind: Literal["WebStatsTableQuery"] = "WebStatsTableQuery" - properties: List[EventPropertyFilter] + properties: List[Union[EventPropertyFilter, HogQLPropertyFilter]] response: Optional[WebStatsTableQueryResponse] = None @@ -761,7 +761,7 @@ class WebTopClicksQuery(BaseModel): ) dateRange: Optional[DateRange] = None kind: Literal["WebTopClicksQuery"] = "WebTopClicksQuery" - properties: List[EventPropertyFilter] + properties: List[Union[EventPropertyFilter, HogQLPropertyFilter]] response: Optional[WebTopClicksQueryResponse] = None From 2e3aa89eba86974ac40416add4506297688ba3f9 Mon Sep 17 00:00:00 2001 From: Robbie Coomber Date: Mon, 16 Oct 2023 12:06:14 +0100 Subject: [PATCH 11/13] Use cpp parser --- posthog/hogql_queries/web_analytics/overview_stats.py | 7 ++++--- posthog/hogql_queries/web_analytics/stats_table.py | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/posthog/hogql_queries/web_analytics/overview_stats.py b/posthog/hogql_queries/web_analytics/overview_stats.py index e059ff33770ee..fa1fd74f2eaf7 100644 --- a/posthog/hogql_queries/web_analytics/overview_stats.py +++ b/posthog/hogql_queries/web_analytics/overview_stats.py @@ -17,9 +17,9 @@ class WebOverviewStatsQueryRunner(WebAnalyticsQueryRunner): def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery: with self.timings.measure("date_expr"): # TODO use the date range, with a previous period, trends query does this so look at that for insp - start = parse_expr("today() - 14") - mid = parse_expr("today() - 7") - end = parse_expr("today()") + start = parse_expr("today() - 14", backend="cpp") + mid = parse_expr("today() - 7", backend="cpp") + end = parse_expr("today()", backend="cpp") with self.timings.measure("overview_stats_query"): overview_stats_query = parse_select( """ @@ -42,6 +42,7 @@ def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery: """, timings=self.timings, placeholders={"start": start, "mid": mid, "end": end, "event_properties": self.event_properties()}, + backend="cpp", ) return overview_stats_query diff --git a/posthog/hogql_queries/web_analytics/stats_table.py b/posthog/hogql_queries/web_analytics/stats_table.py index e70cf4a115651..12e739413f2d1 100644 --- a/posthog/hogql_queries/web_analytics/stats_table.py +++ b/posthog/hogql_queries/web_analytics/stats_table.py @@ -27,6 +27,7 @@ def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery: "session_having": self.session_having(), "breakdown_by": self.bounce_breakdown(), }, + backend="cpp", ) with self.timings.measure("counts_query"): counts_query = parse_select( @@ -36,6 +37,7 @@ def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery: "counts_where": self.events_where(), "breakdown_by": self.counts_breakdown(), }, + backend="cpp", ) with self.timings.measure("top_pages_query"): top_sources_query = parse_select( @@ -60,6 +62,7 @@ def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery: "counts_query": counts_query, "bounce_rate_query": bounce_rate_query, }, + backend="cpp", ) return top_sources_query From 5de12140a4dbb0b0094654e8a11857109712fce5 Mon Sep 17 00:00:00 2001 From: Robbie Coomber Date: Mon, 16 Oct 2023 12:07:20 +0100 Subject: [PATCH 12/13] Move web analytics to the right navbar section --- .../src/layout/navigation/SideBar/SideBar.tsx | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/frontend/src/layout/navigation/SideBar/SideBar.tsx b/frontend/src/layout/navigation/SideBar/SideBar.tsx index 7594457385193..4983ab4222ad1 100644 --- a/frontend/src/layout/navigation/SideBar/SideBar.tsx +++ b/frontend/src/layout/navigation/SideBar/SideBar.tsx @@ -176,6 +176,14 @@ function Pages(): JSX.Element { onClick: hideSideBarMobile, }} /> + + } + identifier={Scene.WebAnalytics} + to={urls.webAnalytics()} + highlight="alpha" + /> + } identifier={Scene.Replay} to={urls.replay()} />
Feature Management
@@ -200,15 +208,6 @@ function Pages(): JSX.Element { to={urls.surveys()} highlight="beta" /> - - - } - identifier={Scene.WebAnalytics} - to={urls.webAnalytics()} - highlight="alpha" - /> -
Data
Date: Mon, 16 Oct 2023 12:28:17 +0100 Subject: [PATCH 13/13] Fix typing issues --- .../web_analytics/web_analytics_query_runner.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py b/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py index 01e778f1b7c67..8a9d1e0b72d9b 100644 --- a/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py +++ b/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py @@ -13,6 +13,7 @@ WebTopClicksQuery, WebOverviewStatsQuery, WebStatsTableQuery, + HogQLPropertyFilter, ) WebQueryNode = Union[ @@ -38,10 +39,13 @@ def query_date_range(self): @cached_property def pathname_property_filter(self) -> Optional[EventPropertyFilter]: - return next((p for p in self.query.properties if p.key == "$pathname"), None) + for p in self.query.properties: + if isinstance(p, EventPropertyFilter) and p.key == "$pathname": + return p + return None @cached_property - def property_filters_without_pathname(self) -> List[EventPropertyFilter]: + def property_filters_without_pathname(self) -> List[Union[EventPropertyFilter, HogQLPropertyFilter]]: return [p for p in self.query.properties if p.key != "$pathname"] def session_where(self):