feat(web-analytics): Add backend version of web analytics queries (#1…

…7629) * Add web top sources query to the backend * Get top source query working * Add other queries and share code between query runners * Fix ABC warnings * Tidy up folder structure of query runners * Rename verbose class name * Revert method order * Rename web analytics classes to have web in the name * Update posthog/hogql_queries/query_runner.py Co-authored-by: Marius Andra <[email protected]> * Improve query runner type names * Fix broken import * Fix response types * Add comment explaining why we set the hash key --------- Co-authored-by: Marius Andra <[email protected]>
PostHog · Oct 2, 2023 · 217496f · 217496f
1 parent d9eac2c
commit 217496f
Show file tree

Hide file tree

Showing 24 changed files with 813 additions and 267 deletions.
diff --git a/frontend/src/queries/nodes/DataTable/DataTable.tsx b/frontend/src/queries/nodes/DataTable/DataTable.tsx
@@ -511,9 +511,9 @@ export function DataTable({ uniqueKey, query, setQuery, context, cachedResults }
                             }
                             footer={
                                 canLoadNextData &&
-                                ((response as any).results.length > 0 || !responseLoading) && (
-                                    <LoadNext query={query.source} />
-                                )
+                                ((response as any).results.length > 0 ||
+                                    (response as any).result.length > 0 ||
+                                    !responseLoading) && <LoadNext query={query.source} />
                             }
                         />
                     )}

diff --git a/frontend/src/queries/nodes/DataTable/dataTableLogic.ts b/frontend/src/queries/nodes/DataTable/dataTableLogic.ts
@@ -138,9 +138,15 @@ export const dataTableLogic = kea<dataTableLogicType>([
                     }))
                 }
 
-                return response && 'results' in response && Array.isArray(response.results)
-                    ? response.results.map((result: any) => ({ result })) ?? null
+                const results = !response
+                    ? null
+                    : 'results' in response && Array.isArray(response.results)
+                    ? response.results
+                    : 'result' in response && Array.isArray(response.result)
+                    ? response.result
                     : null
+
+                return results ? results.map((result: any) => ({ result })) ?? null : null
             },
         ],
         queryWithDefaults: [

diff --git a/frontend/src/queries/nodes/DataTable/queryFeatures.ts b/frontend/src/queries/nodes/DataTable/queryFeatures.ts
@@ -1,4 +1,11 @@
-import { isEventsQuery, isHogQLQuery, isPersonsNode } from '~/queries/utils'
+import {
+    isEventsQuery,
+    isHogQLQuery,
+    isPersonsNode,
+    isWebTopClicksQuery,
+    isWebTopPagesQuery,
+    isWebTopSourcesQuery,
+} from '~/queries/utils'
 import { Node } from '~/queries/schema'
 
 export enum QueryFeature {
@@ -40,5 +47,10 @@ export function getQueryFeatures(query: Node): Set<QueryFeature> {
         features.add(QueryFeature.personsSearch)
     }
 
+    if (isWebTopSourcesQuery(query) || isWebTopPagesQuery(query) || isWebTopClicksQuery(query)) {
+        features.add(QueryFeature.columnsInResponse)
+        features.add(QueryFeature.resultIsArrayOfArrays)
+    }
+
     return features
 }
diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json
@@ -95,6 +95,15 @@
                 },
                 {
                     "$ref": "#/definitions/TimeToSeeDataSessionsQuery"
+                },
+                {
+                    "$ref": "#/definitions/WebTopSourcesQuery"
+                },
+                {
+                    "$ref": "#/definitions/WebTopClicksQuery"
+                },
+                {
+                    "$ref": "#/definitions/WebTopPagesQuery"
                 }
             ]
         },
@@ -389,6 +398,15 @@
                         },
                         {
                             "$ref": "#/definitions/TimeToSeeDataSessionsQuery"
+                        },
+                        {
+                            "$ref": "#/definitions/WebTopSourcesQuery"
+                        },
+                        {
+                            "$ref": "#/definitions/WebTopClicksQuery"
+                        },
+                        {
+                            "$ref": "#/definitions/WebTopPagesQuery"
                         }
                     ],
                     "description": "Source of the events"
@@ -2269,6 +2287,169 @@
             },
             "required": ["result"],
             "type": "object"
+        },
+        "WebAnalyticsFilters": {},
+        "WebTopClicksQuery": {
+            "additionalProperties": false,
+            "properties": {
+                "dateRange": {
+                    "$ref": "#/definitions/DateRange"
+                },
+                "filters": {
+                    "$ref": "#/definitions/WebAnalyticsFilters"
+                },
+                "kind": {
+                    "const": "WebTopClicksQuery",
+                    "type": "string"
+                },
+                "response": {
+                    "$ref": "#/definitions/WebTopClicksQueryResponse"
+                }
+            },
+            "required": ["kind", "filters"],
+            "type": "object"
+        },
+        "WebTopClicksQueryResponse": {
+            "additionalProperties": false,
+            "properties": {
+                "columns": {
+                    "items": {},
+                    "type": "array"
+                },
+                "is_cached": {
+                    "type": "boolean"
+                },
+                "last_refresh": {
+                    "type": "string"
+                },
+                "next_allowed_client_refresh": {
+                    "type": "string"
+                },
+                "result": {
+                    "items": {},
+                    "type": "array"
+                },
+                "timings": {
+                    "items": {
+                        "$ref": "#/definitions/QueryTiming"
+                    },
+                    "type": "array"
+                },
+                "types": {
+                    "items": {},
+                    "type": "array"
+                }
+            },
+            "required": ["result"],
+            "type": "object"
+        },
+        "WebTopPagesQuery": {
+            "additionalProperties": false,
+            "properties": {
+                "dateRange": {
+                    "$ref": "#/definitions/DateRange"
+                },
+                "filters": {
+                    "$ref": "#/definitions/WebAnalyticsFilters"
+                },
+                "kind": {
+                    "const": "WebTopPagesQuery",
+                    "type": "string"
+                },
+                "response": {
+                    "$ref": "#/definitions/WebTopPagesQueryResponse"
+                }
+            },
+            "required": ["kind", "filters"],
+            "type": "object"
+        },
+        "WebTopPagesQueryResponse": {
+            "additionalProperties": false,
+            "properties": {
+                "columns": {
+                    "items": {},
+                    "type": "array"
+                },
+                "is_cached": {
+                    "type": "boolean"
+                },
+                "last_refresh": {
+                    "type": "string"
+                },
+                "next_allowed_client_refresh": {
+                    "type": "string"
+                },
+                "result": {
+                    "items": {},
+                    "type": "array"
+                },
+                "timings": {
+                    "items": {
+                        "$ref": "#/definitions/QueryTiming"
+                    },
+                    "type": "array"
+                },
+                "types": {
+                    "items": {},
+                    "type": "array"
+                }
+            },
+            "required": ["result"],
+            "type": "object"
+        },
+        "WebTopSourcesQuery": {
+            "additionalProperties": false,
+            "properties": {
+                "dateRange": {
+                    "$ref": "#/definitions/DateRange"
+                },
+                "filters": {
+                    "$ref": "#/definitions/WebAnalyticsFilters"
+                },
+                "kind": {
+                    "const": "WebTopSourcesQuery",
+                    "type": "string"
+                },
+                "response": {
+                    "$ref": "#/definitions/WebTopSourcesQueryResponse"
+                }
+            },
+            "required": ["kind", "filters"],
+            "type": "object"
+        },
+        "WebTopSourcesQueryResponse": {
+            "additionalProperties": false,
+            "properties": {
+                "columns": {
+                    "items": {},
+                    "type": "array"
+                },
+                "is_cached": {
+                    "type": "boolean"
+                },
+                "last_refresh": {
+                    "type": "string"
+                },
+                "next_allowed_client_refresh": {
+                    "type": "string"
+                },
+                "result": {
+                    "items": {},
+                    "type": "array"
+                },
+                "timings": {
+                    "items": {
+                        "$ref": "#/definitions/QueryTiming"
+                    },
+                    "type": "array"
+                },
+                "types": {
+                    "items": {},
+                    "type": "array"
+                }
+            },
+            "required": ["result"],
+            "type": "object"
         }
     }
 }
diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts
@@ -57,6 +57,11 @@ export enum NodeKind {
     StickinessQuery = 'StickinessQuery',
     LifecycleQuery = 'LifecycleQuery',
 
+    // Web analytics queries
+    WebTopSourcesQuery = 'WebTopSourcesQuery',
+    WebTopPagesQuery = 'WebTopPagesQuery',
+    WebTopClicksQuery = 'WebTopClicksQuery',
+
     // Time to see data
     TimeToSeeDataSessionsQuery = 'TimeToSeeDataSessionsQuery',
     TimeToSeeDataQuery = 'TimeToSeeDataQuery',
@@ -75,6 +80,9 @@ export type AnyDataNode =
     | HogQLQuery
     | HogQLMetadata
     | TimeToSeeDataSessionsQuery
+    | WebTopSourcesQuery
+    | WebTopClicksQuery
+    | WebTopPagesQuery
 
 export type QuerySchema =
     // Data nodes (see utils.ts)
@@ -277,7 +285,15 @@ export type HasPropertiesNode = EventsNode | EventsQuery | PersonsNode
 export interface DataTableNode extends Node, DataTableNodeViewProps {
     kind: NodeKind.DataTableNode
     /** Source of the events */
-    source: EventsNode | EventsQuery | PersonsNode | HogQLQuery | TimeToSeeDataSessionsQuery
+    source:
+        | EventsNode
+        | EventsQuery
+        | PersonsNode
+        | HogQLQuery
+        | TimeToSeeDataSessionsQuery
+        | WebTopSourcesQuery
+        | WebTopClicksQuery
+        | WebTopPagesQuery
 
     /** Columns shown in the table, unless the `source` provides them. */
     columns?: HogQLExpression[]
@@ -483,6 +499,45 @@ export interface LifecycleQuery extends InsightsQueryBase {
     response?: LifecycleQueryResponse
 }
 
+export type WebAnalyticsFilters = any
+
+export interface WebAnalyticsQueryBase {
+    dateRange?: DateRange
+}
+
+export interface WebTopSourcesQuery extends WebAnalyticsQueryBase {
+    kind: NodeKind.WebTopSourcesQuery
+    filters: WebAnalyticsFilters
+    response?: WebTopSourcesQueryResponse
+}
+export interface WebTopSourcesQueryResponse extends QueryResponse {
+    result: unknown[]
+    types?: unknown[]
+    columns?: unknown[]
+}
+
+export interface WebTopClicksQuery extends WebAnalyticsQueryBase {
+    kind: NodeKind.WebTopClicksQuery
+    filters: WebAnalyticsFilters
+    response?: WebTopClicksQueryResponse
+}
+export interface WebTopClicksQueryResponse extends QueryResponse {
+    result: unknown[]
+    types?: unknown[]
+    columns?: unknown[]
+}
+
+export interface WebTopPagesQuery extends WebAnalyticsQueryBase {
+    kind: NodeKind.WebTopPagesQuery
+    filters: WebAnalyticsFilters
+    response?: WebTopPagesQueryResponse
+}
+export interface WebTopPagesQueryResponse extends QueryResponse {
+    result: unknown[]
+    types?: unknown[]
+    columns?: unknown[]
+}
+
 export type InsightQueryNode =
     | TrendsQuery
     | FunnelsQuery

diff --git a/frontend/src/queries/utils.ts b/frontend/src/queries/utils.ts
@@ -26,6 +26,9 @@ import {
     TimeToSeeDataJSONNode,
     DatabaseSchemaQuery,
     SavedInsightNode,
+    WebTopSourcesQuery,
+    WebTopClicksQuery,
+    WebTopPagesQuery,
 } from '~/queries/schema'
 import { TaxonomicFilterGroupType, TaxonomicFilterValue } from 'lib/components/TaxonomicFilter/types'
 import { dayjs } from 'lib/dayjs'
@@ -90,6 +93,18 @@ export function isHogQLQuery(node?: Node | null): node is HogQLQuery {
     return node?.kind === NodeKind.HogQLQuery
 }
 
+export function isWebTopSourcesQuery(node?: Node | null): node is WebTopSourcesQuery {
+    return node?.kind === NodeKind.WebTopSourcesQuery
+}
+
+export function isWebTopClicksQuery(node?: Node | null): node is WebTopClicksQuery {
+    return node?.kind === NodeKind.WebTopClicksQuery
+}
+
+export function isWebTopPagesQuery(node?: Node | null): node is WebTopPagesQuery {
+    return node?.kind === NodeKind.WebTopPagesQuery
+}
+
 export function containsHogQLQuery(node?: Node | null): boolean {
     if (!node) {
         return false

diff --git a/frontend/src/scenes/organization/Settings/VerifiedDomains/SSOSelect.tsx b/frontend/src/scenes/organization/Settings/VerifiedDomains/SSOSelect.tsx
@@ -5,7 +5,7 @@ import { SSO_PROVIDER_NAMES } from 'lib/constants'
 import { preflightLogic } from 'scenes/PreflightCheck/preflightLogic'
 import { SSOProvider } from '~/types'
 
-interface SSOSelectInterface {
+export interface SSOSelectInterface {
     value: SSOProvider | ''
     loading: boolean
     onChange: (value: SSOProvider | '') => void