feat(hogql): scaffold for funnels query runner (#19910)

frontend/src/lib/constants.tsx

@@ -173,6 +173,7 @@ export const FEATURE_FLAGS = {
     HOGQL_INSIGHTS_RETENTION: 'hogql-insights-retention', // owner: @webjunkie
     HOGQL_INSIGHTS_TRENDS: 'hogql-insights-trends', // owner: @Gilbert09
     HOGQL_INSIGHTS_STICKINESS: 'hogql-insights-stickiness', // owner: @Gilbert09
+    HOGQL_INSIGHTS_FUNNELS: 'hogql-insights-funnels', // owner: @thmsobrmlr
     HOGQL_INSIGHT_LIVE_COMPARE: 'hogql-insight-live-compare', // owner: @mariusandra
     BI_VIZ: 'bi_viz', // owner: @Gilbert09
     WEBHOOKS_DENYLIST: 'webhooks-denylist', // owner: #team-pipeline

frontend/src/queries/nodes/DataNode/dataNodeLogic.ts

@@ -320,26 +320,10 @@ export const dataNodeLogic = kea<dataNodeLogicType>([
             () => [(_, props) => props.cachedResults ?? null],
             (cachedResults: AnyResponseType | null): boolean => !!cachedResults,
         ],
-        hogQLInsightsLifecycleFlagEnabled: [
-            (s) => [s.featureFlags],
-            (featureFlags) => !!featureFlags[FEATURE_FLAGS.HOGQL_INSIGHTS_LIFECYCLE],
-        ],
-        hogQLInsightsPathsFlagEnabled: [
-            (s) => [s.featureFlags],
-            (featureFlags) => !!featureFlags[FEATURE_FLAGS.HOGQL_INSIGHTS_PATHS],
-        ],
         hogQLInsightsRetentionFlagEnabled: [
             (s) => [s.featureFlags],
             (featureFlags) => !!featureFlags[FEATURE_FLAGS.HOGQL_INSIGHTS_RETENTION],
         ],
-        hogQLInsightsTrendsFlagEnabled: [
-            (s) => [s.featureFlags],
-            (featureFlags) => !!featureFlags[FEATURE_FLAGS.HOGQL_INSIGHTS_TRENDS],
-        ],
-        hogQLInsightsStickinessFlagEnabled: [
-            (s) => [s.featureFlags],
-            (featureFlags) => !!featureFlags[FEATURE_FLAGS.HOGQL_INSIGHTS_STICKINESS],
-        ],
         query: [(_, p) => [p.query], (query) => query],
         newQuery: [
             (s, p) => [p.query, s.response],

frontend/src/queries/query.ts

@@ -25,6 +25,7 @@ import {
     isDataTableNode,
     isDataVisualizationNode,
     isEventsQuery,
+    isFunnelsQuery,
     isHogQLQuery,
     isInsightQueryNode,
     isInsightVizNode,
@@ -163,6 +164,9 @@ export async function query<N extends DataNode = DataNode>(
     const hogQLInsightsStickinessFlagEnabled = Boolean(
         featureFlagLogic.findMounted()?.values.featureFlags?.[FEATURE_FLAGS.HOGQL_INSIGHTS_STICKINESS]
     )
+    const hogQLInsightsFunnelsFlagEnabled = Boolean(
+        featureFlagLogic.findMounted()?.values.featureFlags?.[FEATURE_FLAGS.HOGQL_INSIGHTS_FUNNELS]
+    )
     const hogQLInsightsLiveCompareEnabled = Boolean(
         featureFlagLogic.findMounted()?.values.featureFlags?.[FEATURE_FLAGS.HOGQL_INSIGHT_LIVE_COMPARE]
     )
@@ -208,7 +212,8 @@ export async function query<N extends DataNode = DataNode>(
                 (hogQLInsightsPathsFlagEnabled && isPathsQuery(queryNode)) ||
                 (hogQLInsightsRetentionFlagEnabled && isRetentionQuery(queryNode)) ||
                 (hogQLInsightsTrendsFlagEnabled && isTrendsQuery(queryNode)) ||
-                (hogQLInsightsStickinessFlagEnabled && isStickinessQuery(queryNode))
+                (hogQLInsightsStickinessFlagEnabled && isStickinessQuery(queryNode)) ||
+                (hogQLInsightsFunnelsFlagEnabled && isFunnelsQuery(queryNode))
             ) {
                 if (hogQLInsightsLiveCompareEnabled) {
                     let legacyResponse

frontend/src/queries/schema.json

@@ -1484,6 +1484,37 @@
             "required": ["kind", "series"],
             "type": "object"
         },
+        "FunnelsQueryResponse": {
+            "additionalProperties": false,
+            "properties": {
+                "hogql": {
+                    "type": "string"
+                },
+                "is_cached": {
+                    "type": "boolean"
+                },
+                "last_refresh": {
+                    "type": "string"
+                },
+                "next_allowed_client_refresh": {
+                    "type": "string"
+                },
+                "results": {
+                    "items": {
+                        "type": "object"
+                    },
+                    "type": "array"
+                },
+                "timings": {
+                    "items": {
+                        "$ref": "#/definitions/QueryTiming"
+                    },
+                    "type": "array"
+                }
+            },
+            "required": ["results"],
+            "type": "object"
+        },
         "GoalLine": {
             "additionalProperties": false,
             "properties": {

frontend/src/queries/schema.ts

@@ -573,6 +573,10 @@ export interface FunnelsQuery extends InsightsQueryBase {
     breakdownFilter?: BreakdownFilter
 }
 
+export interface FunnelsQueryResponse extends QueryResponse {
+    results: Record<string, any>[]
+}
+
 /** `RetentionFilterType` minus everything inherited from `FilterType` */
 export type RetentionFilterLegacy = Omit<RetentionFilterType, keyof FilterType>
 

posthog/api/services/query.py

@@ -14,6 +14,7 @@
 from posthog.queries.time_to_see_data.serializers import SessionEventsQuerySerializer, SessionsQuerySerializer
 from posthog.queries.time_to_see_data.sessions import get_session_events, get_sessions
 from posthog.schema import (
+    FunnelsQuery,
     HogQLMetadata,
     HogQLQuery,
     EventsQuery,
@@ -36,11 +37,12 @@
 logger = structlog.get_logger(__name__)
 
 QUERY_WITH_RUNNER = (
-    LifecycleQuery
-    | PathsQuery
+    TrendsQuery
+    | FunnelsQuery
     | RetentionQuery
+    | PathsQuery
     | StickinessQuery
-    | TrendsQuery
+    | LifecycleQuery
     | WebOverviewQuery
     | WebTopClicksQuery
     | WebStatsTableQuery

posthog/hogql_queries/insights/funnels/funnels_query_runner.py

@@ -0,0 +1,100 @@
+from datetime import timedelta
+from math import ceil
+from typing import List, Optional, Any, Dict, cast
+
+from django.utils.timezone import datetime
+from posthog.caching.insights_api import (
+    BASE_MINIMUM_INSIGHT_REFRESH_INTERVAL,
+    REDUCED_MINIMUM_INSIGHT_REFRESH_INTERVAL,
+)
+from posthog.caching.utils import is_stale
+
+from posthog.hogql import ast
+from posthog.hogql.constants import LimitContext
+from posthog.hogql.parser import parse_select
+from posthog.hogql.query import execute_hogql_query
+from posthog.hogql.timings import HogQLTimings
+from posthog.hogql_queries.query_runner import QueryRunner
+from posthog.hogql_queries.utils.query_date_range import QueryDateRange
+from posthog.models import Team
+from posthog.models.filters.mixins.utils import cached_property
+from posthog.schema import (
+    FunnelsQuery,
+    FunnelsQueryResponse,
+    HogQLQueryModifiers,
+)
+
+
+class FunnelsQueryRunner(QueryRunner):
+    query: FunnelsQuery
+    query_type = FunnelsQuery
+
+    def __init__(
+        self,
+        query: FunnelsQuery | Dict[str, Any],
+        team: Team,
+        timings: Optional[HogQLTimings] = None,
+        modifiers: Optional[HogQLQueryModifiers] = None,
+        limit_context: Optional[LimitContext] = None,
+    ):
+        super().__init__(query, team=team, timings=timings, modifiers=modifiers, limit_context=limit_context)
+
+    def _is_stale(self, cached_result_package):
+        date_to = self.query_date_range.date_to()
+        interval = self.query_date_range.interval_name
+        return is_stale(self.team, date_to, interval, cached_result_package)
+
+    def _refresh_frequency(self):
+        date_to = self.query_date_range.date_to()
+        date_from = self.query_date_range.date_from()
+        interval = self.query_date_range.interval_name
+
+        delta_days: Optional[int] = None
+        if date_from and date_to:
+            delta = date_to - date_from
+            delta_days = ceil(delta.total_seconds() / timedelta(days=1).total_seconds())
+
+        refresh_frequency = BASE_MINIMUM_INSIGHT_REFRESH_INTERVAL
+        if interval == "hour" or (delta_days is not None and delta_days <= 7):
+            # The interval is shorter for short-term insights
+            refresh_frequency = REDUCED_MINIMUM_INSIGHT_REFRESH_INTERVAL
+
+        return refresh_frequency
+
+    def to_query(self) -> ast.SelectQuery:
+        select_query = parse_select(
+            """
+                SELECT 1
+            """,
+            placeholders={},
+        )
+
+        return cast(ast.SelectQuery, select_query)
+
+    def calculate(self):
+        query = self.to_query()
+
+        res: List[Dict[str, Any]] = []
+        timings = []
+
+        response = execute_hogql_query(
+            query_type="FunnelsQuery",
+            query=query,
+            team=self.team,
+            timings=self.timings,
+            modifiers=self.modifiers,
+        )
+
+        if response.timings is not None:
+            timings.extend(response.timings)
+
+        return FunnelsQueryResponse(results=res, timings=timings)
+
+    @cached_property
+    def query_date_range(self):
+        return QueryDateRange(
+            date_range=self.query.dateRange,
+            team=self.team,
+            interval=self.query.interval,
+            now=datetime.now(),
+        )

posthog/hogql_queries/query_runner.py

@@ -17,22 +17,23 @@
 from posthog.metrics import LABEL_TEAM_ID
 from posthog.models import Team
 from posthog.schema import (
-    QueryTiming,
-    SessionsTimelineQuery,
-    StickinessQuery,
     TrendsQuery,
+    FunnelsQuery,
+    RetentionQuery,
+    PathsQuery,
+    StickinessQuery,
     LifecycleQuery,
-    WebTopClicksQuery,
+    HogQLQuery,
     WebOverviewQuery,
+    WebTopClicksQuery,
+    WebStatsTableQuery,
+    QueryTiming,
+    SessionsTimelineQuery,
     ActorsQuery,
     EventsQuery,
-    WebStatsTableQuery,
-    HogQLQuery,
     InsightActorsQuery,
     DashboardFilter,
     HogQLQueryModifiers,
-    RetentionQuery,
-    PathsQuery,
     SamplingRate,
 )
 from posthog.utils import generate_cache_key, get_safe_cache
@@ -79,16 +80,17 @@ class CachedQueryResponse(QueryResponse):
 
 
 RunnableQueryNode = Union[
+    TrendsQuery,
+    FunnelsQuery,
+    RetentionQuery,
+    PathsQuery,
+    StickinessQuery,
+    LifecycleQuery,
     ActorsQuery,
     EventsQuery,
     HogQLQuery,
     InsightActorsQuery,
-    LifecycleQuery,
-    PathsQuery,
-    RetentionQuery,
     SessionsTimelineQuery,
-    StickinessQuery,
-    TrendsQuery,
     WebOverviewQuery,
     WebStatsTableQuery,
     WebTopClicksQuery,
@@ -110,16 +112,6 @@ def get_query_runner(
     else:
         raise ValueError(f"Can't get a runner for an unknown query type: {query}")
 
-    if kind == "LifecycleQuery":
-        from .insights.lifecycle_query_runner import LifecycleQueryRunner
-
-        return LifecycleQueryRunner(
-            query=cast(LifecycleQuery | Dict[str, Any], query),
-            team=team,
-            timings=timings,
-            limit_context=limit_context,
-            modifiers=modifiers,
-        )
     if kind == "TrendsQuery":
         from .insights.trends.trends_query_runner import TrendsQueryRunner
 
@@ -130,11 +122,11 @@ def get_query_runner(
             limit_context=limit_context,
             modifiers=modifiers,
         )
-    if kind == "StickinessQuery":
-        from .insights.stickiness_query_runner import StickinessQueryRunner
+    if kind == "FunnelsQuery":
+        from .insights.funnels.funnels_query_runner import FunnelsQueryRunner
 
-        return StickinessQueryRunner(
-            query=cast(StickinessQuery | Dict[str, Any], query),
+        return FunnelsQueryRunner(
+            query=cast(FunnelsQuery | Dict[str, Any], query),
             team=team,
             timings=timings,
             limit_context=limit_context,
@@ -160,6 +152,26 @@ def get_query_runner(
             limit_context=limit_context,
             modifiers=modifiers,
         )
+    if kind == "StickinessQuery":
+        from .insights.stickiness_query_runner import StickinessQueryRunner
+
+        return StickinessQueryRunner(
+            query=cast(StickinessQuery | Dict[str, Any], query),
+            team=team,
+            timings=timings,
+            limit_context=limit_context,
+            modifiers=modifiers,
+        )
+    if kind == "LifecycleQuery":
+        from .insights.lifecycle_query_runner import LifecycleQueryRunner
+
+        return LifecycleQueryRunner(
+            query=cast(LifecycleQuery | Dict[str, Any], query),
+            team=team,
+            timings=timings,
+            limit_context=limit_context,
+            modifiers=modifiers,
+        )
     if kind == "EventsQuery":
         from .events_query_runner import EventsQueryRunner
 

posthog/schema.py

@@ -972,6 +972,18 @@ class FunnelsFilter(BaseModel):
     layout: Optional[FunnelLayout] = None
 
 
+class FunnelsQueryResponse(BaseModel):
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    hogql: Optional[str] = None
+    is_cached: Optional[bool] = None
+    last_refresh: Optional[str] = None
+    next_allowed_client_refresh: Optional[str] = None
+    results: List[Dict[str, Any]]
+    timings: Optional[List[QueryTiming]] = None
+
+
 class GroupPropertyFilter(BaseModel):
     model_config = ConfigDict(
         extra="forbid",