feat(hogql): replace filters with query backend side (#21945)

posthog/api/insight.py

@@ -54,8 +54,11 @@
 )
 from posthog.hogql.errors import ExposedHogQLError
 from posthog.hogql.timings import HogQLTimings
-from posthog.hogql_queries.apply_dashboard_filters import DATA_TABLE_LIKE_NODE_KINDS
-from posthog.hogql_queries.legacy_compatibility.feature_flag import should_use_hogql_backend_in_insight_serialization
+from posthog.hogql_queries.apply_dashboard_filters import WRAPPER_NODE_KINDS
+from posthog.hogql_queries.legacy_compatibility.feature_flag import (
+    hogql_insights_replace_filters,
+    should_use_hogql_backend_in_insight_serialization,
+)
 from posthog.hogql_queries.legacy_compatibility.filter_to_query import filter_to_query
 from posthog.kafka_client.topics import KAFKA_METRICS_TIME_TO_SEE_DATA
 from posthog.models import DashboardTile, Filter, Insight, User
@@ -206,11 +209,17 @@ def to_representation(self, instance):
 
         representation["dashboards"] = [tile["dashboard_id"] for tile in representation["dashboard_tiles"]]
 
-        filters = instance.dashboard_filters()
+        if hogql_insights_replace_filters(instance.team) and (
+            instance.query is not None or instance.query_from_filters is not None
+        ):
+            representation["filters"] = {}
+            representation["query"] = instance.query or instance.query_from_filters
+        else:
+            filters = instance.dashboard_filters()
 
-        if not filters.get("date_from") and not instance.query:
-            filters.update({"date_from": f"-{DEFAULT_DATE_FROM_DAYS}d"})
-        representation["filters"] = filters
+            if not filters.get("date_from") and not instance.query:
+                filters.update({"date_from": f"-{DEFAULT_DATE_FROM_DAYS}d"})
+            representation["filters"] = filters
 
         return representation
 
@@ -506,11 +515,22 @@ def to_representation(self, instance: Insight):
             representation["dashboards"] = [tile["dashboard_id"] for tile in representation["dashboard_tiles"]]
 
         dashboard: Optional[Dashboard] = self.context.get("dashboard")
-        representation["filters"] = instance.dashboard_filters(dashboard=dashboard)
-        representation["query"] = instance.get_effective_query(dashboard=dashboard)
+        if hogql_insights_replace_filters(instance.team) and (
+            instance.query is not None or instance.query_from_filters is not None
+        ):
+            from posthog.hogql_queries.apply_dashboard_filters import apply_dashboard_filters
 
-        if "insight" not in representation["filters"] and not representation["query"]:
-            representation["filters"]["insight"] = "TRENDS"
+            query = instance.query or instance.query_from_filters
+            if dashboard:
+                query = apply_dashboard_filters(query, dashboard.filters, instance.team)
+            representation["filters"] = {}
+            representation["query"] = query
+        else:
+            representation["filters"] = instance.dashboard_filters(dashboard=dashboard)
+            representation["query"] = instance.get_effective_query(dashboard=dashboard)
+
+            if "insight" not in representation["filters"] and not representation["query"]:
+                representation["filters"]["insight"] = "TRENDS"
 
         representation["filters_hash"] = self.insight_result(instance).cache_key
 
@@ -713,14 +733,10 @@ def _filter_request(self, request: request.Request, queryset: QuerySet) -> Query
                 insight = request.GET[INSIGHT]
                 if insight == "JSON":
                     queryset = queryset.filter(query__isnull=False)
-                    queryset = queryset.exclude(
-                        query__kind__in=DATA_TABLE_LIKE_NODE_KINDS, query__source__kind="HogQLQuery"
-                    )
+                    queryset = queryset.exclude(query__kind__in=WRAPPER_NODE_KINDS, query__source__kind="HogQLQuery")
                 elif insight == "SQL":
                     queryset = queryset.filter(query__isnull=False)
-                    queryset = queryset.filter(
-                        query__kind__in=DATA_TABLE_LIKE_NODE_KINDS, query__source__kind="HogQLQuery"
-                    )
+                    queryset = queryset.filter(query__kind__in=WRAPPER_NODE_KINDS, query__source__kind="HogQLQuery")
                 else:
                     queryset = queryset.filter(query__isnull=True)
                     queryset = queryset.filter(filters__insight=insight)

posthog/hogql_queries/apply_dashboard_filters.py

@@ -3,14 +3,14 @@
 from posthog.models import Team
 from posthog.schema import DashboardFilter, NodeKind
 
-DATA_TABLE_LIKE_NODE_KINDS = [NodeKind.DataTableNode, NodeKind.DataVisualizationNode]
+WRAPPER_NODE_KINDS = [NodeKind.DataTableNode, NodeKind.DataVisualizationNode, NodeKind.InsightVizNode]
 
 
 # Apply the filters from the django-style Dashboard object
 def apply_dashboard_filters(query: dict, filters: dict, team: Team) -> dict:
     kind = query.get("kind", None)
 
-    if kind in DATA_TABLE_LIKE_NODE_KINDS:
+    if kind in WRAPPER_NODE_KINDS:
         source = apply_dashboard_filters(query["source"], filters, team)
         return {**query, "source": source}
 

posthog/hogql_queries/legacy_compatibility/feature_flag.py

@@ -1,6 +1,6 @@
 import posthoganalytics
 from django.conf import settings
-from posthog.models.user import User
+from posthog.models import User, Team
 
 
 def should_use_hogql_backend_in_insight_serialization(user: User) -> bool:
@@ -14,3 +14,18 @@ def should_use_hogql_backend_in_insight_serialization(user: User) -> bool:
         only_evaluate_locally=True,
         send_feature_flag_events=False,
     )
+
+
+def hogql_insights_replace_filters(team: Team) -> bool:
+    return posthoganalytics.feature_enabled(
+        "hogql-insights-replace-filters",
+        str(team.uuid),
+        groups={"organization": str(team.organization_id)},
+        group_properties={
+            "organization": {
+                "id": str(team.organization_id),
+            }
+        },
+        only_evaluate_locally=True,
+        send_feature_flag_events=False,
+    )

posthog/models/insight.py

@@ -1,5 +1,7 @@
+from functools import cached_property
 from typing import Optional
 
+from sentry_sdk import capture_exception
 import structlog
 from django.contrib.postgres.fields import ArrayField
 from django.db import models
@@ -112,6 +114,15 @@ def caching_state(self):
                 return state
         return None
 
+    @cached_property
+    def query_from_filters(self):
+        from posthog.hogql_queries.legacy_compatibility.filter_to_query import filter_to_query
+
+        try:
+            return {"kind": "InsightVizNode", "source": filter_to_query(self.filters).model_dump(), "full": True}
+        except Exception as e:
+            capture_exception(e)
+
     class Meta:
         db_table = "posthog_dashboarditem"
         unique_together = ("team", "short_id")