feat(hogql): query modifiers (#17946)

ee/clickhouse/queries/test/__snapshots__/test_lifecycle.ambr

@@ -501,7 +501,7 @@
              AND event = '$pageview'
              AND timestamp >= toDateTime(dateTrunc('day', toDateTime('2021-04-28 00:00:00', 'UTC'))) - INTERVAL 1 day
              AND timestamp < toDateTime(dateTrunc('day', toDateTime('2021-05-05 23:59:59', 'UTC'))) + INTERVAL 1 day
-             AND (like(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_props, 'email'), ''), 'null'), '^"|"$', ''), '%test.com'))
+             AND (like(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_properties, 'email'), ''), 'null'), '^"|"$', ''), '%test.com'))
            GROUP BY pdi.person_id)
         GROUP BY start_of_period,
                  status)
@@ -576,7 +576,7 @@
              AND event = '$pageview'
              AND timestamp >= toDateTime(dateTrunc('day', toDateTime('2021-04-28 00:00:00', 'UTC'))) - INTERVAL 1 day
              AND timestamp < toDateTime(dateTrunc('day', toDateTime('2021-05-05 23:59:59', 'UTC'))) + INTERVAL 1 day
-             AND (like(nullIf(nullIf(pmat_email, ''), 'null'), '%test.com'))
+             AND (like(nullIf(nullIf(mat_pp_email, ''), 'null'), '%test.com'))
            GROUP BY pdi.person_id)
         GROUP BY start_of_period,
                  status)

frontend/src/queries/schema.json

@@ -1213,6 +1213,9 @@
                     "const": "HogQLQuery",
                     "type": "string"
                 },
+                "modifiers": {
+                    "$ref": "#/definitions/HogQLQueryModifiers"
+                },
                 "query": {
                     "type": "string"
                 },
@@ -1228,6 +1231,16 @@
             "required": ["kind", "query"],
             "type": "object"
         },
+        "HogQLQueryModifiers": {
+            "additionalProperties": false,
+            "description": "HogQL Query Options are automatically set per team. However, they can be overriden in the query.",
+            "properties": {
+                "personsOnEventsMode": {
+                    "type": "string"
+                }
+            },
+            "type": "object"
+        },
         "HogQLQueryResponse": {
             "additionalProperties": false,
             "properties": {
@@ -1241,6 +1254,9 @@
                 "hogql": {
                     "type": "string"
                 },
+                "modifiers": {
+                    "$ref": "#/definitions/HogQLQueryModifiers"
+                },
                 "query": {
                     "type": "string"
                 },

frontend/src/queries/schema.ts

@@ -130,6 +130,11 @@ export interface DataNode extends Node {
     response?: Record<string, any>
 }
 
+/** HogQL Query Options are automatically set per team. However, they can be overriden in the query. */
+export interface HogQLQueryModifiers {
+    personsOnEventsMode?: string
+}
+
 export interface HogQLQueryResponse {
     query?: string
     hogql?: string
@@ -138,6 +143,7 @@ export interface HogQLQueryResponse {
     types?: any[]
     columns?: any[]
     timings?: QueryTiming[]
+    modifiers?: HogQLQueryModifiers
 }
 
 /** Filters object that will be converted to a HogQL {filters} placeholder */
@@ -152,6 +158,7 @@ export interface HogQLQuery extends DataNode {
     filters?: HogQLFilters
     /** Constant values that can be referenced with the {placeholder} syntax in the query */
     values?: Record<string, any>
+    modifiers?: HogQLQueryModifiers
     response?: HogQLQueryResponse
 }
 

frontend/src/scenes/debug/HogQLDebug.tsx

@@ -7,6 +7,8 @@ import { dataNodeLogic, DataNodeLogicProps } from '~/queries/nodes/DataNode/data
 import { ElapsedTime, Timings } from '~/queries/nodes/DataNode/ElapsedTime'
 import { CodeSnippet, Language } from 'lib/components/CodeSnippet'
 import { CodeEditor } from 'lib/components/CodeEditors'
+import { LemonSelect } from 'lib/lemon-ui/LemonSelect'
+import { LemonLabel } from 'lib/lemon-ui/LemonLabel'
 
 interface HogQLDebugProps {
     query: HogQLQuery
@@ -23,6 +25,25 @@ export function HogQLDebug({ query, setQuery }: HogQLDebugProps): JSX.Element {
                     <DateRange key="date-range" query={query} setQuery={setQuery} />
                     <EventPropertyFilters key="event-property" query={query} setQuery={setQuery} />
                 </div>
+                <div className="flex">
+                    <LemonLabel>
+                        POE:
+                        <LemonSelect
+                            options={[
+                                { value: 'disabled', label: 'Disabled' },
+                                { value: 'v1_enabled', label: 'V1 Enabled' },
+                                { value: 'v2_enabled', label: 'V2 Enabled' },
+                            ]}
+                            onChange={(value) =>
+                                setQuery({
+                                    ...query,
+                                    modifiers: { ...query.modifiers, personsOnEventsMode: value },
+                                } as HogQLQuery)
+                            }
+                            value={(query.modifiers ?? response?.modifiers)?.personsOnEventsMode}
+                        />
+                    </LemonLabel>
+                </div>
                 {dataLoading ? (
                     <>
                         <h2>Running query...</h2>

posthog/api/query.py

@@ -25,6 +25,7 @@
 from posthog.hogql.database.database import create_hogql_database, serialize_database
 from posthog.hogql.errors import HogQLException
 from posthog.hogql.metadata import get_hogql_metadata
+from posthog.hogql.modifiers import create_default_modifiers_for_team
 from posthog.hogql.query import execute_hogql_query
 
 from posthog.hogql_queries.query_runner import get_query_runner
@@ -236,6 +237,7 @@ def process_query(
             query=hogql_query.query,
             team=team,
             filters=hogql_query.filters,
+            modifiers=hogql_query.modifiers,
             placeholders=values,
             default_limit=default_limit,
         )
@@ -245,7 +247,7 @@ def process_query(
         metadata_response = get_hogql_metadata(query=metadata_query, team=team)
         return _unwrap_pydantic_dict(metadata_response)
     elif query_kind == "DatabaseSchemaQuery":
-        database = create_hogql_database(team.pk)
+        database = create_hogql_database(team.pk, modifiers=create_default_modifiers_for_team(team))
         return serialize_database(database)
     elif query_kind == "TimeToSeeDataSessionsQuery":
         sessions_query_serializer = SessionsQuerySerializer(data=query_json)

posthog/api/test/__snapshots__/test_insight.ambr

@@ -3,7 +3,7 @@
   /* user_id:0 request:_snapshot_ */
   SELECT groupArray(value)
   FROM
-    (SELECT array(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_props, 'fish'), ''), 'null'), '^"|"$', '')) AS value,
+    (SELECT array(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_properties, 'fish'), ''), 'null'), '^"|"$', '')) AS value,
             count(*) as count
      FROM events e
      INNER JOIN
@@ -79,7 +79,7 @@
                            if(step_0 = 1, timestamp, null) as latest_0,
                            if(event = 'user did things', 1, 0) as step_1,
                            if(step_1 = 1, timestamp, null) as latest_1,
-                           array(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_props, 'fish'), ''), 'null'), '^"|"$', '')) AS prop_basic,
+                           array(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_properties, 'fish'), ''), 'null'), '^"|"$', '')) AS prop_basic,
                            prop_basic as prop,
                            argMinIf(prop, timestamp, notEmpty(arrayFilter(x -> notEmpty(x), prop))) over (PARTITION by aggregation_target) as prop_vals
                     FROM events e
@@ -170,7 +170,7 @@
                    AND toTimeZone(timestamp, 'UTC') >= toDateTime('2012-01-08 00:00:00', 'UTC')
                    AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC')
                    AND ((and(ifNull(less(toInt64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, 'int_value'), ''), 'null'), '^"|"$', '')), 10), 0), 1))
-                        AND (like(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_props, 'fish'), ''), 'null'), '^"|"$', ''), '%fish%')))
+                        AND (like(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_properties, 'fish'), ''), 'null'), '^"|"$', ''), '%fish%')))
                    AND (step_0 = 1
                         OR step_1 = 1) ))
            WHERE step_0 = 1 ))
@@ -215,11 +215,11 @@
                         person.person_props as person_props ,
                         if(event = 'user signed up'
                            AND (and(ifNull(less(toInt64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, 'int_value'), ''), 'null'), '^"|"$', '')), 10), 0), 1)
-                                AND like(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_props, 'fish'), ''), 'null'), '^"|"$', ''), '%fish%')), 1, 0) as step_0,
+                                AND like(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_properties, 'fish'), ''), 'null'), '^"|"$', ''), '%fish%')), 1, 0) as step_0,
                         if(step_0 = 1, timestamp, null) as latest_0,
                         if(event = 'user did things'
                            AND (and(ifNull(less(toInt64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, 'int_value'), ''), 'null'), '^"|"$', '')), 10), 0), 1)
-                                AND like(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_props, 'fish'), ''), 'null'), '^"|"$', ''), '%fish%')), 1, 0) as step_1,
+                                AND like(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_properties, 'fish'), ''), 'null'), '^"|"$', ''), '%fish%')), 1, 0) as step_1,
                         if(step_1 = 1, timestamp, null) as latest_1
                  FROM events e
                  INNER JOIN
@@ -438,7 +438,7 @@
           AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2012-01-08 00:00:00', 'UTC')), 'UTC')
           AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC')
           AND ((and(ifNull(greater(toInt64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, 'int_value'), ''), 'null'), '^"|"$', '')), 10), 0), 1))
-               AND (like(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_props, 'fish'), ''), 'null'), '^"|"$', ''), '%fish%')))
+               AND (like(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_properties, 'fish'), ''), 'null'), '^"|"$', ''), '%fish%')))
         GROUP BY date)
      GROUP BY day_start
      ORDER BY day_start)
@@ -506,7 +506,7 @@
           AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2012-01-08 00:00:00', 'UTC')), 'UTC')
           AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC')
           AND ((and(ifNull(greater(toInt64OrNull(nullIf(nullIf(events.mat_int_value, ''), 'null')), 10), 0), 1))
-               AND (like(nullIf(nullIf(pmat_fish, ''), 'null'), '%fish%')))
+               AND (like(nullIf(nullIf(mat_pp_fish, ''), 'null'), '%fish%')))
         GROUP BY date)
      GROUP BY day_start
      ORDER BY day_start)
@@ -548,7 +548,7 @@
           AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2012-01-08 00:00:00', 'UTC')), 'UTC')
           AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC')
           AND (and(ifNull(less(toInt64OrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties, 'int_value'), ''), 'null'), '^"|"$', '')), 10), 0), 1)
-               AND like(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_props, 'fish'), ''), 'null'), '^"|"$', ''), '%fish%'))
+               AND like(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person_properties, 'fish'), ''), 'null'), '^"|"$', ''), '%fish%'))
         GROUP BY date)
      GROUP BY day_start
      ORDER BY day_start)
@@ -590,7 +590,7 @@
           AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2012-01-08 00:00:00', 'UTC')), 'UTC')
           AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC')
           AND (and(ifNull(less(toInt64OrNull(nullIf(nullIf(events.mat_int_value, ''), 'null')), 10), 0), 1)
-               AND like(nullIf(nullIf(pmat_fish, ''), 'null'), '%fish%'))
+               AND like(nullIf(nullIf(mat_pp_fish, ''), 'null'), '%fish%'))
         GROUP BY date)
      GROUP BY day_start
      ORDER BY day_start)

posthog/hogql/ai.py

@@ -7,6 +7,7 @@
 from posthog.hogql.printer import print_ast
 from .database.database import create_hogql_database, serialize_database
 from posthog.utils import get_instance_region
+from .query import create_default_modifiers_for_team
 
 if TYPE_CHECKING:
     from posthog.models import User, Team
@@ -52,7 +53,12 @@ class PromptUnclear(Exception):
 
 def write_sql_from_prompt(prompt: str, *, current_query: Optional[str] = None, team: "Team", user: "User") -> str:
     database = create_hogql_database(team.pk)
-    context = HogQLContext(team_id=team.pk, enable_select_queries=True, database=database)
+    context = HogQLContext(
+        team_id=team.pk,
+        enable_select_queries=True,
+        database=database,
+        modifiers=create_default_modifiers_for_team(team),
+    )
     serialized_database = serialize_database(database)
     schema_description = "\n\n".join(
         (

posthog/hogql/context.py

@@ -2,8 +2,7 @@
 from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Any
 
 from posthog.hogql.timings import HogQLTimings
-from posthog.utils import PersonOnEventsMode
-from posthog.schema import HogQLNotice
+from posthog.schema import HogQLNotice, HogQLQueryModifiers
 
 if TYPE_CHECKING:
     from posthog.hogql.database.database import Database
@@ -29,8 +28,6 @@ class HogQLContext:
     values: Dict = field(default_factory=dict)
     # Are we small part of a non-HogQL query? If so, use custom syntax for accessed person properties.
     within_non_hogql_query: bool = False
-    # Do we need to join the persons table or not. Has effect if within_non_hogql_query = True
-    person_on_events_mode: PersonOnEventsMode = PersonOnEventsMode.V1_ENABLED
     # Enable full SELECT queries and subqueries in ClickHouse
     enable_select_queries: bool = False
     # Do we apply a limit of MAX_SELECT_RETURNED_ROWS=10000 to the topmost select query?
@@ -44,6 +41,8 @@ class HogQLContext:
     notices: List["HogQLNotice"] = field(default_factory=list)
     # Timings in seconds for different parts of the HogQL query
     timings: HogQLTimings = field(default_factory=HogQLTimings)
+    # Modifications requested by the HogQL client
+    modifiers: HogQLQueryModifiers = field(default_factory=HogQLQueryModifiers)
 
     def add_value(self, value: Any) -> str:
         key = f"hogql_val_{len(self.values)}"

posthog/hogql/database/database.py

@@ -35,6 +35,7 @@
 from posthog.hogql.errors import HogQLException
 from posthog.models.group_type_mapping import GroupTypeMapping
 from posthog.models.team.team import WeekStartDay
+from posthog.schema import HogQLQueryModifiers
 from posthog.utils import PersonOnEventsMode
 
 
@@ -108,13 +109,15 @@ def add_warehouse_tables(self, **field_definitions: Any):
             setattr(self, f_name, f_def)
 
 
-def create_hogql_database(team_id: int) -> Database:
+def create_hogql_database(team_id: int, modifiers: Optional[HogQLQueryModifiers] = None) -> Database:
     from posthog.models import Team
+    from posthog.hogql.query import create_default_modifiers_for_team
     from posthog.warehouse.models import DataWarehouseTable, DataWarehouseSavedQuery, DataWarehouseViewLink
 
     team = Team.objects.get(pk=team_id)
+    modifiers = create_default_modifiers_for_team(team, modifiers)
     database = Database(timezone=team.timezone, week_start_day=team.week_start_day)
-    if team.person_on_events_mode != PersonOnEventsMode.DISABLED:
+    if modifiers.personsOnEventsMode != PersonOnEventsMode.DISABLED:
         # TODO: split PoE v1 and v2 once SQL Expression fields are supported #15180
         database.events.fields["person"] = FieldTraverser(chain=["poe"])
         database.events.fields["person_id"] = StringDatabaseField(name="person_id")

posthog/hogql/database/test/test_s3_table.py

@@ -2,6 +2,7 @@
 from posthog.hogql.database.database import create_hogql_database
 from posthog.hogql.parser import parse_select
 from posthog.hogql.printer import print_ast
+from posthog.hogql.query import create_default_modifiers_for_team
 from posthog.test.base import BaseTest
 from posthog.hogql.database.test.tables import create_aapl_stock_s3_table
 from posthog.hogql.errors import HogQLException
@@ -12,7 +13,12 @@ def _init_database(self):
         self.database = create_hogql_database(self.team.pk)
         self.database.aapl_stock = create_aapl_stock_s3_table()
         self.database.aapl_stock_2 = create_aapl_stock_s3_table(name="aapl_stock_2")
-        self.context = HogQLContext(team_id=self.team.pk, enable_select_queries=True, database=self.database)
+        self.context = HogQLContext(
+            team_id=self.team.pk,
+            enable_select_queries=True,
+            database=self.database,
+            modifiers=create_default_modifiers_for_team(self.team),
+        )
 
     def _select(self, query: str, dialect: str = "clickhouse") -> str:
         return print_ast(parse_select(query), self.context, dialect=dialect)

posthog/hogql/database/test/test_saved_query.py

@@ -2,6 +2,7 @@
 from posthog.hogql.database.database import create_hogql_database
 from posthog.hogql.parser import parse_select
 from posthog.hogql.printer import print_ast
+from posthog.hogql.query import create_default_modifiers_for_team
 from posthog.test.base import BaseTest
 from posthog.hogql.database.test.tables import (
     create_aapl_stock_table_view,
@@ -20,7 +21,12 @@ def _init_database(self):
         self.database.aapl_stock = create_aapl_stock_s3_table()
         self.database.aapl_stock_nested_view = create_nested_aapl_stock_view()
         self.database.aapl_stock_self = create_aapl_stock_table_self_referencing()
-        self.context = HogQLContext(team_id=self.team.pk, enable_select_queries=True, database=self.database)
+        self.context = HogQLContext(
+            team_id=self.team.pk,
+            enable_select_queries=True,
+            database=self.database,
+            modifiers=create_default_modifiers_for_team(self.team),
+        )
 
     def _select(self, query: str, dialect: str = "clickhouse") -> str:
         return print_ast(parse_select(query), self.context, dialect=dialect)

posthog/hogql/database/test/test_view.py

@@ -2,6 +2,7 @@
 from posthog.hogql.database.database import create_hogql_database
 from posthog.hogql.parser import parse_select
 from posthog.hogql.printer import print_ast
+from posthog.hogql.query import create_default_modifiers_for_team
 from posthog.test.base import BaseTest
 from posthog.hogql.database.test.tables import (
     create_aapl_stock_table_view,
@@ -20,7 +21,12 @@ def _init_database(self):
         self.database.aapl_stock = create_aapl_stock_s3_table()
         self.database.aapl_stock_nested_view = create_nested_aapl_stock_view()
         self.database.aapl_stock_self = create_aapl_stock_table_self_referencing()
-        self.context = HogQLContext(team_id=self.team.pk, enable_select_queries=True, database=self.database)
+        self.context = HogQLContext(
+            team_id=self.team.pk,
+            enable_select_queries=True,
+            database=self.database,
+            modifiers=create_default_modifiers_for_team(self.team),
+        )
 
     def _select(self, query: str, dialect: str = "clickhouse") -> str:
         return print_ast(parse_select(query), self.context, dialect=dialect)