feat(hogql): settings at the query level (#17823)

posthog/api/test/__snapshots__/test_query.ambr

@@ -12,7 +12,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_event_property_filter.1
@@ -29,7 +29,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_event_property_filter.2
@@ -47,7 +47,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_event_property_filter_materialized
@@ -64,7 +64,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_event_property_filter_materialized.1
@@ -81,7 +81,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_event_property_filter_materialized.2
@@ -99,7 +99,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_events_query_all_time_date
@@ -112,7 +112,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_events_query_all_time_date.1
@@ -125,7 +125,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_events_query_all_time_date.2
@@ -138,7 +138,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_full_hogql_query
@@ -152,7 +152,7 @@
   ORDER BY toTimeZone(events.timestamp, 'UTC') ASC
   LIMIT 100 SETTINGS readonly=2,
                      max_execution_time=60,
-                     allow_experimental_object_type=True
+                     allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_full_hogql_query_materialized
@@ -166,7 +166,7 @@
   ORDER BY toTimeZone(events.timestamp, 'UTC') ASC
   LIMIT 100 SETTINGS readonly=2,
                      max_execution_time=60,
-                     allow_experimental_object_type=True
+                     allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_full_hogql_query_view
@@ -180,7 +180,7 @@
   ORDER BY toTimeZone(events.timestamp, 'UTC') ASC
   LIMIT 100 SETTINGS readonly=2,
                      max_execution_time=60,
-                     allow_experimental_object_type=True
+                     allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_full_hogql_query_view.1
@@ -198,7 +198,7 @@
      ORDER BY toTimeZone(events.timestamp, 'UTC') ASC) AS event_view
   LIMIT 100 SETTINGS readonly=2,
                      max_execution_time=60,
-                     allow_experimental_object_type=True
+                     allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_hogql_property_filter
@@ -215,7 +215,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_hogql_property_filter.1
@@ -232,7 +232,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_hogql_property_filter.2
@@ -249,7 +249,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_hogql_property_filter.3
@@ -266,7 +266,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_hogql_property_filter_materialized
@@ -283,7 +283,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_hogql_property_filter_materialized.1
@@ -300,7 +300,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_hogql_property_filter_materialized.2
@@ -317,7 +317,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_hogql_property_filter_materialized.3
@@ -334,7 +334,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_person_property_filter
@@ -359,13 +359,13 @@
      FROM person
      WHERE equals(person.team_id, 2)
      GROUP BY person.id
-     HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0)) AS events__pdi__person ON equals(events__pdi.person_id, events__pdi__person.id)
+     HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__pdi__person ON equals(events__pdi.person_id, events__pdi__person.id)
   WHERE and(equals(events.team_id, 2), ifNull(equals(events__pdi__person.properties___email, '[email protected]'), 0), ifNull(less(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-10 12:14:05.000000', 6, 'UTC')), 0), ifNull(greater(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-09 12:14:00.000000', 6, 'UTC')), 0))
   ORDER BY events.event ASC
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_person_property_filter_materialized
@@ -390,13 +390,13 @@
      FROM person
      WHERE equals(person.team_id, 2)
      GROUP BY person.id
-     HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0)) AS events__pdi__person ON equals(events__pdi.person_id, events__pdi__person.id)
+     HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__pdi__person ON equals(events__pdi.person_id, events__pdi__person.id)
   WHERE and(equals(events.team_id, 2), ifNull(equals(events__pdi__person.properties___email, '[email protected]'), 0), ifNull(less(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-10 12:14:05.000000', 6, 'UTC')), 0), ifNull(greater(toTimeZone(events.timestamp, 'UTC'), toDateTime64('2020-01-09 12:14:00.000000', 6, 'UTC')), 0))
   ORDER BY events.event ASC
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_property_filter_aggregations
@@ -411,7 +411,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_property_filter_aggregations.1
@@ -427,7 +427,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_property_filter_aggregations_materialized
@@ -442,7 +442,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_property_filter_aggregations_materialized.1
@@ -458,7 +458,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_select_event_person
@@ -473,7 +473,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_select_hogql_expressions
@@ -489,7 +489,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_select_hogql_expressions.1
@@ -503,7 +503,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_select_hogql_expressions.2
@@ -518,7 +518,7 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---
 # name: TestQuery.test_select_hogql_expressions.3
@@ -533,6 +533,6 @@
   LIMIT 101
   OFFSET 0 SETTINGS readonly=2,
                     max_execution_time=60,
-                    allow_experimental_object_type=True
+                    allow_experimental_object_type=1
   '
 ---

posthog/hogql/ast.py

@@ -3,7 +3,7 @@
 from dataclasses import dataclass, field
 
 from posthog.hogql.base import Type, Expr, CTE, ConstantType, UnknownType
-from posthog.hogql.constants import ConstantDataType
+from posthog.hogql.constants import ConstantDataType, HogQLQuerySettings
 from posthog.hogql.database.models import (
     FieldTraverser,
     LazyJoin,
@@ -537,6 +537,7 @@ class SelectQuery(Expr):
     limit_by: Optional[List[Expr]] = None
     limit_with_ties: Optional[bool] = None
     offset: Optional[Expr] = None
+    settings: Optional[HogQLQuerySettings] = None
 
 
 @dataclass(kw_only=True)

posthog/hogql/constants.py

@@ -22,10 +22,15 @@
 # Max limit for all SELECT queries, and the default for CSV exports.
 MAX_SELECT_RETURNED_ROWS = 10000
 
-# Settings applied on top of all HogQL queries.
-class HogQLSettings(BaseModel):
+# Settings applied at the SELECT level
+class HogQLQuerySettings(BaseModel):
     model_config = ConfigDict(extra="forbid")
+    optimize_aggregation_in_order: Optional[bool] = None
+
 
+# Settings applied on top of all HogQL queries.
+class HogQLGlobalSettings(HogQLQuerySettings):
+    model_config = ConfigDict(extra="forbid")
     readonly: Optional[int] = 2
     max_execution_time: Optional[int] = 60
     allow_experimental_object_type: Optional[bool] = True

posthog/hogql/database/schema/persons.py

@@ -1,5 +1,6 @@
 from typing import Dict, List
 
+from posthog.hogql.constants import HogQLQuerySettings
 from posthog.hogql.database.argmax import argmax_select
 from posthog.hogql.database.models import (
     Table,
@@ -30,13 +31,15 @@
 
 
 def select_from_persons_table(requested_fields: Dict[str, List[str]]):
-    return argmax_select(
+    select = argmax_select(
         table_name="raw_persons",
         select_fields=requested_fields,
         group_fields=["id"],
         argmax_field="version",
         deleted_field="is_deleted",
     )
+    select.settings = HogQLQuerySettings(optimize_aggregation_in_order=True)
+    return select
 
 
 def join_with_persons_table(from_table: str, to_table: str, requested_fields: Dict[str, List[str]]):

posthog/hogql/database/test/test_database.py

@@ -60,5 +60,5 @@ def test_database_with_warehouse_tables(self, patch_execute):
 
         self.assertEqual(
             response.clickhouse,
-            f"SELECT whatever.id FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_3_sensitive)s, %(hogql_val_4_sensitive)s, %(hogql_val_1)s, %(hogql_val_2)s) AS whatever LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=True",
+            f"SELECT whatever.id FROM s3Cluster('posthog', %(hogql_val_0_sensitive)s, %(hogql_val_3_sensitive)s, %(hogql_val_4_sensitive)s, %(hogql_val_1)s, %(hogql_val_2)s) AS whatever LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1",
         )

posthog/hogql/functions/test/test_cohort.py

@@ -41,7 +41,7 @@ def test_in_cohort_dynamic(self):
         )
         self.assertEqual(
             response.clickhouse,
-            f"SELECT events.event FROM events WHERE and(equals(events.team_id, {self.team.pk}), in(events.person_id, (SELECT cohortpeople.person_id FROM cohortpeople WHERE and(equals(cohortpeople.team_id, {self.team.pk}), equals(cohortpeople.cohort_id, {cohort.pk})) GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0))), equals(events.event, %(hogql_val_0)s)) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=True",
+            f"SELECT events.event FROM events WHERE and(equals(events.team_id, {self.team.pk}), in(events.person_id, (SELECT cohortpeople.person_id FROM cohortpeople WHERE and(equals(cohortpeople.team_id, {self.team.pk}), equals(cohortpeople.cohort_id, {cohort.pk})) GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0))), equals(events.event, %(hogql_val_0)s)) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1",
         )
         self.assertEqual(
             response.hogql,
@@ -62,7 +62,7 @@ def test_in_cohort_static(self):
         )
         self.assertEqual(
             response.clickhouse,
-            f"SELECT events.event FROM events WHERE and(equals(events.team_id, {self.team.pk}), in(events.person_id, (SELECT person_static_cohort.person_id FROM person_static_cohort WHERE and(equals(person_static_cohort.team_id, {self.team.pk}), equals(person_static_cohort.cohort_id, {cohort.pk}))))) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=True",
+            f"SELECT events.event FROM events WHERE and(equals(events.team_id, {self.team.pk}), in(events.person_id, (SELECT person_static_cohort.person_id FROM person_static_cohort WHERE and(equals(person_static_cohort.team_id, {self.team.pk}), equals(person_static_cohort.cohort_id, {cohort.pk}))))) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1",
         )
         self.assertEqual(
             response.hogql,
@@ -82,7 +82,7 @@ def test_in_cohort_strings(self):
         )
         self.assertEqual(
             response.clickhouse,
-            f"SELECT events.event FROM events WHERE and(equals(events.team_id, {self.team.pk}), in(events.person_id, (SELECT person_static_cohort.person_id FROM person_static_cohort WHERE and(equals(person_static_cohort.team_id, {self.team.pk}), equals(person_static_cohort.cohort_id, {cohort.pk}))))) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=True",
+            f"SELECT events.event FROM events WHERE and(equals(events.team_id, {self.team.pk}), in(events.person_id, (SELECT person_static_cohort.person_id FROM person_static_cohort WHERE and(equals(person_static_cohort.team_id, {self.team.pk}), equals(person_static_cohort.cohort_id, {cohort.pk}))))) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1",
         )
         self.assertEqual(
             response.hogql,