diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index 34346526bdad2..3f541bb4d9788 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -1852,6 +1852,10 @@ "HogQLMetadata": { "additionalProperties": false, "properties": { + "debug": { + "description": "Enable more verbose output, usually run from the /debug page", + "type": "boolean" + }, "expr": { "description": "HogQL expression to validate (use `select` or `expr`, but not both)", "type": "string" @@ -1861,7 +1865,8 @@ "description": "Query within which \"expr\" is validated. Defaults to \"select * from events\"" }, "filters": { - "$ref": "#/definitions/HogQLFilters" + "$ref": "#/definitions/HogQLFilters", + "description": "Extra filters applied to query via {filters}" }, "kind": { "const": "HogQLMetadata", @@ -2046,6 +2051,10 @@ "limit": { "type": "integer" }, + "metadata": { + "$ref": "#/definitions/HogQLMetadataResponse", + "description": "Query metadata output" + }, "modifiers": { "$ref": "#/definitions/HogQLQueryModifiers", "description": "Modifiers used when performing the query" @@ -3365,6 +3374,10 @@ "limit": { "type": "integer" }, + "metadata": { + "$ref": "#/definitions/HogQLMetadataResponse", + "description": "Query metadata output" + }, "modifiers": { "$ref": "#/definitions/HogQLQueryModifiers", "description": "Modifiers used when performing the query" diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index 98189913e031b..1907b59e769dd 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -187,6 +187,8 @@ export interface HogQLQueryResponse { timings?: QueryTiming[] /** Query explanation output */ explain?: string[] + /** Query metadata output */ + metadata?: HogQLMetadataResponse /** Modifiers used when performing the query */ modifiers?: HogQLQueryModifiers hasMore?: boolean @@ -242,7 +244,10 @@ export interface HogQLMetadata extends DataNode { exprSource?: AnyDataNode /** Table to validate the expression against */ table?: string + /** Extra filters applied to query via {filters} */ filters?: HogQLFilters + /** Enable more verbose output, usually run from the /debug page */ + debug?: boolean response?: HogQLMetadataResponse } diff --git a/frontend/src/scenes/debug/HogQLDebug.tsx b/frontend/src/scenes/debug/HogQLDebug.tsx index a0202c5fac2bc..ad3efb7e2fd72 100644 --- a/frontend/src/scenes/debug/HogQLDebug.tsx +++ b/frontend/src/scenes/debug/HogQLDebug.tsx @@ -3,6 +3,7 @@ import { CodeEditor } from 'lib/components/CodeEditors' import { CodeSnippet, Language } from 'lib/components/CodeSnippet' import { LemonLabel } from 'lib/lemon-ui/LemonLabel' import { LemonSelect } from 'lib/lemon-ui/LemonSelect' +import { LemonTable } from 'lib/lemon-ui/LemonTable' import { dataNodeLogic, DataNodeLogicProps } from '~/queries/nodes/DataNode/dataNodeLogic' import { DateRange } from '~/queries/nodes/DataNode/DateRange' @@ -17,10 +18,40 @@ interface HogQLDebugProps { query: HogQLQuery setQuery: (query: DataNode) => void } + +function toLineColumn(hogql: string, position: number): { line: number; column: number } { + const lines = hogql.split('\n') + let line = 0 + let column = 0 + for (let i = 0; i < lines.length; i++) { + if (position < lines[i].length) { + line = i + 1 + column = position + 1 + break + } + position -= lines[i].length + 1 + } + return { line, column } +} + +function toLine(hogql: string, position: number): number { + return toLineColumn(hogql, position).line +} + +function toColumn(hogql: string, position: number): number { + return toLineColumn(hogql, position).column +} + export function HogQLDebug({ query, setQuery, queryKey }: HogQLDebugProps): JSX.Element { const dataNodeLogicProps: DataNodeLogicProps = { query, key: queryKey } - const { dataLoading, response, responseErrorObject, elapsedTime } = useValues(dataNodeLogic(dataNodeLogicProps)) - const clickHouseTime = (response as HogQLQueryResponse)?.timings?.find(({ k }) => k === './clickhouse_execute')?.t + const { + dataLoading, + response: _response, + responseErrorObject, + elapsedTime, + } = useValues(dataNodeLogic(dataNodeLogicProps)) + const response = _response as HogQLQueryResponse | null + const clickHouseTime = response?.timings?.find(({ k }) => k === './clickhouse_execute')?.t return ( @@ -141,6 +172,39 @@ export function HogQLDebug({ query, setQuery, queryKey }: HogQLDebugProps): JSX. ) : null} + {response?.metadata ? ( + <> +

Metadata

+ ({ + type: 'error', + line: toLine(response.hogql ?? '', error.start ?? 0), + column: toColumn(response.hogql ?? '', error.start ?? 0), + ...error, + })), + ...response.metadata.warnings.map((warn) => ({ + type: 'warning', + line: toLine(response.hogql ?? '', warn.start ?? 0), + column: toColumn(response.hogql ?? '', warn.start ?? 0), + ...warn, + })), + ...response.metadata.notices.map((notice) => ({ + type: 'notice', + line: toLine(response.hogql ?? '', notice.start ?? 0), + column: toColumn(response.hogql ?? '', notice.start ?? 0), + ...notice, + })), + ].sort((a, b) => (a.start ?? 0) - (b.start ?? 0))} + columns={[ + { title: 'Line', dataIndex: 'line', key: 'line', width: '40px' }, + { title: 'Column', dataIndex: 'column', key: 'column', width: '40px' }, + { title: 'Type', dataIndex: 'type', key: 'type', width: '80px' }, + { title: 'Message', dataIndex: 'message', key: 'message' }, + ]} + /> + + ) : null} {response?.explain ? ( <>

Explained ClickHouseSQL

diff --git a/mypy-baseline.txt b/mypy-baseline.txt index bfe1cd86efc26..91d6e4406701f 100644 --- a/mypy-baseline.txt +++ b/mypy-baseline.txt @@ -359,7 +359,6 @@ posthog/hogql/filters.py:0: note: PEP 484 prohibits implicit Optional. According posthog/hogql/filters.py:0: note: Use https://github.com/hauntsaninja/no_implicit_optional to automatically upgrade your codebase posthog/hogql/query.py:0: error: Incompatible types in assignment (expression has type "None", variable has type "str | SelectQuery | SelectUnionQuery") [assignment] posthog/hogql/query.py:0: error: Incompatible types in assignment (expression has type "Expr", variable has type "SelectQuery | SelectUnionQuery") [assignment] -posthog/hogql/query.py:0: error: Argument 2 to "replace_filters" has incompatible type "HogQLFilters | None"; expected "HogQLFilters" [arg-type] posthog/hogql/query.py:0: error: Incompatible types in assignment (expression has type "Expr", variable has type "SelectQuery | SelectUnionQuery") [assignment] posthog/hogql/query.py:0: error: Argument 1 to "get_default_limit_for_context" has incompatible type "LimitContext | None"; expected "LimitContext" [arg-type] posthog/hogql/query.py:0: error: "SelectQuery" has no attribute "select_queries" [attr-defined] @@ -788,6 +787,16 @@ posthog/api/property_definition.py:0: error: Item "AnonymousUser" of "User | Ano posthog/api/property_definition.py:0: error: Item "None" of "Organization | Any | None" has no attribute "is_feature_available" [union-attr] posthog/api/dashboards/dashboard_templates.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] posthog/temporal/tests/batch_exports/test_batch_exports.py:0: error: TypedDict key must be a string literal; expected one of ("_timestamp", "created_at", "distinct_id", "elements", "elements_chain", ...) [literal-required] +posthog/queries/app_metrics/test/test_app_metrics.py:0: error: Argument 3 to "AppMetricsErrorDetailsQuery" has incompatible type "AppMetricsRequestSerializer"; expected "AppMetricsErrorsRequestSerializer" [arg-type] +posthog/queries/app_metrics/test/test_app_metrics.py:0: error: Argument 3 to "AppMetricsErrorDetailsQuery" has incompatible type "AppMetricsRequestSerializer"; expected "AppMetricsErrorsRequestSerializer" [arg-type] +posthog/queries/app_metrics/test/test_app_metrics.py:0: error: Argument 3 to "AppMetricsErrorDetailsQuery" has incompatible type "AppMetricsRequestSerializer"; expected "AppMetricsErrorsRequestSerializer" [arg-type] +posthog/batch_exports/http.py:0: error: Unsupported right operand type for in ("object") [operator] +posthog/api/plugin.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] +posthog/api/plugin.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] +posthog/warehouse/external_data_source/source.py:0: error: Incompatible types in assignment (expression has type "int", target has type "str") [assignment] +posthog/warehouse/external_data_source/source.py:0: error: Incompatible types in assignment (expression has type "int", target has type "str") [assignment] +posthog/warehouse/external_data_source/source.py:0: error: Incompatible types in assignment (expression has type "dict[str, Collection[str]]", variable has type "StripeSourcePayload") [assignment] +posthog/warehouse/external_data_source/source.py:0: error: Argument 1 to "_create_source" has incompatible type "StripeSourcePayload"; expected "dict[Any, Any]" [arg-type] posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py:0: error: Need type annotation for "_execute_calls" (hint: "_execute_calls: List[] = ...") [var-annotated] posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py:0: error: Need type annotation for "_execute_async_calls" (hint: "_execute_async_calls: List[] = ...") [var-annotated] posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py:0: error: Need type annotation for "_cursors" (hint: "_cursors: List[] = ...") [var-annotated] @@ -854,16 +863,6 @@ posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py:0: error: posthog/temporal/tests/batch_exports/test_backfill_batch_export.py:0: error: Argument "name" to "acreate_batch_export" has incompatible type "object"; expected "str" [arg-type] posthog/temporal/tests/batch_exports/test_backfill_batch_export.py:0: error: Argument "destination_data" to "acreate_batch_export" has incompatible type "object"; expected "dict[Any, Any]" [arg-type] posthog/temporal/tests/batch_exports/test_backfill_batch_export.py:0: error: Argument "interval" to "acreate_batch_export" has incompatible type "object"; expected "str" [arg-type] -posthog/queries/app_metrics/test/test_app_metrics.py:0: error: Argument 3 to "AppMetricsErrorDetailsQuery" has incompatible type "AppMetricsRequestSerializer"; expected "AppMetricsErrorsRequestSerializer" [arg-type] -posthog/queries/app_metrics/test/test_app_metrics.py:0: error: Argument 3 to "AppMetricsErrorDetailsQuery" has incompatible type "AppMetricsRequestSerializer"; expected "AppMetricsErrorsRequestSerializer" [arg-type] -posthog/queries/app_metrics/test/test_app_metrics.py:0: error: Argument 3 to "AppMetricsErrorDetailsQuery" has incompatible type "AppMetricsRequestSerializer"; expected "AppMetricsErrorsRequestSerializer" [arg-type] -posthog/batch_exports/http.py:0: error: Unsupported right operand type for in ("object") [operator] -posthog/api/plugin.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] -posthog/api/plugin.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] -posthog/warehouse/external_data_source/source.py:0: error: Incompatible types in assignment (expression has type "int", target has type "str") [assignment] -posthog/warehouse/external_data_source/source.py:0: error: Incompatible types in assignment (expression has type "int", target has type "str") [assignment] -posthog/warehouse/external_data_source/source.py:0: error: Incompatible types in assignment (expression has type "dict[str, Collection[str]]", variable has type "StripeSourcePayload") [assignment] -posthog/warehouse/external_data_source/source.py:0: error: Argument 1 to "_create_source" has incompatible type "StripeSourcePayload"; expected "dict[Any, Any]" [arg-type] posthog/temporal/tests/batch_exports/test_redshift_batch_export_workflow.py:0: error: Unsupported left operand type for + ("None") [operator] posthog/temporal/tests/batch_exports/test_redshift_batch_export_workflow.py:0: note: Left operand is of type "Any | None" posthog/temporal/tests/batch_exports/test_redshift_batch_export_workflow.py:0: error: Incompatible types in assignment (expression has type "str | int", variable has type "int") [assignment] diff --git a/posthog/hogql/context.py b/posthog/hogql/context.py index 1b43362790f16..68692323e059d 100644 --- a/posthog/hogql/context.py +++ b/posthog/hogql/context.py @@ -46,6 +46,8 @@ class HogQLContext: timings: HogQLTimings = field(default_factory=HogQLTimings) # Modifications requested by the HogQL client modifiers: HogQLQueryModifiers = field(default_factory=HogQLQueryModifiers) + # Enables more verbose output for debugging + debug: bool = False def add_value(self, value: Any) -> str: key = f"hogql_val_{len(self.values)}" diff --git a/posthog/hogql/filters.py b/posthog/hogql/filters.py index 32ce707d0c647..aebe2016367d6 100644 --- a/posthog/hogql/filters.py +++ b/posthog/hogql/filters.py @@ -12,7 +12,7 @@ from posthog.utils import relative_date_parse -def replace_filters(node: ast.Expr, filters: HogQLFilters, team: Team) -> ast.Expr: +def replace_filters(node: ast.Expr, filters: Optional[HogQLFilters], team: Team) -> ast.Expr: return ReplaceFilters(filters, team).visit(node) diff --git a/posthog/hogql/metadata.py b/posthog/hogql/metadata.py index a4f29619579a7..05a6c35cd64ae 100644 --- a/posthog/hogql/metadata.py +++ b/posthog/hogql/metadata.py @@ -30,7 +30,7 @@ def get_hogql_metadata( try: if isinstance(query.expr, str): - context = HogQLContext(team_id=team.pk, modifiers=query_modifiers) + context = HogQLContext(team_id=team.pk, modifiers=query_modifiers, debug=query.debug or False) if query.exprSource is not None: source_query = get_query_runner(query.exprSource, team).to_query() translate_hogql(query.expr, context=context, metadata_source=source_query) @@ -41,6 +41,7 @@ def get_hogql_metadata( team_id=team.pk, modifiers=query_modifiers, enable_select_queries=True, + debug=query.debug or False, ) select_ast = parse_select(query.select) diff --git a/posthog/hogql/query.py b/posthog/hogql/query.py index b3f90f2a1c303..e7bc3f7984205 100644 --- a/posthog/hogql/query.py +++ b/posthog/hogql/query.py @@ -20,7 +20,7 @@ from posthog.models.team import Team from posthog.clickhouse.query_tagging import tag_queries from posthog.client import sync_execute -from posthog.schema import HogQLQueryResponse, HogQLFilters, HogQLQueryModifiers +from posthog.schema import HogQLQueryResponse, HogQLFilters, HogQLQueryModifiers, HogQLMetadata, HogQLMetadataResponse INCREASED_MAX_EXECUTION_TIME = 600 @@ -169,6 +169,7 @@ def execute_hogql_query( else: raise e + metadata: Optional[HogQLMetadataResponse] = None if explain and error is None: # If the query errored, explain will fail as well. with timings.measure("explain"): explain_results = sync_execute( @@ -180,6 +181,10 @@ def execute_hogql_query( readonly=True, ) explain_output = [str(r[0]) for r in explain_results[0]] + with timings.measure("metadata"): + from posthog.hogql.metadata import get_hogql_metadata + + metadata = get_hogql_metadata(HogQLMetadata(select=hogql, debug=True), team) else: explain_output = None @@ -194,4 +199,5 @@ def execute_hogql_query( types=types, modifiers=query_modifiers, explain=explain_output, + metadata=metadata, ) diff --git a/posthog/hogql/test/test_metadata.py b/posthog/hogql/test/test_metadata.py index f2f95089334aa..466e75f98cb80 100644 --- a/posthog/hogql/test/test_metadata.py +++ b/posthog/hogql/test/test_metadata.py @@ -8,13 +8,14 @@ class TestMetadata(ClickhouseTestMixin, APIBaseTest): maxDiff = None - def _expr(self, query: str, table: str = "events") -> HogQLMetadataResponse: + def _expr(self, query: str, table: str = "events", debug=True) -> HogQLMetadataResponse: return get_hogql_metadata( query=HogQLMetadata( kind="HogQLMetadata", expr=query, exprSource=HogQLQuery(kind="HogQLQuery", query=f"select * from {table}"), response=None, + debug=debug, ), team=self.team, ) @@ -190,7 +191,7 @@ def test_metadata_in_cohort(self): }, ) - def test_metadata_property_type_notice(self): + def test_metadata_property_type_notice_debug(self): try: from ee.clickhouse.materialized_columns.analyze import materialize except ModuleNotFoundError: @@ -211,13 +212,49 @@ def test_metadata_property_type_notice(self): "inputSelect": None, "notices": [ { - "message": "Event property 'string' is of type 'String'", + "message": "Event property 'string' is of type 'String'. This property is not materialized 🐢.", "start": 11, "end": 17, "fix": None, }, { - "message": "⚡️Event property 'number' is of type 'Float'", + "message": "Event property 'number' is of type 'Float'. This property is materialized ⚡️.", + "start": 32, + "end": 38, + "fix": None, + }, + ], + }, + ) + + def test_metadata_property_type_notice_no_debug(self): + try: + from ee.clickhouse.materialized_columns.analyze import materialize + except ModuleNotFoundError: + # EE not available? Assume we're good + self.assertEqual(1 + 2, 3) + return + materialize("events", "number") + + PropertyDefinition.objects.create(team=self.team, name="string", property_type="String") + PropertyDefinition.objects.create(team=self.team, name="number", property_type="Numeric") + metadata = self._expr("properties.string || properties.number", debug=False) + self.assertEqual( + metadata.dict(), + metadata.dict() + | { + "isValid": True, + "inputExpr": "properties.string || properties.number", + "inputSelect": None, + "notices": [ + { + "message": "Event property 'string' is of type 'String'.", + "start": 11, + "end": 17, + "fix": None, + }, + { + "message": "Event property 'number' is of type 'Float'.", "start": 32, "end": 38, "fix": None, diff --git a/posthog/hogql/transforms/property_types.py b/posthog/hogql/transforms/property_types.py index 2d58a10684a77..5b99761ca2517 100644 --- a/posthog/hogql/transforms/property_types.py +++ b/posthog/hogql/transforms/property_types.py @@ -180,9 +180,12 @@ def _add_property_notice( else: materialized_column = self._get_materialized_column("events", property_name, "properties") - message = f"{property_type.capitalize()} property '{property_name}' is of type '{field_type}'" - if materialized_column: - message = "⚡️" + message + message = f"{property_type.capitalize()} property '{property_name}' is of type '{field_type}'." + if self.context.debug: + if materialized_column: + message += " This property is materialized ⚡️." + else: + message += " This property is not materialized 🐢." self._add_notice(node=node, message=message) diff --git a/posthog/schema.py b/posthog/schema.py index 7e064d4b0dfb7..5e63ca43359e3 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -1130,6 +1130,7 @@ class HogQLQueryResponse(BaseModel): hasMore: Optional[bool] = None hogql: Optional[str] = Field(default=None, description="Generated HogQL query") limit: Optional[int] = None + metadata: Optional[HogQLMetadataResponse] = Field(default=None, description="Query metadata output") modifiers: Optional[HogQLQueryModifiers] = Field( default=None, description="Modifiers used when performing the query" ) @@ -1264,6 +1265,7 @@ class QueryResponseAlternative7(BaseModel): hasMore: Optional[bool] = None hogql: Optional[str] = Field(default=None, description="Generated HogQL query") limit: Optional[int] = None + metadata: Optional[HogQLMetadataResponse] = Field(default=None, description="Query metadata output") modifiers: Optional[HogQLQueryModifiers] = Field( default=None, description="Modifiers used when performing the query" ) @@ -2549,6 +2551,9 @@ class HogQLMetadata(BaseModel): model_config = ConfigDict( extra="forbid", ) + debug: Optional[bool] = Field( + default=None, description="Enable more verbose output, usually run from the /debug page" + ) expr: Optional[str] = Field( default=None, description="HogQL expression to validate (use `select` or `expr`, but not both)" ) @@ -2570,7 +2575,7 @@ class HogQLMetadata(BaseModel): WebTopClicksQuery, ] ] = Field(default=None, description='Query within which "expr" is validated. Defaults to "select * from events"') - filters: Optional[HogQLFilters] = None + filters: Optional[HogQLFilters] = Field(default=None, description="Extra filters applied to query via {filters}") kind: Literal["HogQLMetadata"] = "HogQLMetadata" response: Optional[HogQLMetadataResponse] = Field(default=None, description="Cached query response") select: Optional[str] = Field(