diff --git a/posthog/hogql_queries/legacy_compatibility/filter_to_query.py b/posthog/hogql_queries/legacy_compatibility/filter_to_query.py index 91e1cdaa75b4b..49ed19f24fd09 100644 --- a/posthog/hogql_queries/legacy_compatibility/filter_to_query.py +++ b/posthog/hogql_queries/legacy_compatibility/filter_to_query.py @@ -1,12 +1,10 @@ +import json +from typing import List, Dict from posthog.models.entity.entity import Entity as BackendEntity -from posthog.models.filters import AnyInsightFilter -from posthog.models.filters.filter import Filter as LegacyFilter -from posthog.models.filters.path_filter import PathFilter as LegacyPathFilter -from posthog.models.filters.retention_filter import RetentionFilter as LegacyRetentionFilter -from posthog.models.filters.stickiness_filter import StickinessFilter as LegacyStickinessFilter from posthog.schema import ( ActionsNode, BreakdownFilter, + ChartDisplayType, DateRange, EventsNode, FunnelExclusion, @@ -27,30 +25,129 @@ from posthog.types import InsightQueryNode -def entity_to_node(entity: BackendEntity) -> EventsNode | ActionsNode: +def is_property_with_operator(property: Dict): + return property.get("type") not in ("cohort", "hogql") + + +def clean_property(property: Dict): + cleaned_property = {**property} + + # fix type typo + if cleaned_property.get("type") == "events": + cleaned_property["type"] = "event" + + # fix value key typo + if cleaned_property.get("values") is not None and cleaned_property.get("value") is None: + cleaned_property["value"] = cleaned_property.pop("values") + + # convert precalculated and static cohorts to cohorts + if cleaned_property.get("type") in ("precalculated-cohort", "static-cohort"): + cleaned_property["type"] = "cohort" + + # fix invalid property key for cohorts + if cleaned_property.get("type") == "cohort" and cleaned_property.get("key") != "id": + cleaned_property["key"] = "id" + + # set a default operator for properties that support it, but don't have an operator set + if is_property_with_operator(cleaned_property) and cleaned_property.get("operator") is None: + cleaned_property["operator"] = "exact" + + # remove the operator for properties that don't support it, but have it set + if not is_property_with_operator(cleaned_property) and cleaned_property.get("operator") is not None: + del cleaned_property["operator"] + + # remove none from values + if isinstance(cleaned_property.get("value"), List): + cleaned_property["value"] = list(filter(lambda x: x is not None, cleaned_property.get("value"))) + + # remove keys without concrete value + cleaned_property = {key: value for key, value in cleaned_property.items() if value is not None} + + return cleaned_property + + +# old style dict properties +def is_old_style_properties(properties): + return isinstance(properties, Dict) and len(properties) == 1 and properties.get("type") not in ("AND", "OR") + + +def transform_old_style_properties(properties): + key = list(properties.keys())[0] + value = list(properties.values())[0] + key_split = key.split("__") + return [ + { + "key": key_split[0], + "value": value, + "operator": key_split[1] if len(key_split) > 1 else "exact", + "type": "event", + } + ] + + +def clean_entity_properties(properties: List[Dict] | None): + if properties is None: + return None + elif is_old_style_properties(properties): + return transform_old_style_properties(properties) + else: + return list(map(clean_property, properties)) + + +def clean_property_group_filter_value(value: Dict): + if value.get("type") in ("AND", "OR"): + value["values"] = map(clean_property_group_filter_value, value.get("values")) + return value + else: + return clean_property(value) + + +def clean_properties(properties: Dict): + properties["values"] = map(clean_property_group_filter_value, properties.get("values")) + return properties + + +def clean_display(display: str): + if display not in ChartDisplayType.__members__: + return None + else: + return display + + +def entity_to_node(entity: BackendEntity, include_properties: bool, include_math: bool) -> EventsNode | ActionsNode: shared = { "name": entity.name, "custom_name": entity.custom_name, - "properties": entity._data.get("properties", None), - "math": entity.math, - "math_property": entity.math_property, - "math_hogql": entity.math_hogql, - "math_group_type_index": entity.math_group_type_index, } + if include_properties: + shared = { + **shared, + "properties": clean_entity_properties(entity._data.get("properties", None)), + } + + if include_math: + shared = { + **shared, + "math": entity.math, + "math_property": entity.math_property, + "math_hogql": entity.math_hogql, + "math_group_type_index": entity.math_group_type_index, + } + if entity.type == "actions": return ActionsNode(id=entity.id, **shared) else: return EventsNode(event=entity.id, **shared) -def to_base_entity_dict(entity: BackendEntity): +def to_base_entity_dict(entity: Dict): return { - "type": entity.type, - "id": entity.id, - "name": entity.name, - "custom_name": entity.custom_name, - "order": entity.order, + "type": entity.get("type"), + "id": entity.get("id"), + "name": entity.get("name"), + "custom_name": entity.get("custom_name"), + "order": entity.get("order"), } @@ -64,187 +161,256 @@ def to_base_entity_dict(entity: BackendEntity): } -def _date_range(filter: AnyInsightFilter): - return {"dateRange": DateRange(**filter.date_to_dict())} +def _date_range(filter: Dict): + date_range = DateRange(date_from=filter.get("date_from"), date_to=filter.get("date_to")) + + if len(date_range.model_dump(exclude_defaults=True)) == 0: + return {} + return {"dateRange": date_range} -def _interval(filter: AnyInsightFilter): - if filter.insight == "RETENTION" or filter.insight == "PATHS": + +def _interval(filter: Dict): + if _insight_type(filter) == "RETENTION" or _insight_type(filter) == "PATHS": return {} - return {"interval": filter.interval} + if filter.get("interval") == "minute": + return {"interval": "hour"} + + return {"interval": filter.get("interval")} -def _series(filter: AnyInsightFilter): - if filter.insight == "RETENTION" or filter.insight == "PATHS": + +def _series(filter: Dict): + if _insight_type(filter) == "RETENTION" or _insight_type(filter) == "PATHS": return {} - return {"series": map(entity_to_node, filter.entities)} + # remove templates gone wrong + if filter.get("events") is not None: + filter["events"] = [event for event in filter.get("events") if not (isinstance(event, str))] + + include_math = True + include_properties = True + if _insight_type(filter) == "LIFECYCLE": + include_math = False + + return { + "series": [ + entity_to_node(entity, include_properties, include_math) + for entity in _entities(filter) + if entity.id is not None + ] + } + + +def _entities(filter: Dict): + processed_entities: List[BackendEntity] = [] + + # add actions + actions = filter.get("actions", []) + if isinstance(actions, str): + actions = json.loads(actions) + processed_entities.extend([BackendEntity({**entity, "type": "actions"}) for entity in actions]) + + # add events + events = filter.get("events", []) + if isinstance(events, str): + events = json.loads(events) + processed_entities.extend([BackendEntity({**entity, "type": "events"}) for entity in events]) + + # order by order + processed_entities.sort(key=lambda entity: entity.order if entity.order else -1) + + # set sequential index values on entities + for index, entity in enumerate(processed_entities): + entity.index = index + + return processed_entities -def _sampling_factor(filter: AnyInsightFilter): - return {"samplingFactor": filter.sampling_factor} +def _sampling_factor(filter: Dict): + return {"samplingFactor": filter.get("sampling_factor")} -def _filter_test_accounts(filter: AnyInsightFilter): - return {"filterTestAccounts": filter.filter_test_accounts} +def _filter_test_accounts(filter: Dict): + return {"filterTestAccounts": filter.get("filter_test_accounts")} -def _properties(filter: AnyInsightFilter): - raw_properties = filter._data.get("properties", None) + +def _properties(filter: Dict): + raw_properties = filter.get("properties", None) if raw_properties is None or len(raw_properties) == 0: return {} elif isinstance(raw_properties, list): raw_properties = {"type": "AND", "values": [{"type": "AND", "values": raw_properties}]} - return {"properties": PropertyGroupFilter(**raw_properties)} + return {"properties": PropertyGroupFilter(**clean_properties(raw_properties))} + elif is_old_style_properties(raw_properties): + raw_properties = transform_old_style_properties(raw_properties) + raw_properties = {"type": "AND", "values": [{"type": "AND", "values": raw_properties}]} + return {"properties": PropertyGroupFilter(**clean_properties(raw_properties))} else: - return {"properties": PropertyGroupFilter(**raw_properties)} + return {"properties": PropertyGroupFilter(**clean_properties(raw_properties))} -def _breakdown_filter(filter: AnyInsightFilter): - if filter.insight != "TRENDS" and filter.insight != "FUNNELS": +def _breakdown_filter(_filter: Dict): + if _insight_type(_filter) != "TRENDS" and _insight_type(_filter) != "FUNNELS": + return {} + + # early return for broken breakdown filters + if _filter.get("breakdown_type") == "undefined" and not isinstance(_filter.get("breakdown"), str): return {} breakdownFilter = { - "breakdown_type": filter.breakdown_type, - "breakdown": filter.breakdown, - "breakdown_normalize_url": filter.breakdown_normalize_url, - "breakdown_group_type_index": filter.breakdown_group_type_index, - "breakdown_histogram_bin_count": filter.breakdown_histogram_bin_count if filter.insight == "TRENDS" else None, + "breakdown_type": _filter.get("breakdown_type"), + "breakdown": _filter.get("breakdown"), + "breakdown_normalize_url": _filter.get("breakdown_normalize_url"), + "breakdown_group_type_index": _filter.get("breakdown_group_type_index"), + "breakdown_histogram_bin_count": _filter.get("breakdown_histogram_bin_count") + if _insight_type(_filter) == "TRENDS" + else None, } - if filter.breakdowns is not None: - if len(filter.breakdowns) == 1: - breakdownFilter["breakdown_type"] = filter.breakdowns[0].get("type", None) - breakdownFilter["breakdown"] = filter.breakdowns[0].get("property", None) + # fix breakdown typo + if breakdownFilter["breakdown_type"] == "events": + breakdownFilter["breakdown_type"] = "event" + + if _filter.get("breakdowns") is not None: + if len(_filter.get("breakdowns")) == 1: + breakdownFilter["breakdown_type"] = _filter.get("breakdowns")[0].get("type", None) + breakdownFilter["breakdown"] = _filter.get("breakdowns")[0].get("property", None) else: raise Exception("Could not convert multi-breakdown property `breakdowns` - found more than one breakdown") if breakdownFilter["breakdown"] is not None and breakdownFilter["breakdown_type"] is None: breakdownFilter["breakdown_type"] = "event" + if isinstance(breakdownFilter["breakdown"], list): + breakdownFilter["breakdown"] = list(filter(lambda x: x is not None, breakdownFilter["breakdown"])) + + if len(BreakdownFilter(**breakdownFilter).model_dump(exclude_defaults=True)) == 0: + return {} + return {"breakdown": BreakdownFilter(**breakdownFilter)} -def _group_aggregation_filter(filter: AnyInsightFilter): - if isinstance(filter, LegacyStickinessFilter): +def _group_aggregation_filter(filter: Dict): + if _insight_type(filter) == "STICKINESS": return {} - return {"aggregation_group_type_index": filter.aggregation_group_type_index} + return {"aggregation_group_type_index": filter.get("aggregation_group_type_index")} -def _insight_filter(filter: AnyInsightFilter): - if filter.insight == "TRENDS" and isinstance(filter, LegacyFilter): - return { +def _insight_filter(filter: Dict): + if _insight_type(filter) == "TRENDS": + insight_filter = { "trendsFilter": TrendsFilter( - smoothing_intervals=filter.smoothing_intervals, - # show_legend=filter.show_legend, - # hidden_legend_indexes=cleanHiddenLegendIndexes(filter.hidden_legend_keys), - compare=filter.compare, - aggregation_axis_format=filter.aggregation_axis_format, - aggregation_axis_prefix=filter.aggregation_axis_prefix, - aggregation_axis_postfix=filter.aggregation_axis_postfix, - formula=filter.formula, - shown_as=filter.shown_as, - display=filter.display, - # show_values_on_series=filter.show_values_on_series, - # show_percent_stack_view=filter.show_percent_stack_view, + smoothing_intervals=filter.get("smoothing_intervals"), + # show_legend=filter.get('show_legend'), + # hidden_legend_indexes=cleanHiddenLegendIndexes(filter.get('hidden_legend_keys')), + compare=filter.get("compare"), + aggregation_axis_format=filter.get("aggregation_axis_format"), + aggregation_axis_prefix=filter.get("aggregation_axis_prefix"), + aggregation_axis_postfix=filter.get("aggregation_axis_postfix"), + formula=filter.get("formula"), + shown_as=filter.get("shown_as"), + display=clean_display(filter.get("display")), + show_values_on_series=filter.get("show_values_on_series"), + show_percent_stack_view=filter.get("show_percent_stack_view"), ) } - elif filter.insight == "FUNNELS" and isinstance(filter, LegacyFilter): - return { + elif _insight_type(filter) == "FUNNELS": + insight_filter = { "funnelsFilter": FunnelsFilter( - funnel_viz_type=filter.funnel_viz_type, - funnel_order_type=filter.funnel_order_type, - funnel_from_step=filter.funnel_from_step, - funnel_to_step=filter.funnel_to_step, - funnel_window_interval_unit=filter.funnel_window_interval_unit, - funnel_window_interval=filter.funnel_window_interval, - # funnel_step_reference=filter.funnel_step_reference, - breakdown_attribution_type=filter.breakdown_attribution_type, - breakdown_attribution_value=filter.breakdown_attribution_value, - bin_count=filter.bin_count, + funnel_viz_type=filter.get("funnel_viz_type"), + funnel_order_type=filter.get("funnel_order_type"), + funnel_from_step=filter.get("funnel_from_step"), + funnel_to_step=filter.get("funnel_to_step"), + funnel_window_interval_unit=filter.get("funnel_window_interval_unit"), + funnel_window_interval=filter.get("funnel_window_interval"), + funnel_step_reference=filter.get("funnel_step_reference"), + breakdown_attribution_type=filter.get("breakdown_attribution_type"), + breakdown_attribution_value=filter.get("breakdown_attribution_value"), + bin_count=filter.get("bin_count"), exclusions=[ FunnelExclusion( **to_base_entity_dict(entity), - funnel_from_step=entity.funnel_from_step, - funnel_to_step=entity.funnel_to_step, + funnel_from_step=entity.get("funnel_from_step"), + funnel_to_step=entity.get("funnel_to_step"), ) - for entity in filter.exclusions + for entity in filter.get("exclusions", []) ], - layout=filter.layout, - # hidden_legend_breakdowns: cleanHiddenLegendSeries(filters.hidden_legend_keys), - funnel_aggregate_by_hogql=filter.funnel_aggregate_by_hogql, + layout=filter.get("layout"), + # hidden_legend_breakdowns: cleanHiddenLegendSeries(filter.get('hidden_legend_keys')), + funnel_aggregate_by_hogql=filter.get("funnel_aggregate_by_hogql"), ), } - elif filter.insight == "RETENTION" and isinstance(filter, LegacyRetentionFilter): - return { + elif _insight_type(filter) == "RETENTION": + insight_filter = { "retentionFilter": RetentionFilter( - retention_type=filter.retention_type, - # retention_reference=filter.retention_reference, - total_intervals=filter.total_intervals, - returning_entity=to_base_entity_dict(filter.returning_entity), - target_entity=to_base_entity_dict(filter.target_entity), - period=filter.period, + retention_type=filter.get("retention_type"), + retention_reference=filter.get("retention_reference"), + total_intervals=filter.get("total_intervals"), + returning_entity=to_base_entity_dict(filter.get("returning_entity")) + if filter.get("returning_entity") is not None + else None, + target_entity=to_base_entity_dict(filter.get("target_entity")) + if filter.get("target_entity") is not None + else None, + period=filter.get("period"), ) } - elif filter.insight == "PATHS" and isinstance(filter, LegacyPathFilter): - return { + elif _insight_type(filter) == "PATHS": + insight_filter = { "pathsFilter": PathsFilter( - # path_type=filter.path_type, # legacy - paths_hogql_expression=filter.paths_hogql_expression, - include_event_types=filter._data.get("include_event_types"), - start_point=filter.start_point, - end_point=filter.end_point, - path_groupings=filter.path_groupings, - exclude_events=filter.exclude_events, - step_limit=filter.step_limit, - path_replacements=filter.path_replacements, - local_path_cleaning_filters=filter.local_path_cleaning_filters, - edge_limit=filter.edge_limit, - min_edge_weight=filter.min_edge_weight, - max_edge_weight=filter.max_edge_weight, - funnel_paths=filter.funnel_paths, - funnel_filter=filter._data.get("funnel_filter"), + # path_type=filter.get('path_type'), # legacy + paths_hogql_expression=filter.get("paths_hogql_expression"), + include_event_types=filter.get("include_event_types"), + start_point=filter.get("start_point"), + end_point=filter.get("end_point"), + path_groupings=filter.get("path_groupings"), + exclude_events=filter.get("exclude_events"), + step_limit=filter.get("step_limit"), + path_replacements=filter.get("path_replacements"), + local_path_cleaning_filters=filter.get("local_path_cleaning_filters"), + edge_limit=filter.get("edge_limit"), + min_edge_weight=filter.get("min_edge_weight"), + max_edge_weight=filter.get("max_edge_weight"), + funnel_paths=filter.get("funnel_paths"), + funnel_filter=filter.get("funnel_filter"), ) } - elif filter.insight == "LIFECYCLE": - return { + elif _insight_type(filter) == "LIFECYCLE": + insight_filter = { "lifecycleFilter": LifecycleFilter( - shown_as=filter.shown_as, - # toggledLifecycles=filter.toggledLifecycles, - # show_values_on_series=filter.show_values_on_series, + shown_as=filter.get("shown_as"), + # toggledLifecycles=filter.get('toggledLifecycles'), + show_values_on_series=filter.get("show_values_on_series"), ) } - elif filter.insight == "STICKINESS" and isinstance(filter, LegacyStickinessFilter): - return { + elif _insight_type(filter) == "STICKINESS": + insight_filter = { "stickinessFilter": StickinessFilter( - compare=filter.compare, - shown_as=filter.shown_as, - # show_legend=filter.show_legend, - # hidden_legend_indexes: cleanHiddenLegendIndexes(filters.hidden_legend_keys), - # show_values_on_series=filter.show_values_on_series, + compare=filter.get("compare"), + shown_as=filter.get("shown_as"), + # show_legend=filter.get('show_legend'), + # hidden_legend_indexes: cleanHiddenLegendIndexes(filter.get('hidden_legend_keys')), + show_values_on_series=filter.get("show_values_on_series"), ) } else: - raise Exception(f"Invalid insight type {filter.insight}.") - - -def filter_to_query(filter: AnyInsightFilter) -> InsightQueryNode: - if (filter.insight == "TRENDS" or filter.insight == "FUNNELS" or filter.insight == "LIFECYCLE") and isinstance( - filter, LegacyFilter - ): - matching_filter_type = True - elif filter.insight == "RETENTION" and isinstance(filter, LegacyRetentionFilter): - matching_filter_type = True - elif filter.insight == "PATHS" and isinstance(filter, LegacyPathFilter): - matching_filter_type = True - elif filter.insight == "STICKINESS" and isinstance(filter, LegacyStickinessFilter): - matching_filter_type = True - else: - matching_filter_type = False + raise Exception(f"Invalid insight type {filter.get('insight')}.") + + if len(list(insight_filter.values())[0].model_dump(exclude_defaults=True)) == 0: + return {} + + return insight_filter - if not matching_filter_type: - raise Exception(f"Filter type {type(filter)} does not match insight type {filter.insight}") - Query = insight_to_query_type[filter.insight] +def _insight_type(filter: Dict) -> str: + if filter.get("insight") == "SESSIONS": + return "TRENDS" + return filter.get("insight", "TRENDS") + + +def filter_to_query(filter: Dict) -> InsightQueryNode: + Query = insight_to_query_type[_insight_type(filter)] data = { **_date_range(filter), @@ -259,3 +425,15 @@ def filter_to_query(filter: AnyInsightFilter) -> InsightQueryNode: } return Query(**data) + + +def filter_str_to_query(filters: str) -> InsightQueryNode: + filter = json.loads(filters) + # we have insights that have been serialized to json twice in the database + # due to people misunderstanding our api + if isinstance(filter, str): + filter = json.loads(filter) + # we also have insights wrapped in an additional array + elif isinstance(filter, list): + filter = filter[0] + return filter_to_query(filter) diff --git a/posthog/hogql_queries/legacy_compatibility/test/test_filter_to_query.py b/posthog/hogql_queries/legacy_compatibility/test/test_filter_to_query.py index 6359b9e3e808d..bad08471ea241 100644 --- a/posthog/hogql_queries/legacy_compatibility/test/test_filter_to_query.py +++ b/posthog/hogql_queries/legacy_compatibility/test/test_filter_to_query.py @@ -1,9 +1,5 @@ import pytest from posthog.hogql_queries.legacy_compatibility.filter_to_query import filter_to_query -from posthog.models.filters.filter import Filter as LegacyFilter -from posthog.models.filters.path_filter import PathFilter as LegacyPathFilter -from posthog.models.filters.retention_filter import RetentionFilter as LegacyRetentionFilter -from posthog.models.filters.stickiness_filter import StickinessFilter as LegacyStickinessFilter from posthog.schema import ( ActionsNode, AggregationAxisFormat, @@ -300,6 +296,257 @@ "filter_test_accounts": True, } +# real world regression tests +insight_18 = { + "actions": [ + { + "id": 2760, + "math": "total", + "name": "Pageviews", + "type": "actions", + "order": 0, + "properties": [{"key": "$browser", "type": "event", "value": "Chrome", "operator": None}], + "math_property": None, + } + ], + "display": "ActionsBar", + "insight": "LIFECYCLE", + "interval": "day", + "shown_as": "Lifecycle", +} +insight_19 = { + "events": [ + { + "id": "created change", + "math": "total", + "name": "created change", + "type": "events", + "order": 0, + "properties": [{"key": "id", "type": "cohort", "value": 2208, "operator": None}], + "custom_name": None, + "math_property": None, + } + ], + "display": "ActionsLineGraph", + "insight": "LIFECYCLE", + "interval": "day", + "shown_as": "Lifecycle", +} +insight_20 = { + "events": [ + { + "id": "$pageview", + "math": "total", + "name": "$pageview", + "type": "events", + "order": 0, + "properties": [], + "math_property": None, + } + ], + "display": "ActionsLineGraph", + "insight": "LIFECYCLE", + "interval": "day", + "shown_as": "Lifecycle", + "properties": [{"key": "id", "type": "cohort", "value": 929, "operator": "exact"}], +} +insight_21 = { + "actions": [ + { + "id": 4317, + "math": "total", + "name": "Some name", + "type": "actions", + "order": 0, + "properties": [], + "custom_name": None, + "math_property": None, + } + ], + "display": "ActionsLineGraph", + "insight": "LIFECYCLE", + "interval": "day", + "shown_as": "Lifecycle", + "properties": [{"key": "id", "type": "precalculated-cohort", "value": 760, "operator": None}], + "funnel_window_days": 14, +} +insight_22 = { + "actions": [ + { + "id": "10184", + "math": None, + "name": "Some name", + "type": "actions", + "order": 0, + "properties": [], + "math_property": None, + } + ], + "display": "ActionsLineGraph", + "insight": "TRENDS", + "interval": "day", + "breakdown_type": "undefined", +} +insight_23 = { + "events": [{"id": "$pageview", "name": "$pageview", "type": "events", "order": 0}], + "display": "ActionsLineGraph", + "insight": "TRENDS", + "interval": "day", + "shown_as": "Volume", + "breakdown": False, + "properties": [{"key": "$current_url", "type": "event", "value": "https://example.com/", "operator": "icontains"}], + "breakdown_type": "undefined", +} +insight_24 = { + "events": [ + { + "id": "$pageview", + "math": None, + "name": "$pageview", + "type": "events", + "order": 0, + "properties": [], + "math_property": None, + } + ], + "display": "ActionsLineGrap[…]ccounts=false", + "insight": "TRENDS", + "interval": "day", + "date_from": "-90d", +} +insight_25 = { + "events": [ + { + "id": "$pageview", + "math": None, + "name": "$pageview", + "type": "events", + "order": 2, + "properties": [{"key": "$host", "type": "event", "value": [None], "operator": "exact"}], + "math_property": None, + }, + ], + "insight": "TRENDS", +} +insight_26 = { + "events": [ + { + "id": "$pageview", + "name": "$pageview", + "type": "events", + }, + ], + "insight": "TRENDS", +} +insight_27 = { + "actions": [ + { + "id": None, + "math": None, + "name": None, + "type": "actions", + "order": None, + "properties": [], + "math_property": None, + } + ], + "insight": "TRENDS", +} +insight_28 = { + "events": [ + { + "id": "$pageview", + "type": "events", + } + ], + "insight": "TRENDS", + "breakdown": [None], + "breakdown_type": "cohort", +} +insight_29 = { + "events": [ + "{EXAMPLE_VARIABLE}", + { + "id": "$pageview", + "math": "dau", + "name": "$pageview", + "type": "events", + "order": 1, + "properties": [ + {"key": "$current_url", "type": "event", "value": "posthog.com/signup$", "operator": "regex"} + ], + "custom_name": "Views on signup page", + }, + ], + "insight": "TRENDS", +} +insight_30 = { + "events": [ + { + "id": "$pageview", + "name": "$pageview", + } + ], + "insight": "TRENDS", + "breakdown": "$geoip_country_code", + "breakdown_type": "events", + "breakdown_group_type_index": 0, +} +insight_31 = { + "events": [{"id": "$autocapture", "math": "total", "name": "$autocapture", "type": "events", "order": 0}], + "insight": "STICKINESS", + "entity_type": "events", +} +insight_32 = { + "events": [ + { + "id": "$pageview", + "math": "dau", + "name": "$pageview", + "type": "events", + "order": None, + "properties": [], + "math_property": None, + } + ], + "insight": "STICKINESS", + "interval": "minute", + "shown_as": "Stickiness", + "date_from": "dStart", +} +insight_33 = { + "period": "Week", + "display": "ActionsTable", + "insight": "RETENTION", + "properties": [ + {"key": "id", "type": "precalculated-cohort", "value": 71, "operator": None}, + {"key": "id", "type": "static-cohort", "value": 696, "operator": None}, + ], + "target_entity": { + "id": 4912, + "math": None, + "name": None, + "type": "actions", + "order": None, + "properties": [], + "custom_name": None, + "math_property": None, + }, + "retention_type": "retention_first_time", + "total_intervals": 11, + "returning_entity": { + "id": 3410, + "math": None, + "name": None, + "type": "actions", + "order": None, + "properties": [], + "custom_name": None, + "math_property": None, + }, +} + + test_insights = [ insight_0, insight_1, @@ -319,20 +566,27 @@ insight_15, insight_16, insight_17, + insight_18, + insight_19, + insight_20, + insight_21, + insight_22, + insight_23, + insight_24, + insight_25, + insight_26, + insight_27, + insight_28, + insight_29, + insight_30, + insight_31, + insight_32, + insight_33, ] -@pytest.mark.parametrize("insight", test_insights) -def test_base_insights(insight): - """smoke test (i.e. filter_to_query should not throw) for real world insights""" - if insight.get("insight") == "RETENTION": - filter = LegacyRetentionFilter(data=insight) - elif insight.get("insight") == "PATHS": - filter = LegacyPathFilter(data=insight) - elif insight.get("insight") == "STICKINESS": - filter = LegacyStickinessFilter(data=insight) - else: - filter = LegacyFilter(data=insight) +@pytest.mark.parametrize("filter", test_insights) +def test_base_insights(filter: dict): filter_to_query(filter) @@ -405,6 +659,11 @@ def test_base_insights(insight): {"type": "OR", "values": [{}]}, ], } +properties_10 = [{"key": "id", "type": "cohort", "value": 71, "operator": None}] +properties_11 = [{"key": [498], "type": "cohort", "value": 498, "operator": None}] +properties_12 = [{"key": "userId", "type": "event", "values": ["63ffaeae99ac3c4240976d60"], "operator": "exact"}] +properties_13 = {"plan": "premium"} +properties_14 = {"$current_url__icontains": "signin"} test_properties = [ properties_0, @@ -417,157 +676,100 @@ def test_base_insights(insight): properties_7, properties_8, properties_9, + properties_10, + properties_11, + properties_12, + properties_13, + properties_14, ] @pytest.mark.parametrize("properties", test_properties) def test_base_properties(properties): """smoke test (i.e. filter_to_query should not throw) for real world properties""" - filter = LegacyFilter(data={"properties": properties}) - filter_to_query(filter) + filter_to_query({"properties": properties}) class TestFilterToQuery(BaseTest): def test_base_trend(self): - filter = LegacyFilter(data={}) + filter = {} query = filter_to_query(filter) self.assertEqual(query.kind, "TrendsQuery") def test_full_trend(self): - filter = LegacyFilter(data={}) + filter = {} query = filter_to_query(filter) self.assertEqual( query.model_dump(exclude_defaults=True), - { - "dateRange": {"date_from": "-7d"}, - "interval": "day", - "series": [], - "filterTestAccounts": False, - "breakdown": {"breakdown_normalize_url": False}, - "trendsFilter": { - "compare": False, - "display": ChartDisplayType.ActionsLineGraph, - "smoothing_intervals": 1, - }, - }, + {"series": []}, ) def test_base_funnel(self): - filter = LegacyFilter(data={"insight": "FUNNELS"}) + filter = {"insight": "FUNNELS"} query = filter_to_query(filter) self.assertEqual(query.kind, "FunnelsQuery") def test_base_retention_query(self): - filter = LegacyFilter(data={"insight": "RETENTION"}) - - with pytest.raises(Exception) as exception: - filter_to_query(filter) - - self.assertEqual( - str(exception.value), - "Filter type does not match insight type RETENTION", - ) - - def test_base_retention_query_from_retention_filter(self): - filter = LegacyRetentionFilter(data={}) + filter = {"insight": "RETENTION"} query = filter_to_query(filter) self.assertEqual(query.kind, "RetentionQuery") def test_base_paths_query(self): - filter = LegacyFilter(data={"insight": "PATHS"}) - - with pytest.raises(Exception) as exception: - filter_to_query(filter) - - self.assertEqual( - str(exception.value), - "Filter type does not match insight type PATHS", - ) - - def test_base_path_query_from_path_filter(self): - filter = LegacyPathFilter(data={}) + filter = {"insight": "PATHS"} query = filter_to_query(filter) self.assertEqual(query.kind, "PathsQuery") def test_base_lifecycle_query(self): - filter = LegacyFilter(data={"insight": "LIFECYCLE"}) + filter = {"insight": "LIFECYCLE"} query = filter_to_query(filter) self.assertEqual(query.kind, "LifecycleQuery") def test_base_stickiness_query(self): - filter = LegacyFilter(data={"insight": "STICKINESS"}) - - with pytest.raises(Exception) as exception: - filter_to_query(filter) - - self.assertEqual( - str(exception.value), - "Filter type does not match insight type STICKINESS", - ) - - def test_base_stickiness_query_from_stickiness_filter(self): - filter = LegacyStickinessFilter(data={}, team=self.team) + filter = {"insight": "STICKINESS"} query = filter_to_query(filter) self.assertEqual(query.kind, "StickinessQuery") - def test_date_range_default(self): - filter = LegacyFilter(data={}) - - query = filter_to_query(filter) - - self.assertEqual(query.dateRange.date_from, "-7d") - self.assertEqual(query.dateRange.date_to, None) - - def test_date_range_custom(self): - filter = LegacyFilter(data={"date_from": "-14d", "date_to": "-7d"}) + def test_date_range(self): + filter = {"date_from": "-14d", "date_to": "-7d"} query = filter_to_query(filter) self.assertEqual(query.dateRange.date_from, "-14d") self.assertEqual(query.dateRange.date_to, "-7d") - def test_interval_default(self): - filter = LegacyFilter(data={}) - - query = filter_to_query(filter) - - self.assertEqual(query.interval, "day") - - def test_interval_custom(self): - filter = LegacyFilter(data={"interval": "hour"}) + def test_interval(self): + filter = {"interval": "hour"} query = filter_to_query(filter) self.assertEqual(query.interval, "hour") def test_series_default(self): - filter = LegacyFilter(data={}) + filter = {} query = filter_to_query(filter) self.assertEqual(query.series, []) def test_series_custom(self): - filter = LegacyFilter( - data={ - "events": [{"id": "$pageview"}, {"id": "$pageview", "math": "dau"}], - "actions": [{"id": 1}, {"id": 1, "math": "dau"}], - } - ) + filter = { + "events": [{"id": "$pageview"}, {"id": "$pageview", "math": "dau"}], + "actions": [{"id": 1}, {"id": 1, "math": "dau"}], + } query = filter_to_query(filter) @@ -582,12 +784,10 @@ def test_series_custom(self): ) def test_series_order(self): - filter = LegacyFilter( - data={ - "events": [{"id": "$pageview", "order": 1}, {"id": "$pageview", "math": "dau", "order": 2}], - "actions": [{"id": 1, "order": 3}, {"id": 1, "math": "dau", "order": 0}], - } - ) + filter = { + "events": [{"id": "$pageview", "order": 1}, {"id": "$pageview", "math": "dau", "order": 2}], + "actions": [{"id": 1, "order": 3}, {"id": 1, "math": "dau", "order": 0}], + } query = filter_to_query(filter) @@ -602,21 +802,19 @@ def test_series_order(self): ) def test_series_math(self): - filter = LegacyFilter( - data={ - "events": [ - {"id": "$pageview", "math": "dau"}, # base math type - {"id": "$pageview", "math": "median", "math_property": "$math_prop"}, # property math type - {"id": "$pageview", "math": "avg_count_per_actor"}, # count per actor math type - {"id": "$pageview", "math": "unique_group", "math_group_type_index": 0}, # unique group - { - "id": "$pageview", - "math": "hogql", - "math_hogql": "avg(toInt(properties.$session_id)) + 1000", - }, # hogql - ] - } - ) + filter = { + "events": [ + {"id": "$pageview", "math": "dau"}, # base math type + {"id": "$pageview", "math": "median", "math_property": "$math_prop"}, # property math type + {"id": "$pageview", "math": "avg_count_per_actor"}, # count per actor math type + {"id": "$pageview", "math": "unique_group", "math_group_type_index": 0}, # unique group + { + "id": "$pageview", + "math": "hogql", + "math_hogql": "avg(toInt(properties.$session_id)) + 1000", + }, # hogql + ] + } query = filter_to_query(filter) @@ -639,55 +837,53 @@ def test_series_math(self): ) def test_series_properties(self): - filter = LegacyFilter( - data={ - "events": [ - {"id": "$pageview", "properties": []}, # smoke test - { - "id": "$pageview", - "properties": [{"key": "success", "type": "event", "value": ["true"], "operator": "exact"}], - }, - { - "id": "$pageview", - "properties": [{"key": "email", "type": "person", "value": "is_set", "operator": "is_set"}], - }, - { - "id": "$pageview", - "properties": [{"key": "text", "value": ["some text"], "operator": "exact", "type": "element"}], - }, - { - "id": "$pageview", - "properties": [{"key": "$session_duration", "value": 1, "operator": "gt", "type": "session"}], - }, - {"id": "$pageview", "properties": [{"key": "id", "value": 2, "type": "cohort"}]}, - { - "id": "$pageview", - "properties": [ - { - "key": "name", - "value": ["Hedgebox Inc."], - "operator": "exact", - "type": "group", - "group_type_index": 2, - } - ], - }, - { - "id": "$pageview", - "properties": [ - {"key": "dateDiff('minute', timestamp, now()) < 30", "type": "hogql", "value": None} - ], - }, - { - "id": "$pageview", - "properties": [ - {"key": "$referring_domain", "type": "event", "value": "google", "operator": "icontains"}, - {"key": "utm_source", "type": "event", "value": "is_not_set", "operator": "is_not_set"}, - ], - }, - ] - } - ) + filter = { + "events": [ + {"id": "$pageview", "properties": []}, # smoke test + { + "id": "$pageview", + "properties": [{"key": "success", "type": "event", "value": ["true"], "operator": "exact"}], + }, + { + "id": "$pageview", + "properties": [{"key": "email", "type": "person", "value": "is_set", "operator": "is_set"}], + }, + { + "id": "$pageview", + "properties": [{"key": "text", "value": ["some text"], "operator": "exact", "type": "element"}], + }, + { + "id": "$pageview", + "properties": [{"key": "$session_duration", "value": 1, "operator": "gt", "type": "session"}], + }, + {"id": "$pageview", "properties": [{"key": "id", "value": 2, "type": "cohort"}]}, + { + "id": "$pageview", + "properties": [ + { + "key": "name", + "value": ["Hedgebox Inc."], + "operator": "exact", + "type": "group", + "group_type_index": 2, + } + ], + }, + { + "id": "$pageview", + "properties": [ + {"key": "dateDiff('minute', timestamp, now()) < 30", "type": "hogql", "value": None} + ], + }, + { + "id": "$pageview", + "properties": [ + {"key": "$referring_domain", "type": "event", "value": "google", "operator": "icontains"}, + {"key": "utm_source", "type": "event", "value": "is_not_set", "operator": "is_not_set"}, + ], + }, + ] + } query = filter_to_query(filter) @@ -746,48 +942,46 @@ def test_series_properties(self): ) def test_breakdown(self): - filter = LegacyFilter(data={"breakdown_type": "event", "breakdown": "$browser"}) + filter = {"breakdown_type": "event", "breakdown": "$browser"} query = filter_to_query(filter) self.assertEqual( query.breakdown, - BreakdownFilter(breakdown_type=BreakdownType.event, breakdown="$browser", breakdown_normalize_url=False), + BreakdownFilter(breakdown_type=BreakdownType.event, breakdown="$browser"), ) def test_breakdown_converts_multi(self): - filter = LegacyFilter(data={"breakdowns": [{"type": "event", "property": "$browser"}]}) + filter = {"breakdowns": [{"type": "event", "property": "$browser"}]} query = filter_to_query(filter) self.assertEqual( query.breakdown, - BreakdownFilter(breakdown_type=BreakdownType.event, breakdown="$browser", breakdown_normalize_url=False), + BreakdownFilter(breakdown_type=BreakdownType.event, breakdown="$browser"), ) def test_breakdown_type_default(self): - filter = LegacyFilter(data={"breakdown": "some_prop"}) + filter = {"breakdown": "some_prop"} query = filter_to_query(filter) self.assertEqual( query.breakdown, - BreakdownFilter(breakdown_type=BreakdownType.event, breakdown="some_prop", breakdown_normalize_url=False), + BreakdownFilter(breakdown_type=BreakdownType.event, breakdown="some_prop"), ) def test_trends_filter(self): - filter = LegacyFilter( - data={ - "smoothing_intervals": 2, - "compare": True, - "aggregation_axis_format": "duration_ms", - "aggregation_axis_prefix": "pre", - "aggregation_axis_postfix": "post", - "formula": "A + B", - "shown_as": "Volume", - "display": "ActionsAreaGraph", - } - ) + filter = { + "smoothing_intervals": 2, + "compare": True, + "aggregation_axis_format": "duration_ms", + "aggregation_axis_prefix": "pre", + "aggregation_axis_postfix": "post", + "formula": "A + B", + "shown_as": "Volume", + "display": "ActionsAreaGraph", + } query = filter_to_query(filter) @@ -806,45 +1000,43 @@ def test_trends_filter(self): ) def test_funnels_filter(self): - filter = LegacyFilter( - data={ - "insight": "FUNNELS", - "funnel_viz_type": "steps", - "funnel_window_interval_unit": "hour", - "funnel_window_interval": 13, - "breakdown_attribution_type": "step", - "breakdown_attribution_value": 2, - "funnel_order_type": "strict", - "funnel_aggregate_by_hogql": "person_id", - "exclusions": [ - { - "id": "$pageview", - "type": "events", - "order": 0, - "name": "$pageview", - "funnel_from_step": 1, - "funnel_to_step": 2, - } - ], - "bin_count": 15, # used in time to convert: number of bins to show in histogram - "funnel_from_step": 1, # used in time to convert: initial step index to compute time to convert - "funnel_to_step": 2, # used in time to convert: ending step index to compute time to convert - # - # frontend only params - # "layout": layout, - # "funnel_step_reference": "previous", # whether conversion shown in graph should be across all steps or just from the previous step - # hidden_legend_keys # used to toggle visibilities in table and legend - # - # persons endpoint only params - # "funnel_step_breakdown": funnel_step_breakdown, # used in steps breakdown: persons modal - # "funnel_correlation_person_entity":funnel_correlation_person_entity, - # "funnel_correlation_person_converted":funnel_correlation_person_converted, # success or failure counts - # "entrance_period_start": entrance_period_start, # this and drop_off is used for funnels time conversion date for the persons modal - # "drop_off": drop_off, - # "funnel_step": funnel_step, - # "funnel_custom_steps": funnel_custom_steps, - } - ) + filter = { + "insight": "FUNNELS", + "funnel_viz_type": "steps", + "funnel_window_interval_unit": "hour", + "funnel_window_interval": 13, + "breakdown_attribution_type": "step", + "breakdown_attribution_value": 2, + "funnel_order_type": "strict", + "funnel_aggregate_by_hogql": "person_id", + "exclusions": [ + { + "id": "$pageview", + "type": "events", + "order": 0, + "name": "$pageview", + "funnel_from_step": 1, + "funnel_to_step": 2, + } + ], + "bin_count": 15, # used in time to convert: number of bins to show in histogram + "funnel_from_step": 1, # used in time to convert: initial step index to compute time to convert + "funnel_to_step": 2, # used in time to convert: ending step index to compute time to convert + # + # frontend only params + # "layout": layout, + # "funnel_step_reference": "previous", # whether conversion shown in graph should be across all steps or just from the previous step + # hidden_legend_keys # used to toggle visibilities in table and legend + # + # persons endpoint only params + # "funnel_step_breakdown": funnel_step_breakdown, # used in steps breakdown: persons modal + # "funnel_correlation_person_entity":funnel_correlation_person_entity, + # "funnel_correlation_person_converted":funnel_correlation_person_converted, # success or failure counts + # "entrance_period_start": entrance_period_start, # this and drop_off is used for funnels time conversion date for the persons modal + # "drop_off": drop_off, + # "funnel_step": funnel_step, + # "funnel_custom_steps": funnel_custom_steps, + } query = filter_to_query(filter) @@ -876,16 +1068,15 @@ def test_funnels_filter(self): ) def test_retention_filter(self): - filter = LegacyRetentionFilter( - data={ - "retention_type": "retention_first_time", - # retention_reference="previous", - "total_intervals": 12, - "returning_entity": {"id": "$pageview", "name": "$pageview", "type": "events"}, - "target_entity": {"id": "$pageview", "name": "$pageview", "type": "events"}, - "period": "Week", - } - ) + filter = { + "insight": "RETENTION", + "retention_type": "retention_first_time", + # retention_reference="previous", + "total_intervals": 12, + "returning_entity": {"id": "$pageview", "name": "$pageview", "type": "events"}, + "target_entity": {"id": "$pageview", "name": "$pageview", "type": "events"}, + "period": "Week", + } query = filter_to_query(filter) @@ -913,34 +1104,33 @@ def test_retention_filter(self): ) def test_paths_filter(self): - filter = LegacyPathFilter( - data={ - "include_event_types": ["$pageview", "hogql"], - "start_point": "http://localhost:8000/events", - "end_point": "http://localhost:8000/home", - "paths_hogql_expression": "event", - "edge_limit": 50, - "min_edge_weight": 10, - "max_edge_weight": 20, - "local_path_cleaning_filters": [{"alias": "merchant", "regex": "\\/merchant\\/\\d+\\/dashboard$"}], - "path_replacements": True, - "exclude_events": ["http://localhost:8000/events"], - "step_limit": 5, - "path_groupings": ["/merchant/*/payment"], - "funnel_paths": "funnel_path_between_steps", - "funnel_filter": { - "insight": "FUNNELS", - "events": [ - {"type": "events", "id": "$pageview", "order": 0, "name": "$pageview", "math": "total"}, - {"type": "events", "id": None, "order": 1, "math": "total"}, - ], - "funnel_viz_type": "steps", - "exclusions": [], - "filter_test_accounts": True, - "funnel_step": 2, - }, - } - ) + filter = { + "insight": "PATHS", + "include_event_types": ["$pageview", "hogql"], + "start_point": "http://localhost:8000/events", + "end_point": "http://localhost:8000/home", + "paths_hogql_expression": "event", + "edge_limit": 50, + "min_edge_weight": 10, + "max_edge_weight": 20, + "local_path_cleaning_filters": [{"alias": "merchant", "regex": "\\/merchant\\/\\d+\\/dashboard$"}], + "path_replacements": True, + "exclude_events": ["http://localhost:8000/events"], + "step_limit": 5, + "path_groupings": ["/merchant/*/payment"], + "funnel_paths": "funnel_path_between_steps", + "funnel_filter": { + "insight": "FUNNELS", + "events": [ + {"type": "events", "id": "$pageview", "order": 0, "name": "$pageview", "math": "total"}, + {"type": "events", "id": None, "order": 1, "math": "total"}, + ], + "funnel_viz_type": "steps", + "exclusions": [], + "filter_test_accounts": True, + "funnel_step": 2, + }, + } query = filter_to_query(filter) @@ -977,9 +1167,7 @@ def test_paths_filter(self): ) def test_stickiness_filter(self): - filter = LegacyStickinessFilter( - data={"insight": "STICKINESS", "compare": True, "shown_as": "Stickiness"}, team=self.team - ) + filter = {"insight": "STICKINESS", "compare": True, "shown_as": "Stickiness"} query = filter_to_query(filter) @@ -989,12 +1177,10 @@ def test_stickiness_filter(self): ) def test_lifecycle_filter(self): - filter = LegacyFilter( - data={ - "insight": "LIFECYCLE", - "shown_as": "Lifecycle", - } - ) + filter = { + "insight": "LIFECYCLE", + "shown_as": "Lifecycle", + } query = filter_to_query(filter) diff --git a/posthog/models/filters/filter.py b/posthog/models/filters/filter.py index 816e1a846d7fe..e0549650981e6 100644 --- a/posthog/models/filters/filter.py +++ b/posthog/models/filters/filter.py @@ -1,6 +1,5 @@ from .base_filter import BaseFilter from .mixins.common import ( - AggregationAxisMixin, BreakdownMixin, BreakdownValueMixin, ClientQueryIdMixin, @@ -89,7 +88,6 @@ class Filter( UpdatedAfterMixin, ClientQueryIdMixin, SampleMixin, - AggregationAxisMixin, BaseFilter, ): """ diff --git a/posthog/models/filters/mixins/common.py b/posthog/models/filters/mixins/common.py index b7303ea3e3ebf..bbb727407c6be 100644 --- a/posthog/models/filters/mixins/common.py +++ b/posthog/models/filters/mixins/common.py @@ -592,21 +592,3 @@ def sampling_factor(self) -> Optional[float]: @include_dict def sampling_factor_to_dict(self): return {SAMPLING_FACTOR: self.sampling_factor or ""} - - -class AggregationAxisMixin(BaseParamMixin): - """ - Aggregation Axis. Only used frontend side. - """ - - @cached_property - def aggregation_axis_format(self) -> Optional[str]: - return self._data.get("aggregation_axis_format", None) - - @cached_property - def aggregation_axis_prefix(self) -> Optional[str]: - return self._data.get("aggregation_axis_prefix", None) - - @cached_property - def aggregation_axis_postfix(self) -> Optional[str]: - return self._data.get("aggregation_axis_postfix", None)