diff --git a/mypy-baseline.txt b/mypy-baseline.txt index c095bc7fff997..781ad2980830b 100644 --- a/mypy-baseline.txt +++ b/mypy-baseline.txt @@ -97,7 +97,7 @@ posthog/hogql/database/database.py:0: error: "FieldOrTable" has no attribute "fi posthog/hogql/database/database.py:0: error: "FieldOrTable" has no attribute "fields" [attr-defined] posthog/hogql/database/database.py:0: error: "FieldOrTable" has no attribute "fields" [attr-defined] posthog/hogql/database/database.py:0: error: "FieldOrTable" has no attribute "fields" [attr-defined] -posthog/hogql/database/database.py:0: error: Incompatible types (expression has type "Literal['view', 'lazy_table']", TypedDict item "type" has type "Literal['integer', 'float', 'string', 'datetime', 'date', 'boolean', 'array', 'json', 'lazy_table', 'virtual_table', 'field_traverser']") [typeddict-item] +posthog/hogql/database/database.py:0: error: Incompatible types (expression has type "Literal['view', 'lazy_table']", TypedDict item "type" has type "Literal['integer', 'float', 'string', 'datetime', 'date', 'boolean', 'array', 'json', 'lazy_table', 'virtual_table', 'field_traverser', 'expression']") [typeddict-item] posthog/warehouse/models/datawarehouse_saved_query.py:0: error: Argument 1 to "create_hogql_database" has incompatible type "int | None"; expected "int" [arg-type] posthog/warehouse/models/datawarehouse_saved_query.py:0: error: Incompatible types in assignment (expression has type "Expr", variable has type "SelectQuery | SelectUnionQuery") [assignment] posthog/models/user.py:0: error: Incompatible types in assignment (expression has type "None", base class "AbstractUser" defined the type as "CharField[str | int | Combinable, str]") [assignment] @@ -419,7 +419,6 @@ posthog/api/feature_flag.py:0: error: Item "Sequence[Any]" of "Any | Sequence[An posthog/api/feature_flag.py:0: error: Item "None" of "Any | Sequence[Any] | None" has no attribute "filters" [union-attr] posthog/api/survey.py:0: error: Incompatible types in assignment (expression has type "Any | Sequence[Any] | None", variable has type "Survey | None") [assignment] posthog/api/user.py:0: error: "User" has no attribute "social_auth" [attr-defined] -ee/api/role.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] ee/api/dashboard_collaborator.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] ee/api/test/base.py:0: error: Incompatible types in assignment (expression has type "None", variable has type "License") [assignment] ee/api/test/base.py:0: error: "setUpTestData" undefined in superclass [misc] @@ -594,6 +593,8 @@ posthog/hogql/test/test_resolver.py:0: error: Incompatible types in assignment ( posthog/hogql/test/test_resolver.py:0: error: "TestResolver" has no attribute "snapshot" [attr-defined] posthog/hogql/test/test_resolver.py:0: error: Incompatible types in assignment (expression has type "Expr", variable has type "SelectQuery") [assignment] posthog/hogql/test/test_resolver.py:0: error: "TestResolver" has no attribute "snapshot" [attr-defined] +posthog/hogql/test/test_resolver.py:0: error: Incompatible types in assignment (expression has type "Expr", variable has type "SelectQuery") [assignment] +posthog/hogql/test/test_resolver.py:0: error: "TestResolver" has no attribute "snapshot" [attr-defined] posthog/hogql/test/test_resolver.py:0: error: Item "SelectUnionQueryType" of "SelectQueryType | SelectUnionQueryType | None" has no attribute "columns" [union-attr] posthog/hogql/test/test_resolver.py:0: error: Item "None" of "SelectQueryType | SelectUnionQueryType | None" has no attribute "columns" [union-attr] posthog/hogql/test/test_resolver.py:0: error: "FieldOrTable" has no attribute "fields" [attr-defined] @@ -649,7 +650,7 @@ posthog/hogql/functions/test/test_cohort.py:0: error: "TestCohort" has no attrib posthog/hogql/database/schema/test/test_channel_type.py:0: error: Value of type "list[Any] | None" is not indexable [index] posthog/hogql/database/schema/test/test_channel_type.py:0: error: Value of type "list[Any] | None" is not indexable [index] posthog/api/organization_member.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] -ee/api/feature_flag_role_access.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] +ee/api/role.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] ee/clickhouse/views/insights.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] posthog/queries/trends/test/test_person.py:0: error: "str" has no attribute "get" [attr-defined] posthog/queries/trends/test/test_person.py:0: error: Invalid index type "int" for "HttpResponse"; expected type "str | bytes" [index] @@ -754,6 +755,7 @@ posthog/api/property_definition.py:0: error: Incompatible types in assignment (e posthog/api/property_definition.py:0: error: Item "AnonymousUser" of "User | AnonymousUser" has no attribute "organization" [union-attr] posthog/api/property_definition.py:0: error: Item "None" of "Organization | Any | None" has no attribute "is_feature_available" [union-attr] posthog/api/dashboards/dashboard_templates.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] +ee/api/feature_flag_role_access.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] posthog/temporal/tests/batch_exports/test_run_updates.py:0: error: Unused "type: ignore" comment [unused-ignore] posthog/temporal/tests/batch_exports/test_run_updates.py:0: error: Unused "type: ignore" comment [unused-ignore] posthog/temporal/tests/batch_exports/test_run_updates.py:0: error: Item "None" of "BatchExportRun | None" has no attribute "data_interval_start" [union-attr] diff --git a/posthog/hogql/autocomplete.py b/posthog/hogql/autocomplete.py index 80e88cb7a3ea5..a7339a80fafd5 100644 --- a/posthog/hogql/autocomplete.py +++ b/posthog/hogql/autocomplete.py @@ -1,7 +1,7 @@ from copy import copy, deepcopy from typing import Callable, Dict, List, Optional, cast from posthog.hogql.context import HogQLContext -from posthog.hogql.database.database import create_hogql_database +from posthog.hogql.database.database import Database, create_hogql_database from posthog.hogql.database.models import ( BooleanDatabaseField, DatabaseField, @@ -239,6 +239,10 @@ def append_table_field_to_response(table: Table, suggestions: List[AutocompleteC details: List[str | None] = [] table_fields = list(table.fields.items()) for field_name, field_or_table in table_fields: + # Skip over hidden fields + if isinstance(field_or_table, ast.DatabaseField) and field_or_table.hidden: + continue + keys.append(field_name) details.append(convert_field_or_table_to_type_string(field_or_table)) @@ -278,11 +282,17 @@ def extend_responses( # TODO: Support ast.SelectUnionQuery nodes -def get_hogql_autocomplete(query: HogQLAutocomplete, team: Team) -> HogQLAutocompleteResponse: +def get_hogql_autocomplete( + query: HogQLAutocomplete, team: Team, database_arg: Optional[Database] = None +) -> HogQLAutocompleteResponse: response = HogQLAutocompleteResponse(suggestions=[], incomplete_list=False) timings = HogQLTimings() - database = create_hogql_database(team_id=team.pk, team_arg=team) + if database_arg is not None: + database = database_arg + else: + database = create_hogql_database(team_id=team.pk, team_arg=team) + context = HogQLContext(team_id=team.pk, team=team, database=database) original_query_select = copy(query.select) diff --git a/posthog/hogql/database/database.py b/posthog/hogql/database/database.py index 52a65644f76cf..6909211070e59 100644 --- a/posthog/hogql/database/database.py +++ b/posthog/hogql/database/database.py @@ -309,6 +309,7 @@ class _SerializedFieldBase(TypedDict): "lazy_table", "virtual_table", "field_traverser", + "expression", ] @@ -346,6 +347,9 @@ def serialize_fields(field_input, context: HogQLContext) -> List[SerializedField if field_key == "team_id": pass elif isinstance(field, DatabaseField): + if field.hidden: + continue + if isinstance(field, IntegerDatabaseField): field_output.append({"key": field_key, "type": "integer"}) elif isinstance(field, FloatDatabaseField): @@ -362,6 +366,8 @@ def serialize_fields(field_input, context: HogQLContext) -> List[SerializedField field_output.append({"key": field_key, "type": "json"}) elif isinstance(field, StringArrayDatabaseField): field_output.append({"key": field_key, "type": "array"}) + elif isinstance(field, ExpressionField): + field_output.append({"key": field_key, "type": "expression"}) elif isinstance(field, LazyJoin): is_view = isinstance(field.resolve_table(context), SavedQuery) field_output.append( diff --git a/posthog/hogql/database/models.py b/posthog/hogql/database/models.py index e95a26614bed8..95a00595c6472 100644 --- a/posthog/hogql/database/models.py +++ b/posthog/hogql/database/models.py @@ -24,6 +24,7 @@ class DatabaseField(FieldOrTable): name: str array: Optional[bool] = None nullable: Optional[bool] = None + hidden: bool = False class IntegerDatabaseField(DatabaseField): @@ -95,15 +96,11 @@ def get_asterisk(self): for key, field in self.fields.items(): if key in fields_to_avoid: continue - if ( - isinstance(field, Table) - or isinstance(field, LazyJoin) - or isinstance(field, FieldTraverser) - or isinstance(field, ExpressionField) - ): + if isinstance(field, Table) or isinstance(field, LazyJoin) or isinstance(field, FieldTraverser): pass # ignore virtual tables and columns for now elif isinstance(field, DatabaseField): - asterisk[key] = field + if not field.hidden: # Skip over hidden fields + asterisk[key] = field else: raise HogQLException(f"Unknown field type {type(field).__name__} for asterisk") return asterisk diff --git a/posthog/hogql/database/test/__snapshots__/test_database.ambr b/posthog/hogql/database/test/__snapshots__/test_database.ambr index 21c60457a1fd3..db4dfc8f6df9f 100644 --- a/posthog/hogql/database/test/__snapshots__/test_database.ambr +++ b/posthog/hogql/database/test/__snapshots__/test_database.ambr @@ -269,6 +269,14 @@ "distinct_id", "person_id" ] + }, + { + "key": "$virt_initial_referring_domain_type", + "type": "expression" + }, + { + "key": "$virt_initial_channel_type", + "type": "expression" } ], "person_distinct_ids": [ @@ -1112,6 +1120,14 @@ "distinct_id", "person_id" ] + }, + { + "key": "$virt_initial_referring_domain_type", + "type": "expression" + }, + { + "key": "$virt_initial_channel_type", + "type": "expression" } ], "person_distinct_ids": [ diff --git a/posthog/hogql/database/test/test_database.py b/posthog/hogql/database/test/test_database.py index da17e15c03107..99810ca54e58a 100644 --- a/posthog/hogql/database/test/test_database.py +++ b/posthog/hogql/database/test/test_database.py @@ -134,7 +134,7 @@ def test_database_expression_fields(self): query = print_ast(parse_select(sql), context, dialect="clickhouse") assert ( query - == "SELECT number AS number FROM (SELECT numbers.number AS number FROM numbers(2) AS numbers) LIMIT 10000" + == "SELECT number AS number, expression AS expression, double AS double FROM (SELECT numbers.number AS number, plus(1, 1) AS expression, multiply(numbers.number, 2) AS double FROM numbers(2) AS numbers) LIMIT 10000" ), query def test_database_warehouse_joins(self): diff --git a/posthog/hogql/test/__snapshots__/test_resolver.ambr b/posthog/hogql/test/__snapshots__/test_resolver.ambr index 1b086d067a621..74ac2f12adc82 100644 --- a/posthog/hogql/test/__snapshots__/test_resolver.ambr +++ b/posthog/hogql/test/__snapshots__/test_resolver.ambr @@ -582,6 +582,321 @@ } ''' # --- +# name: TestResolver.test_asterisk_expander_hidden_field + ''' + { + select: [ + { + alias: "uuid" + expr: { + chain: [ + "uuid" + ] + type: { + name: "uuid" + table_type: { + table: { + fields: { + $group_0: {}, + $group_1: {}, + $group_2: {}, + $group_3: {}, + $group_4: {}, + $session_id: {}, + $window_id: {}, + created_at: {}, + distinct_id: {}, + elements_chain: {}, + event: {}, + goe_0: {}, + goe_1: {}, + goe_2: {}, + goe_3: {}, + goe_4: {}, + group_0: {}, + group_1: {}, + group_2: {}, + group_3: {}, + group_4: {}, + hidden_field: {}, + pdi: {}, + person: {}, + person_id: {}, + poe: {}, + properties: {}, + session: {}, + team_id: {}, + timestamp: {}, + uuid: {} + } + } + } + } + } + hidden: True + type: { + alias: "uuid" + type: + } + }, + { + alias: "event" + expr: { + chain: [ + "event" + ] + type: { + name: "event" + table_type: + } + } + hidden: True + type: { + alias: "event" + type: + } + }, + { + alias: "properties" + expr: { + chain: [ + "properties" + ] + type: { + name: "properties" + table_type: + } + } + hidden: True + type: { + alias: "properties" + type: + } + }, + { + alias: "timestamp" + expr: { + chain: [ + "timestamp" + ] + type: { + name: "timestamp" + table_type: + } + } + hidden: True + type: { + alias: "timestamp" + type: + } + }, + { + alias: "distinct_id" + expr: { + chain: [ + "distinct_id" + ] + type: { + name: "distinct_id" + table_type: + } + } + hidden: True + type: { + alias: "distinct_id" + type: + } + }, + { + alias: "elements_chain" + expr: { + chain: [ + "elements_chain" + ] + type: { + name: "elements_chain" + table_type: + } + } + hidden: True + type: { + alias: "elements_chain" + type: + } + }, + { + alias: "created_at" + expr: { + chain: [ + "created_at" + ] + type: { + name: "created_at" + table_type: + } + } + hidden: True + type: { + alias: "created_at" + type: + } + }, + { + alias: "$session_id" + expr: { + chain: [ + "$session_id" + ] + type: { + name: "$session_id" + table_type: + } + } + hidden: True + type: { + alias: "$session_id" + type: + } + }, + { + alias: "$window_id" + expr: { + chain: [ + "$window_id" + ] + type: { + name: "$window_id" + table_type: + } + } + hidden: True + type: { + alias: "$window_id" + type: + } + }, + { + alias: "$group_0" + expr: { + chain: [ + "$group_0" + ] + type: { + name: "$group_0" + table_type: + } + } + hidden: True + type: { + alias: "$group_0" + type: + } + }, + { + alias: "$group_1" + expr: { + chain: [ + "$group_1" + ] + type: { + name: "$group_1" + table_type: + } + } + hidden: True + type: { + alias: "$group_1" + type: + } + }, + { + alias: "$group_2" + expr: { + chain: [ + "$group_2" + ] + type: { + name: "$group_2" + table_type: + } + } + hidden: True + type: { + alias: "$group_2" + type: + } + }, + { + alias: "$group_3" + expr: { + chain: [ + "$group_3" + ] + type: { + name: "$group_3" + table_type: + } + } + hidden: True + type: { + alias: "$group_3" + type: + } + }, + { + alias: "$group_4" + expr: { + chain: [ + "$group_4" + ] + type: { + name: "$group_4" + table_type: + } + } + hidden: True + type: { + alias: "$group_4" + type: + } + } + ] + select_from: { + table: { + chain: [ + "events" + ] + type: + } + type: + } + type: { + aliases: {} + anonymous_tables: [] + columns: { + $group_0: , + $group_1: , + $group_2: , + $group_3: , + $group_4: , + $session_id: , + $window_id: , + created_at: , + distinct_id: , + elements_chain: , + event: , + properties: , + timestamp: , + uuid: + } + ctes: {} + tables: { + events: + } + } + } + ''' +# --- # name: TestResolver.test_asterisk_expander_select_union ''' { diff --git a/posthog/hogql/test/test_autocomplete.py b/posthog/hogql/test/test_autocomplete.py index 46eb8a1cd0394..bf92abf6e359e 100644 --- a/posthog/hogql/test/test_autocomplete.py +++ b/posthog/hogql/test/test_autocomplete.py @@ -1,4 +1,7 @@ +from typing import Optional from posthog.hogql.autocomplete import get_hogql_autocomplete +from posthog.hogql.database.database import Database, create_hogql_database +from posthog.hogql.database.models import StringDatabaseField from posthog.hogql.database.schema.events import EventsTable from posthog.hogql.database.schema.persons import PERSONS_FIELDS from posthog.models.property_definition import PropertyDefinition @@ -21,9 +24,11 @@ def _create_properties(self): type=PropertyDefinition.Type.PERSON, ) - def _query_response(self, query: str, start: int, end: int) -> HogQLAutocompleteResponse: + def _query_response( + self, query: str, start: int, end: int, database: Optional[Database] = None + ) -> HogQLAutocompleteResponse: autocomplete = HogQLAutocomplete(kind="HogQLAutocomplete", select=query, startPosition=start, endPosition=end) - return get_hogql_autocomplete(query=autocomplete, team=self.team) + return get_hogql_autocomplete(query=autocomplete, team=self.team, database_arg=database) def test_autocomplete(self): query = "select * from events" @@ -226,3 +231,13 @@ def test_autocomplete_non_existing_alias(self): results = self._query_response(query=query, start=9, end=9) assert len(results.suggestions) == 0 + + def test_autocomplete_events_hidden_field(self): + database = create_hogql_database(team_id=self.team.pk, team_arg=self.team) + database.events.fields["event"] = StringDatabaseField(name="event", hidden=True) + + query = "select from events" + results = self._query_response(query=query, start=7, end=7, database=database) + + for suggestion in results.suggestions: + assert suggestion.label != "event" diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.py index de448811b2584..88b7b2c50e0b5 100644 --- a/posthog/hogql/test/test_resolver.py +++ b/posthog/hogql/test/test_resolver.py @@ -10,6 +10,7 @@ from posthog.hogql.context import HogQLContext from posthog.hogql.database.database import create_hogql_database from posthog.hogql.database.models import ( + ExpressionField, FieldTraverser, StringJSONDatabaseField, StringDatabaseField, @@ -252,6 +253,15 @@ def test_asterisk_expander_subquery(self): node = resolve_types(node, self.context, dialect="clickhouse") assert pretty_dataclasses(node) == self.snapshot + @pytest.mark.usefixtures("unittest_snapshot") + def test_asterisk_expander_hidden_field(self): + self.database.events.fields["hidden_field"] = ExpressionField( + name="hidden_field", hidden=True, expr=ast.Field(chain=["event"]) + ) + node = self._select("select * from events") + node = resolve_types(node, self.context, dialect="clickhouse") + assert pretty_dataclasses(node) == self.snapshot + @pytest.mark.usefixtures("unittest_snapshot") def test_asterisk_expander_subquery_alias(self): node = self._select("select x.* from (select 1 as a, 2 as b) x") diff --git a/posthog/hogql/visitor.py b/posthog/hogql/visitor.py index 9acb6fec87db7..c11856169297f 100644 --- a/posthog/hogql/visitor.py +++ b/posthog/hogql/visitor.py @@ -247,6 +247,9 @@ def visit_window_frame_expr(self, node: ast.WindowFrameExpr): def visit_join_constraint(self, node: ast.JoinConstraint): self.visit(node.expr) + def visit_expression_field_type(self, node: ast.ExpressionFieldType): + pass + def visit_hogqlx_tag(self, node: ast.HogQLXTag): for attribute in node.attributes: self.visit(attribute) diff --git a/posthog/warehouse/models/external_table_definitions.py b/posthog/warehouse/models/external_table_definitions.py new file mode 100644 index 0000000000000..6522271a35783 --- /dev/null +++ b/posthog/warehouse/models/external_table_definitions.py @@ -0,0 +1,285 @@ +from typing import Dict +from posthog.hogql import ast +from posthog.hogql.database.models import ( + BooleanDatabaseField, + FieldOrTable, + IntegerDatabaseField, + StringDatabaseField, + StringJSONDatabaseField, +) + + +external_tables: Dict[str, Dict[str, FieldOrTable]] = { + "stripe_customer": { + "id": StringDatabaseField(name="id"), + "name": StringDatabaseField(name="name"), + "email": StringDatabaseField(name="email"), + "phone": StringDatabaseField(name="phone"), + "object": StringDatabaseField(name="object"), + "address": StringJSONDatabaseField(name="address"), + "balance": IntegerDatabaseField(name="balance"), + "__created": IntegerDatabaseField(name="created", hidden=True), + "created_at": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__created"])]), name="created_at" + ), + "currency": StringDatabaseField(name="currency"), + "discount": StringJSONDatabaseField(name="discount"), + "livemode": BooleanDatabaseField(name="livemode"), + "metadata": StringJSONDatabaseField(name="metadata"), + "shipping": StringJSONDatabaseField(name="shipping"), + "delinquent": BooleanDatabaseField(name="delinquent"), + "tax_exempt": StringDatabaseField(name="tax_exempt"), + "description": StringDatabaseField(name="description"), + "default_source": StringDatabaseField(name="default_source"), + "invoice_prefix": StringDatabaseField(name="invoice_prefix"), + "invoice_settings": StringJSONDatabaseField(name="invoice_settings"), + "preferred_locales": StringJSONDatabaseField(name="preferred_locales"), + "next_invoice_sequence": IntegerDatabaseField(name="next_invoice_sequence"), + "__dlt_id": StringDatabaseField(name="_dlt_id", hidden=True), + "__dlt_load_id": StringDatabaseField(name="_dlt_load_id", hidden=True), + }, + "stripe_invoice": { + "id": StringDatabaseField(name="id"), + "tax": IntegerDatabaseField(name="tax"), + "paid": BooleanDatabaseField(name="paid"), + "lines": StringJSONDatabaseField(name="lines"), + "total": IntegerDatabaseField(name="total"), + "charge": StringDatabaseField(name="charge"), + "issuer": StringJSONDatabaseField(name="issuer"), + "number": StringDatabaseField(name="number"), + "object": StringDatabaseField(name="object"), + "status": StringDatabaseField(name="status"), + "__created": IntegerDatabaseField(name="created", hidden=True), + "created_at": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__created"])]), name="created_at" + ), + "currency": StringDatabaseField(name="currency"), + "customer_id": StringDatabaseField(name="customer"), + "discount": StringJSONDatabaseField(name="discount"), + "due_date": IntegerDatabaseField(name="due_date"), + "livemode": BooleanDatabaseField(name="livemode"), + "metadata": StringJSONDatabaseField(name="metadata"), + "subtotal": IntegerDatabaseField(name="subtotal"), + "attempted": BooleanDatabaseField(name="attempted"), + "discounts": StringJSONDatabaseField(name="discounts"), + "rendering": StringJSONDatabaseField(name="rendering"), + "amount_due": IntegerDatabaseField(name="amount_due"), + "__period_start": IntegerDatabaseField(name="period_start", hidden=True), + "period_start_at": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__period_start"])]), name="period_start_at" + ), + "__period_end": IntegerDatabaseField(name="period_end", hidden=True), + "period_end_at": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__period_end"])]), name="period_end_at" + ), + "amount_paid": IntegerDatabaseField(name="amount_paid"), + "description": StringDatabaseField(name="description"), + "invoice_pdf": StringDatabaseField(name="invoice_pdf"), + "account_name": StringDatabaseField(name="account_name"), + "auto_advance": BooleanDatabaseField(name="auto_advance"), + "__effective_at": IntegerDatabaseField(name="effective_at", hidden=True), + "effective_at": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__effective_at"])]), name="effective_at" + ), + "subscription_id": StringDatabaseField(name="subscription"), + "attempt_count": IntegerDatabaseField(name="attempt_count"), + "automatic_tax": StringJSONDatabaseField(name="automatic_tax"), + "customer_name": StringDatabaseField(name="customer_name"), + "billing_reason": StringDatabaseField(name="billing_reason"), + "customer_email": StringDatabaseField(name="customer_email"), + "ending_balance": IntegerDatabaseField(name="ending_balance"), + "payment_intent": StringDatabaseField(name="payment_intent"), + "account_country": StringDatabaseField(name="account_country"), + "amount_shipping": IntegerDatabaseField(name="amount_shipping"), + "amount_remaining": IntegerDatabaseField(name="amount_remaining"), + "customer_address": StringJSONDatabaseField(name="customer_address"), + "customer_tax_ids": StringJSONDatabaseField(name="customer_tax_ids"), + "paid_out_of_band": BooleanDatabaseField(name="paid_out_of_band"), + "payment_settings": StringJSONDatabaseField(name="payment_settings"), + "starting_balance": IntegerDatabaseField(name="starting_balance"), + "collection_method": StringDatabaseField(name="collection_method"), + "default_tax_rates": StringJSONDatabaseField(name="default_tax_rates"), + "total_tax_amounts": StringJSONDatabaseField(name="total_tax_amounts"), + "hosted_invoice_url": StringDatabaseField(name="hosted_invoice_url"), + "status_transitions": StringJSONDatabaseField(name="status_transitions"), + "customer_tax_exempt": StringDatabaseField(name="customer_tax_exempt"), + "total_excluding_tax": IntegerDatabaseField(name="total_excluding_tax"), + "subscription_details": StringJSONDatabaseField(name="subscription_details"), + "__webhooks_delivered_at": IntegerDatabaseField(name="webhooks_delivered_at", hidden=True), + "webhooks_delivered_at": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__webhooks_delivered_at"])]), + name="webhooks_delivered_at", + ), + "subtotal_excluding_tax": IntegerDatabaseField(name="subtotal_excluding_tax"), + "total_discount_amounts": StringJSONDatabaseField(name="total_discount_amounts"), + "pre_payment_credit_notes_amount": IntegerDatabaseField(name="pre_payment_credit_notes_amount"), + "post_payment_credit_notes_amount": IntegerDatabaseField(name="post_payment_credit_notes_amount"), + "__dlt_id": StringDatabaseField(name="_dlt_id", hidden=True), + "__dlt_load_id": StringDatabaseField(name="_dlt_load_id", hidden=True), + }, + "stripe_charge": { + "id": StringDatabaseField(name="id"), + "paid": BooleanDatabaseField(name="paid"), + "amount": IntegerDatabaseField(name="amount"), + "object": StringDatabaseField(name="object"), + "source": StringJSONDatabaseField(name="source"), + "status": StringDatabaseField(name="status"), + "__created": IntegerDatabaseField(name="created", hidden=True), + "created_at": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__created"])]), name="created_at" + ), + "invoice_id": StringDatabaseField(name="invoice"), + "outcome": StringJSONDatabaseField(name="outcome"), + "captured": BooleanDatabaseField(name="captured"), + "currency": StringDatabaseField(name="currency"), + "customer_id": StringDatabaseField(name="customer"), + "disputed": BooleanDatabaseField(name="disputed"), + "livemode": BooleanDatabaseField(name="livemode"), + "metadata": StringJSONDatabaseField(name="metadata"), + "refunded": BooleanDatabaseField(name="refunded"), + "description": StringDatabaseField(name="description"), + "receipt_url": StringDatabaseField(name="receipt_url"), + "failure_code": StringDatabaseField(name="failure_code"), + "fraud_details": StringJSONDatabaseField(name="fraud_details"), + "radar_options": StringJSONDatabaseField(name="radar_options"), + "receipt_email": StringDatabaseField(name="receipt_email"), + "payment_intent_id": StringDatabaseField(name="payment_intent"), + "payment_method_id": StringDatabaseField(name="payment_method"), + "amount_captured": IntegerDatabaseField(name="amount_captured"), + "amount_refunded": IntegerDatabaseField(name="amount_refunded"), + "billing_details": StringJSONDatabaseField(name="billing_details"), + "failure_message": StringDatabaseField(name="failure_message"), + "balance_transaction_id": StringDatabaseField(name="balance_transaction"), + "statement_descriptor": StringDatabaseField(name="statement_descriptor"), + "payment_method_details": StringJSONDatabaseField(name="payment_method_details"), + "calculated_statement_descriptor": StringDatabaseField(name="calculated_statement_descriptor"), + "__dlt_id": StringDatabaseField(name="_dlt_id", hidden=True), + "__dlt_load_id": StringDatabaseField(name="_dlt_load_id", hidden=True), + }, + "stripe_price": { + "id": StringDatabaseField(name="id"), + "type": StringDatabaseField(name="type"), + "active": BooleanDatabaseField(name="active"), + "object": StringDatabaseField(name="object"), + "__created": IntegerDatabaseField(name="created", hidden=True), + "created_at": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__created"])]), name="created_at" + ), + "product_id": StringDatabaseField(name="product"), + "currency": StringDatabaseField(name="currency"), + "livemode": BooleanDatabaseField(name="livemode"), + "metadata": StringJSONDatabaseField(name="metadata"), + "nickname": StringDatabaseField(name="nickname"), + "recurring": StringJSONDatabaseField(name="recurring"), + "tiers_mode": StringDatabaseField(name="tiers_mode"), + "unit_amount": IntegerDatabaseField(name="unit_amount"), + "tax_behavior": StringDatabaseField(name="tax_behavior"), + "billing_scheme": StringDatabaseField(name="billing_scheme"), + "unit_amount_decimal": StringDatabaseField(name="unit_amount_decimal"), + "__dlt_id": StringDatabaseField(name="_dlt_id", hidden=True), + "__dlt_load_id": StringDatabaseField(name="_dlt_load_id", hidden=True), + }, + "stripe_product": { + "id": StringDatabaseField(name="id"), + "name": StringDatabaseField(name="name"), + "type": StringDatabaseField(name="type"), + "active": BooleanDatabaseField(name="active"), + "images": StringJSONDatabaseField(name="images"), + "object": StringDatabaseField(name="object"), + "__created": IntegerDatabaseField(name="created", hidden=True), + "created_at": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__created"])]), name="created_at" + ), + "__updated": IntegerDatabaseField(name="updated", hidden=True), + "updated_at": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__updated"])]), name="updated_at" + ), + "features": StringJSONDatabaseField(name="features"), + "livemode": BooleanDatabaseField(name="livemode"), + "metadata": StringJSONDatabaseField(name="metadata"), + "tax_code": StringDatabaseField(name="tax_code"), + "attributes": StringJSONDatabaseField(name="attributes"), + "description": StringDatabaseField(name="description"), + "default_price_id": StringDatabaseField(name="default_price"), + "__dlt_load_id": StringDatabaseField(name="_dlt_load_id", hidden=True), + "__dlt_id": StringDatabaseField(name="_dlt_id", hidden=True), + }, + "stripe_subscription": { + "id": StringDatabaseField(name="id"), + "plan": StringJSONDatabaseField(name="plan"), + "items": StringJSONDatabaseField(name="items"), + "object": StringDatabaseField(name="object"), + "status": StringDatabaseField(name="status"), + "__created": IntegerDatabaseField(name="created", hidden=True), + "created_at": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__created"])]), name="created_at" + ), + "currency": StringDatabaseField(name="currency"), + "customer_id": StringDatabaseField(name="customer"), + "__ended_at": IntegerDatabaseField(name="ended_at", hidden=True), + "ended_at": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__ended_at"])]), name="ended_at" + ), + "livemode": BooleanDatabaseField(name="livemode"), + "metadata": StringJSONDatabaseField(name="metadata"), + "quantity": IntegerDatabaseField(name="quantity"), + "__start_date": IntegerDatabaseField(name="start_date", hidden=True), + "start_date": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__start_date"])]), name="start_date" + ), + "__canceled_at": IntegerDatabaseField(name="canceled_at", hidden=True), + "canceled_at": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__canceled_at"])]), name="canceled_at" + ), + "automatic_tax": StringJSONDatabaseField(name="automatic_tax"), + "latest_invoice_id": StringDatabaseField(name="latest_invoice"), + "trial_settings": StringJSONDatabaseField(name="trial_settings"), + "invoice_settings": StringJSONDatabaseField(name="invoice_settings"), + "payment_settings": StringJSONDatabaseField(name="payment_settings"), + "collection_method": StringDatabaseField(name="collection_method"), + "default_tax_rates": StringJSONDatabaseField(name="default_tax_rates"), + "__current_period_start": IntegerDatabaseField(name="current_period_start", hidden=True), + "current_period_start": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__current_period_start"])]), + name="current_period_start", + ), + "__current_period_end": IntegerDatabaseField(name="current_period_end", hidden=True), + "current_period_end": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__current_period_end"])]), + name="current_period_end", + ), + "__billing_cycle_anchor": IntegerDatabaseField(name="billing_cycle_anchor", hidden=True), + "billing_cycle_anchor": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__billing_cycle_anchor"])]), + name="billing_cycle_anchor", + ), + "cancel_at_period_end": BooleanDatabaseField(name="cancel_at_period_end"), + "cancellation_details": StringJSONDatabaseField(name="cancellation_details"), + "__dlt_id": StringDatabaseField(name="_dlt_id", hidden=True), + "__dlt_load_id": StringDatabaseField(name="_dlt_load_id", hidden=True), + }, + "stripe_balancetransaction": { + "id": StringDatabaseField(name="id"), + "fee": IntegerDatabaseField(name="fee"), + "net": IntegerDatabaseField(name="net"), + "type": StringDatabaseField(name="type"), + "amount": IntegerDatabaseField(name="amount"), + "object": StringDatabaseField(name="object"), + "source_id": StringDatabaseField(name="source"), + "status": StringDatabaseField(name="status"), + "__created": IntegerDatabaseField(name="created", hidden=True), + "created_at": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__created"])]), name="created_at" + ), + "currency": StringDatabaseField(name="currency"), + "description": StringDatabaseField(name="description"), + "fee_details": StringJSONDatabaseField(name="fee_details"), + "__available_on": IntegerDatabaseField(name="available_on", hidden=True), + "available_on": ast.ExpressionField( + expr=ast.Call(name="fromUnixTimestamp", args=[ast.Field(chain=["__available_on"])]), name="available_on" + ), + "reporting_category": StringDatabaseField(name="reporting_category"), + "__dlt_id": StringDatabaseField(name="_dlt_id", hidden=True), + "__dlt_load_id": StringDatabaseField(name="_dlt_load_id", hidden=True), + }, +} diff --git a/posthog/warehouse/models/table.py b/posthog/warehouse/models/table.py index 5fbe84b3f34d9..91c6f61709d6e 100644 --- a/posthog/warehouse/models/table.py +++ b/posthog/warehouse/models/table.py @@ -7,6 +7,7 @@ BooleanDatabaseField, DateDatabaseField, DateTimeDatabaseField, + FieldOrTable, IntegerDatabaseField, StringArrayDatabaseField, StringDatabaseField, @@ -27,6 +28,7 @@ from uuid import UUID from sentry_sdk import capture_exception from posthog.warehouse.util import database_sync_to_async +from .external_table_definitions import external_tables CLICKHOUSE_HOGQL_MAPPING = { "UUID": StringDatabaseField, @@ -98,6 +100,13 @@ class TableFormat(models.TextChoices): __repr__ = sane_repr("name") + def table_name_without_prefix(self) -> str: + if self.external_data_source is not None and self.external_data_source.prefix is not None: + prefix = self.external_data_source.prefix + else: + prefix = "" + return self.name[len(prefix) :] + def get_columns(self, safe_expose_ch_error=True) -> Dict[str, str]: try: result = sync_execute( @@ -126,7 +135,7 @@ def hogql_definition(self) -> S3Table: if not self.columns: raise Exception("Columns must be fetched and saved to use in HogQL.") - fields = {} + fields: Dict[str, FieldOrTable] = {} structure = [] for column, type in self.columns.items(): # Support for 'old' style columns @@ -153,6 +162,9 @@ def hogql_definition(self) -> S3Table: fields[column] = hogql_type(name=column) + # Replace fields with any redefined fields if they exist + fields = external_tables.get(self.table_name_without_prefix(), fields) + return S3Table( name=self.name, url=self.url_pattern,