diff --git a/.github/workflows/alert-on-failed-automerge.yml b/.github/workflows/alert-on-failed-automerge.yml index af9fde9675ed4..31582b50d11c1 100644 --- a/.github/workflows/alert-on-failed-automerge.yml +++ b/.github/workflows/alert-on-failed-automerge.yml @@ -9,7 +9,7 @@ on: jobs: notify_on_failure: if: ${{ github.event.check_suite.conclusion == 'failure' }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout code diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml index 20a3dbf314649..4b2a41b91384f 100644 --- a/.github/workflows/automerge.yml +++ b/.github/workflows/automerge.yml @@ -23,7 +23,7 @@ on: jobs: automerge: name: Automerge if requested - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: IS_POSTHOG_BOT_AVAILABLE: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN != '' }} steps: diff --git a/.github/actions/browserslist-update/action.yml b/.github/workflows/browserslist-update.yml similarity index 59% rename from .github/actions/browserslist-update/action.yml rename to .github/workflows/browserslist-update.yml index 28fe56bee93b3..9d07dba8fc762 100644 --- a/.github/actions/browserslist-update/action.yml +++ b/.github/workflows/browserslist-update.yml @@ -2,7 +2,7 @@ name: Update Browserslist database on: schedule: - - cron: '0 0 5,15,25 * *' + - cron: '0 12 * * MON' workflow_dispatch: permissions: @@ -17,16 +17,24 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Configure git run: | - # Setup for committing using built-in token. See https://github.com/actions/checkout#push-a-commit-using-the-built-in-token - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --global user.email "action@github.com" + git config --global user.name "Browserslist Update Action" + + - name: Install pnpm + uses: pnpm/action-setup@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 18.12.1 + - name: Update Browserslist database and create PR if applies uses: c2corg/browserslist-update-action@v2 with: - base_branch: main + github_token: ${{ secrets.GITHUB_TOKEN }} commit_message: 'build: update Browserslist db' title: 'build: update Browserslist db' - body: 'Auto-generated by [browserslist-update-action](https://github.com/c2corg/browserslist-update-action/)' labels: 'dependencies, automerge' diff --git a/.github/workflows/ci-backend-update-test-timing.yml b/.github/workflows/ci-backend-update-test-timing.yml index 39a1993119701..eb1c36329ce6e 100644 --- a/.github/workflows/ci-backend-update-test-timing.yml +++ b/.github/workflows/ci-backend-update-test-timing.yml @@ -19,7 +19,7 @@ env: jobs: django: name: Run Django tests and save test durations - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml index 943b6611006d7..410d4deb18461 100644 --- a/.github/workflows/ci-backend.yml +++ b/.github/workflows/ci-backend.yml @@ -43,7 +43,7 @@ jobs: # Job to decide if we should run backend ci # See https://github.com/dorny/paths-filter#conditional-execution for more details changes: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 5 name: Determine need to run backend checks # Set job outputs to values from filter step @@ -94,7 +94,7 @@ jobs: timeout-minutes: 30 name: Python code quality checks - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: # If this run wasn't initiated by the bot (meaning: snapshot update) and we've determined @@ -153,7 +153,7 @@ jobs: timeout-minutes: 10 name: Validate Django and CH migrations - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 @@ -228,7 +228,7 @@ jobs: timeout-minutes: 30 name: Django tests – ${{ matrix.segment }} (persons-on-events ${{ matrix.person-on-events && 'on' || 'off' }}), Py ${{ matrix.python-version }}, ${{ matrix.clickhouse-server-image }} (${{matrix.group}}/${{ matrix.concurrency }}) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -328,7 +328,7 @@ jobs: matrix: clickhouse-server-image: ['clickhouse/clickhouse-server:24.8.7.41'] if: needs.changes.outputs.backend == 'true' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: 'Checkout repo' uses: actions/checkout@v3 @@ -382,7 +382,7 @@ jobs: calculate-running-time: name: Calculate running time needs: [django, async-migrations] - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: # Run on pull requests to PostHog/posthog + on PostHog/posthog outside of PRs - but never on forks needs.changes.outputs.backend == 'true' && ( diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml index 97deeffe6bd2a..4ec28981e5f1d 100644 --- a/.github/workflows/ci-e2e.yml +++ b/.github/workflows/ci-e2e.yml @@ -14,7 +14,7 @@ concurrency: jobs: changes: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 5 name: Determine need to run E2E checks # Set job outputs to values from filter step @@ -53,7 +53,7 @@ jobs: chunks: needs: changes name: Cypress preparation - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 5 outputs: chunks: ${{ steps.chunk.outputs.chunks }} @@ -67,7 +67,7 @@ jobs: container: name: Build and cache container image - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 60 needs: [changes] permissions: @@ -91,7 +91,7 @@ jobs: cypress: name: Cypress E2E tests (${{ strategy.job-index }}) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 60 needs: [chunks, changes, container] permissions: @@ -279,7 +279,7 @@ jobs: calculate-running-time: name: Calculate running time - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: [cypress] if: needs.changes.outputs.shouldTriggerCypress == 'true' && github.event.pull_request.head.repo.full_name == 'PostHog/posthog' diff --git a/.github/workflows/ci-frontend.yml b/.github/workflows/ci-frontend.yml index d466e0d3cdb82..f59c7e8eef790 100644 --- a/.github/workflows/ci-frontend.yml +++ b/.github/workflows/ci-frontend.yml @@ -16,7 +16,7 @@ jobs: # we skip each step individually, so they are still reported as success # because many of them are required for CI checks to be green changes: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 5 name: Determine need to run frontend checks outputs: @@ -54,7 +54,7 @@ jobs: name: Code quality checks needs: changes # kea typegen and typescript:check need some more oomph - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: # we need at least one thing to run to make sure we include everything for required jobs - uses: actions/checkout@v3 @@ -121,7 +121,7 @@ jobs: minimum-change-threshold: 1000 jest: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: changes name: Jest test (${{ matrix.segment }} - ${{ matrix.chunk }}) diff --git a/.github/workflows/ci-hobby.yml b/.github/workflows/ci-hobby.yml index 0025e656c8204..73d29cbdad746 100644 --- a/.github/workflows/ci-hobby.yml +++ b/.github/workflows/ci-hobby.yml @@ -21,7 +21,7 @@ concurrency: jobs: changes: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # this is a slow one timeout-minutes: 30 name: Setup DO Hobby Instance and test diff --git a/.github/workflows/ci-hog.yml b/.github/workflows/ci-hog.yml index 7fee499a801f1..69a3bc3d5f133 100644 --- a/.github/workflows/ci-hog.yml +++ b/.github/workflows/ci-hog.yml @@ -20,7 +20,7 @@ jobs: # Job to decide if we should run backend ci # See https://github.com/dorny/paths-filter#conditional-execution for more details changes: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 5 name: Determine need to run Hog checks # Set job outputs to values from filter step @@ -50,7 +50,7 @@ jobs: timeout-minutes: 30 name: Hog tests - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: # If this run wasn't initiated by the bot (meaning: snapshot update) and we've determined diff --git a/.github/workflows/ci-plugin-server.yml b/.github/workflows/ci-plugin-server.yml index ecd13c789ddf8..30ca845cd89b6 100644 --- a/.github/workflows/ci-plugin-server.yml +++ b/.github/workflows/ci-plugin-server.yml @@ -24,7 +24,7 @@ jobs: # Job to decide if we should run plugin server ci # See https://github.com/dorny/paths-filter#conditional-execution for more details changes: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 5 name: Determine need to run plugin server checks outputs: @@ -53,7 +53,7 @@ jobs: name: Code quality needs: changes if: needs.changes.outputs.plugin-server == 'true' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 defaults: run: working-directory: 'plugin-server' @@ -81,7 +81,7 @@ jobs: tests: name: Plugin Server Tests (${{matrix.shard}}) needs: changes - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -193,7 +193,7 @@ jobs: functional-tests: name: Functional tests needs: changes - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 env: REDIS_URL: 'redis://localhost' diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a914e2a95225e..56f1ed3bf0330 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -31,7 +31,7 @@ jobs: # - https://gh.io/supported-runners-and-hardware-resources # - https://gh.io/using-larger-runners (GitHub.com only) # Consider using larger runners or machines with greater resources for possible analysis time improvements. - runs-on: 'ubuntu-latest' + runs-on: 'ubuntu-24.04' timeout-minutes: 15 permissions: # required for all workflows diff --git a/.github/workflows/codespaces.yml b/.github/workflows/codespaces.yml index b725c032ddacd..06f796951dbe2 100644 --- a/.github/workflows/codespaces.yml +++ b/.github/workflows/codespaces.yml @@ -21,7 +21,7 @@ on: jobs: build: name: Build Codespaces image - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 # Build on master and PRs with the label 'codespaces-build' only if: ${{ github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'codespaces-build') }} diff --git a/.github/workflows/container-images-cd.yml b/.github/workflows/container-images-cd.yml index e7fe1a1608b73..b393214ec72da 100644 --- a/.github/workflows/container-images-cd.yml +++ b/.github/workflows/container-images-cd.yml @@ -22,7 +22,7 @@ jobs: posthog_build: name: Build and push PostHog if: github.repository == 'PostHog/posthog' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: id-token: write # allow issuing OIDC tokens for this workflow run contents: read # allow at least reading the repo contents, add other permissions if necessary diff --git a/.github/workflows/container-images-ci.yml b/.github/workflows/container-images-ci.yml index c690c11dd2663..7b434a7cb546d 100644 --- a/.github/workflows/container-images-ci.yml +++ b/.github/workflows/container-images-ci.yml @@ -13,7 +13,7 @@ concurrency: jobs: posthog_build: name: Build Docker image - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: id-token: write # allow issuing OIDC tokens for this workflow run contents: read # allow at least reading the repo contents, add other permissions if necessary @@ -51,7 +51,7 @@ jobs: lint: name: Lint changed Dockerfiles - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Check out uses: actions/checkout@v3 diff --git a/.github/workflows/copy-clickhouse-udfs.yml b/.github/workflows/copy-clickhouse-udfs.yml index 3dc6fce3ade07..b55d66bc30e8b 100644 --- a/.github/workflows/copy-clickhouse-udfs.yml +++ b/.github/workflows/copy-clickhouse-udfs.yml @@ -9,7 +9,7 @@ on: jobs: trigger_udfs_workflow: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Trigger UDFs Workflow uses: benc-uk/workflow-dispatch@v1 diff --git a/.github/workflows/foss-sync.yml b/.github/workflows/foss-sync.yml index 82334bfe89f5a..b8edfe63210b9 100644 --- a/.github/workflows/foss-sync.yml +++ b/.github/workflows/foss-sync.yml @@ -10,7 +10,7 @@ jobs: repo-sync: name: Sync posthog-foss with posthog if: github.repository == 'PostHog/posthog' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Sync repositories 1 to 1 - master branch uses: PostHog/git-sync@v3 diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 37c4e75ddb9bc..4fc7344a97216 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -7,7 +7,7 @@ on: jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout code diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index 3efcc6c9523de..2f700829f2921 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -10,7 +10,7 @@ on: jobs: lint-pr: name: Validate PR title against Conventional Commits - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: amannn/action-semantic-pull-request@v5 env: diff --git a/.github/workflows/livestream-docker-image.yml b/.github/workflows/livestream-docker-image.yml index 7023ee98c03f1..5efde377f5866 100644 --- a/.github/workflows/livestream-docker-image.yml +++ b/.github/workflows/livestream-docker-image.yml @@ -10,7 +10,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: read @@ -59,7 +59,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} deploy: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: build steps: - name: get deployer token diff --git a/.github/workflows/pr-cleanup.yml b/.github/workflows/pr-cleanup.yml index 26f46533847f7..0c0586e8ec0b5 100644 --- a/.github/workflows/pr-cleanup.yml +++ b/.github/workflows/pr-cleanup.yml @@ -9,7 +9,7 @@ on: jobs: deploy_preview_cleanup: name: Deploy Preview Cleanup - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: ${{ contains(github.event.pull_request.labels.*.name, 'deploy') }} permissions: diff --git a/.github/workflows/pr-deploy.yml b/.github/workflows/pr-deploy.yml index 26896c7566091..807b1823dc17b 100644 --- a/.github/workflows/pr-deploy.yml +++ b/.github/workflows/pr-deploy.yml @@ -10,7 +10,7 @@ on: jobs: deploy_preview: name: Deploy preview environment - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: id-token: write diff --git a/.github/workflows/release-hogvm.yml b/.github/workflows/release-hogvm.yml index 3d173bcca4591..6c8480202b24d 100644 --- a/.github/workflows/release-hogvm.yml +++ b/.github/workflows/release-hogvm.yml @@ -9,7 +9,7 @@ on: jobs: release: name: Publish - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 defaults: run: working-directory: hogvm/typescript diff --git a/.github/workflows/rust-docker-build.yml b/.github/workflows/rust-docker-build.yml index 1535867f572f0..4237599d9b311 100644 --- a/.github/workflows/rust-docker-build.yml +++ b/.github/workflows/rust-docker-build.yml @@ -118,7 +118,7 @@ jobs: deploy: name: deploy ${{ matrix.release }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: build if: github.ref == 'refs/heads/master' strategy: diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 0dd62fcd25f7f..29268601c4db4 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -13,7 +13,7 @@ jobs: # Job to decide if we should run rust ci # See https://github.com/dorny/paths-filter#conditional-execution for more details changes: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 5 if: github.repository == 'PostHog/posthog' name: Determine need to run Rust checks diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index 9f2797cca668a..635e01600286c 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -7,7 +7,7 @@ jobs: stale: # Only unleash the stale bot on PostHog/posthog, as there's no POSTHOG_BOT_GITHUB_TOKEN token on forks if: ${{ github.repository == 'PostHog/posthog' }} - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/stale@v9 with: diff --git a/.github/workflows/storybook-chromatic.yml b/.github/workflows/storybook-chromatic.yml index 809328e296c90..7d5d68e1ac2a3 100644 --- a/.github/workflows/storybook-chromatic.yml +++ b/.github/workflows/storybook-chromatic.yml @@ -18,7 +18,7 @@ concurrency: jobs: storybook-chromatic: name: Publish to Chromatic - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 15 if: github.event.pull_request.head.repo.full_name == github.repository # Don't run on forks outputs: @@ -50,7 +50,7 @@ jobs: visual-regression: name: Visual regression tests - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 container: image: mcr.microsoft.com/playwright:v1.45.0 @@ -211,7 +211,7 @@ jobs: visual-regression-summary: name: Summarize visual regression tests - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 5 needs: visual-regression if: always() # Run even if visual-regression fails for one (or more) of the browsers diff --git a/.github/workflows/storybook-deploy.yml b/.github/workflows/storybook-deploy.yml index 9a04f8e582ced..33520a355a244 100644 --- a/.github/workflows/storybook-deploy.yml +++ b/.github/workflows/storybook-deploy.yml @@ -8,7 +8,7 @@ on: jobs: storybook-deployment: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: github.repository == 'PostHog/posthog' steps: - name: Check out PostHog/posthog repo diff --git a/.github/workflows/vector-docker-build-deploy.yml b/.github/workflows/vector-docker-build-deploy.yml index a1d97f846f298..d78e2e9b2d089 100644 --- a/.github/workflows/vector-docker-build-deploy.yml +++ b/.github/workflows/vector-docker-build-deploy.yml @@ -74,7 +74,7 @@ jobs: platforms: linux/arm64,linux/amd64 deploy: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: build if: github.ref == 'refs/heads/master' steps: diff --git a/ee/benchmarks/helpers.py b/ee/benchmarks/helpers.py index 285a1dc97ee9f..8535e6adef47d 100644 --- a/ee/benchmarks/helpers.py +++ b/ee/benchmarks/helpers.py @@ -14,7 +14,7 @@ django.setup() -from posthog.clickhouse.materialized_columns import get_enabled_materialized_columns # noqa: E402 +from ee.clickhouse.materialized_columns.columns import get_enabled_materialized_columns # noqa: E402 from posthog import client # noqa: E402 from posthog.clickhouse.query_tagging import reset_query_tags, tag_queries # noqa: E402 from posthog.models.utils import UUIDT # noqa: E402 diff --git a/ee/clickhouse/materialized_columns/analyze.py b/ee/clickhouse/materialized_columns/analyze.py index 43a1e83256912..bfae76ef2432c 100644 --- a/ee/clickhouse/materialized_columns/analyze.py +++ b/ee/clickhouse/materialized_columns/analyze.py @@ -171,6 +171,7 @@ def materialize_properties_task( backfill_period_days: int = MATERIALIZE_COLUMNS_BACKFILL_PERIOD_DAYS, dry_run: bool = False, team_id_to_analyze: Optional[int] = None, + is_nullable: bool = False, ) -> None: """ Creates materialized columns for event and person properties based off of slow queries @@ -203,7 +204,7 @@ def materialize_properties_task( logger.info(f"Materializing column. table={table}, property_name={property_name}") if not dry_run: - materialize(table, property_name, table_column=table_column) + materialize(table, property_name, table_column=table_column, is_nullable=is_nullable) properties[table].append((property_name, table_column)) if backfill_period_days > 0 and not dry_run: diff --git a/ee/clickhouse/materialized_columns/columns.py b/ee/clickhouse/materialized_columns/columns.py index c9624bf96bacd..66be7d75f0e1d 100644 --- a/ee/clickhouse/materialized_columns/columns.py +++ b/ee/clickhouse/materialized_columns/columns.py @@ -1,33 +1,35 @@ from __future__ import annotations +import logging import re -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterable, Iterator from copy import copy from dataclasses import dataclass, replace from datetime import timedelta -from typing import Any, Literal, NamedTuple, TypeVar, cast +from typing import Any, Literal, TypeVar, cast from clickhouse_driver import Client from django.utils.timezone import now +from posthog.cache_utils import cache_for from posthog.clickhouse.client.connection import default_client from posthog.clickhouse.cluster import ClickhouseCluster, ConnectionInfo, FuturesMap, HostInfo from posthog.clickhouse.kafka_engine import trim_quotes_expr from posthog.clickhouse.materialized_columns import ColumnName, TablesWithMaterializedColumns from posthog.client import sync_execute from posthog.models.event.sql import EVENTS_DATA_TABLE -from posthog.models.instance_setting import get_instance_setting from posthog.models.person.sql import PERSONS_TABLE from posthog.models.property import PropertyName, TableColumn, TableWithProperties from posthog.models.utils import generate_random_short_suffix from posthog.settings import CLICKHOUSE_DATABASE, CLICKHOUSE_PER_TEAM_SETTINGS, TEST + +logger = logging.getLogger(__name__) + T = TypeVar("T") DEFAULT_TABLE_COLUMN: Literal["properties"] = "properties" -TRIM_AND_EXTRACT_PROPERTY = trim_quotes_expr("JSONExtractRaw({table_column}, %(property)s)") - SHORT_TABLE_COLUMN_NAME = { "properties": "p", "group_properties": "gp", @@ -40,15 +42,36 @@ } -class MaterializedColumn(NamedTuple): +@dataclass +class MaterializedColumn: name: ColumnName details: MaterializedColumnDetails + is_nullable: bool + + @property + def type(self) -> str: + if self.is_nullable: + return "Nullable(String)" + else: + return "String" + + def get_expression_and_parameters(self) -> tuple[str, dict[str, Any]]: + if self.is_nullable: + return ( + f"JSONExtract({self.details.table_column}, %(property_name)s, %(property_type)s)", + {"property_name": self.details.property_name, "property_type": self.type}, + ) + else: + return ( + trim_quotes_expr(f"JSONExtractRaw({self.details.table_column}, %(property)s)"), + {"property": self.details.property_name}, + ) @staticmethod def get_all(table: TablesWithMaterializedColumns) -> Iterator[MaterializedColumn]: rows = sync_execute( """ - SELECT name, comment + SELECT name, comment, type like 'Nullable(%%)' as is_nullable FROM system.columns WHERE database = %(database)s AND table = %(table)s @@ -58,8 +81,8 @@ def get_all(table: TablesWithMaterializedColumns) -> Iterator[MaterializedColumn {"database": CLICKHOUSE_DATABASE, "table": table}, ) - for name, comment in rows: - yield MaterializedColumn(name, MaterializedColumnDetails.from_column_comment(comment)) + for name, comment, is_nullable in rows: + yield MaterializedColumn(name, MaterializedColumnDetails.from_column_comment(comment), is_nullable) @staticmethod def get(table: TablesWithMaterializedColumns, column_name: ColumnName) -> MaterializedColumn: @@ -111,22 +134,24 @@ def from_column_comment(cls, comment: str) -> MaterializedColumnDetails: def get_materialized_columns( table: TablesWithMaterializedColumns, - exclude_disabled_columns: bool = False, -) -> dict[tuple[PropertyName, TableColumn], ColumnName]: - if not get_instance_setting("MATERIALIZED_COLUMNS_ENABLED"): - return {} - +) -> dict[tuple[PropertyName, TableColumn], MaterializedColumn]: return { - (column.details.property_name, column.details.table_column): column.name + (column.details.property_name, column.details.table_column): column for column in MaterializedColumn.get_all(table) - if not (exclude_disabled_columns and column.details.is_disabled) } +@cache_for(timedelta(minutes=15)) +def get_enabled_materialized_columns( + table: TablesWithMaterializedColumns, +) -> dict[tuple[PropertyName, TableColumn], MaterializedColumn]: + return {k: column for k, column in get_materialized_columns(table).items() if not column.details.is_disabled} + + def get_cluster() -> ClickhouseCluster: extra_hosts = [] for host_config in map(copy, CLICKHOUSE_PER_TEAM_SETTINGS.values()): - extra_hosts.append(ConnectionInfo(host_config.pop("host"), host_config.pop("port", None))) + extra_hosts.append(ConnectionInfo(host_config.pop("host"))) assert len(host_config) == 0, f"unexpected values: {host_config!r}" return ClickhouseCluster(default_client(), extra_hosts=extra_hosts) @@ -161,6 +186,10 @@ def map_data_nodes(self, cluster: ClickhouseCluster, fn: Callable[[Client], T]) } +def get_minmax_index_name(column: str) -> str: + return f"minmax_{column}" + + @dataclass class CreateColumnOnDataNodesTask: table: str @@ -169,20 +198,17 @@ class CreateColumnOnDataNodesTask: add_column_comment: bool def execute(self, client: Client) -> None: + expression, parameters = self.column.get_expression_and_parameters() actions = [ - f""" - ADD COLUMN IF NOT EXISTS {self.column.name} VARCHAR - MATERIALIZED {TRIM_AND_EXTRACT_PROPERTY.format(table_column=self.column.details.table_column)} - """, + f"ADD COLUMN IF NOT EXISTS {self.column.name} {self.column.type} MATERIALIZED {expression}", ] - parameters = {"property": self.column.details.property_name} if self.add_column_comment: actions.append(f"COMMENT COLUMN {self.column.name} %(comment)s") parameters["comment"] = self.column.details.as_column_comment() if self.create_minmax_index: - index_name = f"minmax_{self.column.name}" + index_name = get_minmax_index_name(self.column.name) actions.append(f"ADD INDEX IF NOT EXISTS {index_name} {self.column.name} TYPE minmax GRANULARITY 1") client.execute( @@ -201,7 +227,7 @@ def execute(self, client: Client) -> None: client.execute( f""" ALTER TABLE {self.table} - ADD COLUMN IF NOT EXISTS {self.column.name} VARCHAR, + ADD COLUMN IF NOT EXISTS {self.column.name} {self.column.type}, COMMENT COLUMN {self.column.name} %(comment)s """, {"comment": self.column.details.as_column_comment()}, @@ -215,6 +241,7 @@ def materialize( column_name: ColumnName | None = None, table_column: TableColumn = DEFAULT_TABLE_COLUMN, create_minmax_index=not TEST, + is_nullable: bool = False, ) -> ColumnName | None: if (property, table_column) in get_materialized_columns(table): if TEST: @@ -235,6 +262,7 @@ def materialize( property_name=property, is_disabled=False, ), + is_nullable=is_nullable, ) table_info.map_data_nodes( @@ -261,64 +289,106 @@ def materialize( @dataclass class UpdateColumnCommentTask: table: str - column: MaterializedColumn + columns: list[MaterializedColumn] def execute(self, client: Client) -> None: + actions = [] + parameters = {} + for i, column in enumerate(self.columns): + parameter_name = f"comment_{i}" + actions.append(f"COMMENT COLUMN {column.name} %({parameter_name})s") + parameters[parameter_name] = column.details.as_column_comment() + client.execute( - f"ALTER TABLE {self.table} COMMENT COLUMN {self.column.name} %(comment)s", - {"comment": self.column.details.as_column_comment()}, + f"ALTER TABLE {self.table} " + ", ".join(actions), + parameters, settings={"alter_sync": 2 if TEST else 1}, ) -def update_column_is_disabled(table: TablesWithMaterializedColumns, column_name: str, is_disabled: bool) -> None: +def update_column_is_disabled( + table: TablesWithMaterializedColumns, column_names: Iterable[str], is_disabled: bool +) -> None: cluster = get_cluster() table_info = tables[table] + columns = [MaterializedColumn.get(table, column_name) for column_name in column_names] + cluster.map_all_hosts( UpdateColumnCommentTask( table_info.read_table, - MaterializedColumn( - name=column_name, - details=replace( - MaterializedColumn.get(table, column_name).details, - is_disabled=is_disabled, - ), - ), + [replace(column, details=replace(column.details, is_disabled=is_disabled)) for column in columns], ).execute ).result() +def check_index_exists(client: Client, table: str, index: str) -> bool: + [(count,)] = client.execute( + """ + SELECT count() + FROM system.data_skipping_indices + WHERE database = currentDatabase() AND table = %(table)s AND name = %(name)s + """, + {"table": table, "name": index}, + ) + assert 1 >= count >= 0 + return bool(count) + + +def check_column_exists(client: Client, table: str, column: str) -> bool: + [(count,)] = client.execute( + """ + SELECT count() + FROM system.columns + WHERE database = currentDatabase() AND table = %(table)s AND name = %(name)s + """, + {"table": table, "name": column}, + ) + assert 1 >= count >= 0 + return bool(count) + + @dataclass class DropColumnTask: table: str - column_name: str + column_names: list[str] try_drop_index: bool def execute(self, client: Client) -> None: - # XXX: copy/pasted from create task - if self.try_drop_index: - index_name = f"minmax_{self.column_name}" + actions = [] + + for column_name in self.column_names: + if self.try_drop_index: + index_name = get_minmax_index_name(column_name) + drop_index_action = f"DROP INDEX IF EXISTS {index_name}" + if check_index_exists(client, self.table, index_name): + actions.append(drop_index_action) + else: + logger.info("Skipping %r, nothing to do...", drop_index_action) + + drop_column_action = f"DROP COLUMN IF EXISTS {column_name}" + if check_column_exists(client, self.table, column_name): + actions.append(drop_column_action) + else: + logger.info("Skipping %r, nothing to do...", drop_column_action) + + if actions: client.execute( - f"ALTER TABLE {self.table} DROP INDEX IF EXISTS {index_name}", + f"ALTER TABLE {self.table} " + ", ".join(actions), settings={"alter_sync": 2 if TEST else 1}, ) - client.execute( - f"ALTER TABLE {self.table} DROP COLUMN IF EXISTS {self.column_name}", - settings={"alter_sync": 2 if TEST else 1}, - ) - -def drop_column(table: TablesWithMaterializedColumns, column_name: str) -> None: +def drop_column(table: TablesWithMaterializedColumns, column_names: Iterable[str]) -> None: cluster = get_cluster() table_info = tables[table] + column_names = [*column_names] if isinstance(table_info, ShardedTableInfo): cluster.map_all_hosts( DropColumnTask( table_info.dist_table, - column_name, + column_names, try_drop_index=False, # no indexes on distributed tables ).execute ).result() @@ -327,7 +397,7 @@ def drop_column(table: TablesWithMaterializedColumns, column_name: str) -> None: cluster, DropColumnTask( table_info.data_table, - column_name, + column_names, try_drop_index=True, ).execute, ).result() @@ -345,12 +415,13 @@ def execute(self, client: Client) -> None: # Note that for this to work all inserts should list columns explicitly # Improve this if https://github.com/ClickHouse/ClickHouse/issues/27730 ever gets resolved for column in self.columns: + expression, parameters = column.get_expression_and_parameters() client.execute( f""" ALTER TABLE {self.table} - MODIFY COLUMN {column.name} VARCHAR DEFAULT {TRIM_AND_EXTRACT_PROPERTY.format(table_column=column.details.table_column)} + MODIFY COLUMN {column.name} {column.type} DEFAULT {expression} """, - {"property": column.details.property_name}, + parameters, settings=self.test_settings, ) @@ -420,10 +491,10 @@ def _materialized_column_name( prefix += f"{SHORT_TABLE_COLUMN_NAME[table_column]}_" property_str = re.sub("[^0-9a-zA-Z$]", "_", property) - existing_materialized_columns = set(get_materialized_columns(table).values()) + existing_materialized_column_names = {column.name for column in get_materialized_columns(table).values()} suffix = "" - while f"{prefix}{property_str}{suffix}" in existing_materialized_columns: + while f"{prefix}{property_str}{suffix}" in existing_materialized_column_names: suffix = "_" + generate_random_short_suffix() return f"{prefix}{property_str}{suffix}" diff --git a/ee/clickhouse/materialized_columns/test/test_analyze.py b/ee/clickhouse/materialized_columns/test/test_analyze.py index 6fdb0fb05cb0e..3b225ab670f92 100644 --- a/ee/clickhouse/materialized_columns/test/test_analyze.py +++ b/ee/clickhouse/materialized_columns/test/test_analyze.py @@ -49,9 +49,9 @@ def test_mat_columns(self, patch_backfill, patch_materialize): materialize_properties_task() patch_materialize.assert_has_calls( [ - call("events", "materialize_me", table_column="properties"), - call("events", "materialize_me2", table_column="properties"), - call("events", "materialize_person_prop", table_column="person_properties"), - call("events", "materialize_me3", table_column="properties"), + call("events", "materialize_me", table_column="properties", is_nullable=False), + call("events", "materialize_me2", table_column="properties", is_nullable=False), + call("events", "materialize_person_prop", table_column="person_properties", is_nullable=False), + call("events", "materialize_me3", table_column="properties", is_nullable=False), ] ) diff --git a/ee/clickhouse/materialized_columns/test/test_columns.py b/ee/clickhouse/materialized_columns/test/test_columns.py index 4cbbef0c4a416..ba2739bf34760 100644 --- a/ee/clickhouse/materialized_columns/test/test_columns.py +++ b/ee/clickhouse/materialized_columns/test/test_columns.py @@ -1,5 +1,6 @@ from datetime import timedelta from time import sleep +from collections.abc import Iterable from unittest import TestCase from unittest.mock import patch @@ -10,11 +11,13 @@ MaterializedColumnDetails, backfill_materialized_columns, drop_column, + get_enabled_materialized_columns, get_materialized_columns, materialize, update_column_is_disabled, ) -from posthog.clickhouse.materialized_columns import TablesWithMaterializedColumns, get_enabled_materialized_columns +from ee.tasks.materialized_columns import mark_all_materialized +from posthog.clickhouse.materialized_columns import TablesWithMaterializedColumns from posthog.client import sync_execute from posthog.conftest import create_clickhouse_tables from posthog.constants import GROUP_TYPES_LIMIT @@ -142,11 +145,11 @@ def test_materialized_column_naming(self, mock_choice): ("$foO();ääsqlinject", "properties"): "mat_$foO_____sqlinject_YYYY", ("$foO_____sqlinject", "properties"): "mat_$foO_____sqlinject_ZZZZ", }, - get_materialized_columns("events"), + {k: column.name for k, column in get_materialized_columns("events").items()}, ) self.assertEqual( - get_materialized_columns("person"), + {k: column.name for k, column in get_materialized_columns("person").items()}, {("SoMePrOp", "properties"): "pmat_SoMePrOp"}, ) @@ -241,20 +244,26 @@ def test_backfilling_data(self): def test_column_types(self): materialize("events", "myprop", create_minmax_index=True) + materialize("events", "myprop_nullable", create_minmax_index=True, is_nullable=True) - expr = "replaceRegexpAll(JSONExtractRaw(properties, 'myprop'), '^\"|\"$', '')" - self.assertEqual(("MATERIALIZED", expr), self._get_column_types("mat_myprop")) + expr_nonnullable = "replaceRegexpAll(JSONExtractRaw(properties, 'myprop'), '^\"|\"$', '')" + expr_nullable = "JSONExtract(properties, 'myprop_nullable', 'Nullable(String)')" + self.assertEqual(("String", "MATERIALIZED", expr_nonnullable), self._get_column_types("mat_myprop")) + self.assertEqual( + ("Nullable(String)", "MATERIALIZED", expr_nullable), self._get_column_types("mat_myprop_nullable") + ) - backfill_materialized_columns("events", [("myprop", "properties")], timedelta(days=50)) - self.assertEqual(("DEFAULT", expr), self._get_column_types("mat_myprop")) + backfill_materialized_columns( + "events", [("myprop", "properties"), ("myprop_nullable", "properties")], timedelta(days=50) + ) + self.assertEqual(("String", "DEFAULT", expr_nonnullable), self._get_column_types("mat_myprop")) + self.assertEqual(("Nullable(String)", "DEFAULT", expr_nullable), self._get_column_types("mat_myprop_nullable")) - try: - from ee.tasks.materialized_columns import mark_all_materialized - except ImportError: - pass - else: - mark_all_materialized() - self.assertEqual(("MATERIALIZED", expr), self._get_column_types("mat_myprop")) + mark_all_materialized() + self.assertEqual(("String", "MATERIALIZED", expr_nonnullable), self._get_column_types("mat_myprop")) + self.assertEqual( + ("Nullable(String)", "MATERIALIZED", expr_nullable), self._get_column_types("mat_myprop_nullable") + ) def _count_materialized_rows(self, column): return sync_execute( @@ -284,7 +293,7 @@ def _get_count_of_mutations_running(self) -> int: def _get_column_types(self, column: str): return sync_execute( """ - SELECT default_kind, default_expression + SELECT type, default_kind, default_expression FROM system.columns WHERE database = %(database)s AND table = %(table)s AND name = %(column)s """, @@ -297,42 +306,121 @@ def _get_column_types(self, column: str): def test_lifecycle(self): table: TablesWithMaterializedColumns = "events" - property: PropertyName = "myprop" + property_names = ["foo", "bar"] source_column: TableColumn = "properties" - # create the materialized column - destination_column = materialize(table, property, table_column=source_column, create_minmax_index=True) - assert destination_column is not None + # create materialized columns + materialized_columns = {} + for property_name in property_names: + destination_column = materialize(table, property_name, table_column=source_column, create_minmax_index=True) + if destination_column is not None: + materialized_columns[property_name] = destination_column + + assert set(property_names) == materialized_columns.keys() + + # ensure they exist everywhere + for property_name, destination_column in materialized_columns.items(): + key = (property_name, source_column) + assert get_materialized_columns(table)[key].name == destination_column + assert MaterializedColumn.get(table, destination_column) == MaterializedColumn( + destination_column, + MaterializedColumnDetails(source_column, property_name, is_disabled=False), + is_nullable=False, + ) - # ensure it exists everywhere - key = (property, source_column) - assert get_materialized_columns(table)[key] == destination_column - assert MaterializedColumn.get(table, destination_column) == MaterializedColumn( - destination_column, - MaterializedColumnDetails(source_column, property, is_disabled=False), + # disable them and ensure updates apply as needed + update_column_is_disabled(table, materialized_columns.values(), is_disabled=True) + for property_name, destination_column in materialized_columns.items(): + key = (property_name, source_column) + assert get_materialized_columns(table)[key].name == destination_column + assert MaterializedColumn.get(table, destination_column) == MaterializedColumn( + destination_column, + MaterializedColumnDetails(source_column, property_name, is_disabled=True), + is_nullable=False, + ) + + # re-enable them and ensure updates apply as needed + update_column_is_disabled(table, materialized_columns.values(), is_disabled=False) + for property_name, destination_column in materialized_columns.items(): + key = (property_name, source_column) + assert get_materialized_columns(table)[key].name == destination_column + assert MaterializedColumn.get(table, destination_column) == MaterializedColumn( + destination_column, + MaterializedColumnDetails(source_column, property_name, is_disabled=False), + is_nullable=False, + ) + + # drop them and ensure updates apply as needed + drop_column(table, materialized_columns.values()) + for property_name, destination_column in materialized_columns.items(): + key = (property_name, source_column) + assert key not in get_materialized_columns(table) + with self.assertRaises(ValueError): + MaterializedColumn.get(table, destination_column) + + def _get_latest_mutation_id(self, table: str) -> str: + [(mutation_id,)] = sync_execute( + """ + SELECT max(mutation_id) + FROM system.mutations + WHERE + database = currentDatabase() + AND table = %(table)s + """, + {"table": table}, ) + return mutation_id + + def _get_mutations_since_id(self, table: str, id: str) -> Iterable[str]: + return [ + command + for (command,) in sync_execute( + """ + SELECT command + FROM system.mutations + WHERE + database = currentDatabase() + AND table = %(table)s + AND mutation_id > %(mutation_id)s + ORDER BY mutation_id + """, + {"table": table, "mutation_id": id}, + ) + ] - # disable it and ensure updates apply as needed - update_column_is_disabled(table, destination_column, is_disabled=True) - assert get_materialized_columns(table)[key] == destination_column - assert key not in get_materialized_columns(table, exclude_disabled_columns=True) - assert MaterializedColumn.get(table, destination_column) == MaterializedColumn( - destination_column, - MaterializedColumnDetails(source_column, property, is_disabled=True), + def test_drop_optimized_no_index(self): + table: TablesWithMaterializedColumns = ( + "person" # little bit easier than events because no shard awareness needed ) + property: PropertyName = "myprop" + source_column: TableColumn = "properties" + + destination_column = materialize(table, property, table_column=source_column, create_minmax_index=False) + assert destination_column is not None + + latest_mutation_id_before_drop = self._get_latest_mutation_id(table) - # re-enable it and ensure updates apply as needed - update_column_is_disabled(table, destination_column, is_disabled=False) - assert get_materialized_columns(table, exclude_disabled_columns=False)[key] == destination_column - assert get_materialized_columns(table, exclude_disabled_columns=True)[key] == destination_column - assert MaterializedColumn.get(table, destination_column) == MaterializedColumn( - destination_column, - MaterializedColumnDetails(source_column, property, is_disabled=False), + drop_column(table, destination_column) + + mutations_ran = self._get_mutations_since_id(table, latest_mutation_id_before_drop) + assert not any("DROP INDEX" in mutation for mutation in mutations_ran) + + def test_drop_optimized_no_column(self): + table: TablesWithMaterializedColumns = ( + "person" # little bit easier than events because no shard awareness needed ) + property: PropertyName = "myprop" + source_column: TableColumn = "properties" + + # create the materialized column + destination_column = materialize(table, property, table_column=source_column, create_minmax_index=False) + assert destination_column is not None + + sync_execute(f"ALTER TABLE {table} DROP COLUMN {destination_column}", settings={"alter_sync": 1}) + + latest_mutation_id_before_drop = self._get_latest_mutation_id(table) - # drop it and ensure updates apply as needed drop_column(table, destination_column) - assert key not in get_materialized_columns(table, exclude_disabled_columns=False) - assert key not in get_materialized_columns(table, exclude_disabled_columns=True) - with self.assertRaises(ValueError): - MaterializedColumn.get(table, destination_column) + + mutations_ran = self._get_mutations_since_id(table, latest_mutation_id_before_drop) + assert not any("DROP COLUMN" in mutation for mutation in mutations_ran) diff --git a/ee/clickhouse/models/test/test_cohort.py b/ee/clickhouse/models/test/test_cohort.py index 8af41154c48a5..1600584169a28 100644 --- a/ee/clickhouse/models/test/test_cohort.py +++ b/ee/clickhouse/models/test/test_cohort.py @@ -1,4 +1,5 @@ from datetime import datetime, timedelta +from typing import Optional from django.utils import timezone from freezegun import freeze_time @@ -8,12 +9,13 @@ from posthog.models.action import Action from posthog.models.cohort import Cohort from posthog.models.cohort.sql import GET_COHORTPEOPLE_BY_COHORT_ID -from posthog.models.cohort.util import format_filter_query, get_person_ids_by_cohort_id +from posthog.models.cohort.util import format_filter_query from posthog.models.filters import Filter from posthog.models.organization import Organization from posthog.models.person import Person from posthog.models.property.util import parse_prop_grouped_clauses from posthog.models.team import Team +from posthog.queries.person_distinct_id_query import get_team_distinct_ids_query from posthog.queries.util import PersonPropertiesMode from posthog.schema import PersonsOnEventsMode from posthog.test.base import ( @@ -25,6 +27,7 @@ snapshot_clickhouse_insert_cohortpeople_queries, snapshot_clickhouse_queries, ) +from posthog.models.person.sql import GET_LATEST_PERSON_SQL, GET_PERSON_IDS_BY_FILTER def _create_action(**kwargs): @@ -34,12 +37,44 @@ def _create_action(**kwargs): return action +def get_person_ids_by_cohort_id( + team_id: int, + cohort_id: int, + limit: Optional[int] = None, + offset: Optional[int] = None, +): + from posthog.models.property.util import parse_prop_grouped_clauses + + filter = Filter(data={"properties": [{"key": "id", "value": cohort_id, "type": "cohort"}]}) + filter_query, filter_params = parse_prop_grouped_clauses( + team_id=team_id, + property_group=filter.property_groups, + table_name="pdi", + hogql_context=filter.hogql_context, + ) + + results = sync_execute( + GET_PERSON_IDS_BY_FILTER.format( + person_query=GET_LATEST_PERSON_SQL, + distinct_query=filter_query, + query="", + GET_TEAM_PERSON_DISTINCT_IDS=get_team_distinct_ids_query(team_id), + offset="OFFSET %(offset)s" if offset else "", + limit="ORDER BY _timestamp ASC LIMIT %(limit)s" if limit else "", + ), + {**filter_params, "team_id": team_id, "offset": offset, "limit": limit}, + ) + + return [str(row[0]) for row in results] + + class TestCohort(ClickhouseTestMixin, BaseTest): - def _get_cohortpeople(self, cohort: Cohort): + def _get_cohortpeople(self, cohort: Cohort, *, team_id: Optional[int] = None): + team_id = team_id or cohort.team_id return sync_execute( GET_COHORTPEOPLE_BY_COHORT_ID, { - "team_id": self.team.pk, + "team_id": team_id, "cohort_id": cohort.pk, "version": cohort.version, }, @@ -452,7 +487,7 @@ def test_cohort_get_person_ids_by_cohort_id(self): name="cohort1", ) - results = get_person_ids_by_cohort_id(self.team, cohort.id) + results = get_person_ids_by_cohort_id(self.team.pk, cohort.id) self.assertEqual(len(results), 2) self.assertIn(str(user1.uuid), results) self.assertIn(str(user3.uuid), results) @@ -468,7 +503,7 @@ def test_insert_by_distinct_id_or_email(self): cohort = Cohort.objects.create(team=self.team, groups=[], is_static=True) cohort.insert_users_by_list(["1", "123"]) cohort = Cohort.objects.get() - results = get_person_ids_by_cohort_id(self.team, cohort.id) + results = get_person_ids_by_cohort_id(self.team.pk, cohort.id) self.assertEqual(len(results), 2) self.assertEqual(cohort.is_calculating, False) @@ -483,12 +518,12 @@ def test_insert_by_distinct_id_or_email(self): #  If we accidentally call calculate_people it shouldn't erase people cohort.calculate_people_ch(pending_version=0) - results = get_person_ids_by_cohort_id(self.team, cohort.id) + results = get_person_ids_by_cohort_id(self.team.pk, cohort.id) self.assertEqual(len(results), 3) # if we add people again, don't increase the number of people in cohort cohort.insert_users_by_list(["123"]) - results = get_person_ids_by_cohort_id(self.team, cohort.id) + results = get_person_ids_by_cohort_id(self.team.pk, cohort.id) self.assertEqual(len(results), 3) @snapshot_clickhouse_insert_cohortpeople_queries @@ -1370,3 +1405,45 @@ def test_cohort_versioning(self): # Should have p1 in this cohort even if version is different results = self._get_cohortpeople(cohort1) self.assertEqual(len(results), 1) + + def test_calculate_people_ch_in_multiteam_project(self): + # Create another team in the same project + team2 = Team.objects.create(organization=self.organization, project=self.team.project) + + # Create people in team 1 + _person1_team1 = _create_person( + team_id=self.team.pk, + distinct_ids=["person1"], + properties={"$some_prop": "else"}, + ) + person2_team1 = _create_person( + team_id=self.team.pk, + distinct_ids=["person2"], + properties={"$some_prop": "something"}, + ) + # Create people in team 2 with same property + person1_team2 = _create_person( + team_id=team2.pk, + distinct_ids=["person1_team2"], + properties={"$some_prop": "something"}, + ) + _person2_team2 = _create_person( + team_id=team2.pk, + distinct_ids=["person2_team2"], + properties={"$some_prop": "else"}, + ) + # Create cohort in team 2 (but same project as team 1) + shared_cohort = Cohort.objects.create( + team=team2, + groups=[{"properties": [{"key": "$some_prop", "value": "something", "type": "person"}]}], + name="shared cohort", + ) + # Calculate cohort + shared_cohort.calculate_people_ch(pending_version=0) + + # Verify shared_cohort is now calculated for both teams + results_team1 = self._get_cohortpeople(shared_cohort, team_id=self.team.pk) + results_team2 = self._get_cohortpeople(shared_cohort, team_id=team2.pk) + + self.assertCountEqual([r[0] for r in results_team1], [person2_team1.uuid]) + self.assertCountEqual([r[0] for r in results_team2], [person1_team2.uuid]) diff --git a/ee/clickhouse/queries/funnels/funnel_correlation.py b/ee/clickhouse/queries/funnels/funnel_correlation.py index 3e5b69005d689..0b909c84b398e 100644 --- a/ee/clickhouse/queries/funnels/funnel_correlation.py +++ b/ee/clickhouse/queries/funnels/funnel_correlation.py @@ -13,7 +13,7 @@ from ee.clickhouse.queries.column_optimizer import EnterpriseColumnOptimizer from ee.clickhouse.queries.groups_join_query import GroupsJoinQuery -from posthog.clickhouse.materialized_columns import get_enabled_materialized_columns +from posthog.clickhouse.materialized_columns import get_materialized_column_for_property from posthog.constants import ( AUTOCAPTURE_EVENT, TREND_FILTER_TYPE_ACTIONS, @@ -156,8 +156,6 @@ def properties_to_include(self) -> list[str]: ): # When dealing with properties, make sure funnel response comes with properties # so we don't have to join on persons/groups to get these properties again - mat_event_cols = get_enabled_materialized_columns("events") - for property_name in cast(list, self._filter.correlation_property_names): if self._filter.aggregation_group_type_index is not None: continue # We don't support group properties on events at this time @@ -165,10 +163,11 @@ def properties_to_include(self) -> list[str]: if "$all" == property_name: return [f"person_properties"] - possible_mat_col = mat_event_cols.get((property_name, "person_properties")) - - if possible_mat_col is not None: - props_to_include.append(possible_mat_col) + possible_mat_col = get_materialized_column_for_property( + "events", "person_properties", property_name + ) + if possible_mat_col is not None and not possible_mat_col.is_nullable: + props_to_include.append(possible_mat_col.name) else: props_to_include.append(f"person_properties") diff --git a/ee/clickhouse/queries/funnels/test/test_funnel_correlations_persons.py b/ee/clickhouse/queries/funnels/test/test_funnel_correlations_persons.py index 4617ffde3c2d5..c6954e15eed9b 100644 --- a/ee/clickhouse/queries/funnels/test/test_funnel_correlations_persons.py +++ b/ee/clickhouse/queries/funnels/test/test_funnel_correlations_persons.py @@ -251,6 +251,7 @@ def test_create_funnel_correlation_cohort(self, _insert_cohort_from_insight_filt "funnel_correlation_person_entity": "{'id': 'positively_related', 'type': 'events'}", "funnel_correlation_person_converted": "TrUe", }, + self.team.pk, ) insert_cohort_from_insight_filter(cohort_id, params) diff --git a/ee/clickhouse/queries/related_actors_query.py b/ee/clickhouse/queries/related_actors_query.py index 4b6198c222710..99817998d7119 100644 --- a/ee/clickhouse/queries/related_actors_query.py +++ b/ee/clickhouse/queries/related_actors_query.py @@ -41,7 +41,7 @@ def __init__( def run(self) -> list[SerializedActor]: results: list[SerializedActor] = [] results.extend(self._query_related_people()) - for group_type_mapping in GroupTypeMapping.objects.filter(team_id=self.team.pk): + for group_type_mapping in GroupTypeMapping.objects.filter(project_id=self.team.project_id): results.extend(self._query_related_groups(group_type_mapping.group_type_index)) return results diff --git a/ee/clickhouse/views/groups.py b/ee/clickhouse/views/groups.py index 4970a770854a2..be692dc597525 100644 --- a/ee/clickhouse/views/groups.py +++ b/ee/clickhouse/views/groups.py @@ -35,7 +35,9 @@ class GroupsTypesViewSet(TeamAndOrgViewSetMixin, mixins.ListModelMixin, viewsets @action(detail=False, methods=["PATCH"], name="Update group types metadata") def update_metadata(self, request: request.Request, *args, **kwargs): for row in cast(list[dict], request.data): - instance = GroupTypeMapping.objects.get(team=self.team, group_type_index=row["group_type_index"]) + instance = GroupTypeMapping.objects.get( + project_id=self.team.project_id, group_type_index=row["group_type_index"] + ) serializer = self.get_serializer(instance, data=row) serializer.is_valid(raise_exception=True) serializer.save() diff --git a/ee/clickhouse/views/test/test_clickhouse_path_person.py b/ee/clickhouse/views/test/test_clickhouse_path_person.py index 48fc8a2475c06..597aa6dffc5e2 100644 --- a/ee/clickhouse/views/test/test_clickhouse_path_person.py +++ b/ee/clickhouse/views/test/test_clickhouse_path_person.py @@ -97,6 +97,7 @@ def test_create_paths_cohort(self, _insert_cohort_from_insight_filter): "date_from": "2021-05-01", "date_to": "2021-05-10", }, + self.team.pk, ) insert_cohort_from_insight_filter(cohort_id, params) diff --git a/ee/hogai/schema_generator/nodes.py b/ee/hogai/schema_generator/nodes.py index c5e7ffbba85c4..f2d383d5c1e30 100644 --- a/ee/hogai/schema_generator/nodes.py +++ b/ee/hogai/schema_generator/nodes.py @@ -113,7 +113,7 @@ def router(self, state: AssistantState): @cached_property def _group_mapping_prompt(self) -> str: - groups = GroupTypeMapping.objects.filter(team=self._team).order_by("group_type_index") + groups = GroupTypeMapping.objects.filter(project_id=self._team.project_id).order_by("group_type_index") if not groups: return "The user has not defined any groups." diff --git a/ee/hogai/taxonomy_agent/nodes.py b/ee/hogai/taxonomy_agent/nodes.py index d499269a8ca95..025058a51eec1 100644 --- a/ee/hogai/taxonomy_agent/nodes.py +++ b/ee/hogai/taxonomy_agent/nodes.py @@ -179,7 +179,7 @@ def _events_prompt(self) -> str: @cached_property def _team_group_types(self) -> list[str]: return list( - GroupTypeMapping.objects.filter(team=self._team) + GroupTypeMapping.objects.filter(project_id=self._team.project_id) .order_by("group_type_index") .values_list("group_type", flat=True) ) diff --git a/ee/hogai/taxonomy_agent/toolkit.py b/ee/hogai/taxonomy_agent/toolkit.py index 2af39253b9e68..dc8a0e092c2e6 100644 --- a/ee/hogai/taxonomy_agent/toolkit.py +++ b/ee/hogai/taxonomy_agent/toolkit.py @@ -169,7 +169,7 @@ def render_text_description(self) -> str: @property def _groups(self): - return GroupTypeMapping.objects.filter(team=self._team).order_by("group_type_index") + return GroupTypeMapping.objects.filter(project_id=self._team.project_id).order_by("group_type_index") @cached_property def _entity_names(self) -> list[str]: diff --git a/ee/hogai/utils.py b/ee/hogai/utils.py index 278e2f4076495..559a369df83c8 100644 --- a/ee/hogai/utils.py +++ b/ee/hogai/utils.py @@ -31,7 +31,7 @@ class Conversation(BaseModel): - messages: list[RootAssistantMessage] = Field(..., min_length=1, max_length=20) + messages: list[RootAssistantMessage] = Field(..., min_length=1, max_length=50) session_id: str diff --git a/ee/management/commands/materialize_columns.py b/ee/management/commands/materialize_columns.py index c1ca3b3fd2287..5ddbf55dea2b7 100644 --- a/ee/management/commands/materialize_columns.py +++ b/ee/management/commands/materialize_columns.py @@ -1,3 +1,4 @@ +import argparse import logging from django.core.management.base import BaseCommand @@ -69,8 +70,14 @@ def add_arguments(self, parser): default=MATERIALIZE_COLUMNS_MAX_AT_ONCE, help="Max number of columns to materialize via single invocation. Same as MATERIALIZE_COLUMNS_MAX_AT_ONCE env variable.", ) + parser.add_argument( + "--nullable", + action=argparse.BooleanOptionalAction, + default=True, + dest="is_nullable", + ) - def handle(self, *args, **options): + def handle(self, *, is_nullable: bool, **options): logger.setLevel(logging.INFO) if options["dry_run"]: @@ -90,6 +97,7 @@ def handle(self, *args, **options): ], backfill_period_days=options["backfill_period"], dry_run=options["dry_run"], + is_nullable=is_nullable, ) else: materialize_properties_task( @@ -99,4 +107,5 @@ def handle(self, *args, **options): backfill_period_days=options["backfill_period"], dry_run=options["dry_run"], team_id_to_analyze=options["analyze_team_id"], + is_nullable=is_nullable, ) diff --git a/ee/management/commands/update_materialized_column.py b/ee/management/commands/update_materialized_column.py index b45444eb4831b..bb55a61545dd6 100644 --- a/ee/management/commands/update_materialized_column.py +++ b/ee/management/commands/update_materialized_column.py @@ -1,7 +1,7 @@ import logging from typing import Any -from collections.abc import Callable +from collections.abc import Callable, Iterable from django.core.management.base import BaseCommand, CommandParser from posthog.clickhouse.materialized_columns import ColumnName, TablesWithMaterializedColumns @@ -9,9 +9,9 @@ logger = logging.getLogger(__name__) -COLUMN_OPERATIONS: dict[str, Callable[[TablesWithMaterializedColumns, ColumnName], Any]] = { - "enable": lambda table, column_name: update_column_is_disabled(table, column_name, is_disabled=False), - "disable": lambda table, column_name: update_column_is_disabled(table, column_name, is_disabled=True), +COLUMN_OPERATIONS: dict[str, Callable[[TablesWithMaterializedColumns, Iterable[ColumnName]], Any]] = { + "enable": lambda table, column_names: update_column_is_disabled(table, column_names, is_disabled=False), + "disable": lambda table, column_names: update_column_is_disabled(table, column_names, is_disabled=True), "drop": drop_column, } @@ -20,10 +20,12 @@ class Command(BaseCommand): def add_arguments(self, parser: CommandParser) -> None: parser.add_argument("operation", choices=COLUMN_OPERATIONS.keys()) parser.add_argument("table") - parser.add_argument("column_name") + parser.add_argument("column_names", nargs="+", metavar="column") - def handle(self, operation: str, table: TablesWithMaterializedColumns, column_name: ColumnName, **options): - logger.info("Running %r for %r.%r...", operation, table, column_name) + def handle( + self, operation: str, table: TablesWithMaterializedColumns, column_names: Iterable[ColumnName], **options + ): + logger.info("Running %r on %r for %r...", operation, table, column_names) fn = COLUMN_OPERATIONS[operation] - fn(table, column_name) + fn(table, column_names) logger.info("Success!") diff --git a/ee/tasks/materialized_columns.py b/ee/tasks/materialized_columns.py index d05cdddc0b0ca..98091c3b1d00a 100644 --- a/ee/tasks/materialized_columns.py +++ b/ee/tasks/materialized_columns.py @@ -1,50 +1,49 @@ +from collections.abc import Iterator +from dataclasses import dataclass from celery.utils.log import get_task_logger +from clickhouse_driver import Client -from ee.clickhouse.materialized_columns.columns import ( - TRIM_AND_EXTRACT_PROPERTY, - get_materialized_columns, -) +from ee.clickhouse.materialized_columns.columns import MaterializedColumn, get_cluster, tables as table_infos from posthog.client import sync_execute -from posthog.settings import CLICKHOUSE_CLUSTER, CLICKHOUSE_DATABASE +from posthog.settings import CLICKHOUSE_DATABASE from posthog.clickhouse.materialized_columns import ColumnName, TablesWithMaterializedColumns logger = get_task_logger(__name__) -def mark_all_materialized() -> None: - if any_ongoing_mutations(): - logger.info("There are running mutations, skipping marking as materialized") - return - - for ( - table, - property_name, - table_column, - column_name, - ) in get_materialized_columns_with_default_expression(): - updated_table = "sharded_events" if table == "events" else table - - # :TRICKY: On cloud, we ON CLUSTER updates to events/sharded_events but not to persons. Why? ¯\_(ツ)_/¯ - execute_on_cluster = f"ON CLUSTER '{CLICKHOUSE_CLUSTER}'" if table == "events" else "" - - sync_execute( - f""" - ALTER TABLE {updated_table} - {execute_on_cluster} - MODIFY COLUMN - {column_name} VARCHAR MATERIALIZED {TRIM_AND_EXTRACT_PROPERTY.format(table_column=table_column)} - """, - {"property": property_name}, +@dataclass +class MarkMaterializedTask: + table: str + column: MaterializedColumn + + def execute(self, client: Client) -> None: + expression, parameters = self.column.get_expression_and_parameters() + client.execute( + f"ALTER TABLE {self.table} MODIFY COLUMN {self.column.name} {self.column.type} MATERIALIZED {expression}", + parameters, ) -def get_materialized_columns_with_default_expression(): - tables: list[TablesWithMaterializedColumns] = ["events", "person"] - for table in tables: - materialized_columns = get_materialized_columns(table) - for (property_name, table_column), column_name in materialized_columns.items(): - if is_default_expression(table, column_name): - yield table, property_name, table_column, column_name +def mark_all_materialized() -> None: + cluster = get_cluster() + + for table_name, column in get_materialized_columns_with_default_expression(): + table_info = table_infos[table_name] + table_info.map_data_nodes( + cluster, + MarkMaterializedTask( + table_info.data_table, + column, + ).execute, + ).result() + + +def get_materialized_columns_with_default_expression() -> Iterator[tuple[str, MaterializedColumn]]: + table_names: list[TablesWithMaterializedColumns] = ["events", "person"] + for table_name in table_names: + for column in MaterializedColumn.get_all(table_name): + if is_default_expression(table_name, column.name): + yield table_name, column def any_ongoing_mutations() -> bool: diff --git a/ee/tasks/test/test_calculate_cohort.py b/ee/tasks/test/test_calculate_cohort.py index c5264bbe12631..ed0dd3e4290cc 100644 --- a/ee/tasks/test/test_calculate_cohort.py +++ b/ee/tasks/test/test_calculate_cohort.py @@ -45,6 +45,7 @@ def test_create_stickiness_cohort(self, _insert_cohort_from_insight_filter): "stickiness_days": "1", "label": "$pageview", }, + self.team.pk, ) insert_cohort_from_insight_filter( @@ -118,6 +119,7 @@ def test_create_trends_cohort(self, _insert_cohort_from_insight_filter): "date_to": "2021-01-01", "label": "$pageview", }, + self.team.pk, ) insert_cohort_from_insight_filter( cohort_id, @@ -228,6 +230,7 @@ def test_create_trends_cohort_arg_test(self, _insert_cohort_from_insight_filter) "interval": "day", "properties": '[{"key": "$domain", "value": "app.posthog.com", "operator": "icontains", "type": "event"}]', }, + self.team.pk, ) insert_cohort_from_insight_filter( cohort_id, @@ -357,6 +360,7 @@ def test_create_funnels_cohort(self, _insert_cohort_from_insight_filter): "date_to": "2021-01-07", "funnel_step": "1", }, + self.team.pk, ) insert_cohort_from_insight_filter(cohort_id, params) @@ -445,6 +449,7 @@ def _create_events(data, event="$pageview"): "entity_order": "0", "lifecycle_type": "returning", }, + self.team.pk, ) insert_cohort_from_insight_filter( @@ -507,6 +512,7 @@ def _create_events(data, event="$pageview"): "entity_order": "0", "lifecycle_type": "dormant", }, + self.team.pk, ) self.assertEqual(_insert_cohort_from_insight_filter.call_count, 2) diff --git a/frontend/__snapshots__/components-cards-insight-details--data-table-events-query--dark.png b/frontend/__snapshots__/components-cards-insight-details--data-table-events-query--dark.png index 1f129b49b9cf8..24cc8f4b2f6b6 100644 Binary files a/frontend/__snapshots__/components-cards-insight-details--data-table-events-query--dark.png and b/frontend/__snapshots__/components-cards-insight-details--data-table-events-query--dark.png differ diff --git a/frontend/__snapshots__/components-cards-insight-details--data-table-events-query--light.png b/frontend/__snapshots__/components-cards-insight-details--data-table-events-query--light.png index 3077ca6a36368..64fd850a0fc25 100644 Binary files a/frontend/__snapshots__/components-cards-insight-details--data-table-events-query--light.png and b/frontend/__snapshots__/components-cards-insight-details--data-table-events-query--light.png differ diff --git a/frontend/__snapshots__/components-cards-insight-details--data-table-hog-ql-query--dark.png b/frontend/__snapshots__/components-cards-insight-details--data-table-hog-ql-query--dark.png index 1f129b49b9cf8..24cc8f4b2f6b6 100644 Binary files a/frontend/__snapshots__/components-cards-insight-details--data-table-hog-ql-query--dark.png and b/frontend/__snapshots__/components-cards-insight-details--data-table-hog-ql-query--dark.png differ diff --git a/frontend/__snapshots__/components-cards-insight-details--data-table-hog-ql-query--light.png b/frontend/__snapshots__/components-cards-insight-details--data-table-hog-ql-query--light.png index 3077ca6a36368..64fd850a0fc25 100644 Binary files a/frontend/__snapshots__/components-cards-insight-details--data-table-hog-ql-query--light.png and b/frontend/__snapshots__/components-cards-insight-details--data-table-hog-ql-query--light.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--dark.png b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--dark.png index 64260538a2c7b..ee27b11bed568 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--dark.png and b/frontend/__snapshots__/scenes-app-insights--funnel-top-to-bottom-breakdown-edit--dark.png differ diff --git a/frontend/__snapshots__/scenes-other-settings--settings-project--dark.png b/frontend/__snapshots__/scenes-other-settings--settings-project--dark.png index 636ef7cab1bf5..b630543a63f45 100644 Binary files a/frontend/__snapshots__/scenes-other-settings--settings-project--dark.png and b/frontend/__snapshots__/scenes-other-settings--settings-project--dark.png differ diff --git a/frontend/__snapshots__/scenes-other-settings--settings-project--light.png b/frontend/__snapshots__/scenes-other-settings--settings-project--light.png index 60db13f851093..3a2fe7a7a7229 100644 Binary files a/frontend/__snapshots__/scenes-other-settings--settings-project--light.png and b/frontend/__snapshots__/scenes-other-settings--settings-project--light.png differ diff --git a/frontend/__snapshots__/scenes-other-settings--settings-project-with-replay-features--dark.png b/frontend/__snapshots__/scenes-other-settings--settings-project-with-replay-features--dark.png index 61441f05dddaa..d5450bc6cd3df 100644 Binary files a/frontend/__snapshots__/scenes-other-settings--settings-project-with-replay-features--dark.png and b/frontend/__snapshots__/scenes-other-settings--settings-project-with-replay-features--dark.png differ diff --git a/frontend/__snapshots__/scenes-other-settings--settings-project-with-replay-features--light.png b/frontend/__snapshots__/scenes-other-settings--settings-project-with-replay-features--light.png index 9a9851e2b50b5..5dae24aa10f4a 100644 Binary files a/frontend/__snapshots__/scenes-other-settings--settings-project-with-replay-features--light.png and b/frontend/__snapshots__/scenes-other-settings--settings-project-with-replay-features--light.png differ diff --git a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-all-options--dark.png b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-all-options--dark.png index 636ef7cab1bf5..b630543a63f45 100644 Binary files a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-all-options--dark.png and b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-all-options--dark.png differ diff --git a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-all-options--light.png b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-all-options--light.png index 60db13f851093..3a2fe7a7a7229 100644 Binary files a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-all-options--light.png and b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-all-options--light.png differ diff --git a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-password-only--dark.png b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-password-only--dark.png index 636ef7cab1bf5..b630543a63f45 100644 Binary files a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-password-only--dark.png and b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-password-only--dark.png differ diff --git a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-password-only--light.png b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-password-only--light.png index 60db13f851093..3a2fe7a7a7229 100644 Binary files a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-password-only--light.png and b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-password-only--light.png differ diff --git a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-github--dark.png b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-github--dark.png index 636ef7cab1bf5..b630543a63f45 100644 Binary files a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-github--dark.png and b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-github--dark.png differ diff --git a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-github--light.png b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-github--light.png index 60db13f851093..3a2fe7a7a7229 100644 Binary files a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-github--light.png and b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-github--light.png differ diff --git a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-google--dark.png b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-google--dark.png index 636ef7cab1bf5..b630543a63f45 100644 Binary files a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-google--dark.png and b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-google--dark.png differ diff --git a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-google--light.png b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-google--light.png index 60db13f851093..3a2fe7a7a7229 100644 Binary files a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-google--light.png and b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-google--light.png differ diff --git a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-saml--dark.png b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-saml--dark.png index 636ef7cab1bf5..b630543a63f45 100644 Binary files a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-saml--dark.png and b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-saml--dark.png differ diff --git a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-saml--light.png b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-saml--light.png index 60db13f851093..3a2fe7a7a7229 100644 Binary files a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-saml--light.png and b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-enforced-saml--light.png differ diff --git a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-only--dark.png b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-only--dark.png index 636ef7cab1bf5..b630543a63f45 100644 Binary files a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-only--dark.png and b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-only--dark.png differ diff --git a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-only--light.png b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-only--light.png index 60db13f851093..3a2fe7a7a7229 100644 Binary files a/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-only--light.png and b/frontend/__snapshots__/scenes-other-settings--settings-session-timeout-sso-only--light.png differ diff --git a/frontend/src/lib/components/DefinitionPopover/DefinitionPopoverContents.tsx b/frontend/src/lib/components/DefinitionPopover/DefinitionPopoverContents.tsx index f1d7410f53ca0..e9ef0b32bbb80 100644 --- a/frontend/src/lib/components/DefinitionPopover/DefinitionPopoverContents.tsx +++ b/frontend/src/lib/components/DefinitionPopover/DefinitionPopoverContents.tsx @@ -24,6 +24,7 @@ import { DataWarehouseTableForInsight } from 'scenes/data-warehouse/types' import { ActionType, CohortType, EventDefinition, PropertyDefinition } from '~/types' +import { HogQLDropdown } from '../HogQLDropdown/HogQLDropdown' import { taxonomicFilterLogic } from '../TaxonomicFilter/taxonomicFilterLogic' import { TZLabel } from '../TZLabel' @@ -291,8 +292,21 @@ function DefinitionView({ group }: { group: TaxonomicFilterGroup }): JSX.Element label: column.name + ' (' + column.type + ')', value: column.name, })) + const hogqlOption = { label: 'HogQL Expression', value: '' } const itemValue = localDefinition ? group?.getValue?.(localDefinition) : null + const isUsingHogQLExpression = (value: string | undefined): boolean => { + if (value === undefined) { + return false + } + const column = Object.values(_definition.fields ?? {}).find((n) => n.name == value) + return !column + } + + const distinct_id_field_value = + 'distinct_id_field' in localDefinition ? localDefinition.distinct_id_field : undefined + const timestamp_field_value = 'timestamp_field' in localDefinition ? localDefinition.timestamp_field : undefined + return (
@@ -310,23 +324,33 @@ function DefinitionView({ group }: { group: TaxonomicFilterGroup }): JSX.Element Distinct ID field setLocalDefinition({ distinct_id_field: value })} /> + {isUsingHogQLExpression(distinct_id_field_value) && ( + setLocalDefinition({ distinct_id_field: value })} + /> + )} setLocalDefinition({ timestamp_field: value })} /> + {isUsingHogQLExpression(timestamp_field_value) && ( + setLocalDefinition({ timestamp_field: value })} + /> + )}
void +}): JSX.Element => { + const [isHogQLDropdownVisible, setIsHogQLDropdownVisible] = useState(false) + + return ( +
+ setIsHogQLDropdownVisible(false)} + overlay={ + // eslint-disable-next-line react/forbid-dom-props +
+ { + onHogQLValueChange(currentValue) + setIsHogQLDropdownVisible(false) + }} + /> +
+ } + > + setIsHogQLDropdownVisible(!isHogQLDropdownVisible)} + > + {hogQLValue} + + +
+ ) +} diff --git a/frontend/src/scenes/data-warehouse/ViewLinkModal.tsx b/frontend/src/scenes/data-warehouse/ViewLinkModal.tsx index 3004b8ee60daf..e888ee9abb21d 100644 --- a/frontend/src/scenes/data-warehouse/ViewLinkModal.tsx +++ b/frontend/src/scenes/data-warehouse/ViewLinkModal.tsx @@ -5,7 +5,6 @@ import { LemonButton, LemonCheckbox, LemonDivider, - LemonDropdown, LemonInput, LemonModal, LemonSelect, @@ -14,12 +13,12 @@ import { import { useActions, useValues } from 'kea' import { Field, Form } from 'kea-forms' import { CodeSnippet, Language } from 'lib/components/CodeSnippet' -import { HogQLEditor } from 'lib/components/HogQLEditor/HogQLEditor' +import { HogQLDropdown } from 'lib/components/HogQLDropdown/HogQLDropdown' import { IconSwapHoriz } from 'lib/lemon-ui/icons' import { useState } from 'react' import { viewLinkLogic } from 'scenes/data-warehouse/viewLinkLogic' -import { DatabaseSchemaField, NodeKind } from '~/queries/schema' +import { DatabaseSchemaField } from '~/queries/schema' export function ViewLinkModal(): JSX.Element { const { isJoinTableModalOpen } = useValues(viewLinkLogic) @@ -122,6 +121,7 @@ export function ViewLinkForm(): JSX.Element { /> {sourceIsUsingHogQLExpression && ( {joiningIsUsingHogQLExpression && ( void -}): JSX.Element => { - const [isHogQLDropdownVisible, setIsHogQLDropdownVisible] = useState(false) - - return ( -
- setIsHogQLDropdownVisible(false)} - overlay={ - // eslint-disable-next-line react/forbid-dom-props -
- { - onHogQLValueChange(currentValue) - setIsHogQLDropdownVisible(false) - }} - /> -
- } - > - setIsHogQLDropdownVisible(!isHogQLDropdownVisible)} - > - {hogQLValue} - - -
- ) -} - interface KeyLabelProps { column: DatabaseSchemaField } diff --git a/frontend/src/scenes/data-warehouse/new/sourceWizardLogic.tsx b/frontend/src/scenes/data-warehouse/new/sourceWizardLogic.tsx index a924b0ba594b7..f8e1a5a131205 100644 --- a/frontend/src/scenes/data-warehouse/new/sourceWizardLogic.tsx +++ b/frontend/src/scenes/data-warehouse/new/sourceWizardLogic.tsx @@ -513,18 +513,60 @@ export const SOURCE_DETAILS: Record = { placeholder: 'COMPUTE_WAREHOUSE', }, { - name: 'user', - label: 'User', - type: 'text', - required: true, - placeholder: 'user', - }, - { - name: 'password', - label: 'Password', - type: 'password', + type: 'select', + name: 'auth_type', + label: 'Authentication type', required: true, - placeholder: '', + defaultValue: 'password', + options: [ + { + label: 'Password', + value: 'password', + fields: [ + { + name: 'username', + label: 'Username', + type: 'text', + required: true, + placeholder: 'User1', + }, + { + name: 'password', + label: 'Password', + type: 'password', + required: true, + placeholder: '', + }, + ], + }, + { + label: 'Key pair', + value: 'keypair', + fields: [ + { + name: 'username', + label: 'Username', + type: 'text', + required: true, + placeholder: 'User1', + }, + { + name: 'private_key', + label: 'Private key', + type: 'textarea', + required: true, + placeholder: '', + }, + { + name: 'passphrase', + label: 'Passphrase', + type: 'password', + required: false, + placeholder: '', + }, + ], + }, + ], }, { name: 'role', diff --git a/frontend/src/scenes/experiments/ExperimentView/SummaryTable.tsx b/frontend/src/scenes/experiments/ExperimentView/SummaryTable.tsx index b859dae72e071..4ba16ded0e86c 100644 --- a/frontend/src/scenes/experiments/ExperimentView/SummaryTable.tsx +++ b/frontend/src/scenes/experiments/ExperimentView/SummaryTable.tsx @@ -323,6 +323,13 @@ export function SummaryTable(): JSX.Element { ], }, ] + if (experiment.filters.insight === InsightType.FUNNELS) { + if (experiment.filters?.events?.[0]) { + filters.push(experiment.filters.events[0]) + } else if (experiment.filters?.actions?.[0]) { + filters.push(experiment.filters.actions[0]) + } + } const filterGroup: Partial = { filter_group: { type: FilterLogicalOperator.And, diff --git a/frontend/src/scenes/experiments/Metrics/PrimaryGoalFunnels.tsx b/frontend/src/scenes/experiments/Metrics/PrimaryGoalFunnels.tsx index aefca698f5ad7..50468541a0d9b 100644 --- a/frontend/src/scenes/experiments/Metrics/PrimaryGoalFunnels.tsx +++ b/frontend/src/scenes/experiments/Metrics/PrimaryGoalFunnels.tsx @@ -1,6 +1,7 @@ import { LemonLabel } from '@posthog/lemon-ui' import { LemonInput } from '@posthog/lemon-ui' import { useActions, useValues } from 'kea' +import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types' import { TestAccountFilterSwitch } from 'lib/components/TestAccountFiltersSwitch' import { EXPERIMENT_DEFAULT_DURATION, FEATURE_FLAGS } from 'lib/constants' import { LemonBanner } from 'lib/lemon-ui/LemonBanner' @@ -31,6 +32,12 @@ export function PrimaryGoalFunnels(): JSX.Element { const metricIdx = 0 const currentMetric = experiment.metrics[metricIdx] as ExperimentFunnelsQuery + const actionFilterProps = { + ...commonActionFilterProps, + // Remove data warehouse from the list because it's not supported in experiments + actionsTaxonomicGroupTypes: [TaxonomicFilterGroupType.Events, TaxonomicFilterGroupType.Actions], + } + return ( <>
@@ -107,7 +114,7 @@ export function PrimaryGoalFunnels(): JSX.Element { seriesIndicatorType="numeric" sortable={true} showNestedArrow={true} - {...commonActionFilterProps} + {...actionFilterProps} />
([ }, ...state.slice(index + 1), ], + setThreadLoaded: (state) => state.filter((message) => !isReasoningMessage(message)), }, ], threadLoading: [ diff --git a/frontend/src/scenes/onboarding/sdks/sdk-install-instructions/flutter.tsx b/frontend/src/scenes/onboarding/sdks/sdk-install-instructions/flutter.tsx index d9c2860b784fc..1017f86e4a4a6 100644 --- a/frontend/src/scenes/onboarding/sdks/sdk-install-instructions/flutter.tsx +++ b/frontend/src/scenes/onboarding/sdks/sdk-install-instructions/flutter.tsx @@ -189,3 +189,26 @@ export function SDKInstallFlutterInstructions(props: FlutterSetupProps): JSX.Ele ) } + +export function SDKInstallFlutterTrackScreenInstructions(): JSX.Element { + return ( + <> +

+ With the PosthogObserver Observer, + PostHog will try to record all screen changes automatically. +

+

+ If you want to manually send a new screen capture event, use the screen function. +

+ {`import 'package:posthog_flutter/posthog_flutter.dart'; + +await Posthog().screen( + screenName: 'Dashboard', + properties: { + 'background': 'blue', + 'hero': 'superhog' + }); +`} + + ) +} diff --git a/frontend/src/scenes/onboarding/sdks/session-replay/SessionReplaySDKInstructions.tsx b/frontend/src/scenes/onboarding/sdks/session-replay/SessionReplaySDKInstructions.tsx index 7eaef80918243..29641d978af38 100644 --- a/frontend/src/scenes/onboarding/sdks/session-replay/SessionReplaySDKInstructions.tsx +++ b/frontend/src/scenes/onboarding/sdks/session-replay/SessionReplaySDKInstructions.tsx @@ -12,6 +12,7 @@ import { AngularInstructions, AstroInstructions, BubbleInstructions, + FlutterInstructions, FramerInstructions, HTMLSnippetInstructions, iOSInstructions, @@ -20,12 +21,11 @@ import { NuxtJSInstructions, ReactInstructions, RemixInstructions, + RNInstructions, SvelteInstructions, VueInstructions, WebflowInstructions, } from '.' -import { FlutterInstructions } from './flutter' -import { RNInstructions } from './react-native' export const SessionReplaySDKInstructions: SDKInstructionsMap = { [SDKKey.JS_WEB]: JSWebInstructions, diff --git a/frontend/src/scenes/onboarding/sdks/session-replay/index.tsx b/frontend/src/scenes/onboarding/sdks/session-replay/index.tsx index 3a5aadfe8909d..a769433142bec 100644 --- a/frontend/src/scenes/onboarding/sdks/session-replay/index.tsx +++ b/frontend/src/scenes/onboarding/sdks/session-replay/index.tsx @@ -2,6 +2,7 @@ export * from './android' export * from './angular' export * from './astro' export * from './bubble' +export * from './flutter' export * from './framer' export * from './html-snippet' export * from './ios' @@ -9,6 +10,7 @@ export * from './js-web' export * from './next-js' export * from './nuxt' export * from './react' +export * from './react-native' export * from './remix' export * from './svelte' export * from './vue' diff --git a/frontend/src/scenes/onboarding/sdks/web-analytics/WebAnalyticsSDKInstructions.tsx b/frontend/src/scenes/onboarding/sdks/web-analytics/WebAnalyticsSDKInstructions.tsx index 4c6e9a7953123..4457c6dfb7b3d 100644 --- a/frontend/src/scenes/onboarding/sdks/web-analytics/WebAnalyticsSDKInstructions.tsx +++ b/frontend/src/scenes/onboarding/sdks/web-analytics/WebAnalyticsSDKInstructions.tsx @@ -1,11 +1,11 @@ import { SDKInstructionsMap, SDKKey } from '~/types' -import { FlutterInstructions } from '../session-replay/flutter' import { AndroidInstructions, AngularInstructions, AstroInstructions, BubbleInstructions, + FlutterInstructions, FramerInstructions, HTMLSnippetInstructions, iOSInstructions, diff --git a/frontend/src/scenes/onboarding/sdks/web-analytics/flutter.tsx b/frontend/src/scenes/onboarding/sdks/web-analytics/flutter.tsx new file mode 100644 index 0000000000000..7496820b35ad1 --- /dev/null +++ b/frontend/src/scenes/onboarding/sdks/web-analytics/flutter.tsx @@ -0,0 +1,13 @@ +import { WebAnalyticsMobileFinalSteps } from 'scenes/onboarding/sdks/web-analytics/FinalSteps' + +import { SDKInstallFlutterInstructions, SDKInstallFlutterTrackScreenInstructions } from '../sdk-install-instructions' + +export function FlutterInstructions(): JSX.Element { + return ( + <> + + + + + ) +} diff --git a/frontend/src/scenes/onboarding/sdks/web-analytics/index.tsx b/frontend/src/scenes/onboarding/sdks/web-analytics/index.tsx index 463169921abcb..a769433142bec 100644 --- a/frontend/src/scenes/onboarding/sdks/web-analytics/index.tsx +++ b/frontend/src/scenes/onboarding/sdks/web-analytics/index.tsx @@ -2,6 +2,7 @@ export * from './android' export * from './angular' export * from './astro' export * from './bubble' +export * from './flutter' export * from './framer' export * from './html-snippet' export * from './ios' diff --git a/frontend/src/scenes/settings/Settings.tsx b/frontend/src/scenes/settings/Settings.tsx index 2c92acb7d59dd..0345733ec2db4 100644 --- a/frontend/src/scenes/settings/Settings.tsx +++ b/frontend/src/scenes/settings/Settings.tsx @@ -10,6 +10,7 @@ import { IconChevronRight, IconLink } from 'lib/lemon-ui/icons' import { capitalizeFirstLetter, inStorybookTestRunner } from 'lib/utils' import React from 'react' import { teamLogic } from 'scenes/teamLogic' +import { urls } from 'scenes/urls' import { settingsLogic } from './settingsLogic' import { SettingsLogicProps } from './types' @@ -56,7 +57,16 @@ export function Settings({ {levels.map((level) => (
  • selectLevel(level)} + to={urls.settings(level)} + onClick={ + // Outside of /settings, we want to select the level without navigating + props.logicKey === 'settingsScene' + ? (e) => { + selectLevel(level) + e.preventDefault() + } + : undefined + } size="small" fullWidth active={selectedLevel === level && !selectedSectionId} @@ -70,7 +80,16 @@ export function Settings({ .map((section) => (
  • selectSection(section.id, section.level)} + to={urls.settings(section.id)} + onClick={ + // Outside of /settings, we want to select the level without navigating + props.logicKey === 'settingsScene' + ? (e) => { + selectSection(section.id, section.level) + e.preventDefault() + } + : undefined + } size="small" fullWidth active={selectedSectionId === section.id} @@ -116,7 +135,7 @@ export function Settings({ } function SettingsRenderer(props: SettingsLogicProps): JSX.Element { - const { settings } = useValues(settingsLogic(props)) + const { settings, selectedLevel, selectedSectionId } = useValues(settingsLogic(props)) const { selectSetting } = useActions(settingsLogic(props)) return ( @@ -126,7 +145,20 @@ function SettingsRenderer(props: SettingsLogicProps): JSX.Element {

    {x.title} - } size="small" onClick={() => selectSetting?.(x.id)} /> + } + size="small" + to={urls.settings(selectedSectionId ?? selectedLevel, x.id)} + onClick={ + // Outside of /settings, we want to select the level without navigating + props.logicKey === 'settingsScene' + ? (e) => { + selectSetting(x.id) + e.preventDefault() + } + : undefined + } + />

    {x.description &&

    {x.description}

    } diff --git a/frontend/src/scenes/settings/SettingsScene.stories.tsx b/frontend/src/scenes/settings/SettingsScene.stories.tsx index 8ebdd09f15b14..6584b2a93d717 100644 --- a/frontend/src/scenes/settings/SettingsScene.stories.tsx +++ b/frontend/src/scenes/settings/SettingsScene.stories.tsx @@ -46,7 +46,7 @@ export const SettingsProject: StoryFn = () => { return } SettingsProject.parameters = { - testOptions: { waitForSelector: '.Settings__sections button' }, + testOptions: { waitForSelector: '.Settings__sections a' }, } export const SettingsProjectWithReplayFeatures: StoryFn = () => { @@ -61,7 +61,7 @@ export const SettingsProjectWithReplayFeatures: StoryFn = () => { return } SettingsProjectWithReplayFeatures.parameters = { - testOptions: { waitForSelector: '.Settings__sections button' }, + testOptions: { waitForSelector: '.Settings__sections a' }, } export const SettingsUser: StoryFn = () => { @@ -71,7 +71,7 @@ export const SettingsUser: StoryFn = () => { return } SettingsUser.parameters = { - testOptions: { waitForSelector: '.Settings__sections button' }, + testOptions: { waitForSelector: '.Settings__sections a' }, } export const SettingsOrganization: StoryFn = () => { @@ -81,7 +81,7 @@ export const SettingsOrganization: StoryFn = () => { return } SettingsOrganization.parameters = { - testOptions: { waitForSelector: '.Settings__sections button' }, + testOptions: { waitForSelector: '.Settings__sections a' }, } export const SettingsWebVitals: StoryFn = () => { @@ -91,7 +91,7 @@ export const SettingsWebVitals: StoryFn = () => { return } SettingsOrganization.parameters = { - testOptions: { waitForSelector: '.Settings__sections button' }, + testOptions: { waitForSelector: '.Settings__sections a' }, } function TimeSensitiveSettings(props: { @@ -133,40 +133,40 @@ export const SettingsSessionTimeoutAllOptions: StoryFn = () => { return } SettingsSessionTimeoutAllOptions.parameters = { - testOptions: { waitForSelector: '.Settings__sections button' }, + testOptions: { waitForSelector: '.Settings__sections a' }, } export const SettingsSessionTimeoutPasswordOnly: StoryFn = () => { return } SettingsSessionTimeoutPasswordOnly.parameters = { - testOptions: { waitForSelector: '.Settings__sections button' }, + testOptions: { waitForSelector: '.Settings__sections a' }, } export const SettingsSessionTimeoutSsoOnly: StoryFn = () => { return } SettingsSessionTimeoutSsoOnly.parameters = { - testOptions: { waitForSelector: '.Settings__sections button' }, + testOptions: { waitForSelector: '.Settings__sections a' }, } export const SettingsSessionTimeoutSsoEnforcedGithub: StoryFn = () => { return } SettingsSessionTimeoutSsoEnforcedGithub.parameters = { - testOptions: { waitForSelector: '.Settings__sections button' }, + testOptions: { waitForSelector: '.Settings__sections a' }, } export const SettingsSessionTimeoutSsoEnforcedGoogle: StoryFn = () => { return } SettingsSessionTimeoutSsoEnforcedGoogle.parameters = { - testOptions: { waitForSelector: '.Settings__sections button' }, + testOptions: { waitForSelector: '.Settings__sections a' }, } export const SettingsSessionTimeoutSsoEnforcedSaml: StoryFn = () => { return } SettingsSessionTimeoutSsoEnforcedSaml.parameters = { - testOptions: { waitForSelector: '.Settings__sections button' }, + testOptions: { waitForSelector: '.Settings__sections a' }, } diff --git a/frontend/src/scenes/settings/environment/SessionRecordingIngestionSettings.tsx b/frontend/src/scenes/settings/environment/SessionRecordingIngestionSettings.tsx index bd4a79e41c4ec..f645c195e5440 100644 --- a/frontend/src/scenes/settings/environment/SessionRecordingIngestionSettings.tsx +++ b/frontend/src/scenes/settings/environment/SessionRecordingIngestionSettings.tsx @@ -21,6 +21,7 @@ import { FEATURE_FLAGS, SESSION_REPLAY_MINIMUM_DURATION_OPTIONS } from 'lib/cons import { IconCancel } from 'lib/lemon-ui/icons' import { LemonField } from 'lib/lemon-ui/LemonField' import { LemonLabel } from 'lib/lemon-ui/LemonLabel/LemonLabel' +import { SupportedPlatforms } from 'scenes/settings/environment/SessionRecordingSettings' import { sessionReplayIngestionControlLogic } from 'scenes/settings/environment/sessionReplayIngestionControlLogic' import { teamLogic } from 'scenes/teamLogic' import { userLogic } from 'scenes/userLogic' @@ -378,6 +379,8 @@ export function SessionRecordingIngestionSettings(): JSX.Element | null {

    + + {samplingControlFeatureEnabled && ( <>
    diff --git a/frontend/src/scenes/settings/environment/SessionRecordingSettings.tsx b/frontend/src/scenes/settings/environment/SessionRecordingSettings.tsx index ecc30da766820..4b53a895a9996 100644 --- a/frontend/src/scenes/settings/environment/SessionRecordingSettings.tsx +++ b/frontend/src/scenes/settings/environment/SessionRecordingSettings.tsx @@ -1,5 +1,15 @@ -import { IconPlus } from '@posthog/icons' -import { LemonBanner, LemonButton, LemonDialog, LemonSwitch, LemonTag, Link } from '@posthog/lemon-ui' +import { IconCheck, IconInfo, IconPlus, IconX } from '@posthog/icons' +import { + LemonBanner, + LemonButton, + LemonDialog, + LemonDivider, + LemonSwitch, + LemonTag, + Link, + Tooltip, +} from '@posthog/lemon-ui' +import clsx from 'clsx' import { useActions, useValues } from 'kea' import { AuthorizedUrlList } from 'lib/components/AuthorizedUrlList/AuthorizedUrlList' import { AuthorizedUrlListType } from 'lib/components/AuthorizedUrlList/authorizedUrlListLogic' @@ -8,11 +18,84 @@ import { PropertySelect } from 'lib/components/PropertySelect/PropertySelect' import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types' import { IconSelectEvents } from 'lib/lemon-ui/icons' import { LemonLabel } from 'lib/lemon-ui/LemonLabel/LemonLabel' -import { objectsEqual } from 'lib/utils' +import { isObject, objectsEqual } from 'lib/utils' +import { ReactNode } from 'react' import { teamLogic } from 'scenes/teamLogic' import { SessionRecordingAIConfig } from '~/types' +interface SupportedPlatformProps { + note?: ReactNode + label: string + supported: boolean +} + +function SupportedPlatform(props: SupportedPlatformProps): JSX.Element { + const node = ( +
    + {props.note ? : props.supported ? : } {props.label} +
    + ) + if (props.note) { + return {node} + } + return node +} + +export function SupportedPlatforms(props: { + web?: boolean | { note?: ReactNode } + android?: boolean | { note?: ReactNode } + ios?: boolean | { note?: ReactNode } + reactNative?: boolean | { note?: ReactNode } + flutter?: boolean | { note?: ReactNode } +}): JSX.Element { + return ( +
    + Supported platforms: + + + + + + + + + + + + + + +
    + ) +} + function LogCaptureSettings(): JSX.Element { const { updateCurrentTeam } = useActions(teamLogic) const { currentTeam } = useValues(teamLogic) @@ -20,6 +103,7 @@ function LogCaptureSettings(): JSX.Element { return (

    Log capture

    +

    This setting controls if browser console logs will be captured as a part of recordings. The console logs will be shown in the recording player to help you debug any issues. @@ -52,6 +136,19 @@ function CanvasCaptureSettings(): JSX.Element | null { return (

    Canvas capture

    + + If you're using the `canvaskit` renderer on Flutter Web, you must also enable canvas capture + + ), + }} + web={true} + reactNative={false} + />

    This setting controls if browser canvas elements will be captured as part of recordings.{' '} @@ -111,6 +208,7 @@ export function NetworkCaptureSettings(): JSX.Element { return ( <> +

    This setting controls if performance and network information will be captured alongside recordings. The network requests and timings will be shown in the recording player to help you debug any issues. @@ -140,7 +238,7 @@ export function NetworkCaptureSettings(): JSX.Element { Learn how to mask header and payload values in our docs

    -

    Capture headers and body are only available for JavaScript Web.

    + @@ -220,6 +318,7 @@ export function NetworkCaptureSettings(): JSX.Element { export function ReplayAuthorizedDomains(): JSX.Element { return (
    +

    Use the settings below to restrict the domains where recordings will be captured. If no domains are selected, then there will be no domain restriction. @@ -399,6 +498,7 @@ export function ReplayGeneral(): JSX.Element { return (

    +

    Watch recordings of how users interact with your web app to see what can be improved.{' '} ([ }), reducers(({ props }) => ({ - selectedLevel: [ + selectedLevelRaw: [ (props.settingLevelId ?? 'project') as SettingLevelId, { selectLevel: (_, { level }) => level, selectSection: (_, { level }) => level, }, ], - selectedSectionId: [ + selectedSectionIdRaw: [ (props.sectionId ?? null) as SettingSectionId | null, { selectLevel: () => null, @@ -94,6 +94,40 @@ export const settingsLogic = kea([ return sections }, ], + selectedLevel: [ + (s) => [s.selectedLevelRaw, s.selectedSectionIdRaw, s.featureFlags], + (selectedLevelRaw, selectedSectionIdRaw, featureFlags): SettingLevelId => { + // As of middle of September 2024, `details` and `danger-zone` are the only sections present + // at both Environment and Project levels. Others we want to redirect based on the feature flag. + if ( + !selectedSectionIdRaw || + (!selectedSectionIdRaw.endsWith('-details') && !selectedSectionIdRaw.endsWith('-danger-zone')) + ) { + if (featureFlags[FEATURE_FLAGS.ENVIRONMENTS]) { + return selectedLevelRaw === 'project' ? 'environment' : selectedLevelRaw + } + return selectedLevelRaw === 'environment' ? 'project' : selectedLevelRaw + } + return selectedLevelRaw + }, + ], + selectedSectionId: [ + (s) => [s.selectedSectionIdRaw, s.featureFlags], + (selectedSectionIdRaw, featureFlags): SettingSectionId | null => { + if (!selectedSectionIdRaw) { + return null + } + // As of middle of September 2024, `details` and `danger-zone` are the only sections present + // at both Environment and Project levels. Others we want to redirect based on the feature flag. + if (!selectedSectionIdRaw.endsWith('-details') && !selectedSectionIdRaw.endsWith('-danger-zone')) { + if (featureFlags[FEATURE_FLAGS.ENVIRONMENTS]) { + return selectedSectionIdRaw.replace(/^project/, 'environment') as SettingSectionId + } + return selectedSectionIdRaw.replace(/^environment/, 'project') as SettingSectionId + } + return selectedSectionIdRaw + }, + ], selectedSection: [ (s) => [s.sections, s.selectedSectionId], (sections, selectedSectionId): SettingSection | null => { diff --git a/frontend/src/scenes/settings/settingsSceneLogic.ts b/frontend/src/scenes/settings/settingsSceneLogic.ts index ac429fd3f3f9b..0813661d4c988 100644 --- a/frontend/src/scenes/settings/settingsSceneLogic.ts +++ b/frontend/src/scenes/settings/settingsSceneLogic.ts @@ -51,6 +51,7 @@ export const settingsSceneLogic = kea([ // As of middle of September 2024, `details` and `danger-zone` are the only sections present // at both Environment and Project levels. Others we want to redirect based on the feature flag. + // This is just for URLs, since analogous logic for _rendering_ settings is already in settingsLogic. if (!section.endsWith('-details') && !section.endsWith('-danger-zone')) { if (values.featureFlags[FEATURE_FLAGS.ENVIRONMENTS]) { section = section.replace(/^project/, 'environment') @@ -73,16 +74,20 @@ export const settingsSceneLogic = kea([ })), actionToUrl(({ values }) => ({ + // Replacing history item instead of pushing, so that the environments<>project redirect doesn't affect history selectLevel({ level }) { - return [urls.settings(level), router.values.searchParams, router.values.hashParams] + return [urls.settings(level), router.values.searchParams, router.values.hashParams, { replace: true }] }, selectSection({ section }) { - return [urls.settings(section), router.values.searchParams, router.values.hashParams] + return [urls.settings(section), router.values.searchParams, router.values.hashParams, { replace: true }] }, selectSetting({ setting }) { - const url = urls.settings(values.selectedSectionId ?? values.selectedLevel, setting) - - return [url] + return [ + urls.settings(values.selectedSectionId ?? values.selectedLevel, setting), + undefined, + undefined, + { replace: true }, + ] }, })), ]) diff --git a/mypy-baseline.txt b/mypy-baseline.txt index 8b815fbdb5ec5..78020a95ec1b4 100644 --- a/mypy-baseline.txt +++ b/mypy-baseline.txt @@ -1,67 +1,4 @@ posthog/warehouse/models/ssh_tunnel.py:0: error: Incompatible types in assignment (expression has type "NoEncryption", variable has type "BestAvailableEncryption") [assignment] -posthog/temporal/data_imports/pipelines/sql_database_v2/schema_types.py:0: error: Statement is unreachable [unreachable] -posthog/temporal/data_imports/pipelines/sql_database_v2/schema_types.py:0: error: Non-overlapping equality check (left operand type: "Literal['text', 'double', 'bool', 'timestamp', 'bigint', 'json', 'decimal', 'wei', 'date', 'time'] | None", right operand type: "Literal['interval']") [comparison-overlap] -posthog/temporal/data_imports/pipelines/sql_database_v2/arrow_helpers.py:0: error: Unused "type: ignore" comment [unused-ignore] -posthog/temporal/data_imports/pipelines/sql_database_v2/arrow_helpers.py:0: error: Invalid index type "str | None" for "dict[str, ndarray[Any, dtype[Any]]]"; expected type "str" [index] -posthog/temporal/data_imports/pipelines/sql_database_v2/arrow_helpers.py:0: error: Invalid index type "str | None" for "dict[str, ndarray[Any, dtype[Any]]]"; expected type "str" [index] -posthog/temporal/data_imports/pipelines/sql_database_v2/arrow_helpers.py:0: error: Invalid index type "str | None" for "dict[str, TColumnSchema]"; expected type "str" [index] -posthog/temporal/data_imports/pipelines/sql_database/helpers.py:0: error: Unused "type: ignore" comment [unused-ignore] -posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Item "None" of "Incremental[Any] | None" has no attribute "row_order" [union-attr] -posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Incompatible types in assignment (expression has type "Literal['asc', 'desc'] | Any | None", variable has type "Literal['asc', 'desc']") [assignment] -posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Incompatible types in assignment (expression has type "None", variable has type "Column[Any]") [assignment] -posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Incompatible types in assignment (expression has type "None", variable has type "Literal['asc', 'desc']") [assignment] -posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Item "None" of "dict[str, Any] | None" has no attribute "get" [union-attr] -posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Unused "type: ignore" comment [unused-ignore] -posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Argument "primary_key" to "make_hints" has incompatible type "list[str] | None"; expected "str | Sequence[str] | Callable[[Any], str | Sequence[str]]" [arg-type] -posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Unused "type: ignore" comment [unused-ignore] -posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Unused "type: ignore" comment [unused-ignore] -posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Unused "type: ignore" comment [unused-ignore] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Dict entry 2 has incompatible type "Literal['auto']": "None"; expected "Literal['json_response', 'header_link', 'auto', 'single_page', 'cursor', 'offset', 'page_number']": "type[BasePaginator]" [dict-item] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Incompatible types in assignment (expression has type "None", variable has type "AuthConfigBase") [assignment] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Argument 1 to "get_auth_class" has incompatible type "Literal['bearer', 'api_key', 'http_basic'] | None"; expected "Literal['bearer', 'api_key', 'http_basic']" [arg-type] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Need type annotation for "dependency_graph" [var-annotated] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Incompatible types in assignment (expression has type "None", target has type "ResolvedParam") [assignment] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Incompatible return value type (got "tuple[TopologicalSorter[Any], dict[str, EndpointResource], dict[str, ResolvedParam]]", expected "tuple[Any, dict[str, EndpointResource], dict[str, ResolvedParam | None]]") [return-value] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Unsupported right operand type for in ("str | Endpoint | None") [operator] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Value of type variable "StrOrLiteralStr" of "parse" of "Formatter" cannot be "str | None" [type-var] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Unsupported right operand type for in ("dict[str, ResolveParamConfig | IncrementalParamConfig | Any] | None") [operator] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Unsupported right operand type for in ("dict[str, ResolveParamConfig | IncrementalParamConfig | Any] | None") [operator] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Value of type "dict[str, ResolveParamConfig | IncrementalParamConfig | Any] | None" is not indexable [index] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Item "None" of "dict[str, ResolveParamConfig | IncrementalParamConfig | Any] | None" has no attribute "pop" [union-attr] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Value of type "dict[str, ResolveParamConfig | IncrementalParamConfig | Any] | None" is not indexable [index] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Item "None" of "str | None" has no attribute "format" [union-attr] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Argument 1 to "single_entity_path" has incompatible type "str | None"; expected "str" [arg-type] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Item "None" of "dict[str, ResolveParamConfig | IncrementalParamConfig | Any] | None" has no attribute "items" [union-attr] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Incompatible types in assignment (expression has type "str | None", variable has type "str") [assignment] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Incompatible types in assignment (expression has type "str | None", variable has type "str") [assignment] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Statement is unreachable [unreachable] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Unpacked dict entry 0 has incompatible type "dict[str, Any] | None"; expected "SupportsKeysAndGetItem[str, Any]" [dict-item] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Unpacked dict entry 1 has incompatible type "dict[str, Any] | None"; expected "SupportsKeysAndGetItem[str, Any]" [dict-item] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Unpacked dict entry 0 has incompatible type "dict[str, Any] | None"; expected "SupportsKeysAndGetItem[str, ResolveParamConfig | IncrementalParamConfig | Any]" [dict-item] -posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Unpacked dict entry 1 has incompatible type "dict[str, ResolveParamConfig | IncrementalParamConfig | Any] | None"; expected "SupportsKeysAndGetItem[str, ResolveParamConfig | IncrementalParamConfig | Any]" [dict-item] -posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Not all union combinations were tried because there are too many unions [misc] -posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument 2 to "source" has incompatible type "str | None"; expected "str" [arg-type] -posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument 3 to "source" has incompatible type "str | None"; expected "str" [arg-type] -posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument 4 to "source" has incompatible type "int | None"; expected "int" [arg-type] -posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument 6 to "source" has incompatible type "Schema | None"; expected "Schema" [arg-type] -posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument 7 to "source" has incompatible type "Literal['evolve', 'discard_value', 'freeze', 'discard_row'] | TSchemaContractDict | None"; expected "Literal['evolve', 'discard_value', 'freeze', 'discard_row'] | TSchemaContractDict" [arg-type] -posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument 8 to "source" has incompatible type "type[BaseConfiguration] | None"; expected "type[BaseConfiguration]" [arg-type] -posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument 1 to "build_resource_dependency_graph" has incompatible type "EndpointResourceBase | None"; expected "EndpointResourceBase" [arg-type] -posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Incompatible types in assignment (expression has type "list[str] | None", variable has type "list[str]") [assignment] -posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument 1 to "setup_incremental_object" has incompatible type "dict[str, ResolveParamConfig | IncrementalParamConfig | Any] | None"; expected "dict[str, Any]" [arg-type] -posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument "base_url" to "RESTClient" has incompatible type "str | None"; expected "str" [arg-type] -posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument 1 to "exclude_keys" has incompatible type "dict[str, ResolveParamConfig | IncrementalParamConfig | Any] | None"; expected "Mapping[str, Any]" [arg-type] -posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Incompatible default for argument "resolved_param" (default has type "ResolvedParam | None", argument has type "ResolvedParam") [assignment] -posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] -posthog/temporal/data_imports/pipelines/vitally/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] -posthog/temporal/data_imports/pipelines/vitally/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] -posthog/temporal/data_imports/pipelines/vitally/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] -posthog/temporal/data_imports/pipelines/vitally/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] -posthog/temporal/data_imports/pipelines/vitally/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] -posthog/temporal/data_imports/pipelines/vitally/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] -posthog/temporal/data_imports/pipelines/vitally/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] -posthog/temporal/data_imports/pipelines/vitally/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] -posthog/temporal/data_imports/pipelines/vitally/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] posthog/utils.py:0: error: No overload variant of "asdict" matches argument type "type[DataclassInstance]" [call-overload] posthog/utils.py:0: note: Possible overload variants: posthog/utils.py:0: note: def asdict(obj: DataclassInstance) -> dict[str, Any] @@ -322,7 +259,7 @@ ee/billing/billing_manager.py:0: error: Incompatible types in assignment (expres posthog/models/property/util.py:0: error: Incompatible type for lookup 'pk': (got "str | int | list[str]", expected "str | int") [misc] posthog/models/property/util.py:0: error: Argument 3 to "format_filter_query" has incompatible type "HogQLContext | None"; expected "HogQLContext" [arg-type] posthog/models/property/util.py:0: error: Argument 3 to "format_cohort_subquery" has incompatible type "HogQLContext | None"; expected "HogQLContext" [arg-type] -posthog/models/property/util.py:0: error: Invalid index type "tuple[str, str]" for "dict[tuple[str, Literal['properties', 'group_properties', 'person_properties']], str]"; expected type "tuple[str, Literal['properties', 'group_properties', 'person_properties']]" [index] +posthog/models/property/util.py:0: error: Argument 2 to "get_materialized_column_for_property" has incompatible type "str"; expected "Literal['properties', 'group_properties', 'person_properties']" [arg-type] posthog/models/property/util.py:0: error: Argument 1 to "append" of "list" has incompatible type "str | int"; expected "str" [arg-type] posthog/models/property/util.py:0: error: Argument 1 to "append" of "list" has incompatible type "str | int"; expected "str" [arg-type] posthog/models/property/util.py:0: error: Argument 1 to "append" of "list" has incompatible type "str | int"; expected "str" [arg-type] @@ -332,7 +269,7 @@ posthog/api/documentation.py:0: note: def run_validation(self, data: Any = ...) posthog/api/documentation.py:0: note: Subclass: posthog/api/documentation.py:0: note: def run_validation(self, data: Any) -> Any posthog/queries/trends/util.py:0: error: Argument 1 to "translate_hogql" has incompatible type "str | None"; expected "str" [arg-type] -posthog/queries/column_optimizer/foss_column_optimizer.py:0: error: Argument 1 to "get" of "dict" has incompatible type "tuple[str, str]"; expected "tuple[str, Literal['properties', 'group_properties', 'person_properties']]" [arg-type] +posthog/queries/column_optimizer/foss_column_optimizer.py:0: error: Argument 2 to "get_materialized_column_for_property" has incompatible type "str"; expected "Literal['properties', 'group_properties', 'person_properties']" [arg-type] posthog/hogql/property.py:0: error: Incompatible type for lookup 'id': (got "str | int | list[str]", expected "str | int") [misc] posthog/hogql/property.py:0: error: Incompatible type for lookup 'pk': (got "str | float", expected "str | int") [misc] posthog/api/utils.py:0: error: Incompatible types in assignment (expression has type "type[EventDefinition]", variable has type "type[EnterpriseEventDefinition]") [assignment] @@ -470,6 +407,9 @@ posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: def posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: def [TDltResourceImpl: DltResource] resource(None = ..., /, name: str = ..., table_name: str | Callable[[Any], str] = ..., max_table_nesting: int = ..., write_disposition: Literal['skip', 'append', 'replace', 'merge'] | TWriteDispositionDict | TMergeDispositionDict | TScd2StrategyDict | Callable[[Any], Literal['skip', 'append', 'replace', 'merge'] | TWriteDispositionDict | TMergeDispositionDict | TScd2StrategyDict] = ..., columns: dict[str, TColumnSchema] | Sequence[TColumnSchema] | BaseModel | type[BaseModel] | Callable[[Any], dict[str, TColumnSchema] | Sequence[TColumnSchema] | BaseModel | type[BaseModel]] = ..., primary_key: str | Sequence[str] | Callable[[Any], str | Sequence[str]] = ..., merge_key: str | Sequence[str] | Callable[[Any], str | Sequence[str]] = ..., schema_contract: Literal['evolve', 'discard_value', 'freeze', 'discard_row'] | TSchemaContractDict | Callable[[Any], Literal['evolve', 'discard_value', 'freeze', 'discard_row'] | TSchemaContractDict] = ..., table_format: Literal['iceberg', 'delta', 'hive'] | Callable[[Any], Literal['iceberg', 'delta', 'hive']] = ..., file_format: Literal['preferred', 'jsonl', 'typed-jsonl', 'insert_values', 'parquet', 'csv', 'reference'] | Callable[[Any], Literal['preferred', 'jsonl', 'typed-jsonl', 'insert_values', 'parquet', 'csv', 'reference']] = ..., references: Sequence[TTableReference] | Callable[[Any], Sequence[TTableReference]] = ..., selected: bool = ..., spec: type[BaseConfiguration] = ..., parallelized: bool = ..., _impl_cls: type[TDltResourceImpl] = ...) -> Callable[[Callable[TResourceFunParams, Any]], TDltResourceImpl] posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: def [TDltResourceImpl: DltResource] resource(None = ..., /, name: str | Callable[[Any], str] = ..., table_name: str | Callable[[Any], str] = ..., max_table_nesting: int = ..., write_disposition: Literal['skip', 'append', 'replace', 'merge'] | TWriteDispositionDict | TMergeDispositionDict | TScd2StrategyDict | Callable[[Any], Literal['skip', 'append', 'replace', 'merge'] | TWriteDispositionDict | TMergeDispositionDict | TScd2StrategyDict] = ..., columns: dict[str, TColumnSchema] | Sequence[TColumnSchema] | BaseModel | type[BaseModel] | Callable[[Any], dict[str, TColumnSchema] | Sequence[TColumnSchema] | BaseModel | type[BaseModel]] = ..., primary_key: str | Sequence[str] | Callable[[Any], str | Sequence[str]] = ..., merge_key: str | Sequence[str] | Callable[[Any], str | Sequence[str]] = ..., schema_contract: Literal['evolve', 'discard_value', 'freeze', 'discard_row'] | TSchemaContractDict | Callable[[Any], Literal['evolve', 'discard_value', 'freeze', 'discard_row'] | TSchemaContractDict] = ..., table_format: Literal['iceberg', 'delta', 'hive'] | Callable[[Any], Literal['iceberg', 'delta', 'hive']] = ..., file_format: Literal['preferred', 'jsonl', 'typed-jsonl', 'insert_values', 'parquet', 'csv', 'reference'] | Callable[[Any], Literal['preferred', 'jsonl', 'typed-jsonl', 'insert_values', 'parquet', 'csv', 'reference']] = ..., references: Sequence[TTableReference] | Callable[[Any], Sequence[TTableReference]] = ..., selected: bool = ..., spec: type[BaseConfiguration] = ..., parallelized: bool = ..., _impl_cls: type[TDltResourceImpl] = ..., standalone: Literal[True] = ...) -> Callable[[Callable[TResourceFunParams, Any]], Callable[TResourceFunParams, TDltResourceImpl]] posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: def [TDltResourceImpl: DltResource] resource(list[Any] | tuple[Any] | Iterator[Any], /, name: str = ..., table_name: str | Callable[[Any], str] = ..., max_table_nesting: int = ..., write_disposition: Literal['skip', 'append', 'replace', 'merge'] | TWriteDispositionDict | TMergeDispositionDict | TScd2StrategyDict | Callable[[Any], Literal['skip', 'append', 'replace', 'merge'] | TWriteDispositionDict | TMergeDispositionDict | TScd2StrategyDict] = ..., columns: dict[str, TColumnSchema] | Sequence[TColumnSchema] | BaseModel | type[BaseModel] | Callable[[Any], dict[str, TColumnSchema] | Sequence[TColumnSchema] | BaseModel | type[BaseModel]] = ..., primary_key: str | Sequence[str] | Callable[[Any], str | Sequence[str]] = ..., merge_key: str | Sequence[str] | Callable[[Any], str | Sequence[str]] = ..., schema_contract: Literal['evolve', 'discard_value', 'freeze', 'discard_row'] | TSchemaContractDict | Callable[[Any], Literal['evolve', 'discard_value', 'freeze', 'discard_row'] | TSchemaContractDict] = ..., table_format: Literal['iceberg', 'delta', 'hive'] | Callable[[Any], Literal['iceberg', 'delta', 'hive']] = ..., file_format: Literal['preferred', 'jsonl', 'typed-jsonl', 'insert_values', 'parquet', 'csv', 'reference'] | Callable[[Any], Literal['preferred', 'jsonl', 'typed-jsonl', 'insert_values', 'parquet', 'csv', 'reference']] = ..., references: Sequence[TTableReference] | Callable[[Any], Sequence[TTableReference]] = ..., selected: bool = ..., spec: type[BaseConfiguration] = ..., parallelized: bool = ..., _impl_cls: type[TDltResourceImpl] = ...) -> TDltResourceImpl +posthog/temporal/data_imports/pipelines/sql_database_v2/schema_types.py:0: error: Statement is unreachable [unreachable] +posthog/temporal/data_imports/pipelines/sql_database_v2/schema_types.py:0: error: Non-overlapping equality check (left operand type: "Literal['text', 'double', 'bool', 'timestamp', 'bigint', 'json', 'decimal', 'wei', 'date', 'time'] | None", right operand type: "Literal['interval']") [comparison-overlap] +posthog/temporal/data_imports/pipelines/sql_database/helpers.py:0: error: Unused "type: ignore" comment [unused-ignore] posthog/tasks/test/test_update_survey_iteration.py:0: error: Item "None" of "FeatureFlag | None" has no attribute "filters" [union-attr] posthog/tasks/test/test_stop_surveys_reached_target.py:0: error: No overload variant of "__sub__" of "datetime" matches argument type "None" [operator] posthog/tasks/test/test_stop_surveys_reached_target.py:0: note: Possible overload variants: @@ -603,22 +543,10 @@ posthog/warehouse/data_load/validate_schema.py:0: error: Incompatible types in a posthog/warehouse/data_load/validate_schema.py:0: error: Incompatible types in assignment (expression has type "object", variable has type "str | int | Combinable") [assignment] posthog/warehouse/data_load/validate_schema.py:0: error: Incompatible types in assignment (expression has type "dict[str, dict[str, str | bool]] | dict[str, str]", variable has type "dict[str, dict[str, str]]") [assignment] posthog/warehouse/data_load/source_templates.py:0: error: Incompatible types in assignment (expression has type "str", variable has type "Type") [assignment] -posthog/warehouse/api/external_data_schema.py:0: error: Incompatible return value type (got "str | None", expected "SyncType | None") [return-value] -posthog/warehouse/api/external_data_schema.py:0: error: Argument 1 to "get_sql_schemas_for_source_type" has incompatible type "str"; expected "Type" [arg-type] -posthog/warehouse/api/external_data_schema.py:0: error: No overload variant of "get" of "dict" matches argument type "str" [call-overload] -posthog/warehouse/api/external_data_schema.py:0: note: Possible overload variants: -posthog/warehouse/api/external_data_schema.py:0: note: def get(self, Type, /) -> dict[str, list[IncrementalField]] | None -posthog/warehouse/api/external_data_schema.py:0: note: def get(self, Type, dict[str, list[IncrementalField]], /) -> dict[str, list[IncrementalField]] -posthog/warehouse/api/external_data_schema.py:0: note: def [_T] get(self, Type, _T, /) -> dict[str, list[IncrementalField]] | _T -posthog/warehouse/api/table.py:0: error: Unused "type: ignore" comment [unused-ignore] -posthog/warehouse/api/table.py:0: error: Unused "type: ignore" comment [unused-ignore] -posthog/warehouse/api/table.py:0: error: Unused "type: ignore" comment [unused-ignore] -posthog/temporal/data_imports/workflow_activities/sync_new_schemas.py:0: error: No overload variant of "get" of "dict" matches argument types "str", "tuple[()]" [call-overload] -posthog/temporal/data_imports/workflow_activities/sync_new_schemas.py:0: note: Possible overload variants: -posthog/temporal/data_imports/workflow_activities/sync_new_schemas.py:0: note: def get(self, Type, /) -> Sequence[str] | None -posthog/temporal/data_imports/workflow_activities/sync_new_schemas.py:0: note: def get(self, Type, Sequence[str], /) -> Sequence[str] -posthog/temporal/data_imports/workflow_activities/sync_new_schemas.py:0: note: def [_T] get(self, Type, _T, /) -> Sequence[str] | _T -posthog/temporal/data_imports/workflow_activities/sync_new_schemas.py:0: error: Argument "source_id" to "sync_old_schemas_with_new_schemas" has incompatible type "str"; expected "UUID" [arg-type] +posthog/temporal/data_imports/pipelines/sql_database_v2/arrow_helpers.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/sql_database_v2/arrow_helpers.py:0: error: Invalid index type "str | None" for "dict[str, ndarray[Any, dtype[Any]]]"; expected type "str" [index] +posthog/temporal/data_imports/pipelines/sql_database_v2/arrow_helpers.py:0: error: Invalid index type "str | None" for "dict[str, ndarray[Any, dtype[Any]]]"; expected type "str" [index] +posthog/temporal/data_imports/pipelines/sql_database_v2/arrow_helpers.py:0: error: Invalid index type "str | None" for "dict[str, TColumnSchema]"; expected type "str" [index] posthog/tasks/exports/test/test_csv_exporter.py:0: error: Function is missing a return type annotation [no-untyped-def] posthog/tasks/exports/test/test_csv_exporter.py:0: error: Function is missing a type annotation [no-untyped-def] posthog/tasks/exports/test/test_csv_exporter.py:0: error: Function is missing a type annotation for one or more arguments [no-untyped-def] @@ -664,7 +592,6 @@ posthog/queries/trends/test/test_person.py:0: error: Invalid index type "int" fo posthog/queries/trends/test/test_person.py:0: error: "str" has no attribute "get" [attr-defined] posthog/queries/trends/test/test_person.py:0: error: Invalid index type "int" for "_MonkeyPatchedResponse"; expected type "str" [index] posthog/models/test/test_organization_model.py:0: error: Module "django.utils.timezone" does not explicitly export attribute "timedelta" [attr-defined] -posthog/hogql_queries/test/test_actors_query_runner.py:0: error: Incompatible types in assignment (expression has type "Expr", variable has type "SelectQuery") [assignment] posthog/hogql/test/test_resolver.py:0: error: Item "None" of "JoinExpr | None" has no attribute "next_join" [union-attr] posthog/hogql/test/test_resolver.py:0: error: Item "None" of "JoinExpr | Any | None" has no attribute "constraint" [union-attr] posthog/hogql/test/test_resolver.py:0: error: Item "None" of "JoinConstraint | Any | None" has no attribute "constraint_type" [union-attr] @@ -792,6 +719,22 @@ posthog/temporal/tests/batch_exports/test_batch_exports.py:0: error: TypedDict k posthog/temporal/data_modeling/run_workflow.py:0: error: Dict entry 20 has incompatible type "str": "Literal['complex']"; expected "str": "Literal['text', 'double', 'bool', 'timestamp', 'bigint', 'binary', 'json', 'decimal', 'wei', 'date', 'time']" [dict-item] posthog/temporal/data_modeling/run_workflow.py:0: error: Dict entry 21 has incompatible type "str": "Literal['complex']"; expected "str": "Literal['text', 'double', 'bool', 'timestamp', 'bigint', 'binary', 'json', 'decimal', 'wei', 'date', 'time']" [dict-item] posthog/temporal/data_modeling/run_workflow.py:0: error: Dict entry 22 has incompatible type "str": "Literal['complex']"; expected "str": "Literal['text', 'double', 'bool', 'timestamp', 'bigint', 'binary', 'json', 'decimal', 'wei', 'date', 'time']" [dict-item] +posthog/temporal/data_imports/workflow_activities/sync_new_schemas.py:0: error: No overload variant of "get" of "dict" matches argument types "str", "tuple[()]" [call-overload] +posthog/temporal/data_imports/workflow_activities/sync_new_schemas.py:0: note: Possible overload variants: +posthog/temporal/data_imports/workflow_activities/sync_new_schemas.py:0: note: def get(self, Type, /) -> Sequence[str] | None +posthog/temporal/data_imports/workflow_activities/sync_new_schemas.py:0: note: def get(self, Type, Sequence[str], /) -> Sequence[str] +posthog/temporal/data_imports/workflow_activities/sync_new_schemas.py:0: note: def [_T] get(self, Type, _T, /) -> Sequence[str] | _T +posthog/temporal/data_imports/workflow_activities/sync_new_schemas.py:0: error: Argument "source_id" to "sync_old_schemas_with_new_schemas" has incompatible type "str"; expected "UUID" [arg-type] +posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Item "None" of "Incremental[Any] | None" has no attribute "row_order" [union-attr] +posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Incompatible types in assignment (expression has type "Literal['asc', 'desc'] | Any | None", variable has type "Literal['asc', 'desc']") [assignment] +posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Incompatible types in assignment (expression has type "None", variable has type "Column[Any]") [assignment] +posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Incompatible types in assignment (expression has type "None", variable has type "Literal['asc', 'desc']") [assignment] +posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Item "None" of "dict[str, Any] | None" has no attribute "get" [union-attr] +posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Argument "primary_key" to "make_hints" has incompatible type "list[str] | None"; expected "str | Sequence[str] | Callable[[Any], str | Sequence[str]]" [arg-type] +posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/sql_database_v2/helpers.py:0: error: Unused "type: ignore" comment [unused-ignore] posthog/temporal/data_imports/pipelines/pipeline_sync.py:0: error: "FilesystemDestinationClientConfiguration" has no attribute "delta_jobs_per_write" [attr-defined] posthog/temporal/data_imports/pipelines/pipeline_sync.py:0: error: "type[FilesystemDestinationClientConfiguration]" has no attribute "delta_jobs_per_write" [attr-defined] posthog/temporal/data_imports/pipelines/pipeline_sync.py:0: error: Incompatible types in assignment (expression has type "object", variable has type "DataWarehouseCredential | Combinable | None") [assignment] @@ -826,6 +769,23 @@ posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py:0: posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py:0: error: Need type annotation for "_execute_async_calls" (hint: "_execute_async_calls: list[] = ...") [var-annotated] posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py:0: error: Need type annotation for "_cursors" (hint: "_cursors: list[] = ...") [var-annotated] posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py:0: error: List item 0 has incompatible type "tuple[str, str, int, int, int, int, str, int]"; expected "tuple[str, str, int, int, str, str, str, str]" [list-item] +posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: error: No overload variant of "with_only_columns" of "Select" matches argument type "ReadOnlyColumnCollection[str, Column[Any]]" [call-overload] +posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: Possible overload variants: +posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: def [_T0] with_only_columns(self, TypedColumnsClauseRole[_T0] | SQLCoreOperations[_T0] | type[_T0], /) -> Select[tuple[_T0]] +posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: def [_T0, _T1] with_only_columns(self, TypedColumnsClauseRole[_T0] | SQLCoreOperations[_T0] | type[_T0], TypedColumnsClauseRole[_T1] | SQLCoreOperations[_T1] | type[_T1], /) -> Select[tuple[_T0, _T1]] +posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: def [_T0, _T1, _T2] with_only_columns(self, TypedColumnsClauseRole[_T0] | SQLCoreOperations[_T0] | type[_T0], TypedColumnsClauseRole[_T1] | SQLCoreOperations[_T1] | type[_T1], TypedColumnsClauseRole[_T2] | SQLCoreOperations[_T2] | type[_T2], /) -> Select[tuple[_T0, _T1, _T2]] +posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: def [_T0, _T1, _T2, _T3] with_only_columns(self, TypedColumnsClauseRole[_T0] | SQLCoreOperations[_T0] | type[_T0], TypedColumnsClauseRole[_T1] | SQLCoreOperations[_T1] | type[_T1], TypedColumnsClauseRole[_T2] | SQLCoreOperations[_T2] | type[_T2], TypedColumnsClauseRole[_T3] | SQLCoreOperations[_T3] | type[_T3], /) -> Select[tuple[_T0, _T1, _T2, _T3]] +posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: def [_T0, _T1, _T2, _T3, _T4] with_only_columns(self, TypedColumnsClauseRole[_T0] | SQLCoreOperations[_T0] | type[_T0], TypedColumnsClauseRole[_T1] | SQLCoreOperations[_T1] | type[_T1], TypedColumnsClauseRole[_T2] | SQLCoreOperations[_T2] | type[_T2], TypedColumnsClauseRole[_T3] | SQLCoreOperations[_T3] | type[_T3], TypedColumnsClauseRole[_T4] | SQLCoreOperations[_T4] | type[_T4], /) -> Select[tuple[_T0, _T1, _T2, _T3, _T4]] +posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: def [_T0, _T1, _T2, _T3, _T4, _T5] with_only_columns(self, TypedColumnsClauseRole[_T0] | SQLCoreOperations[_T0] | type[_T0], TypedColumnsClauseRole[_T1] | SQLCoreOperations[_T1] | type[_T1], TypedColumnsClauseRole[_T2] | SQLCoreOperations[_T2] | type[_T2], TypedColumnsClauseRole[_T3] | SQLCoreOperations[_T3] | type[_T3], TypedColumnsClauseRole[_T4] | SQLCoreOperations[_T4] | type[_T4], TypedColumnsClauseRole[_T5] | SQLCoreOperations[_T5] | type[_T5], /) -> Select[tuple[_T0, _T1, _T2, _T3, _T4, _T5]] +posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: def [_T0, _T1, _T2, _T3, _T4, _T5, _T6] with_only_columns(self, TypedColumnsClauseRole[_T0] | SQLCoreOperations[_T0] | type[_T0], TypedColumnsClauseRole[_T1] | SQLCoreOperations[_T1] | type[_T1], TypedColumnsClauseRole[_T2] | SQLCoreOperations[_T2] | type[_T2], TypedColumnsClauseRole[_T3] | SQLCoreOperations[_T3] | type[_T3], TypedColumnsClauseRole[_T4] | SQLCoreOperations[_T4] | type[_T4], TypedColumnsClauseRole[_T5] | SQLCoreOperations[_T5] | type[_T5], TypedColumnsClauseRole[_T6] | SQLCoreOperations[_T6] | type[_T6], /) -> Select[tuple[_T0, _T1, _T2, _T3, _T4, _T5, _T6]] +posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: def [_T0, _T1, _T2, _T3, _T4, _T5, _T6, _T7] with_only_columns(self, TypedColumnsClauseRole[_T0] | SQLCoreOperations[_T0] | type[_T0], TypedColumnsClauseRole[_T1] | SQLCoreOperations[_T1] | type[_T1], TypedColumnsClauseRole[_T2] | SQLCoreOperations[_T2] | type[_T2], TypedColumnsClauseRole[_T3] | SQLCoreOperations[_T3] | type[_T3], TypedColumnsClauseRole[_T4] | SQLCoreOperations[_T4] | type[_T4], TypedColumnsClauseRole[_T5] | SQLCoreOperations[_T5] | type[_T5], TypedColumnsClauseRole[_T6] | SQLCoreOperations[_T6] | type[_T6], TypedColumnsClauseRole[_T7] | SQLCoreOperations[_T7] | type[_T7], /) -> Select[tuple[_T0, _T1, _T2, _T3, _T4, _T5, _T6, _T7]] +posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: def with_only_columns(self, *entities: TypedColumnsClauseRole[Any] | ColumnsClauseRole | SQLCoreOperations[Any] | Literal['*', 1] | type[Any] | Inspectable[_HasClauseElement[Any]] | _HasClauseElement[Any], maintain_column_froms: bool = ..., **Any) -> Select[Any] +posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: error: No overload variant of "resource" matches argument types "Callable[[Engine, Table, int, Literal['sqlalchemy', 'pyarrow', 'pandas', 'connectorx'], Incremental[Any] | None, bool, Callable[[Table], None] | None, Literal['minimal', 'full', 'full_with_precision'], dict[str, Any] | None, Callable[[TypeEngine[Any]], TypeEngine[Any] | type[TypeEngine[Any]] | None] | None, list[str] | None, Callable[[Select[Any], Table], Select[Any]] | None, list[str] | None], Iterator[Any]]", "str", "list[str] | None", "list[str] | None", "dict[str, TColumnSchema]", "Collection[str]", "str" [call-overload] +posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: Possible overload variants: +posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: def [TResourceFunParams`-1, TDltResourceImpl: DltResource] resource(Callable[TResourceFunParams, Any], /, name: str = ..., table_name: str | Callable[[Any], str] = ..., max_table_nesting: int = ..., write_disposition: Literal['skip', 'append', 'replace', 'merge'] | TWriteDispositionDict | TMergeDispositionDict | TScd2StrategyDict | Callable[[Any], Literal['skip', 'append', 'replace', 'merge'] | TWriteDispositionDict | TMergeDispositionDict | TScd2StrategyDict] = ..., columns: dict[str, TColumnSchema] | Sequence[TColumnSchema] | BaseModel | type[BaseModel] | Callable[[Any], dict[str, TColumnSchema] | Sequence[TColumnSchema] | BaseModel | type[BaseModel]] = ..., primary_key: str | Sequence[str] | Callable[[Any], str | Sequence[str]] = ..., merge_key: str | Sequence[str] | Callable[[Any], str | Sequence[str]] = ..., schema_contract: Literal['evolve', 'discard_value', 'freeze', 'discard_row'] | TSchemaContractDict | Callable[[Any], Literal['evolve', 'discard_value', 'freeze', 'discard_row'] | TSchemaContractDict] = ..., table_format: Literal['iceberg', 'delta', 'hive'] | Callable[[Any], Literal['iceberg', 'delta', 'hive']] = ..., file_format: Literal['preferred', 'jsonl', 'typed-jsonl', 'insert_values', 'parquet', 'csv', 'reference'] | Callable[[Any], Literal['preferred', 'jsonl', 'typed-jsonl', 'insert_values', 'parquet', 'csv', 'reference']] = ..., references: Sequence[TTableReference] | Callable[[Any], Sequence[TTableReference]] = ..., selected: bool = ..., spec: type[BaseConfiguration] = ..., parallelized: bool = ..., _impl_cls: type[TDltResourceImpl] = ...) -> TDltResourceImpl +posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: def [TDltResourceImpl: DltResource] resource(None = ..., /, name: str = ..., table_name: str | Callable[[Any], str] = ..., max_table_nesting: int = ..., write_disposition: Literal['skip', 'append', 'replace', 'merge'] | TWriteDispositionDict | TMergeDispositionDict | TScd2StrategyDict | Callable[[Any], Literal['skip', 'append', 'replace', 'merge'] | TWriteDispositionDict | TMergeDispositionDict | TScd2StrategyDict] = ..., columns: dict[str, TColumnSchema] | Sequence[TColumnSchema] | BaseModel | type[BaseModel] | Callable[[Any], dict[str, TColumnSchema] | Sequence[TColumnSchema] | BaseModel | type[BaseModel]] = ..., primary_key: str | Sequence[str] | Callable[[Any], str | Sequence[str]] = ..., merge_key: str | Sequence[str] | Callable[[Any], str | Sequence[str]] = ..., schema_contract: Literal['evolve', 'discard_value', 'freeze', 'discard_row'] | TSchemaContractDict | Callable[[Any], Literal['evolve', 'discard_value', 'freeze', 'discard_row'] | TSchemaContractDict] = ..., table_format: Literal['iceberg', 'delta', 'hive'] | Callable[[Any], Literal['iceberg', 'delta', 'hive']] = ..., file_format: Literal['preferred', 'jsonl', 'typed-jsonl', 'insert_values', 'parquet', 'csv', 'reference'] | Callable[[Any], Literal['preferred', 'jsonl', 'typed-jsonl', 'insert_values', 'parquet', 'csv', 'reference']] = ..., references: Sequence[TTableReference] | Callable[[Any], Sequence[TTableReference]] = ..., selected: bool = ..., spec: type[BaseConfiguration] = ..., parallelized: bool = ..., _impl_cls: type[TDltResourceImpl] = ...) -> Callable[[Callable[TResourceFunParams, Any]], TDltResourceImpl] +posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: def [TDltResourceImpl: DltResource] resource(None = ..., /, name: str | Callable[[Any], str] = ..., table_name: str | Callable[[Any], str] = ..., max_table_nesting: int = ..., write_disposition: Literal['skip', 'append', 'replace', 'merge'] | TWriteDispositionDict | TMergeDispositionDict | TScd2StrategyDict | Callable[[Any], Literal['skip', 'append', 'replace', 'merge'] | TWriteDispositionDict | TMergeDispositionDict | TScd2StrategyDict] = ..., columns: dict[str, TColumnSchema] | Sequence[TColumnSchema] | BaseModel | type[BaseModel] | Callable[[Any], dict[str, TColumnSchema] | Sequence[TColumnSchema] | BaseModel | type[BaseModel]] = ..., primary_key: str | Sequence[str] | Callable[[Any], str | Sequence[str]] = ..., merge_key: str | Sequence[str] | Callable[[Any], str | Sequence[str]] = ..., schema_contract: Literal['evolve', 'discard_value', 'freeze', 'discard_row'] | TSchemaContractDict | Callable[[Any], Literal['evolve', 'discard_value', 'freeze', 'discard_row'] | TSchemaContractDict] = ..., table_format: Literal['iceberg', 'delta', 'hive'] | Callable[[Any], Literal['iceberg', 'delta', 'hive']] = ..., file_format: Literal['preferred', 'jsonl', 'typed-jsonl', 'insert_values', 'parquet', 'csv', 'reference'] | Callable[[Any], Literal['preferred', 'jsonl', 'typed-jsonl', 'insert_values', 'parquet', 'csv', 'reference']] = ..., references: Sequence[TTableReference] | Callable[[Any], Sequence[TTableReference]] = ..., selected: bool = ..., spec: type[BaseConfiguration] = ..., parallelized: bool = ..., _impl_cls: type[TDltResourceImpl] = ..., standalone: Literal[True] = ...) -> Callable[[Callable[TResourceFunParams, Any]], Callable[TResourceFunParams, TDltResourceImpl]] +posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py:0: note: def [TDltResourceImpl: DltResource] resource(list[Any] | tuple[Any] | Iterator[Any], /, name: str = ..., table_name: str | Callable[[Any], str] = ..., max_table_nesting: int = ..., write_disposition: Literal['skip', 'append', 'replace', 'merge'] | TWriteDispositionDict | TMergeDispositionDict | TScd2StrategyDict | Callable[[Any], Literal['skip', 'append', 'replace', 'merge'] | TWriteDispositionDict | TMergeDispositionDict | TScd2StrategyDict] = ..., columns: dict[str, TColumnSchema] | Sequence[TColumnSchema] | BaseModel | type[BaseModel] | Callable[[Any], dict[str, TColumnSchema] | Sequence[TColumnSchema] | BaseModel | type[BaseModel]] = ..., primary_key: str | Sequence[str] | Callable[[Any], str | Sequence[str]] = ..., merge_key: str | Sequence[str] | Callable[[Any], str | Sequence[str]] = ..., schema_contract: Literal['evolve', 'discard_value', 'freeze', 'discard_row'] | TSchemaContractDict | Callable[[Any], Literal['evolve', 'discard_value', 'freeze', 'discard_row'] | TSchemaContractDict] = ..., table_format: Literal['iceberg', 'delta', 'hive'] | Callable[[Any], Literal['iceberg', 'delta', 'hive']] = ..., file_format: Literal['preferred', 'jsonl', 'typed-jsonl', 'insert_values', 'parquet', 'csv', 'reference'] | Callable[[Any], Literal['preferred', 'jsonl', 'typed-jsonl', 'insert_values', 'parquet', 'csv', 'reference']] = ..., references: Sequence[TTableReference] | Callable[[Any], Sequence[TTableReference]] = ..., selected: bool = ..., spec: type[BaseConfiguration] = ..., parallelized: bool = ..., _impl_cls: type[TDltResourceImpl] = ...) -> TDltResourceImpl posthog/migrations/0237_remove_timezone_from_teams.py:0: error: Argument 2 to "RunPython" has incompatible type "Callable[[Migration, Any], None]"; expected "_CodeCallable | None" [arg-type] posthog/migrations/0228_fix_tile_layouts.py:0: error: Argument 2 to "RunPython" has incompatible type "Callable[[Migration, Any], None]"; expected "_CodeCallable | None" [arg-type] posthog/api/plugin_log_entry.py:0: error: Name "timezone.datetime" is not defined [name-defined] @@ -835,6 +795,29 @@ posthog/api/plugin_log_entry.py:0: error: Module "django.utils.timezone" does no posthog/temporal/tests/batch_exports/test_redshift_batch_export_workflow.py:0: error: Incompatible types in assignment (expression has type "str | int", variable has type "int") [assignment] posthog/api/sharing.py:0: error: Item "None" of "list[Any] | None" has no attribute "__iter__" (not iterable) [union-attr] posthog/temporal/data_imports/external_data_job.py:0: error: Argument "status" to "update_external_job_status" has incompatible type "str"; expected "Status" [arg-type] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Dict entry 2 has incompatible type "Literal['auto']": "None"; expected "Literal['json_response', 'header_link', 'auto', 'single_page', 'cursor', 'offset', 'page_number']": "type[BasePaginator]" [dict-item] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Incompatible types in assignment (expression has type "None", variable has type "AuthConfigBase") [assignment] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Argument 1 to "get_auth_class" has incompatible type "Literal['bearer', 'api_key', 'http_basic'] | None"; expected "Literal['bearer', 'api_key', 'http_basic']" [arg-type] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Need type annotation for "dependency_graph" [var-annotated] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Incompatible types in assignment (expression has type "None", target has type "ResolvedParam") [assignment] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Incompatible return value type (got "tuple[TopologicalSorter[Any], dict[str, EndpointResource], dict[str, ResolvedParam]]", expected "tuple[Any, dict[str, EndpointResource], dict[str, ResolvedParam | None]]") [return-value] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Unsupported right operand type for in ("str | Endpoint | None") [operator] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Value of type variable "StrOrLiteralStr" of "parse" of "Formatter" cannot be "str | None" [type-var] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Unsupported right operand type for in ("dict[str, ResolveParamConfig | IncrementalParamConfig | Any] | None") [operator] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Unsupported right operand type for in ("dict[str, ResolveParamConfig | IncrementalParamConfig | Any] | None") [operator] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Value of type "dict[str, ResolveParamConfig | IncrementalParamConfig | Any] | None" is not indexable [index] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Item "None" of "dict[str, ResolveParamConfig | IncrementalParamConfig | Any] | None" has no attribute "pop" [union-attr] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Value of type "dict[str, ResolveParamConfig | IncrementalParamConfig | Any] | None" is not indexable [index] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Item "None" of "str | None" has no attribute "format" [union-attr] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Argument 1 to "single_entity_path" has incompatible type "str | None"; expected "str" [arg-type] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Item "None" of "dict[str, ResolveParamConfig | IncrementalParamConfig | Any] | None" has no attribute "items" [union-attr] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Incompatible types in assignment (expression has type "str | None", variable has type "str") [assignment] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Incompatible types in assignment (expression has type "str | None", variable has type "str") [assignment] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Statement is unreachable [unreachable] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Unpacked dict entry 0 has incompatible type "dict[str, Any] | None"; expected "SupportsKeysAndGetItem[str, Any]" [dict-item] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Unpacked dict entry 1 has incompatible type "dict[str, Any] | None"; expected "SupportsKeysAndGetItem[str, Any]" [dict-item] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Unpacked dict entry 0 has incompatible type "dict[str, Any] | None"; expected "SupportsKeysAndGetItem[str, ResolveParamConfig | IncrementalParamConfig | Any]" [dict-item] +posthog/temporal/data_imports/pipelines/rest_source/config_setup.py:0: error: Unpacked dict entry 1 has incompatible type "dict[str, ResolveParamConfig | IncrementalParamConfig | Any] | None"; expected "SupportsKeysAndGetItem[str, ResolveParamConfig | IncrementalParamConfig | Any]" [dict-item] posthog/api/test/batch_exports/conftest.py:0: error: Signature of "run" incompatible with supertype "Worker" [override] posthog/api/test/batch_exports/conftest.py:0: note: Superclass: posthog/api/test/batch_exports/conftest.py:0: note: def run(self) -> Coroutine[Any, Any, None] @@ -849,6 +832,49 @@ posthog/temporal/tests/external_data/test_external_data_job.py:0: error: Invalid posthog/temporal/tests/external_data/test_external_data_job.py:0: error: Invalid index type "str" for "dict[Type, Sequence[str]]"; expected type "Type" [index] posthog/temporal/tests/external_data/test_external_data_job.py:0: error: Invalid index type "str" for "dict[Type, Sequence[str]]"; expected type "Type" [index] posthog/temporal/tests/data_imports/test_end_to_end.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Not all union combinations were tried because there are too many unions [misc] +posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument 2 to "source" has incompatible type "str | None"; expected "str" [arg-type] +posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument 3 to "source" has incompatible type "str | None"; expected "str" [arg-type] +posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument 4 to "source" has incompatible type "int | None"; expected "int" [arg-type] +posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument 6 to "source" has incompatible type "Schema | None"; expected "Schema" [arg-type] +posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument 7 to "source" has incompatible type "Literal['evolve', 'discard_value', 'freeze', 'discard_row'] | TSchemaContractDict | None"; expected "Literal['evolve', 'discard_value', 'freeze', 'discard_row'] | TSchemaContractDict" [arg-type] +posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument 8 to "source" has incompatible type "type[BaseConfiguration] | None"; expected "type[BaseConfiguration]" [arg-type] +posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument 1 to "build_resource_dependency_graph" has incompatible type "EndpointResourceBase | None"; expected "EndpointResourceBase" [arg-type] +posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Incompatible types in assignment (expression has type "list[str] | None", variable has type "list[str]") [assignment] +posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument 1 to "setup_incremental_object" has incompatible type "dict[str, ResolveParamConfig | IncrementalParamConfig | Any] | None"; expected "dict[str, Any]" [arg-type] +posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument "base_url" to "RESTClient" has incompatible type "str | None"; expected "str" [arg-type] +posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Argument 1 to "exclude_keys" has incompatible type "dict[str, ResolveParamConfig | IncrementalParamConfig | Any] | None"; expected "Mapping[str, Any]" [arg-type] +posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Incompatible default for argument "resolved_param" (default has type "ResolvedParam | None", argument has type "ResolvedParam") [assignment] +posthog/temporal/data_imports/pipelines/rest_source/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/api/test/test_team.py:0: error: "HttpResponse" has no attribute "json" [attr-defined] +posthog/api/test/test_team.py:0: error: "HttpResponse" has no attribute "json" [attr-defined] +posthog/test/test_middleware.py:0: error: Incompatible types in assignment (expression has type "_MonkeyPatchedWSGIResponse", variable has type "_MonkeyPatchedResponse") [assignment] +posthog/temporal/data_imports/pipelines/zendesk/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/zendesk/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/zendesk/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/zendesk/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/zendesk/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/zendesk/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/zendesk/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/zendesk/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/zendesk/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/vitally/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/vitally/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/vitally/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/vitally/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/vitally/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/vitally/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/vitally/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/vitally/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/vitally/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/stripe/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/stripe/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/stripe/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/stripe/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/stripe/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/stripe/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/stripe/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/pipelines/stripe/__init__.py:0: error: Unused "type: ignore" comment [unused-ignore] posthog/management/commands/test/test_create_batch_export_from_app.py:0: error: Incompatible return value type (got "dict[str, Collection[str]]", expected "dict[str, str]") [return-value] posthog/management/commands/test/test_create_batch_export_from_app.py:0: error: Incompatible types in assignment (expression has type "dict[str, Collection[str]]", variable has type "dict[str, str]") [assignment] posthog/management/commands/test/test_create_batch_export_from_app.py:0: error: Unpacked dict entry 1 has incompatible type "str"; expected "SupportsKeysAndGetItem[str, str]" [dict-item] @@ -890,6 +916,22 @@ posthog/api/test/batch_exports/test_update.py:0: error: Value of type "BatchExpo posthog/api/test/batch_exports/test_update.py:0: error: Value of type "BatchExport" is not indexable [index] posthog/api/test/batch_exports/test_update.py:0: error: Value of type "BatchExport" is not indexable [index] posthog/api/test/batch_exports/test_pause.py:0: error: "batch_export_delete_schedule" does not return a value (it only ever returns None) [func-returns-value] +posthog/warehouse/api/external_data_schema.py:0: error: Incompatible return value type (got "str | None", expected "SyncType | None") [return-value] +posthog/warehouse/api/external_data_schema.py:0: error: Argument 1 to "get_sql_schemas_for_source_type" has incompatible type "str"; expected "Type" [arg-type] +posthog/warehouse/api/external_data_schema.py:0: error: No overload variant of "get" of "dict" matches argument type "str" [call-overload] +posthog/warehouse/api/external_data_schema.py:0: note: Possible overload variants: +posthog/warehouse/api/external_data_schema.py:0: note: def get(self, Type, /) -> dict[str, list[IncrementalField]] | None +posthog/warehouse/api/external_data_schema.py:0: note: def get(self, Type, dict[str, list[IncrementalField]], /) -> dict[str, list[IncrementalField]] +posthog/warehouse/api/external_data_schema.py:0: note: def [_T] get(self, Type, _T, /) -> dict[str, list[IncrementalField]] | _T +posthog/warehouse/api/table.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/warehouse/api/table.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/warehouse/api/table.py:0: error: Unused "type: ignore" comment [unused-ignore] +posthog/temporal/data_imports/external_data_job.py:0: error: Argument "status" to "update_external_job_status" has incompatible type "str"; expected "Status" [arg-type] +posthog/temporal/tests/external_data/test_external_data_job.py:0: error: Invalid index type "str" for "dict[Type, Sequence[str]]"; expected type "Type" [index] +posthog/temporal/tests/external_data/test_external_data_job.py:0: error: Invalid index type "str" for "dict[Type, Sequence[str]]"; expected type "Type" [index] +posthog/temporal/tests/external_data/test_external_data_job.py:0: error: Invalid index type "str" for "dict[Type, Sequence[str]]"; expected type "Type" [index] +posthog/temporal/tests/external_data/test_external_data_job.py:0: error: Invalid index type "str" for "dict[Type, Sequence[str]]"; expected type "Type" [index] +posthog/temporal/tests/data_imports/test_end_to_end.py:0: error: Unused "type: ignore" comment [unused-ignore] posthog/api/query.py:0: error: Statement is unreachable [unreachable] posthog/api/test/test_capture.py:0: error: Statement is unreachable [unreachable] posthog/api/test/test_capture.py:0: error: Incompatible return value type (got "_MonkeyPatchedWSGIResponse", expected "HttpResponse") [return-value] diff --git a/package.json b/package.json index ca06eb6507b3d..4da2f9aff7c1a 100644 --- a/package.json +++ b/package.json @@ -161,7 +161,7 @@ "pmtiles": "^2.11.0", "postcss": "^8.4.31", "postcss-preset-env": "^9.3.0", - "posthog-js": "1.194.3", + "posthog-js": "1.194.5", "posthog-js-lite": "3.0.0", "prettier": "^2.8.8", "prop-types": "^15.7.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 30cdc4508d7f0..3097e83e0a10a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -305,8 +305,8 @@ dependencies: specifier: ^9.3.0 version: 9.3.0(postcss@8.4.31) posthog-js: - specifier: 1.194.3 - version: 1.194.3 + specifier: 1.194.5 + version: 1.194.5 posthog-js-lite: specifier: 3.0.0 version: 3.0.0 @@ -17829,8 +17829,8 @@ packages: resolution: {integrity: sha512-dyajjnfzZD1tht4N7p7iwf7nBnR1MjVaVu+MKr+7gBgA39bn28wizCIJZztZPtHy4PY0YwtSGgwfBCuG/hnHgA==} dev: false - /posthog-js@1.194.3: - resolution: {integrity: sha512-/YFpBMqZzRpywa07QeoaIojdrUDijFajT4gZBSCFUBuZA5BN5xr5S1spsvtpT7E4RjkQSVgRvUngI4W19csgQw==} + /posthog-js@1.194.5: + resolution: {integrity: sha512-bYa20TkwzkDsr2y3iCiJNto/bthkYkmHZopIOXzFEw7KeB581Y1WueaOry5MFHEwnpZuomqEmcMQGBAoWvv8VA==} dependencies: core-js: 3.39.0 fflate: 0.4.8 diff --git a/posthog/api/cohort.py b/posthog/api/cohort.py index 2d5d557f52b0b..762e9a5b4a894 100644 --- a/posthog/api/cohort.py +++ b/posthog/api/cohort.py @@ -14,6 +14,7 @@ ) from posthog.models.person.person import PersonDistinctId from posthog.models.property.property import Property, PropertyGroup +from posthog.models.team.team import Team from posthog.queries.base import property_group_to_Q from posthog.metrics import LABEL_TEAM_ID from posthog.renderers import SafeJSONRenderer @@ -22,8 +23,6 @@ from django.conf import settings from django.db.models import QuerySet, Prefetch, prefetch_related_objects, OuterRef, Subquery -from django.db.models.expressions import F -from django.utils import timezone from rest_framework import serializers, viewsets, request, status from posthog.api.utils import action from rest_framework.exceptions import ValidationError @@ -52,7 +51,7 @@ from posthog.hogql.constants import CSV_EXPORT_LIMIT from posthog.event_usage import report_user_action from posthog.hogql.context import HogQLContext -from posthog.models import Cohort, FeatureFlag, User, Person +from posthog.models import Cohort, FeatureFlag, Person from posthog.models.async_deletion import AsyncDeletion, DeletionType from posthog.models.cohort.util import get_dependent_cohorts, print_cohort_hogql_query from posthog.models.cohort import CohortOrEmpty @@ -139,14 +138,14 @@ def _handle_static(self, cohort: Cohort, context: dict, validated_data: dict) -> elif context.get("from_feature_flag_key"): insert_cohort_from_feature_flag.delay(cohort.pk, context["from_feature_flag_key"], self.context["team_id"]) elif validated_data.get("query"): - insert_cohort_from_query.delay(cohort.pk) + insert_cohort_from_query.delay(cohort.pk, self.context["team_id"]) else: filter_data = request.GET.dict() existing_cohort_id = context.get("from_cohort_id") if existing_cohort_id: filter_data = {**filter_data, "from_cohort_id": existing_cohort_id} if filter_data: - insert_cohort_from_insight_filter.delay(cohort.pk, filter_data) + insert_cohort_from_insight_filter.delay(cohort.pk, filter_data, self.context["team_id"]) def create(self, validated_data: dict, *args: Any, **kwargs: Any) -> Cohort: request = self.context["request"] @@ -173,7 +172,7 @@ def _calculate_static_by_csv(self, file, cohort: Cohort) -> None: decoded_file = file.read().decode("utf-8").splitlines() reader = csv.reader(decoded_file) distinct_ids_and_emails = [row[0] for row in reader if len(row) > 0 and row] - calculate_cohort_from_list.delay(cohort.pk, distinct_ids_and_emails) + calculate_cohort_from_list.delay(cohort.pk, distinct_ids_and_emails, team_id=self.context["team_id"]) def validate_query(self, query: Optional[dict]) -> Optional[dict]: if not query: @@ -195,7 +194,7 @@ def validate_filters(self, request_filters: dict): instance = cast(Cohort, self.instance) cohort_id = instance.pk flags: QuerySet[FeatureFlag] = FeatureFlag.objects.filter( - team_id=self.context["team_id"], active=True, deleted=False + team__project_id=self.context["project_id"], active=True, deleted=False ) cohort_used_in_flags = len([flag for flag in flags if cohort_id in flag.get_cohort_ids()]) > 0 @@ -208,7 +207,7 @@ def validate_filters(self, request_filters: dict): ) if prop.type == "cohort": - nested_cohort = Cohort.objects.get(pk=prop.value, team_id=self.context["team_id"]) + nested_cohort = Cohort.objects.get(pk=prop.value, team__project_id=self.context["project_id"]) dependent_cohorts = get_dependent_cohorts(nested_cohort) for dependent_cohort in [nested_cohort, *dependent_cohorts]: if ( @@ -229,7 +228,6 @@ def validate_filters(self, request_filters: dict): def update(self, cohort: Cohort, validated_data: dict, *args: Any, **kwargs: Any) -> Cohort: # type: ignore request = self.context["request"] - user = cast(User, request.user) cohort.name = validated_data.get("name", cohort.name) cohort.description = validated_data.get("description", cohort.description) @@ -240,22 +238,29 @@ def update(self, cohort: Cohort, validated_data: dict, *args: Any, **kwargs: Any is_deletion_change = deleted_state is not None and cohort.deleted != deleted_state if is_deletion_change: + relevant_team_ids = Team.objects.filter(project_id=cohort.team.project_id).values_list("id", flat=True) cohort.deleted = deleted_state if deleted_state: # De-attach from experiments cohort.experiment_set.set([]) - AsyncDeletion.objects.get_or_create( - deletion_type=DeletionType.Cohort_full, - team_id=cohort.team.pk, - key=f"{cohort.pk}_{cohort.version}", - created_by=user, + AsyncDeletion.objects.bulk_create( + [ + AsyncDeletion( + deletion_type=DeletionType.Cohort_full, + team_id=team_id, + # Only appending `team_id` if it's not the same as the cohort's `team_id``, so that + # the migration to environments does not accidentally cause duplicate `AsyncDeletion`s + key=f"{cohort.pk}_{cohort.version}{('_'+team_id) if team_id != cohort.team_id else ''}", + ) + for team_id in relevant_team_ids + ], + ignore_conflicts=True, ) else: AsyncDeletion.objects.filter( deletion_type=DeletionType.Cohort_full, - team_id=cohort.team.pk, - key=f"{cohort.pk}_{cohort.version}", + key__startswith=f"{cohort.pk}_{cohort.version}", # We target this _prefix_, so all teams are covered ).delete() elif not cohort.is_static: cohort.is_calculating = True @@ -475,12 +480,12 @@ def perform_update(self, serializer): class LegacyCohortViewSet(CohortViewSet): - param_derived_from_user_current_team = "project_id" + param_derived_from_user_current_team = "team_id" def will_create_loops(cohort: Cohort) -> bool: # Loops can only be formed when trying to update a Cohort, not when creating one - team_id = cohort.team_id + project_id = cohort.team.project_id # We can model this as a directed graph, where each node is a Cohort and each edge is a reference to another Cohort # There's a loop only if there's a cycle in the directed graph. The "directed" bit is important. @@ -501,7 +506,7 @@ def dfs_loop_helper(current_cohort: Cohort, seen_cohorts, cohorts_on_path): return True elif property.value not in seen_cohorts: try: - nested_cohort = Cohort.objects.get(pk=property.value, team_id=team_id) + nested_cohort = Cohort.objects.get(pk=property.value, team__project_id=project_id) except Cohort.DoesNotExist: raise ValidationError("Invalid Cohort ID in filter") @@ -514,23 +519,21 @@ def dfs_loop_helper(current_cohort: Cohort, seen_cohorts, cohorts_on_path): return dfs_loop_helper(cohort, set(), set()) -def insert_cohort_people_into_pg(cohort: Cohort): +def insert_cohort_people_into_pg(cohort: Cohort, *, team_id: int): ids = sync_execute( - "SELECT person_id FROM {} where team_id = %(team_id)s AND cohort_id = %(cohort_id)s".format( - PERSON_STATIC_COHORT_TABLE - ), - {"cohort_id": cohort.pk, "team_id": cohort.team.pk}, + f"SELECT person_id FROM {PERSON_STATIC_COHORT_TABLE} where team_id = %(team_id)s AND cohort_id = %(cohort_id)s", + {"cohort_id": cohort.pk, "team_id": team_id}, ) - cohort.insert_users_list_by_uuid(items=[str(id[0]) for id in ids]) + cohort.insert_users_list_by_uuid(items=[str(id[0]) for id in ids], team_id=team_id) -def insert_cohort_query_actors_into_ch(cohort: Cohort): - context = HogQLContext(enable_select_queries=True, team_id=cohort.team.pk) - query = print_cohort_hogql_query(cohort, context) - insert_actors_into_cohort_by_query(cohort, query, {}, context) +def insert_cohort_query_actors_into_ch(cohort: Cohort, *, team: Team): + context = HogQLContext(enable_select_queries=True, team_id=team.id) + query = print_cohort_hogql_query(cohort, context, team=team) + insert_actors_into_cohort_by_query(cohort, query, {}, context, team_id=team.id) -def insert_cohort_actors_into_ch(cohort: Cohort, filter_data: dict): +def insert_cohort_actors_into_ch(cohort: Cohort, filter_data: dict, *, team_id: int): from_existing_cohort_id = filter_data.get("from_cohort_id") context: HogQLContext @@ -543,7 +546,7 @@ def insert_cohort_actors_into_ch(cohort: Cohort, filter_data: dict): ORDER BY person_id """ params = { - "team_id": cohort.team.pk, + "team_id": team_id, "from_cohort_id": existing_cohort.pk, "version": existing_cohort.version, } @@ -590,48 +593,36 @@ def insert_cohort_actors_into_ch(cohort: Cohort, filter_data: dict): else: query, params = query_builder.actor_query(limit_actors=False) - insert_actors_into_cohort_by_query(cohort, query, params, context) - - -def insert_actors_into_cohort_by_query(cohort: Cohort, query: str, params: dict[str, Any], context: HogQLContext): - try: - sync_execute( - INSERT_COHORT_ALL_PEOPLE_THROUGH_PERSON_ID.format(cohort_table=PERSON_STATIC_COHORT_TABLE, query=query), - { - "cohort_id": cohort.pk, - "_timestamp": datetime.now(), - "team_id": cohort.team.pk, - **context.values, - **params, - }, - ) - - cohort.is_calculating = False - cohort.last_calculation = timezone.now() - cohort.errors_calculating = 0 - cohort.last_error_at = None - cohort.save(update_fields=["errors_calculating", "last_calculation", "is_calculating", "last_error_at"]) - except Exception as err: - if settings.DEBUG: - raise - cohort.is_calculating = False - cohort.errors_calculating = F("errors_calculating") + 1 - cohort.last_error_at = timezone.now() - cohort.save(update_fields=["errors_calculating", "is_calculating", "last_error_at"]) - capture_exception(err) + insert_actors_into_cohort_by_query(cohort, query, params, context, team_id=team_id) + + +def insert_actors_into_cohort_by_query( + cohort: Cohort, query: str, params: dict[str, Any], context: HogQLContext, *, team_id: int +): + sync_execute( + INSERT_COHORT_ALL_PEOPLE_THROUGH_PERSON_ID.format(cohort_table=PERSON_STATIC_COHORT_TABLE, query=query), + { + "cohort_id": cohort.pk, + "_timestamp": datetime.now(), + "team_id": team_id, + **context.values, + **params, + }, + ) def get_cohort_actors_for_feature_flag(cohort_id: int, flag: str, team_id: int, batchsize: int = 1_000): # :TODO: Find a way to incorporate this into the same code path as feature flag evaluation + team: Team = Team.objects.get(pk=team_id) try: - feature_flag = FeatureFlag.objects.get(team_id=team_id, key=flag) + feature_flag = FeatureFlag.objects.get(team__project_id=team.project_id, key=flag) except FeatureFlag.DoesNotExist: return [] if not feature_flag.active or feature_flag.deleted or feature_flag.aggregation_group_type_index is not None: return [] - cohort = Cohort.objects.get(pk=cohort_id, team_id=team_id) + cohort = Cohort.objects.get(pk=cohort_id, team__project_id=team.project_id) matcher_cache = FlagsMatcherCache(team_id) uuids_to_add_to_cohort = [] cohorts_cache: dict[int, CohortOrEmpty] = {} @@ -640,7 +631,9 @@ def get_cohort_actors_for_feature_flag(cohort_id: int, flag: str, team_id: int, # TODO: Consider disabling flags with cohorts for creating static cohorts # because this is currently a lot more inefficient for flag matching, # as we're required to go to the database for each person. - cohorts_cache = {cohort.pk: cohort for cohort in Cohort.objects.filter(team_id=team_id, deleted=False)} + cohorts_cache = { + cohort.pk: cohort for cohort in Cohort.objects.filter(team__project_id=team.project_id, deleted=False) + } default_person_properties = {} for condition in feature_flag.conditions: @@ -727,7 +720,7 @@ def get_cohort_actors_for_feature_flag(cohort_id: int, flag: str, team_id: int, if len(uuids_to_add_to_cohort) >= batchsize: cohort.insert_users_list_by_uuid( - uuids_to_add_to_cohort, insert_in_clickhouse=True, batchsize=batchsize + uuids_to_add_to_cohort, insert_in_clickhouse=True, batchsize=batchsize, team_id=team_id ) uuids_to_add_to_cohort = [] @@ -735,7 +728,9 @@ def get_cohort_actors_for_feature_flag(cohort_id: int, flag: str, team_id: int, batch_of_persons = queryset[start : start + batchsize] if len(uuids_to_add_to_cohort) > 0: - cohort.insert_users_list_by_uuid(uuids_to_add_to_cohort, insert_in_clickhouse=True, batchsize=batchsize) + cohort.insert_users_list_by_uuid( + uuids_to_add_to_cohort, insert_in_clickhouse=True, batchsize=batchsize, team_id=team_id + ) except Exception as err: if settings.DEBUG or settings.TEST: diff --git a/posthog/api/feature_flag.py b/posthog/api/feature_flag.py index 435ccbe1cf27f..25c71d898950c 100644 --- a/posthog/api/feature_flag.py +++ b/posthog/api/feature_flag.py @@ -746,7 +746,7 @@ def local_evaluation(self, request: request.Request, **kwargs): "group_type_mapping": { str(row.group_type_index): row.group_type for row in GroupTypeMapping.objects.db_manager(DATABASE_FOR_LOCAL_EVALUATION).filter( - team_id=self.team_id + project_id=self.project_id ) }, "cohorts": cohorts, diff --git a/posthog/api/organization.py b/posthog/api/organization.py index c522ca164c0b9..6fe798479dd7b 100644 --- a/posthog/api/organization.py +++ b/posthog/api/organization.py @@ -5,6 +5,8 @@ from django.shortcuts import get_object_or_404 from rest_framework import exceptions, permissions, serializers, viewsets from rest_framework.request import Request +from rest_framework.response import Response +import posthoganalytics from posthog import settings from posthog.api.routing import TeamAndOrgViewSetMixin @@ -12,7 +14,7 @@ from posthog.auth import PersonalAPIKeyAuthentication from posthog.cloud_utils import is_cloud from posthog.constants import INTERNAL_BOT_EMAIL_SUFFIX, AvailableFeature -from posthog.event_usage import report_organization_deleted +from posthog.event_usage import report_organization_deleted, groups from posthog.models import Organization, User from posthog.models.async_deletion import AsyncDeletion, DeletionType from posthog.rbac.user_access_control import UserAccessControlSerializerMixin @@ -240,3 +242,24 @@ def get_serializer_context(self) -> dict[str, Any]: **super().get_serializer_context(), "user_permissions": UserPermissions(cast(User, self.request.user)), } + + def update(self, request: Request, *args: Any, **kwargs: Any) -> Response: + if "enforce_2fa" in request.data: + enforce_2fa_value = request.data["enforce_2fa"] + organization = self.get_object() + user = cast(User, request.user) + + # Add capture event for 2FA enforcement change + posthoganalytics.capture( + str(user.distinct_id), + "organization 2fa enforcement toggled", + properties={ + "enabled": enforce_2fa_value, + "organization_id": str(organization.id), + "organization_name": organization.name, + "user_role": user.organization_memberships.get(organization=organization).level, + }, + groups=groups(organization), + ) + + return super().update(request, *args, **kwargs) diff --git a/posthog/api/project.py b/posthog/api/project.py index 9bc0d91cec45f..b3b808f520430 100644 --- a/posthog/api/project.py +++ b/posthog/api/project.py @@ -191,7 +191,7 @@ def get_effective_membership_level(self, project: Project) -> Optional[Organizat return self.user_permissions.team(team).effective_membership_level def get_has_group_types(self, project: Project) -> bool: - return GroupTypeMapping.objects.filter(team_id=project.id).exists() + return GroupTypeMapping.objects.filter(project_id=project.id).exists() def get_live_events_token(self, project: Project) -> Optional[str]: team = project.teams.get(pk=project.pk) diff --git a/posthog/api/team.py b/posthog/api/team.py index f2486a68fe8a0..b473fc490ec7e 100644 --- a/posthog/api/team.py +++ b/posthog/api/team.py @@ -234,7 +234,7 @@ def get_effective_membership_level(self, team: Team) -> Optional[OrganizationMem return self.user_permissions.team(team).effective_membership_level def get_has_group_types(self, team: Team) -> bool: - return GroupTypeMapping.objects.filter(team_id=team.id).exists() + return GroupTypeMapping.objects.filter(project_id=team.project_id).exists() def get_live_events_token(self, team: Team) -> Optional[str]: return encode_jwt( diff --git a/posthog/api/test/__snapshots__/test_cohort.ambr b/posthog/api/test/__snapshots__/test_cohort.ambr index f1fe8c5d00333..2a4e7cdcc4d86 100644 --- a/posthog/api/test/__snapshots__/test_cohort.ambr +++ b/posthog/api/test/__snapshots__/test_cohort.ambr @@ -98,11 +98,14 @@ # name: TestCohort.test_async_deletion_of_cohort.3 ''' /* user_id:0 celery:posthog.tasks.calculate_cohort.clear_stale_cohort */ - SELECT count() + SELECT team_id, + count() AS stale_people_count FROM cohortpeople - WHERE team_id = 99999 + WHERE team_id IN [1, 2, 3, 4, 5 /* ... */] AND cohort_id = 99999 AND version < 1 + GROUP BY team_id + HAVING stale_people_count > 0 ''' # --- # name: TestCohort.test_async_deletion_of_cohort.4 @@ -163,11 +166,14 @@ # name: TestCohort.test_async_deletion_of_cohort.7 ''' /* user_id:0 celery:posthog.tasks.calculate_cohort.clear_stale_cohort */ - SELECT count() + SELECT team_id, + count() AS stale_people_count FROM cohortpeople - WHERE team_id = 99999 + WHERE team_id IN [1, 2, 3, 4, 5 /* ... */] AND cohort_id = 99999 AND version < 2 + GROUP BY team_id + HAVING stale_people_count > 0 ''' # --- # name: TestCohort.test_async_deletion_of_cohort.8 diff --git a/posthog/api/test/__snapshots__/test_decide.ambr b/posthog/api/test/__snapshots__/test_decide.ambr index 56f6257a978d2..7ab6aaee06295 100644 --- a/posthog/api/test/__snapshots__/test_decide.ambr +++ b/posthog/api/test/__snapshots__/test_decide.ambr @@ -519,7 +519,7 @@ ''' SELECT 1 AS "a" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 LIMIT 1 ''' # --- diff --git a/posthog/api/test/__snapshots__/test_feature_flag.ambr b/posthog/api/test/__snapshots__/test_feature_flag.ambr index b51af7a796f7d..8250f4f667393 100644 --- a/posthog/api/test/__snapshots__/test_feature_flag.ambr +++ b/posthog/api/test/__snapshots__/test_feature_flag.ambr @@ -336,6 +336,69 @@ ''' # --- # name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_iterator + ''' + SELECT "posthog_team"."id", + "posthog_team"."uuid", + "posthog_team"."organization_id", + "posthog_team"."project_id", + "posthog_team"."api_token", + "posthog_team"."app_urls", + "posthog_team"."name", + "posthog_team"."slack_incoming_webhook", + "posthog_team"."created_at", + "posthog_team"."updated_at", + "posthog_team"."anonymize_ips", + "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", + "posthog_team"."ingested_event", + "posthog_team"."autocapture_opt_out", + "posthog_team"."autocapture_web_vitals_opt_in", + "posthog_team"."autocapture_web_vitals_allowed_metrics", + "posthog_team"."autocapture_exceptions_opt_in", + "posthog_team"."autocapture_exceptions_errors_to_ignore", + "posthog_team"."person_processing_opt_out", + "posthog_team"."session_recording_opt_in", + "posthog_team"."session_recording_sample_rate", + "posthog_team"."session_recording_minimum_duration_milliseconds", + "posthog_team"."session_recording_linked_flag", + "posthog_team"."session_recording_network_payload_capture_config", + "posthog_team"."session_recording_url_trigger_config", + "posthog_team"."session_recording_url_blocklist_config", + "posthog_team"."session_recording_event_trigger_config", + "posthog_team"."session_replay_config", + "posthog_team"."survey_config", + "posthog_team"."capture_console_log_opt_in", + "posthog_team"."capture_performance_opt_in", + "posthog_team"."capture_dead_clicks", + "posthog_team"."surveys_opt_in", + "posthog_team"."heatmaps_opt_in", + "posthog_team"."session_recording_version", + "posthog_team"."signup_token", + "posthog_team"."is_demo", + "posthog_team"."access_control", + "posthog_team"."week_start_day", + "posthog_team"."inject_web_apps", + "posthog_team"."test_account_filters", + "posthog_team"."test_account_filters_default_checked", + "posthog_team"."path_cleaning_filters", + "posthog_team"."timezone", + "posthog_team"."data_attributes", + "posthog_team"."person_display_name_properties", + "posthog_team"."live_events_columns", + "posthog_team"."recording_domains", + "posthog_team"."primary_dashboard_id", + "posthog_team"."extra_settings", + "posthog_team"."modifiers", + "posthog_team"."correlation_config", + "posthog_team"."session_recording_retention_period_days", + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" + FROM "posthog_team" + WHERE "posthog_team"."id" = 99999 + LIMIT 21 + ''' +# --- +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_iterator.1 ''' SELECT "posthog_featureflag"."id", "posthog_featureflag"."key", @@ -353,12 +416,13 @@ "posthog_featureflag"."usage_dashboard_id", "posthog_featureflag"."has_enriched_analytics" FROM "posthog_featureflag" + INNER JOIN "posthog_team" ON ("posthog_featureflag"."team_id" = "posthog_team"."id") WHERE ("posthog_featureflag"."key" = 'some-feature2' - AND "posthog_featureflag"."team_id" = 99999) + AND "posthog_team"."project_id" = 99999) LIMIT 21 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_iterator.1 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_iterator.2 ''' SELECT "posthog_cohort"."id", "posthog_cohort"."name", @@ -379,12 +443,13 @@ "posthog_cohort"."is_static", "posthog_cohort"."groups" FROM "posthog_cohort" + INNER JOIN "posthog_team" ON ("posthog_cohort"."team_id" = "posthog_team"."id") WHERE ("posthog_cohort"."id" = 99999 - AND "posthog_cohort"."team_id" = 99999) + AND "posthog_team"."project_id" = 99999) LIMIT 21 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_iterator.2 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_iterator.3 ''' SELECT "posthog_person"."id", "posthog_person"."created_at", @@ -405,7 +470,7 @@ LIMIT 2 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_iterator.3 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_iterator.4 ''' SELECT "posthog_persondistinctid"."id", "posthog_persondistinctid"."team_id", @@ -425,7 +490,7 @@ 5 /* ... */)) ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_iterator.4 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_iterator.5 ''' SELECT "posthog_person"."uuid" FROM "posthog_person" @@ -440,76 +505,6 @@ LIMIT 1))) ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_iterator.5 - ''' - SELECT "posthog_team"."id", - "posthog_team"."uuid", - "posthog_team"."organization_id", - "posthog_team"."project_id", - "posthog_team"."api_token", - "posthog_team"."app_urls", - "posthog_team"."name", - "posthog_team"."slack_incoming_webhook", - "posthog_team"."created_at", - "posthog_team"."updated_at", - "posthog_team"."anonymize_ips", - "posthog_team"."completed_snippet_onboarding", - "posthog_team"."has_completed_onboarding_for", - "posthog_team"."ingested_event", - "posthog_team"."autocapture_opt_out", - "posthog_team"."autocapture_web_vitals_opt_in", - "posthog_team"."autocapture_web_vitals_allowed_metrics", - "posthog_team"."autocapture_exceptions_opt_in", - "posthog_team"."autocapture_exceptions_errors_to_ignore", - "posthog_team"."person_processing_opt_out", - "posthog_team"."session_recording_opt_in", - "posthog_team"."session_recording_sample_rate", - "posthog_team"."session_recording_minimum_duration_milliseconds", - "posthog_team"."session_recording_linked_flag", - "posthog_team"."session_recording_network_payload_capture_config", - "posthog_team"."session_recording_url_trigger_config", - "posthog_team"."session_recording_url_blocklist_config", - "posthog_team"."session_recording_event_trigger_config", - "posthog_team"."session_replay_config", - "posthog_team"."survey_config", - "posthog_team"."capture_console_log_opt_in", - "posthog_team"."capture_performance_opt_in", - "posthog_team"."capture_dead_clicks", - "posthog_team"."surveys_opt_in", - "posthog_team"."heatmaps_opt_in", - "posthog_team"."session_recording_version", - "posthog_team"."signup_token", - "posthog_team"."is_demo", - "posthog_team"."access_control", - "posthog_team"."week_start_day", - "posthog_team"."inject_web_apps", - "posthog_team"."test_account_filters", - "posthog_team"."test_account_filters_default_checked", - "posthog_team"."path_cleaning_filters", - "posthog_team"."timezone", - "posthog_team"."data_attributes", - "posthog_team"."person_display_name_properties", - "posthog_team"."live_events_columns", - "posthog_team"."recording_domains", - "posthog_team"."primary_dashboard_id", - "posthog_team"."extra_settings", - "posthog_team"."modifiers", - "posthog_team"."correlation_config", - "posthog_team"."session_recording_retention_period_days", - "posthog_team"."plugins_opt_in", - "posthog_team"."opt_out_capture", - "posthog_team"."event_names", - "posthog_team"."event_names_with_usage", - "posthog_team"."event_properties", - "posthog_team"."event_properties_with_usage", - "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id", - "posthog_team"."external_data_workspace_last_synced_at" - FROM "posthog_team" - WHERE "posthog_team"."id" = 99999 - LIMIT 21 - ''' -# --- # name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_iterator.6 ''' SELECT "posthog_person"."id", @@ -589,72 +584,6 @@ ''' # --- # name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too - ''' - SELECT "posthog_featureflag"."id", - "posthog_featureflag"."key", - "posthog_featureflag"."name", - "posthog_featureflag"."filters", - "posthog_featureflag"."rollout_percentage", - "posthog_featureflag"."team_id", - "posthog_featureflag"."created_by_id", - "posthog_featureflag"."created_at", - "posthog_featureflag"."deleted", - "posthog_featureflag"."active", - "posthog_featureflag"."rollback_conditions", - "posthog_featureflag"."performed_rollback", - "posthog_featureflag"."ensure_experience_continuity", - "posthog_featureflag"."usage_dashboard_id", - "posthog_featureflag"."has_enriched_analytics" - FROM "posthog_featureflag" - WHERE ("posthog_featureflag"."key" = 'some-feature-new' - AND "posthog_featureflag"."team_id" = 99999) - LIMIT 21 - ''' -# --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.1 - ''' - SELECT "posthog_cohort"."id", - "posthog_cohort"."name", - "posthog_cohort"."description", - "posthog_cohort"."team_id", - "posthog_cohort"."deleted", - "posthog_cohort"."filters", - "posthog_cohort"."query", - "posthog_cohort"."version", - "posthog_cohort"."pending_version", - "posthog_cohort"."count", - "posthog_cohort"."created_by_id", - "posthog_cohort"."created_at", - "posthog_cohort"."is_calculating", - "posthog_cohort"."last_calculation", - "posthog_cohort"."errors_calculating", - "posthog_cohort"."last_error_at", - "posthog_cohort"."is_static", - "posthog_cohort"."groups" - FROM "posthog_cohort" - WHERE ("posthog_cohort"."id" = 99999 - AND "posthog_cohort"."team_id" = 99999) - LIMIT 21 - ''' -# --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.10 - ''' - SELECT "posthog_person"."uuid" - FROM "posthog_person" - WHERE ("posthog_person"."team_id" = 99999 - AND "posthog_person"."uuid" IN ('00000000000040008000000000000000'::uuid, - '00000000000040008000000000000001'::uuid, - '00000000000040008000000000000002'::uuid, - '00000000000040008000000000000003'::uuid) - AND NOT (EXISTS - (SELECT 1 AS "a" - FROM "posthog_cohortpeople" U1 - WHERE (U1."cohort_id" = 99999 - AND U1."person_id" = ("posthog_person"."id")) - LIMIT 1))) - ''' -# --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.11 ''' SELECT "posthog_team"."id", "posthog_team"."uuid", @@ -710,13 +639,6 @@ "posthog_team"."modifiers", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."plugins_opt_in", - "posthog_team"."opt_out_capture", - "posthog_team"."event_names", - "posthog_team"."event_names_with_usage", - "posthog_team"."event_properties", - "posthog_team"."event_properties_with_usage", - "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" @@ -724,32 +646,31 @@ LIMIT 21 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.2 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.1 ''' - SELECT "posthog_cohort"."id", - "posthog_cohort"."name", - "posthog_cohort"."description", - "posthog_cohort"."team_id", - "posthog_cohort"."deleted", - "posthog_cohort"."filters", - "posthog_cohort"."query", - "posthog_cohort"."version", - "posthog_cohort"."pending_version", - "posthog_cohort"."count", - "posthog_cohort"."created_by_id", - "posthog_cohort"."created_at", - "posthog_cohort"."is_calculating", - "posthog_cohort"."last_calculation", - "posthog_cohort"."errors_calculating", - "posthog_cohort"."last_error_at", - "posthog_cohort"."is_static", - "posthog_cohort"."groups" - FROM "posthog_cohort" - WHERE (NOT "posthog_cohort"."deleted" - AND "posthog_cohort"."team_id" = 99999) + SELECT "posthog_featureflag"."id", + "posthog_featureflag"."key", + "posthog_featureflag"."name", + "posthog_featureflag"."filters", + "posthog_featureflag"."rollout_percentage", + "posthog_featureflag"."team_id", + "posthog_featureflag"."created_by_id", + "posthog_featureflag"."created_at", + "posthog_featureflag"."deleted", + "posthog_featureflag"."active", + "posthog_featureflag"."rollback_conditions", + "posthog_featureflag"."performed_rollback", + "posthog_featureflag"."ensure_experience_continuity", + "posthog_featureflag"."usage_dashboard_id", + "posthog_featureflag"."has_enriched_analytics" + FROM "posthog_featureflag" + INNER JOIN "posthog_team" ON ("posthog_featureflag"."team_id" = "posthog_team"."id") + WHERE ("posthog_featureflag"."key" = 'some-feature-new' + AND "posthog_team"."project_id" = 99999) + LIMIT 21 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.3 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.10 ''' SELECT "posthog_person"."id", "posthog_person"."created_at", @@ -786,9 +707,119 @@ AND NOT (("posthog_person"."properties" -> 'key') = 'null'::jsonb)))) ORDER BY "posthog_person"."id" ASC LIMIT 1000 + OFFSET 1000 + ''' +# --- +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.11 + ''' + SELECT "posthog_person"."uuid" + FROM "posthog_person" + WHERE ("posthog_person"."team_id" = 99999 + AND "posthog_person"."uuid" IN ('00000000000040008000000000000000'::uuid, + '00000000000040008000000000000001'::uuid, + '00000000000040008000000000000002'::uuid, + '00000000000040008000000000000003'::uuid) + AND NOT (EXISTS + (SELECT 1 AS "a" + FROM "posthog_cohortpeople" U1 + WHERE (U1."cohort_id" = 99999 + AND U1."person_id" = ("posthog_person"."id")) + LIMIT 1))) + ''' +# --- +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.2 + ''' + SELECT "posthog_cohort"."id", + "posthog_cohort"."name", + "posthog_cohort"."description", + "posthog_cohort"."team_id", + "posthog_cohort"."deleted", + "posthog_cohort"."filters", + "posthog_cohort"."query", + "posthog_cohort"."version", + "posthog_cohort"."pending_version", + "posthog_cohort"."count", + "posthog_cohort"."created_by_id", + "posthog_cohort"."created_at", + "posthog_cohort"."is_calculating", + "posthog_cohort"."last_calculation", + "posthog_cohort"."errors_calculating", + "posthog_cohort"."last_error_at", + "posthog_cohort"."is_static", + "posthog_cohort"."groups" + FROM "posthog_cohort" + INNER JOIN "posthog_team" ON ("posthog_cohort"."team_id" = "posthog_team"."id") + WHERE ("posthog_cohort"."id" = 99999 + AND "posthog_team"."project_id" = 99999) + LIMIT 21 + ''' +# --- +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.3 + ''' + SELECT "posthog_cohort"."id", + "posthog_cohort"."name", + "posthog_cohort"."description", + "posthog_cohort"."team_id", + "posthog_cohort"."deleted", + "posthog_cohort"."filters", + "posthog_cohort"."query", + "posthog_cohort"."version", + "posthog_cohort"."pending_version", + "posthog_cohort"."count", + "posthog_cohort"."created_by_id", + "posthog_cohort"."created_at", + "posthog_cohort"."is_calculating", + "posthog_cohort"."last_calculation", + "posthog_cohort"."errors_calculating", + "posthog_cohort"."last_error_at", + "posthog_cohort"."is_static", + "posthog_cohort"."groups" + FROM "posthog_cohort" + INNER JOIN "posthog_team" ON ("posthog_cohort"."team_id" = "posthog_team"."id") + WHERE (NOT "posthog_cohort"."deleted" + AND "posthog_team"."project_id" = 99999) ''' # --- # name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.4 + ''' + SELECT "posthog_person"."id", + "posthog_person"."created_at", + "posthog_person"."properties_last_updated_at", + "posthog_person"."properties_last_operation", + "posthog_person"."team_id", + "posthog_person"."properties", + "posthog_person"."is_user_id", + "posthog_person"."is_identified", + "posthog_person"."uuid", + "posthog_person"."version" + FROM "posthog_person" + WHERE ("posthog_person"."team_id" = 99999 + AND ((("posthog_person"."properties" -> 'group') = '"none"'::jsonb + AND "posthog_person"."properties" ? 'group' + AND NOT (("posthog_person"."properties" -> 'group') = 'null'::jsonb)) + OR (("posthog_person"."properties" -> 'group2') IN ('1'::jsonb, + '2'::jsonb, + '3'::jsonb) + AND "posthog_person"."properties" ? 'group2' + AND NOT (("posthog_person"."properties" -> 'group2') = 'null'::jsonb)) + OR EXISTS + (SELECT 1 AS "a" + FROM "posthog_cohortpeople" U0 + WHERE (U0."cohort_id" = 99999 + AND U0."cohort_id" = 99999 + AND U0."person_id" = ("posthog_person"."id")) + LIMIT 1) + OR (("posthog_person"."properties" -> 'does-not-exist') = '"none"'::jsonb + AND "posthog_person"."properties" ? 'does-not-exist' + AND NOT (("posthog_person"."properties" -> 'does-not-exist') = 'null'::jsonb)) + OR (("posthog_person"."properties" -> 'key') = '"value"'::jsonb + AND "posthog_person"."properties" ? 'key' + AND NOT (("posthog_person"."properties" -> 'key') = 'null'::jsonb)))) + ORDER BY "posthog_person"."id" ASC + LIMIT 1000 + ''' +# --- +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.5 ''' SELECT "posthog_persondistinctid"."id", "posthog_persondistinctid"."team_id", @@ -808,7 +839,7 @@ 5 /* ... */)) ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.5 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.6 ''' SELECT ("posthog_person"."id" IS NULL OR "posthog_person"."id" IS NULL @@ -827,7 +858,7 @@ AND "posthog_person"."team_id" = 99999) ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.6 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.7 ''' SELECT ("posthog_person"."id" IS NOT NULL OR "posthog_person"."id" IS NULL @@ -846,7 +877,7 @@ AND "posthog_person"."team_id" = 99999) ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.7 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.8 ''' SELECT ("posthog_person"."id" IS NULL OR "posthog_person"."id" IS NOT NULL @@ -865,7 +896,7 @@ AND "posthog_person"."team_id" = 99999) ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.8 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.9 ''' SELECT ("posthog_person"."id" IS NULL OR "posthog_person"."id" IS NULL @@ -884,47 +915,70 @@ AND "posthog_person"."team_id" = 99999) ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_cohort_flag_adds_cohort_props_as_default_too.9 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment ''' - SELECT "posthog_person"."id", - "posthog_person"."created_at", - "posthog_person"."properties_last_updated_at", - "posthog_person"."properties_last_operation", - "posthog_person"."team_id", - "posthog_person"."properties", - "posthog_person"."is_user_id", - "posthog_person"."is_identified", - "posthog_person"."uuid", - "posthog_person"."version" - FROM "posthog_person" - WHERE ("posthog_person"."team_id" = 99999 - AND ((("posthog_person"."properties" -> 'group') = '"none"'::jsonb - AND "posthog_person"."properties" ? 'group' - AND NOT (("posthog_person"."properties" -> 'group') = 'null'::jsonb)) - OR (("posthog_person"."properties" -> 'group2') IN ('1'::jsonb, - '2'::jsonb, - '3'::jsonb) - AND "posthog_person"."properties" ? 'group2' - AND NOT (("posthog_person"."properties" -> 'group2') = 'null'::jsonb)) - OR EXISTS - (SELECT 1 AS "a" - FROM "posthog_cohortpeople" U0 - WHERE (U0."cohort_id" = 99999 - AND U0."cohort_id" = 99999 - AND U0."person_id" = ("posthog_person"."id")) - LIMIT 1) - OR (("posthog_person"."properties" -> 'does-not-exist') = '"none"'::jsonb - AND "posthog_person"."properties" ? 'does-not-exist' - AND NOT (("posthog_person"."properties" -> 'does-not-exist') = 'null'::jsonb)) - OR (("posthog_person"."properties" -> 'key') = '"value"'::jsonb - AND "posthog_person"."properties" ? 'key' - AND NOT (("posthog_person"."properties" -> 'key') = 'null'::jsonb)))) - ORDER BY "posthog_person"."id" ASC - LIMIT 1000 - OFFSET 1000 + SELECT "posthog_team"."id", + "posthog_team"."uuid", + "posthog_team"."organization_id", + "posthog_team"."project_id", + "posthog_team"."api_token", + "posthog_team"."app_urls", + "posthog_team"."name", + "posthog_team"."slack_incoming_webhook", + "posthog_team"."created_at", + "posthog_team"."updated_at", + "posthog_team"."anonymize_ips", + "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", + "posthog_team"."ingested_event", + "posthog_team"."autocapture_opt_out", + "posthog_team"."autocapture_web_vitals_opt_in", + "posthog_team"."autocapture_web_vitals_allowed_metrics", + "posthog_team"."autocapture_exceptions_opt_in", + "posthog_team"."autocapture_exceptions_errors_to_ignore", + "posthog_team"."person_processing_opt_out", + "posthog_team"."session_recording_opt_in", + "posthog_team"."session_recording_sample_rate", + "posthog_team"."session_recording_minimum_duration_milliseconds", + "posthog_team"."session_recording_linked_flag", + "posthog_team"."session_recording_network_payload_capture_config", + "posthog_team"."session_recording_url_trigger_config", + "posthog_team"."session_recording_url_blocklist_config", + "posthog_team"."session_recording_event_trigger_config", + "posthog_team"."session_replay_config", + "posthog_team"."survey_config", + "posthog_team"."capture_console_log_opt_in", + "posthog_team"."capture_performance_opt_in", + "posthog_team"."capture_dead_clicks", + "posthog_team"."surveys_opt_in", + "posthog_team"."heatmaps_opt_in", + "posthog_team"."session_recording_version", + "posthog_team"."signup_token", + "posthog_team"."is_demo", + "posthog_team"."access_control", + "posthog_team"."week_start_day", + "posthog_team"."inject_web_apps", + "posthog_team"."test_account_filters", + "posthog_team"."test_account_filters_default_checked", + "posthog_team"."path_cleaning_filters", + "posthog_team"."timezone", + "posthog_team"."data_attributes", + "posthog_team"."person_display_name_properties", + "posthog_team"."live_events_columns", + "posthog_team"."recording_domains", + "posthog_team"."primary_dashboard_id", + "posthog_team"."extra_settings", + "posthog_team"."modifiers", + "posthog_team"."correlation_config", + "posthog_team"."session_recording_retention_period_days", + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" + FROM "posthog_team" + WHERE "posthog_team"."id" = 99999 + LIMIT 21 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.1 ''' SELECT "posthog_featureflag"."id", "posthog_featureflag"."key", @@ -942,38 +996,32 @@ "posthog_featureflag"."usage_dashboard_id", "posthog_featureflag"."has_enriched_analytics" FROM "posthog_featureflag" + INNER JOIN "posthog_team" ON ("posthog_featureflag"."team_id" = "posthog_team"."id") WHERE ("posthog_featureflag"."key" = 'some-feature2' - AND "posthog_featureflag"."team_id" = 99999) + AND "posthog_team"."project_id" = 99999) LIMIT 21 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.1 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.10 ''' - SELECT "posthog_cohort"."id", - "posthog_cohort"."name", - "posthog_cohort"."description", - "posthog_cohort"."team_id", - "posthog_cohort"."deleted", - "posthog_cohort"."filters", - "posthog_cohort"."query", - "posthog_cohort"."version", - "posthog_cohort"."pending_version", - "posthog_cohort"."count", - "posthog_cohort"."created_by_id", - "posthog_cohort"."created_at", - "posthog_cohort"."is_calculating", - "posthog_cohort"."last_calculation", - "posthog_cohort"."errors_calculating", - "posthog_cohort"."last_error_at", - "posthog_cohort"."is_static", - "posthog_cohort"."groups" - FROM "posthog_cohort" - WHERE ("posthog_cohort"."id" = 99999 - AND "posthog_cohort"."team_id" = 99999) - LIMIT 21 + SELECT "posthog_person"."id", + "posthog_person"."created_at", + "posthog_person"."properties_last_updated_at", + "posthog_person"."properties_last_operation", + "posthog_person"."team_id", + "posthog_person"."properties", + "posthog_person"."is_user_id", + "posthog_person"."is_identified", + "posthog_person"."uuid", + "posthog_person"."version" + FROM "posthog_person" + WHERE ("posthog_person"."team_id" = 99999 + AND ("posthog_person"."properties" -> 'key') IS NOT NULL) + ORDER BY "posthog_person"."id" ASC + LIMIT 1000 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.10 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.11 ''' SELECT "posthog_persondistinctid"."id", "posthog_persondistinctid"."team_id", @@ -993,7 +1041,7 @@ 5 /* ... */)) ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.11 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.12 ''' SELECT "posthog_person"."id", "posthog_person"."created_at", @@ -1013,7 +1061,7 @@ OFFSET 1000 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.12 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.13 ''' SELECT "posthog_person"."uuid" FROM "posthog_person" @@ -1028,77 +1076,34 @@ LIMIT 1))) ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.13 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.2 ''' - SELECT "posthog_team"."id", - "posthog_team"."uuid", - "posthog_team"."organization_id", - "posthog_team"."project_id", - "posthog_team"."api_token", - "posthog_team"."app_urls", - "posthog_team"."name", - "posthog_team"."slack_incoming_webhook", - "posthog_team"."created_at", - "posthog_team"."updated_at", - "posthog_team"."anonymize_ips", - "posthog_team"."completed_snippet_onboarding", - "posthog_team"."has_completed_onboarding_for", - "posthog_team"."ingested_event", - "posthog_team"."autocapture_opt_out", - "posthog_team"."autocapture_web_vitals_opt_in", - "posthog_team"."autocapture_web_vitals_allowed_metrics", - "posthog_team"."autocapture_exceptions_opt_in", - "posthog_team"."autocapture_exceptions_errors_to_ignore", - "posthog_team"."person_processing_opt_out", - "posthog_team"."session_recording_opt_in", - "posthog_team"."session_recording_sample_rate", - "posthog_team"."session_recording_minimum_duration_milliseconds", - "posthog_team"."session_recording_linked_flag", - "posthog_team"."session_recording_network_payload_capture_config", - "posthog_team"."session_recording_url_trigger_config", - "posthog_team"."session_recording_url_blocklist_config", - "posthog_team"."session_recording_event_trigger_config", - "posthog_team"."session_replay_config", - "posthog_team"."survey_config", - "posthog_team"."capture_console_log_opt_in", - "posthog_team"."capture_performance_opt_in", - "posthog_team"."capture_dead_clicks", - "posthog_team"."surveys_opt_in", - "posthog_team"."heatmaps_opt_in", - "posthog_team"."session_recording_version", - "posthog_team"."signup_token", - "posthog_team"."is_demo", - "posthog_team"."access_control", - "posthog_team"."week_start_day", - "posthog_team"."inject_web_apps", - "posthog_team"."test_account_filters", - "posthog_team"."test_account_filters_default_checked", - "posthog_team"."path_cleaning_filters", - "posthog_team"."timezone", - "posthog_team"."data_attributes", - "posthog_team"."person_display_name_properties", - "posthog_team"."live_events_columns", - "posthog_team"."recording_domains", - "posthog_team"."primary_dashboard_id", - "posthog_team"."extra_settings", - "posthog_team"."modifiers", - "posthog_team"."correlation_config", - "posthog_team"."session_recording_retention_period_days", - "posthog_team"."plugins_opt_in", - "posthog_team"."opt_out_capture", - "posthog_team"."event_names", - "posthog_team"."event_names_with_usage", - "posthog_team"."event_properties", - "posthog_team"."event_properties_with_usage", - "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id", - "posthog_team"."external_data_workspace_last_synced_at" - FROM "posthog_team" - WHERE "posthog_team"."id" = 99999 + SELECT "posthog_cohort"."id", + "posthog_cohort"."name", + "posthog_cohort"."description", + "posthog_cohort"."team_id", + "posthog_cohort"."deleted", + "posthog_cohort"."filters", + "posthog_cohort"."query", + "posthog_cohort"."version", + "posthog_cohort"."pending_version", + "posthog_cohort"."count", + "posthog_cohort"."created_by_id", + "posthog_cohort"."created_at", + "posthog_cohort"."is_calculating", + "posthog_cohort"."last_calculation", + "posthog_cohort"."errors_calculating", + "posthog_cohort"."last_error_at", + "posthog_cohort"."is_static", + "posthog_cohort"."groups" + FROM "posthog_cohort" + INNER JOIN "posthog_team" ON ("posthog_cohort"."team_id" = "posthog_team"."id") + WHERE ("posthog_cohort"."id" = 99999 + AND "posthog_team"."project_id" = 99999) LIMIT 21 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.2 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.3 ''' SELECT "posthog_person"."id", "posthog_person"."created_at", @@ -1119,7 +1124,7 @@ LIMIT 1000 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.3 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.4 ''' SELECT "posthog_persondistinctid"."id", "posthog_persondistinctid"."team_id", @@ -1139,7 +1144,7 @@ 5 /* ... */)) ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.4 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.5 ''' SELECT "posthog_person"."id", "posthog_person"."created_at", @@ -1161,7 +1166,7 @@ OFFSET 1000 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.5 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.6 ''' SELECT "posthog_person"."uuid" FROM "posthog_person" @@ -1175,7 +1180,7 @@ LIMIT 1))) ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.6 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.7 ''' SELECT "posthog_team"."id", "posthog_team"."uuid", @@ -1231,13 +1236,6 @@ "posthog_team"."modifiers", "posthog_team"."correlation_config", "posthog_team"."session_recording_retention_period_days", - "posthog_team"."plugins_opt_in", - "posthog_team"."opt_out_capture", - "posthog_team"."event_names", - "posthog_team"."event_names_with_usage", - "posthog_team"."event_properties", - "posthog_team"."event_properties_with_usage", - "posthog_team"."event_properties_numerical", "posthog_team"."external_data_workspace_id", "posthog_team"."external_data_workspace_last_synced_at" FROM "posthog_team" @@ -1245,7 +1243,7 @@ LIMIT 21 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.7 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.8 ''' SELECT "posthog_featureflag"."id", "posthog_featureflag"."key", @@ -1263,12 +1261,13 @@ "posthog_featureflag"."usage_dashboard_id", "posthog_featureflag"."has_enriched_analytics" FROM "posthog_featureflag" + INNER JOIN "posthog_team" ON ("posthog_featureflag"."team_id" = "posthog_team"."id") WHERE ("posthog_featureflag"."key" = 'some-feature-new' - AND "posthog_featureflag"."team_id" = 99999) + AND "posthog_team"."project_id" = 99999) LIMIT 21 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.8 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.9 ''' SELECT "posthog_cohort"."id", "posthog_cohort"."name", @@ -1289,31 +1288,76 @@ "posthog_cohort"."is_static", "posthog_cohort"."groups" FROM "posthog_cohort" + INNER JOIN "posthog_team" ON ("posthog_cohort"."team_id" = "posthog_team"."id") WHERE ("posthog_cohort"."id" = 99999 - AND "posthog_cohort"."team_id" = 99999) + AND "posthog_team"."project_id" = 99999) LIMIT 21 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_default_person_properties_adjustment.9 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag ''' - SELECT "posthog_person"."id", - "posthog_person"."created_at", - "posthog_person"."properties_last_updated_at", - "posthog_person"."properties_last_operation", - "posthog_person"."team_id", - "posthog_person"."properties", - "posthog_person"."is_user_id", - "posthog_person"."is_identified", - "posthog_person"."uuid", - "posthog_person"."version" - FROM "posthog_person" - WHERE ("posthog_person"."team_id" = 99999 - AND ("posthog_person"."properties" -> 'key') IS NOT NULL) - ORDER BY "posthog_person"."id" ASC - LIMIT 1000 + SELECT "posthog_team"."id", + "posthog_team"."uuid", + "posthog_team"."organization_id", + "posthog_team"."project_id", + "posthog_team"."api_token", + "posthog_team"."app_urls", + "posthog_team"."name", + "posthog_team"."slack_incoming_webhook", + "posthog_team"."created_at", + "posthog_team"."updated_at", + "posthog_team"."anonymize_ips", + "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", + "posthog_team"."ingested_event", + "posthog_team"."autocapture_opt_out", + "posthog_team"."autocapture_web_vitals_opt_in", + "posthog_team"."autocapture_web_vitals_allowed_metrics", + "posthog_team"."autocapture_exceptions_opt_in", + "posthog_team"."autocapture_exceptions_errors_to_ignore", + "posthog_team"."person_processing_opt_out", + "posthog_team"."session_recording_opt_in", + "posthog_team"."session_recording_sample_rate", + "posthog_team"."session_recording_minimum_duration_milliseconds", + "posthog_team"."session_recording_linked_flag", + "posthog_team"."session_recording_network_payload_capture_config", + "posthog_team"."session_recording_url_trigger_config", + "posthog_team"."session_recording_url_blocklist_config", + "posthog_team"."session_recording_event_trigger_config", + "posthog_team"."session_replay_config", + "posthog_team"."survey_config", + "posthog_team"."capture_console_log_opt_in", + "posthog_team"."capture_performance_opt_in", + "posthog_team"."capture_dead_clicks", + "posthog_team"."surveys_opt_in", + "posthog_team"."heatmaps_opt_in", + "posthog_team"."session_recording_version", + "posthog_team"."signup_token", + "posthog_team"."is_demo", + "posthog_team"."access_control", + "posthog_team"."week_start_day", + "posthog_team"."inject_web_apps", + "posthog_team"."test_account_filters", + "posthog_team"."test_account_filters_default_checked", + "posthog_team"."path_cleaning_filters", + "posthog_team"."timezone", + "posthog_team"."data_attributes", + "posthog_team"."person_display_name_properties", + "posthog_team"."live_events_columns", + "posthog_team"."recording_domains", + "posthog_team"."primary_dashboard_id", + "posthog_team"."extra_settings", + "posthog_team"."modifiers", + "posthog_team"."correlation_config", + "posthog_team"."session_recording_retention_period_days", + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" + FROM "posthog_team" + WHERE "posthog_team"."id" = 99999 + LIMIT 21 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag.1 ''' SELECT "posthog_featureflag"."id", "posthog_featureflag"."key", @@ -1331,12 +1375,13 @@ "posthog_featureflag"."usage_dashboard_id", "posthog_featureflag"."has_enriched_analytics" FROM "posthog_featureflag" + INNER JOIN "posthog_team" ON ("posthog_featureflag"."team_id" = "posthog_team"."id") WHERE ("posthog_featureflag"."key" = 'some-feature2' - AND "posthog_featureflag"."team_id" = 99999) + AND "posthog_team"."project_id" = 99999) LIMIT 21 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag.1 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag.2 ''' SELECT "posthog_cohort"."id", "posthog_cohort"."name", @@ -1357,12 +1402,13 @@ "posthog_cohort"."is_static", "posthog_cohort"."groups" FROM "posthog_cohort" + INNER JOIN "posthog_team" ON ("posthog_cohort"."team_id" = "posthog_team"."id") WHERE ("posthog_cohort"."id" = 99999 - AND "posthog_cohort"."team_id" = 99999) + AND "posthog_team"."project_id" = 99999) LIMIT 21 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag.2 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag.3 ''' SELECT "posthog_person"."id", "posthog_person"."created_at", @@ -1383,7 +1429,7 @@ LIMIT 1000 ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag.3 +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag.4 ''' SELECT "posthog_persondistinctid"."id", "posthog_persondistinctid"."team_id", @@ -1403,20 +1449,6 @@ 5 /* ... */)) ''' # --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag.4 - ''' - SELECT "posthog_featureflaghashkeyoverride"."feature_flag_key", - "posthog_featureflaghashkeyoverride"."hash_key", - "posthog_featureflaghashkeyoverride"."person_id" - FROM "posthog_featureflaghashkeyoverride" - WHERE ("posthog_featureflaghashkeyoverride"."person_id" IN (1, - 2, - 3, - 4, - 5 /* ... */) - AND "posthog_featureflaghashkeyoverride"."team_id" = 99999) - ''' -# --- # name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag.5 ''' SELECT "posthog_featureflaghashkeyoverride"."feature_flag_key", @@ -1427,128 +1459,72 @@ 2, 3, 4, - 5 /* ... */) - AND "posthog_featureflaghashkeyoverride"."team_id" = 99999) - ''' -# --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag.6 - ''' - SELECT "posthog_featureflaghashkeyoverride"."feature_flag_key", - "posthog_featureflaghashkeyoverride"."hash_key", - "posthog_featureflaghashkeyoverride"."person_id" - FROM "posthog_featureflaghashkeyoverride" - WHERE ("posthog_featureflaghashkeyoverride"."person_id" IN (1, - 2, - 3, - 4, - 5 /* ... */) - AND "posthog_featureflaghashkeyoverride"."team_id" = 99999) - ''' -# --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag.7 - ''' - SELECT "posthog_person"."id", - "posthog_person"."created_at", - "posthog_person"."properties_last_updated_at", - "posthog_person"."properties_last_operation", - "posthog_person"."team_id", - "posthog_person"."properties", - "posthog_person"."is_user_id", - "posthog_person"."is_identified", - "posthog_person"."uuid", - "posthog_person"."version" - FROM "posthog_person" - WHERE ("posthog_person"."team_id" = 99999 - AND ("posthog_person"."properties" -> 'key') = '"value"'::jsonb - AND "posthog_person"."properties" ? 'key' - AND NOT (("posthog_person"."properties" -> 'key') = 'null'::jsonb)) - ORDER BY "posthog_person"."id" ASC - LIMIT 1000 - OFFSET 1000 - ''' -# --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag.8 - ''' - SELECT "posthog_person"."uuid" - FROM "posthog_person" - WHERE ("posthog_person"."team_id" = 99999 - AND "posthog_person"."uuid" IN ('00000000000040008000000000000002'::uuid) - AND NOT (EXISTS - (SELECT 1 AS "a" - FROM "posthog_cohortpeople" U1 - WHERE (U1."cohort_id" = 99999 - AND U1."person_id" = ("posthog_person"."id")) - LIMIT 1))) - ''' -# --- -# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag.9 - ''' - SELECT "posthog_team"."id", - "posthog_team"."uuid", - "posthog_team"."organization_id", - "posthog_team"."project_id", - "posthog_team"."api_token", - "posthog_team"."app_urls", - "posthog_team"."name", - "posthog_team"."slack_incoming_webhook", - "posthog_team"."created_at", - "posthog_team"."updated_at", - "posthog_team"."anonymize_ips", - "posthog_team"."completed_snippet_onboarding", - "posthog_team"."has_completed_onboarding_for", - "posthog_team"."ingested_event", - "posthog_team"."autocapture_opt_out", - "posthog_team"."autocapture_web_vitals_opt_in", - "posthog_team"."autocapture_web_vitals_allowed_metrics", - "posthog_team"."autocapture_exceptions_opt_in", - "posthog_team"."autocapture_exceptions_errors_to_ignore", - "posthog_team"."person_processing_opt_out", - "posthog_team"."session_recording_opt_in", - "posthog_team"."session_recording_sample_rate", - "posthog_team"."session_recording_minimum_duration_milliseconds", - "posthog_team"."session_recording_linked_flag", - "posthog_team"."session_recording_network_payload_capture_config", - "posthog_team"."session_recording_url_trigger_config", - "posthog_team"."session_recording_url_blocklist_config", - "posthog_team"."session_recording_event_trigger_config", - "posthog_team"."session_replay_config", - "posthog_team"."survey_config", - "posthog_team"."capture_console_log_opt_in", - "posthog_team"."capture_performance_opt_in", - "posthog_team"."capture_dead_clicks", - "posthog_team"."surveys_opt_in", - "posthog_team"."heatmaps_opt_in", - "posthog_team"."session_recording_version", - "posthog_team"."signup_token", - "posthog_team"."is_demo", - "posthog_team"."access_control", - "posthog_team"."week_start_day", - "posthog_team"."inject_web_apps", - "posthog_team"."test_account_filters", - "posthog_team"."test_account_filters_default_checked", - "posthog_team"."path_cleaning_filters", - "posthog_team"."timezone", - "posthog_team"."data_attributes", - "posthog_team"."person_display_name_properties", - "posthog_team"."live_events_columns", - "posthog_team"."recording_domains", - "posthog_team"."primary_dashboard_id", - "posthog_team"."extra_settings", - "posthog_team"."modifiers", - "posthog_team"."correlation_config", - "posthog_team"."session_recording_retention_period_days", - "posthog_team"."plugins_opt_in", - "posthog_team"."opt_out_capture", - "posthog_team"."event_names", - "posthog_team"."event_names_with_usage", - "posthog_team"."event_properties", - "posthog_team"."event_properties_with_usage", - "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id", - "posthog_team"."external_data_workspace_last_synced_at" - FROM "posthog_team" - WHERE "posthog_team"."id" = 99999 - LIMIT 21 + 5 /* ... */) + AND "posthog_featureflaghashkeyoverride"."team_id" = 99999) + ''' +# --- +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag.6 + ''' + SELECT "posthog_featureflaghashkeyoverride"."feature_flag_key", + "posthog_featureflaghashkeyoverride"."hash_key", + "posthog_featureflaghashkeyoverride"."person_id" + FROM "posthog_featureflaghashkeyoverride" + WHERE ("posthog_featureflaghashkeyoverride"."person_id" IN (1, + 2, + 3, + 4, + 5 /* ... */) + AND "posthog_featureflaghashkeyoverride"."team_id" = 99999) + ''' +# --- +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag.7 + ''' + SELECT "posthog_featureflaghashkeyoverride"."feature_flag_key", + "posthog_featureflaghashkeyoverride"."hash_key", + "posthog_featureflaghashkeyoverride"."person_id" + FROM "posthog_featureflaghashkeyoverride" + WHERE ("posthog_featureflaghashkeyoverride"."person_id" IN (1, + 2, + 3, + 4, + 5 /* ... */) + AND "posthog_featureflaghashkeyoverride"."team_id" = 99999) + ''' +# --- +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag.8 + ''' + SELECT "posthog_person"."id", + "posthog_person"."created_at", + "posthog_person"."properties_last_updated_at", + "posthog_person"."properties_last_operation", + "posthog_person"."team_id", + "posthog_person"."properties", + "posthog_person"."is_user_id", + "posthog_person"."is_identified", + "posthog_person"."uuid", + "posthog_person"."version" + FROM "posthog_person" + WHERE ("posthog_person"."team_id" = 99999 + AND ("posthog_person"."properties" -> 'key') = '"value"'::jsonb + AND "posthog_person"."properties" ? 'key' + AND NOT (("posthog_person"."properties" -> 'key') = 'null'::jsonb)) + ORDER BY "posthog_person"."id" ASC + LIMIT 1000 + OFFSET 1000 + ''' +# --- +# name: TestCohortGenerationForFeatureFlag.test_creating_static_cohort_with_experience_continuity_flag.9 + ''' + SELECT "posthog_person"."uuid" + FROM "posthog_person" + WHERE ("posthog_person"."team_id" = 99999 + AND "posthog_person"."uuid" IN ('00000000000040008000000000000002'::uuid) + AND NOT (EXISTS + (SELECT 1 AS "a" + FROM "posthog_cohortpeople" U1 + WHERE (U1."cohort_id" = 99999 + AND U1."person_id" = ("posthog_person"."id")) + LIMIT 1))) ''' # --- # name: TestFeatureFlag.test_cant_create_flag_with_data_that_fails_to_query @@ -1668,6 +1644,33 @@ ''' # --- # name: TestFeatureFlag.test_creating_static_cohort.10 + ''' + SELECT "posthog_cohort"."id", + "posthog_cohort"."name", + "posthog_cohort"."description", + "posthog_cohort"."team_id", + "posthog_cohort"."deleted", + "posthog_cohort"."filters", + "posthog_cohort"."query", + "posthog_cohort"."version", + "posthog_cohort"."pending_version", + "posthog_cohort"."count", + "posthog_cohort"."created_by_id", + "posthog_cohort"."created_at", + "posthog_cohort"."is_calculating", + "posthog_cohort"."last_calculation", + "posthog_cohort"."errors_calculating", + "posthog_cohort"."last_error_at", + "posthog_cohort"."is_static", + "posthog_cohort"."groups" + FROM "posthog_cohort" + INNER JOIN "posthog_team" ON ("posthog_cohort"."team_id" = "posthog_team"."id") + WHERE ("posthog_cohort"."id" = 99999 + AND "posthog_team"."project_id" = 99999) + LIMIT 21 + ''' +# --- +# name: TestFeatureFlag.test_creating_static_cohort.11 ''' SELECT "posthog_person"."id", "posthog_person"."created_at", @@ -1688,7 +1691,7 @@ LIMIT 10000 ''' # --- -# name: TestFeatureFlag.test_creating_static_cohort.11 +# name: TestFeatureFlag.test_creating_static_cohort.12 ''' SELECT "posthog_persondistinctid"."id", "posthog_persondistinctid"."team_id", @@ -1708,7 +1711,7 @@ 5 /* ... */)) ''' # --- -# name: TestFeatureFlag.test_creating_static_cohort.12 +# name: TestFeatureFlag.test_creating_static_cohort.13 ''' SELECT "posthog_person"."id", "posthog_person"."created_at", @@ -1730,7 +1733,7 @@ OFFSET 10000 ''' # --- -# name: TestFeatureFlag.test_creating_static_cohort.13 +# name: TestFeatureFlag.test_creating_static_cohort.14 ''' SELECT "posthog_person"."uuid" FROM "posthog_person" @@ -1744,76 +1747,6 @@ LIMIT 1))) ''' # --- -# name: TestFeatureFlag.test_creating_static_cohort.14 - ''' - SELECT "posthog_team"."id", - "posthog_team"."uuid", - "posthog_team"."organization_id", - "posthog_team"."project_id", - "posthog_team"."api_token", - "posthog_team"."app_urls", - "posthog_team"."name", - "posthog_team"."slack_incoming_webhook", - "posthog_team"."created_at", - "posthog_team"."updated_at", - "posthog_team"."anonymize_ips", - "posthog_team"."completed_snippet_onboarding", - "posthog_team"."has_completed_onboarding_for", - "posthog_team"."ingested_event", - "posthog_team"."autocapture_opt_out", - "posthog_team"."autocapture_web_vitals_opt_in", - "posthog_team"."autocapture_web_vitals_allowed_metrics", - "posthog_team"."autocapture_exceptions_opt_in", - "posthog_team"."autocapture_exceptions_errors_to_ignore", - "posthog_team"."person_processing_opt_out", - "posthog_team"."session_recording_opt_in", - "posthog_team"."session_recording_sample_rate", - "posthog_team"."session_recording_minimum_duration_milliseconds", - "posthog_team"."session_recording_linked_flag", - "posthog_team"."session_recording_network_payload_capture_config", - "posthog_team"."session_recording_url_trigger_config", - "posthog_team"."session_recording_url_blocklist_config", - "posthog_team"."session_recording_event_trigger_config", - "posthog_team"."session_replay_config", - "posthog_team"."survey_config", - "posthog_team"."capture_console_log_opt_in", - "posthog_team"."capture_performance_opt_in", - "posthog_team"."capture_dead_clicks", - "posthog_team"."surveys_opt_in", - "posthog_team"."heatmaps_opt_in", - "posthog_team"."session_recording_version", - "posthog_team"."signup_token", - "posthog_team"."is_demo", - "posthog_team"."access_control", - "posthog_team"."week_start_day", - "posthog_team"."inject_web_apps", - "posthog_team"."test_account_filters", - "posthog_team"."test_account_filters_default_checked", - "posthog_team"."path_cleaning_filters", - "posthog_team"."timezone", - "posthog_team"."data_attributes", - "posthog_team"."person_display_name_properties", - "posthog_team"."live_events_columns", - "posthog_team"."recording_domains", - "posthog_team"."primary_dashboard_id", - "posthog_team"."extra_settings", - "posthog_team"."modifiers", - "posthog_team"."correlation_config", - "posthog_team"."session_recording_retention_period_days", - "posthog_team"."plugins_opt_in", - "posthog_team"."opt_out_capture", - "posthog_team"."event_names", - "posthog_team"."event_names_with_usage", - "posthog_team"."event_properties", - "posthog_team"."event_properties_with_usage", - "posthog_team"."event_properties_numerical", - "posthog_team"."external_data_workspace_id", - "posthog_team"."external_data_workspace_last_synced_at" - FROM "posthog_team" - WHERE "posthog_team"."id" = 99999 - LIMIT 21 - ''' -# --- # name: TestFeatureFlag.test_creating_static_cohort.15 ''' SELECT "posthog_team"."id", @@ -2141,6 +2074,69 @@ ''' # --- # name: TestFeatureFlag.test_creating_static_cohort.8 + ''' + SELECT "posthog_team"."id", + "posthog_team"."uuid", + "posthog_team"."organization_id", + "posthog_team"."project_id", + "posthog_team"."api_token", + "posthog_team"."app_urls", + "posthog_team"."name", + "posthog_team"."slack_incoming_webhook", + "posthog_team"."created_at", + "posthog_team"."updated_at", + "posthog_team"."anonymize_ips", + "posthog_team"."completed_snippet_onboarding", + "posthog_team"."has_completed_onboarding_for", + "posthog_team"."ingested_event", + "posthog_team"."autocapture_opt_out", + "posthog_team"."autocapture_web_vitals_opt_in", + "posthog_team"."autocapture_web_vitals_allowed_metrics", + "posthog_team"."autocapture_exceptions_opt_in", + "posthog_team"."autocapture_exceptions_errors_to_ignore", + "posthog_team"."person_processing_opt_out", + "posthog_team"."session_recording_opt_in", + "posthog_team"."session_recording_sample_rate", + "posthog_team"."session_recording_minimum_duration_milliseconds", + "posthog_team"."session_recording_linked_flag", + "posthog_team"."session_recording_network_payload_capture_config", + "posthog_team"."session_recording_url_trigger_config", + "posthog_team"."session_recording_url_blocklist_config", + "posthog_team"."session_recording_event_trigger_config", + "posthog_team"."session_replay_config", + "posthog_team"."survey_config", + "posthog_team"."capture_console_log_opt_in", + "posthog_team"."capture_performance_opt_in", + "posthog_team"."capture_dead_clicks", + "posthog_team"."surveys_opt_in", + "posthog_team"."heatmaps_opt_in", + "posthog_team"."session_recording_version", + "posthog_team"."signup_token", + "posthog_team"."is_demo", + "posthog_team"."access_control", + "posthog_team"."week_start_day", + "posthog_team"."inject_web_apps", + "posthog_team"."test_account_filters", + "posthog_team"."test_account_filters_default_checked", + "posthog_team"."path_cleaning_filters", + "posthog_team"."timezone", + "posthog_team"."data_attributes", + "posthog_team"."person_display_name_properties", + "posthog_team"."live_events_columns", + "posthog_team"."recording_domains", + "posthog_team"."primary_dashboard_id", + "posthog_team"."extra_settings", + "posthog_team"."modifiers", + "posthog_team"."correlation_config", + "posthog_team"."session_recording_retention_period_days", + "posthog_team"."external_data_workspace_id", + "posthog_team"."external_data_workspace_last_synced_at" + FROM "posthog_team" + WHERE "posthog_team"."id" = 99999 + LIMIT 21 + ''' +# --- +# name: TestFeatureFlag.test_creating_static_cohort.9 ''' SELECT "posthog_featureflag"."id", "posthog_featureflag"."key", @@ -2158,34 +2154,9 @@ "posthog_featureflag"."usage_dashboard_id", "posthog_featureflag"."has_enriched_analytics" FROM "posthog_featureflag" + INNER JOIN "posthog_team" ON ("posthog_featureflag"."team_id" = "posthog_team"."id") WHERE ("posthog_featureflag"."key" = 'some-feature' - AND "posthog_featureflag"."team_id" = 99999) - LIMIT 21 - ''' -# --- -# name: TestFeatureFlag.test_creating_static_cohort.9 - ''' - SELECT "posthog_cohort"."id", - "posthog_cohort"."name", - "posthog_cohort"."description", - "posthog_cohort"."team_id", - "posthog_cohort"."deleted", - "posthog_cohort"."filters", - "posthog_cohort"."query", - "posthog_cohort"."version", - "posthog_cohort"."pending_version", - "posthog_cohort"."count", - "posthog_cohort"."created_by_id", - "posthog_cohort"."created_at", - "posthog_cohort"."is_calculating", - "posthog_cohort"."last_calculation", - "posthog_cohort"."errors_calculating", - "posthog_cohort"."last_error_at", - "posthog_cohort"."is_static", - "posthog_cohort"."groups" - FROM "posthog_cohort" - WHERE ("posthog_cohort"."id" = 99999 - AND "posthog_cohort"."team_id" = 99999) + AND "posthog_team"."project_id" = 99999) LIMIT 21 ''' # --- diff --git a/posthog/api/test/test_cohort.py b/posthog/api/test/test_cohort.py index 5e16d6b7bc519..8a32a8b9a5dd0 100644 --- a/posthog/api/test/test_cohort.py +++ b/posthog/api/test/test_cohort.py @@ -1111,7 +1111,7 @@ def _calc(query: str) -> int: self.assertEqual(1, _calc("select 1 from events")) # raises on all other cases - response = self.client.post( + query_post_response = self.client.post( f"/api/projects/{self.team.id}/cohorts", data={ "name": "cohort A", @@ -1122,7 +1122,15 @@ def _calc(query: str) -> int: }, }, ) - self.assertEqual(response.status_code, 500, response.content) + query_get_response = self.client.get( + f"/api/projects/{self.team.id}/cohorts/{query_post_response.json()['id']}/" + ) + + self.assertEqual(query_post_response.status_code, 201) + self.assertEqual(query_get_response.status_code, 200) + self.assertEqual( + query_get_response.json()["errors_calculating"], 1 + ) # Should be because selecting from groups is not allowed @patch("posthog.api.cohort.report_user_action") def test_cohort_with_is_set_filter_missing_value(self, patch_capture): diff --git a/posthog/api/test/test_feature_flag.py b/posthog/api/test/test_feature_flag.py index 86cee8950dfdf..7ffb834c41e4a 100644 --- a/posthog/api/test/test_feature_flag.py +++ b/posthog/api/test/test_feature_flag.py @@ -1259,7 +1259,7 @@ def test_my_flags_is_not_nplus1(self) -> None: format="json", ).json() - with self.assertNumQueries(FuzzyInt(7, 8)): + with self.assertNumQueries(FuzzyInt(8, 9)): response = self.client.get(f"/api/projects/{self.team.id}/feature_flags/my_flags") self.assertEqual(response.status_code, status.HTTP_200_OK) @@ -2229,7 +2229,7 @@ def test_local_evaluation_for_invalid_cohorts(self, mock_capture): self.client.logout() - with self.assertNumQueries(16): + with self.assertNumQueries(18): # 1. SAVEPOINT # 2. SELECT "posthog_personalapikey"."id", # 3. RELEASE SAVEPOINT @@ -2242,10 +2242,12 @@ def test_local_evaluation_for_invalid_cohorts(self, mock_capture): # 10. SELECT "posthog_organizationmembership"."id", # 11. SELECT "posthog_cohort"."id" -- all cohorts # 12. SELECT "posthog_featureflag"."id", "posthog_featureflag"."key", -- all flags - # 13. SELECT "posthog_cohort". id = 99999 - # 14. SELECT "posthog_cohort". id = deleted cohort - # 15. SELECT "posthog_cohort". id = cohort from other team - # 16. SELECT "posthog_grouptypemapping"."id", -- group type mapping + # 13. SELECT "posthog_team"."id", "posthog_team"."uuid", + # 14. SELECT "posthog_cohort". id = 99999 + # 15. SELECT "posthog_team"."id", "posthog_team"."uuid", + # 16. SELECT "posthog_cohort". id = deleted cohort + # 17. SELECT "posthog_cohort". id = cohort from other team + # 18. SELECT "posthog_grouptypemapping"."id", -- group type mapping response = self.client.get( f"/api/feature_flag/local_evaluation?token={self.team.api_token}&send_cohorts", @@ -4230,7 +4232,7 @@ def test_creating_static_cohort_with_deleted_flag(self): name="some cohort", ) - with self.assertNumQueries(1): + with self.assertNumQueries(2): get_cohort_actors_for_feature_flag(cohort.pk, "some-feature", self.team.pk) cohort.refresh_from_db() @@ -4268,7 +4270,7 @@ def test_creating_static_cohort_with_inactive_flag(self): name="some cohort", ) - with self.assertNumQueries(1): + with self.assertNumQueries(2): get_cohort_actors_for_feature_flag(cohort.pk, "some-feature2", self.team.pk) cohort.refresh_from_db() @@ -4307,7 +4309,7 @@ def test_creating_static_cohort_with_group_flag(self): name="some cohort", ) - with self.assertNumQueries(1): + with self.assertNumQueries(2): get_cohort_actors_for_feature_flag(cohort.pk, "some-feature3", self.team.pk) cohort.refresh_from_db() @@ -4339,7 +4341,7 @@ def test_creating_static_cohort_with_no_person_distinct_ids(self): name="some cohort", ) - with self.assertNumQueries(5): + with self.assertNumQueries(6): get_cohort_actors_for_feature_flag(cohort.pk, "some-feature2", self.team.pk) cohort.refresh_from_db() @@ -4357,7 +4359,7 @@ def test_creating_static_cohort_with_non_existing_flag(self): name="some cohort", ) - with self.assertNumQueries(1): + with self.assertNumQueries(2): get_cohort_actors_for_feature_flag(cohort.pk, "some-feature2", self.team.pk) cohort.refresh_from_db() diff --git a/posthog/api/test/test_organization.py b/posthog/api/test/test_organization.py index 2396f78e3c557..143fbe3f524b9 100644 --- a/posthog/api/test/test_organization.py +++ b/posthog/api/test/test_organization.py @@ -1,4 +1,5 @@ from rest_framework import status +from unittest.mock import patch, ANY from posthog.models import Organization, OrganizationMembership, Team from posthog.models.personal_api_key import PersonalAPIKey, hash_key_value @@ -128,7 +129,8 @@ def test_cant_update_plugins_access_level(self): self.organization.refresh_from_db() self.assertEqual(self.organization.plugins_access_level, 3) - def test_enforce_2fa_for_everyone(self): + @patch("posthoganalytics.capture") + def test_enforce_2fa_for_everyone(self, mock_capture): # Only admins should be able to enforce 2fa response = self.client.patch(f"/api/organizations/{self.organization.id}/", {"enforce_2fa": True}) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) @@ -142,6 +144,19 @@ def test_enforce_2fa_for_everyone(self): self.organization.refresh_from_db() self.assertEqual(self.organization.enforce_2fa, True) + # Verify the capture event was called correctly + mock_capture.assert_any_call( + self.user.distinct_id, + "organization 2fa enforcement toggled", + properties={ + "enabled": True, + "organization_id": str(self.organization.id), + "organization_name": self.organization.name, + "user_role": OrganizationMembership.Level.ADMIN, + }, + groups={"instance": ANY, "organization": str(self.organization.id)}, + ) + def test_projects_outside_personal_api_key_scoped_organizations_not_listed(self): other_org, _, _ = Organization.objects.bootstrap(self.user) personal_api_key = generate_random_token_personal() diff --git a/posthog/api/test/test_person.py b/posthog/api/test/test_person.py index 2c9694f6eda6d..29eb3990407d5 100644 --- a/posthog/api/test/test_person.py +++ b/posthog/api/test/test_person.py @@ -873,7 +873,7 @@ def test_pagination_limit(self): create_person(team_id=self.team.pk, version=0) returned_ids = [] - with self.assertNumQueries(10): + with self.assertNumQueries(9): response = self.client.get("/api/person/?limit=10").json() self.assertEqual(len(response["results"]), 9) returned_ids += [x["distinct_ids"][0] for x in response["results"]] diff --git a/posthog/api/test/test_team.py b/posthog/api/test/test_team.py index 0040ddd257e2d..0e40b6a595d36 100644 --- a/posthog/api/test/test_team.py +++ b/posthog/api/test/test_team.py @@ -16,6 +16,7 @@ from posthog.models import ActivityLog, EarlyAccessFeature from posthog.models.async_deletion.async_deletion import AsyncDeletion, DeletionType from posthog.models.dashboard import Dashboard +from posthog.models.group_type_mapping import GroupTypeMapping from posthog.models.instance_setting import get_instance_setting from posthog.models.organization import Organization, OrganizationMembership from posthog.models.personal_api_key import PersonalAPIKey, hash_key_value @@ -87,6 +88,26 @@ def test_retrieve_team(self): self.assertNotIn("event_names_with_usage", response_data) self.assertNotIn("event_properties_with_usage", response_data) + def test_retrieve_team_has_group_types(self): + other_team = Team.objects.create(organization=self.organization, project=self.project) + + response = self.client.get("/api/environments/@current/") + response_data = response.json() + + self.assertEqual(response.status_code, status.HTTP_200_OK, response_data) + self.assertEqual(response_data["has_group_types"], False) + + # Creating a group type in the same project, but different team + GroupTypeMapping.objects.create( + project=self.project, team=other_team, group_type="person", group_type_index=0 + ) + + response = self.client.get("/api/environments/@current/") + response_data = response.json() + + self.assertEqual(response.status_code, status.HTTP_200_OK, response_data) + self.assertEqual(response_data["has_group_types"], True) # Irreleveant that group type has different `team` + def test_cant_retrieve_team_from_another_org(self): org = Organization.objects.create(name="New Org") team = Team.objects.create(organization=org, name="Default project") diff --git a/posthog/clickhouse/cluster.py b/posthog/clickhouse/cluster.py index 3aa67c94ff3b5..75c91db9da75f 100644 --- a/posthog/clickhouse/cluster.py +++ b/posthog/clickhouse/cluster.py @@ -52,7 +52,9 @@ def result( class ConnectionInfo(NamedTuple): address: str - port: int + + def make_pool(self) -> ChPool: + return make_ch_pool(host=self.address) class HostInfo(NamedTuple): @@ -67,10 +69,10 @@ class HostInfo(NamedTuple): class ClickhouseCluster: def __init__(self, bootstrap_client: Client, extra_hosts: Sequence[ConnectionInfo] | None = None) -> None: self.__hosts = [ - HostInfo(ConnectionInfo(host_address, port), shard_num, replica_num) - for (host_address, port, shard_num, replica_num) in bootstrap_client.execute( + HostInfo(ConnectionInfo(host_address), shard_num, replica_num) + for (host_address, shard_num, replica_num) in bootstrap_client.execute( """ - SELECT host_address, port, shard_num, replica_num + SELECT host_address, shard_num, replica_num FROM system.clusters WHERE name = %(name)s ORDER BY shard_num, replica_num @@ -87,7 +89,7 @@ def __init__(self, bootstrap_client: Client, extra_hosts: Sequence[ConnectionInf def __get_task_function(self, host: HostInfo, fn: Callable[[Client], T]) -> Callable[[], T]: pool = self.__pools.get(host) if pool is None: - pool = self.__pools[host] = make_ch_pool(host=host.connection_info.address, port=host.connection_info.port) + pool = self.__pools[host] = host.connection_info.make_pool() def task(): with pool.get_client() as client: diff --git a/posthog/clickhouse/materialized_columns.py b/posthog/clickhouse/materialized_columns.py index 2ff858274ab4d..09b2d8b24c6dc 100644 --- a/posthog/clickhouse/materialized_columns.py +++ b/posthog/clickhouse/materialized_columns.py @@ -1,6 +1,6 @@ -from datetime import timedelta +from typing import Protocol -from posthog.cache_utils import cache_for +from posthog.models.instance_setting import get_instance_setting from posthog.models.property import PropertyName, TableColumn, TableWithProperties from posthog.settings import EE_AVAILABLE @@ -8,19 +8,25 @@ ColumnName = str TablesWithMaterializedColumns = TableWithProperties + +class MaterializedColumn(Protocol): + name: ColumnName + is_nullable: bool + + if EE_AVAILABLE: - from ee.clickhouse.materialized_columns.columns import get_materialized_columns -else: + from ee.clickhouse.materialized_columns.columns import get_enabled_materialized_columns - def get_materialized_columns( - table: TablesWithMaterializedColumns, - exclude_disabled_columns: bool = False, - ) -> dict[tuple[PropertyName, TableColumn], ColumnName]: - return {} + def get_materialized_column_for_property( + table: TablesWithMaterializedColumns, table_column: TableColumn, property_name: PropertyName + ) -> MaterializedColumn | None: + if not get_instance_setting("MATERIALIZED_COLUMNS_ENABLED"): + return None + return get_enabled_materialized_columns(table).get((property_name, table_column)) +else: -@cache_for(timedelta(minutes=15)) -def get_enabled_materialized_columns( - table: TablesWithMaterializedColumns, -) -> dict[tuple[PropertyName, TableColumn], ColumnName]: - return get_materialized_columns(table, exclude_disabled_columns=True) + def get_materialized_column_for_property( + table: TablesWithMaterializedColumns, table_column: TableColumn, property_name: PropertyName + ) -> MaterializedColumn | None: + return None diff --git a/posthog/clickhouse/migrations/0026_fix_materialized_window_and_session_ids.py b/posthog/clickhouse/migrations/0026_fix_materialized_window_and_session_ids.py index 7e7847c570bac..1be2a1c033c66 100644 --- a/posthog/clickhouse/migrations/0026_fix_materialized_window_and_session_ids.py +++ b/posthog/clickhouse/migrations/0026_fix_materialized_window_and_session_ids.py @@ -1,6 +1,6 @@ from infi.clickhouse_orm import migrations -from posthog.clickhouse.materialized_columns import get_materialized_columns +from posthog.clickhouse.materialized_columns import get_materialized_column_for_property from posthog.client import sync_execute from posthog.settings import CLICKHOUSE_CLUSTER @@ -45,9 +45,9 @@ def materialize_session_and_window_id(database): properties = ["$session_id", "$window_id"] for property_name in properties: - materialized_columns = get_materialized_columns("events") + current_materialized_column = get_materialized_column_for_property("events", "properties", property_name) # If the column is not materialized, materialize it - if (property_name, "properties") not in materialized_columns: + if current_materialized_column is None: materialize("events", property_name, property_name) # Now, we need to clean up any potentail inconsistencies with existing column names @@ -71,9 +71,8 @@ def materialize_session_and_window_id(database): # materialized the column or renamed the column, and then ran the 0004_... async migration # before this migration runs. possible_old_column_names = {"mat_" + property_name} - current_materialized_column_name = materialized_columns.get((property_name, "properties"), None) - if current_materialized_column_name is not None and current_materialized_column_name != property_name: - possible_old_column_names.add(current_materialized_column_name) + if current_materialized_column is not None and current_materialized_column.name != property_name: + possible_old_column_names.add(current_materialized_column.name) for possible_old_column_name in possible_old_column_names: ensure_only_new_column_exists(database, "sharded_events", possible_old_column_name, property_name) diff --git a/posthog/demo/matrix/manager.py b/posthog/demo/matrix/manager.py index 0abc17f32ca08..f52a74fa6ba9a 100644 --- a/posthog/demo/matrix/manager.py +++ b/posthog/demo/matrix/manager.py @@ -204,7 +204,7 @@ def _erase_master_team_data(cls): # ) # ] # ) - GroupTypeMapping.objects.filter(team_id=cls.MASTER_TEAM_ID).delete() + GroupTypeMapping.objects.filter(project_id=cls.MASTER_TEAM_ID).delete() def _copy_analytics_data_from_master_team(self, target_team: Team): from posthog.models.event.sql import COPY_EVENTS_BETWEEN_TEAMS @@ -222,11 +222,11 @@ def _copy_analytics_data_from_master_team(self, target_team: Team): sync_execute(COPY_PERSON_DISTINCT_ID2S_BETWEEN_TEAMS, copy_params) sync_execute(COPY_EVENTS_BETWEEN_TEAMS, copy_params) sync_execute(COPY_GROUPS_BETWEEN_TEAMS, copy_params) - GroupTypeMapping.objects.filter(team_id=target_team.pk).delete() + GroupTypeMapping.objects.filter(project_id=target_team.project_id).delete() GroupTypeMapping.objects.bulk_create( ( - GroupTypeMapping(team=target_team, project_id=target_team.project_id, **record) - for record in GroupTypeMapping.objects.filter(team_id=self.MASTER_TEAM_ID).values( + GroupTypeMapping(team_id=target_team.id, project_id=target_team.project_id, **record) + for record in GroupTypeMapping.objects.filter(project_id=self.MASTER_TEAM_ID).values( "group_type", "group_type_index", "name_singular", "name_plural" ) ), diff --git a/posthog/hogql/database/database.py b/posthog/hogql/database/database.py index 37370800f30c3..9ca4500aa2abd 100644 --- a/posthog/hogql/database/database.py +++ b/posthog/hogql/database/database.py @@ -287,7 +287,7 @@ def create_hogql_database( "$virt_initial_channel_type", modifiers.customChannelTypeRules ) - for mapping in GroupTypeMapping.objects.filter(team=team): + for mapping in GroupTypeMapping.objects.filter(project_id=team.project_id): if database.events.fields.get(mapping.group_type) is None: database.events.fields[mapping.group_type] = FieldTraverser(chain=[f"group_{mapping.group_type_index}"]) diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 37fea932f2014..418e2f6354807 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -6,7 +6,11 @@ from typing import Literal, Optional, Union, cast from uuid import UUID -from posthog.clickhouse.materialized_columns import TablesWithMaterializedColumns, get_enabled_materialized_columns +from posthog.clickhouse.materialized_columns import ( + MaterializedColumn, + TablesWithMaterializedColumns, + get_materialized_column_for_property, +) from posthog.clickhouse.property_groups import property_groups from posthog.hogql import ast from posthog.hogql.base import AST, _T_AST @@ -197,6 +201,7 @@ class JoinExprResponse: class PrintableMaterializedColumn: table: Optional[str] column: str + is_nullable: bool def __str__(self) -> str: if self.table is None: @@ -1321,10 +1326,11 @@ def __get_all_materialized_property_sources( field_name = cast(Union[Literal["properties"], Literal["person_properties"]], field.name) materialized_column = self._get_materialized_column(table_name, property_name, field_name) - if materialized_column: + if materialized_column is not None: yield PrintableMaterializedColumn( self.visit(field_type.table_type), - self._print_identifier(materialized_column), + self._print_identifier(materialized_column.name), + is_nullable=materialized_column.is_nullable, ) if self.context.modifiers.propertyGroupsMode in ( @@ -1352,8 +1358,12 @@ def __get_all_materialized_property_sources( materialized_column = self._get_materialized_column("events", property_name, "person_properties") else: materialized_column = self._get_materialized_column("person", property_name, "properties") - if materialized_column: - yield PrintableMaterializedColumn(None, self._print_identifier(materialized_column)) + if materialized_column is not None: + yield PrintableMaterializedColumn( + None, + self._print_identifier(materialized_column.name), + is_nullable=materialized_column.is_nullable, + ) def visit_property_type(self, type: ast.PropertyType): if type.joined_subquery is not None and type.joined_subquery_field_name is not None: @@ -1361,7 +1371,10 @@ def visit_property_type(self, type: ast.PropertyType): materialized_property_source = self.__get_materialized_property_source_for_property_type(type) if materialized_property_source is not None: - if isinstance(materialized_property_source, PrintableMaterializedColumn): + if ( + isinstance(materialized_property_source, PrintableMaterializedColumn) + and not materialized_property_source.is_nullable + ): # TODO: rematerialize all columns to properly support empty strings and "null" string values. if self.context.modifiers.materializationMode == MaterializationMode.LEGACY_NULL_AS_STRING: materialized_property_sql = f"nullIf({materialized_property_source}, '')" @@ -1511,9 +1524,10 @@ def _unsafe_json_extract_trim_quotes(self, unsafe_field: str, unsafe_args: list[ def _get_materialized_column( self, table_name: str, property_name: PropertyName, field_name: TableColumn - ) -> Optional[str]: - materialized_columns = get_enabled_materialized_columns(cast(TablesWithMaterializedColumns, table_name)) - return materialized_columns.get((property_name, field_name), None) + ) -> MaterializedColumn | None: + return get_materialized_column_for_property( + cast(TablesWithMaterializedColumns, table_name), field_name, property_name + ) def _get_timezone(self) -> str: return self.context.database.get_timezone() if self.context.database else "UTC" diff --git a/posthog/hogql/test/test_printer.py b/posthog/hogql/test/test_printer.py index 8d7dad46040ac..4f2422263d0c8 100644 --- a/posthog/hogql/test/test_printer.py +++ b/posthog/hogql/test/test_printer.py @@ -460,14 +460,22 @@ def test_hogql_properties_materialized_json_access(self): self.assertEqual(1 + 2, 3) return - materialize("events", "withmat") context = HogQLContext(team_id=self.team.pk) + materialize("events", "withmat") self.assertEqual( self._expr("properties.withmat.json.yet", context), "replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(nullIf(nullIf(events.mat_withmat, ''), 'null'), %(hogql_val_0)s, %(hogql_val_1)s), ''), 'null'), '^\"|\"$', '')", ) self.assertEqual(context.values, {"hogql_val_0": "json", "hogql_val_1": "yet"}) + context = HogQLContext(team_id=self.team.pk) + materialize("events", "withmat_nullable", is_nullable=True) + self.assertEqual( + self._expr("properties.withmat_nullable.json.yet", context), + "replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.mat_withmat_nullable, %(hogql_val_0)s, %(hogql_val_1)s), ''), 'null'), '^\"|\"$', '')", + ) + self.assertEqual(context.values, {"hogql_val_0": "json", "hogql_val_1": "yet"}) + def test_materialized_fields_and_properties(self): try: from ee.clickhouse.materialized_columns.analyze import materialize @@ -499,6 +507,12 @@ def test_materialized_fields_and_properties(self): "nullIf(nullIf(events.`mat_$browser_______`, ''), 'null')", ) + materialize("events", "nullable_property", is_nullable=True) + self.assertEqual( + self._expr("properties['nullable_property']"), + "events.mat_nullable_property", + ) + def test_property_groups(self): context = HogQLContext( team_id=self.team.pk, diff --git a/posthog/hogql/transforms/property_types.py b/posthog/hogql/transforms/property_types.py index 6dbac74590da6..e561607629f1f 100644 --- a/posthog/hogql/transforms/property_types.py +++ b/posthog/hogql/transforms/property_types.py @@ -1,6 +1,10 @@ -from typing import Literal, Optional, cast +from typing import Literal, cast -from posthog.clickhouse.materialized_columns import TablesWithMaterializedColumns, get_enabled_materialized_columns +from posthog.clickhouse.materialized_columns import ( + MaterializedColumn, + TablesWithMaterializedColumns, + get_materialized_column_for_property, +) from posthog.hogql import ast from posthog.hogql.context import HogQLContext from posthog.hogql.database.models import ( @@ -258,7 +262,7 @@ def _add_property_notice( message = f"{property_type.capitalize()} property '{property_name}' is of type '{field_type}'." if self.context.debug: - if materialized_column: + if materialized_column is not None: message += " This property is materialized ⚡️." else: message += " This property is not materialized 🐢." @@ -277,6 +281,7 @@ def _add_notice(self, node: ast.Field, message: str): def _get_materialized_column( self, table_name: str, property_name: PropertyName, field_name: TableColumn - ) -> Optional[str]: - materialized_columns = get_enabled_materialized_columns(cast(TablesWithMaterializedColumns, table_name)) - return materialized_columns.get((property_name, field_name), None) + ) -> MaterializedColumn | None: + return get_materialized_column_for_property( + cast(TablesWithMaterializedColumns, table_name), field_name, property_name + ) diff --git a/posthog/hogql_queries/actors_query_runner.py b/posthog/hogql_queries/actors_query_runner.py index cec4b7019f212..cde5bd2d6311f 100644 --- a/posthog/hogql_queries/actors_query_runner.py +++ b/posthog/hogql_queries/actors_query_runner.py @@ -3,7 +3,7 @@ from collections.abc import Sequence, Iterator from posthog.hogql import ast -from posthog.hogql.constants import HogQLGlobalSettings +from posthog.hogql.constants import HogQLGlobalSettings, HogQLQuerySettings from posthog.hogql.parser import parse_expr, parse_order_expr from posthog.hogql.property import has_aggregation from posthog.hogql.resolver_utils import extract_select_queries @@ -307,6 +307,7 @@ def to_query(self) -> ast.SelectQuery: having=having, group_by=group_by if has_any_aggregation else None, order_by=order_by, + settings=HogQLQuerySettings(join_algorithm="auto", optimize_aggregation_in_order=True), ) def to_actors_query(self) -> ast.SelectQuery: diff --git a/posthog/hogql_queries/experiments/test/test_experiment_funnels_query_runner.py b/posthog/hogql_queries/experiments/test/test_experiment_funnels_query_runner.py index 98599790f3865..3f3127fcb1b55 100644 --- a/posthog/hogql_queries/experiments/test/test_experiment_funnels_query_runner.py +++ b/posthog/hogql_queries/experiments/test/test_experiment_funnels_query_runner.py @@ -16,6 +16,7 @@ from posthog.constants import ExperimentNoResultsErrorKeys import json from posthog.test.test_journeys import journeys_for +from flaky import flaky class TestExperimentFunnelsQueryRunner(ClickhouseTestMixin, APIBaseTest): @@ -128,6 +129,7 @@ def test_query_runner(self): self.assertIn("control", result.credible_intervals) self.assertIn("test", result.credible_intervals) + @flaky(max_runs=10, min_passes=1) @freeze_time("2020-01-01T12:00:00Z") def test_query_runner_standard_flow(self): feature_flag = self.create_feature_flag() diff --git a/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py b/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py index 8837bfeab8607..4402afde55eec 100644 --- a/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py +++ b/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py @@ -29,6 +29,7 @@ from pyarrow import parquet as pq import pyarrow as pa import json +from flaky import flaky from boto3 import resource from botocore.config import Config @@ -650,6 +651,7 @@ def test_query_runner_with_avg_math(self): prepared_count_query = query_runner.prepared_count_query self.assertEqual(prepared_count_query.series[0].math, "sum") + @flaky(max_runs=10, min_passes=1) @freeze_time("2020-01-01T12:00:00Z") def test_query_runner_standard_flow(self): feature_flag = self.create_feature_flag() diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel.ambr index 2315f2b51ebf6..2f2933fb62433 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel.ambr @@ -193,7 +193,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.created_at DESC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -612,7 +614,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.created_at DESC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -730,7 +734,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.created_at DESC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -848,7 +854,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.created_at DESC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1848,7 +1856,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1989,7 +1999,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2130,7 +2142,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2271,7 +2285,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation.ambr index f95e83e21b1d9..4573056cf6cac 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation.ambr @@ -482,7 +482,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -673,7 +675,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -864,7 +868,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1055,7 +1061,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1408,7 +1416,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1599,7 +1609,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1790,7 +1802,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1981,7 +1995,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2520,7 +2536,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2635,7 +2653,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2750,7 +2770,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2865,7 +2887,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3128,7 +3152,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3243,7 +3269,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3490,7 +3518,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3605,7 +3635,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3720,7 +3752,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3835,7 +3869,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -4098,7 +4134,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -4213,7 +4251,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -4482,7 +4522,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -4604,7 +4646,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -4726,7 +4770,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -4848,7 +4894,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -5264,7 +5312,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -5386,7 +5436,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -5508,7 +5560,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -5630,7 +5684,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -6046,7 +6102,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -6168,7 +6226,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -6290,7 +6350,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -6412,7 +6474,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -6828,7 +6892,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -6950,7 +7016,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -7072,7 +7140,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -7194,7 +7264,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -7610,7 +7682,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -7732,7 +7806,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -7854,7 +7930,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -7976,7 +8054,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_actors.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_actors.ambr index ca6d26d135828..ea2c02c121f49 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_actors.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_actors.ambr @@ -163,7 +163,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -428,7 +430,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -619,7 +623,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_actors_udf.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_actors_udf.ambr index dcec437b05683..f1f604cc85b02 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_actors_udf.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_actors_udf.ambr @@ -107,7 +107,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -239,7 +241,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -374,7 +378,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_udf.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_udf.ambr index 0912fa7845d36..71680063ab927 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_udf.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_udf.ambr @@ -358,7 +358,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -493,7 +495,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -628,7 +632,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -763,7 +769,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1026,7 +1034,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1161,7 +1171,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1296,7 +1308,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1431,7 +1445,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1820,7 +1836,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1887,7 +1905,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1954,7 +1974,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2021,7 +2043,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2202,7 +2226,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2269,7 +2295,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2434,7 +2462,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2501,7 +2531,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2568,7 +2600,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2635,7 +2669,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2816,7 +2852,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2883,7 +2921,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3070,7 +3110,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3144,7 +3186,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3218,7 +3262,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3292,7 +3338,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3592,7 +3640,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3666,7 +3716,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3740,7 +3792,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3814,7 +3868,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -4114,7 +4170,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -4188,7 +4246,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -4262,7 +4322,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -4336,7 +4398,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -4636,7 +4700,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -4710,7 +4776,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -4784,7 +4852,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -4858,7 +4928,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -5158,7 +5230,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -5232,7 +5306,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -5306,7 +5382,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -5380,7 +5458,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_persons.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_persons.ambr index f623ea36204cd..d2d6bbab5f69f 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_persons.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_persons.ambr @@ -162,7 +162,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -349,7 +351,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -536,7 +540,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_persons_udf.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_persons_udf.ambr index 9fbd6af6c74ed..38542d31104b9 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_persons_udf.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_persons_udf.ambr @@ -53,7 +53,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -131,7 +133,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -209,7 +213,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict.ambr index 163abb23ac305..d06597a0b35da 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict.ambr @@ -754,7 +754,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -872,7 +874,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -990,7 +994,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1108,7 +1114,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_persons.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_persons.ambr index f2e6752d368d0..651f296097a7b 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_persons.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_persons.ambr @@ -122,7 +122,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -269,7 +271,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -416,7 +420,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_persons_udf.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_persons_udf.ambr index 846e534decf6a..3a0a96ffa7162 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_persons_udf.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_persons_udf.ambr @@ -53,7 +53,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -131,7 +133,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -209,7 +213,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_udf.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_udf.ambr index f0bbdae5329d3..21eb841990a53 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_udf.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_udf.ambr @@ -554,7 +554,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -637,7 +639,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -720,7 +724,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -803,7 +809,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_actors.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_actors.ambr index e735153b628c3..5eafb6901598c 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_actors.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_actors.ambr @@ -148,7 +148,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -321,7 +323,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -494,7 +498,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_actors_udf.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_actors_udf.ambr index 6e86eda210324..36d25b420b5ee 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_actors_udf.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_actors_udf.ambr @@ -51,7 +51,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -127,7 +129,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -203,7 +207,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_udf.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_udf.ambr index cb6d8db14b8ce..545e7fa4d506c 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_udf.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_udf.ambr @@ -115,7 +115,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.created_at DESC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -419,7 +421,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.created_at DESC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -491,7 +495,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.created_at DESC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -563,7 +569,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.created_at DESC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1258,7 +1266,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1341,7 +1351,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1424,7 +1436,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1507,7 +1521,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_unordered.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_unordered.ambr index 3f5e9e4467e64..c52ab6eb60b8d 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_unordered.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_unordered.ambr @@ -1471,7 +1471,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2310,7 +2312,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2572,7 +2576,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2834,7 +2840,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_unordered_persons.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_unordered_persons.ambr index a4bfbc566ff43..4315f4b9bba92 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_unordered_persons.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_unordered_persons.ambr @@ -266,7 +266,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/test/__snapshots__/test_insight_actors_query_runner.ambr b/posthog/hogql_queries/insights/test/__snapshots__/test_insight_actors_query_runner.ambr index b8db8df7c3613..b2a235c2308e0 100644 --- a/posthog/hogql_queries/insights/test/__snapshots__/test_insight_actors_query_runner.ambr +++ b/posthog/hogql_queries/insights/test/__snapshots__/test_insight_actors_query_runner.ambr @@ -66,7 +66,8 @@ WHERE equals(person.team_id, 99999) GROUP BY person.id HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'US/Pacific'), person.version), plus(now64(6, 'US/Pacific'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) - ORDER BY persons.properties___name ASC) + ORDER BY persons.properties___name ASC SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto') LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -154,7 +155,8 @@ and isNull(toStartOfDay(parseDateTime64BestEffortOrNull('2020-01-12', 6, 'US/Pacific')))), ifNull(equals(status, 'returning'), 0))) AS source))) GROUP BY person.id HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'US/Pacific'), person.version), plus(now64(6, 'US/Pacific'), toIntervalDay(1))), 0))))) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) - ORDER BY persons.properties___name ASC) + ORDER BY persons.properties___name ASC SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto') LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -192,7 +194,8 @@ WHERE equals(groups.team_id, 99999) GROUP BY groups.group_type_index, groups.group_key) AS groups ON equals(groups.key, source.group_key) - ORDER BY groups.properties___name ASC) + ORDER BY groups.properties___name ASC SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto') LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -258,7 +261,8 @@ WHERE ifNull(equals(num_intervals, 2), 0)) AS source))) GROUP BY person.id HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'US/Pacific'), person.version), plus(now64(6, 'US/Pacific'), toIntervalDay(1))), 0))))) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) - ORDER BY persons.properties___name ASC) + ORDER BY persons.properties___name ASC SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto') LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -291,7 +295,8 @@ WHERE equals(groups.team_id, 99999) GROUP BY groups.group_type_index, groups.group_key) AS groups ON equals(groups.key, source.actor_id) - ORDER BY groups.properties___name ASC) + ORDER BY groups.properties___name ASC SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto') LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -357,7 +362,8 @@ GROUP BY actor_id) AS source))) GROUP BY person.id HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'US/Pacific'), person.version), plus(now64(6, 'US/Pacific'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) - ORDER BY persons.properties___name ASC) + ORDER BY persons.properties___name ASC SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto') LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -432,7 +438,8 @@ GROUP BY actor_id) AS source))) GROUP BY person.id HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'US/Pacific'), person.version), plus(now64(6, 'US/Pacific'), toIntervalDay(1))), 0))))) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) - ORDER BY persons.properties___name ASC) + ORDER BY persons.properties___name ASC SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto') LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, diff --git a/posthog/hogql_queries/insights/test/__snapshots__/test_paths_query_runner_ee.ambr b/posthog/hogql_queries/insights/test/__snapshots__/test_paths_query_runner_ee.ambr index 20f2012034bba..905fb297fe4a9 100644 --- a/posthog/hogql_queries/insights/test/__snapshots__/test_paths_query_runner_ee.ambr +++ b/posthog/hogql_queries/insights/test/__snapshots__/test_paths_query_runner_ee.ambr @@ -1301,7 +1301,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1448,7 +1450,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1595,7 +1599,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1744,7 +1750,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1891,7 +1899,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2038,7 +2048,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2187,7 +2199,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2334,7 +2348,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2481,7 +2497,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2723,7 +2741,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -2890,7 +2910,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3043,7 +3065,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3208,7 +3232,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3383,7 +3409,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3723,7 +3751,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -3967,7 +3997,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -4211,7 +4243,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -4455,7 +4489,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -4849,7 +4885,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -4996,7 +5034,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -5225,7 +5265,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -5454,7 +5496,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -5601,7 +5645,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -5748,7 +5794,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -13155,7 +13203,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -13302,7 +13352,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -13449,7 +13501,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -13598,7 +13652,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -13745,7 +13801,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -13892,7 +13950,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -14041,7 +14101,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -14188,7 +14250,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -14335,7 +14399,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -14577,7 +14643,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -14744,7 +14812,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -14897,7 +14967,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -15062,7 +15134,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -15237,7 +15311,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -15577,7 +15653,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -15821,7 +15899,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -16065,7 +16145,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -16309,7 +16391,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -16703,7 +16787,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -16850,7 +16936,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -17079,7 +17167,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -17308,7 +17398,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -17455,7 +17547,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -17602,7 +17696,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY persons.id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/test/__snapshots__/test_retention_query_runner.ambr b/posthog/hogql_queries/insights/test/__snapshots__/test_retention_query_runner.ambr index d726c177939a3..477e07b03d8f9 100644 --- a/posthog/hogql_queries/insights/test/__snapshots__/test_retention_query_runner.ambr +++ b/posthog/hogql_queries/insights/test/__snapshots__/test_retention_query_runner.ambr @@ -63,7 +63,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY length(source.appearances) DESC, source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -164,7 +166,9 @@ groups.group_key) AS groups ON equals(groups.key, source.actor_id) ORDER BY length(source.appearances) DESC, source.actor_id ASC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr index 4b096b060262e..9bd3a90b8d559 100644 --- a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr +++ b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr @@ -42,11 +42,14 @@ # name: TestTrends.test_action_filtering_with_cohort.4 ''' /* celery:posthog.tasks.calculate_cohort.clear_stale_cohort */ - SELECT count() + SELECT team_id, + count() AS stale_people_count FROM cohortpeople - WHERE team_id = 99999 + WHERE team_id IN [1, 2, 3, 4, 5 /* ... */] AND cohort_id = 99999 AND version < 2 + GROUP BY team_id + HAVING stale_people_count > 0 ''' # --- # name: TestTrends.test_action_filtering_with_cohort.5 @@ -138,11 +141,14 @@ # name: TestTrends.test_action_filtering_with_cohort_poe_v2.4 ''' /* celery:posthog.tasks.calculate_cohort.clear_stale_cohort */ - SELECT count() + SELECT team_id, + count() AS stale_people_count FROM cohortpeople - WHERE team_id = 99999 + WHERE team_id IN [1, 2, 3, 4, 5 /* ... */] AND cohort_id = 99999 AND version < 2 + GROUP BY team_id + HAVING stale_people_count > 0 ''' # --- # name: TestTrends.test_action_filtering_with_cohort_poe_v2.5 @@ -278,7 +284,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY source.event_count DESC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, @@ -1422,7 +1430,9 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) ORDER BY source.event_count DESC LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, max_execution_time=60, allow_experimental_object_type=1, format_csv_allow_double_quotes=0, diff --git a/posthog/hogql_queries/insights/trends/trends_query_runner.py b/posthog/hogql_queries/insights/trends/trends_query_runner.py index 668cd8b2afb48..c7de458195b2f 100644 --- a/posthog/hogql_queries/insights/trends/trends_query_runner.py +++ b/posthog/hogql_queries/insights/trends/trends_query_runner.py @@ -219,7 +219,10 @@ def to_actors_query_options(self) -> InsightActorsQueryOptionsResponse: for value in self.query.breakdownFilter.breakdown: if value != "all" and str(value) != "0": res_breakdown.append( - BreakdownItem(label=Cohort.objects.get(pk=int(value), team=self.team).name, value=value) + BreakdownItem( + label=Cohort.objects.get(pk=int(value), team__project_id=self.team.project_id).name, + value=value, + ) ) else: res_breakdown.append(BreakdownItem(label="all users", value="all")) diff --git a/posthog/hogql_queries/test/test_actors_query_runner.py b/posthog/hogql_queries/test/test_actors_query_runner.py index 904c1adad8d9f..36a12166cb589 100644 --- a/posthog/hogql_queries/test/test_actors_query_runner.py +++ b/posthog/hogql_queries/test/test_actors_query_runner.py @@ -1,3 +1,5 @@ +from typing import cast + import pytest from posthog.hogql import ast @@ -66,7 +68,7 @@ def test_default_persons_query(self): runner = self._create_runner(ActorsQuery()) query = runner.to_query() - query = clear_locations(query) + query = cast(ast.SelectQuery, clear_locations(query)) expected = ast.SelectQuery( select=[ ast.Field(chain=["id"]), @@ -78,7 +80,8 @@ def test_default_persons_query(self): where=None, order_by=[ast.OrderExpr(expr=ast.Field(chain=["created_at"]), order="DESC")], ) - assert clear_locations(query) == expected + query.settings = None + assert query == expected response = runner.calculate() assert len(response.results) == 10 diff --git a/posthog/management/commands/generate_demo_data.py b/posthog/management/commands/generate_demo_data.py index ce094620453a1..dae5cca8ffa73 100644 --- a/posthog/management/commands/generate_demo_data.py +++ b/posthog/management/commands/generate_demo_data.py @@ -2,6 +2,7 @@ import logging import secrets from time import monotonic +from typing import Optional from django.core import exceptions from django.core.management.base import BaseCommand @@ -67,13 +68,13 @@ def handle(self, *args, **options): seed = options.get("seed") or secrets.token_hex(16) now = options.get("now") or dt.datetime.now(dt.UTC) existing_team_id = options.get("team_id") - if ( - existing_team_id is not None - and existing_team_id != 0 - and not Team.objects.filter(pk=existing_team_id).exists() - ): - print(f"Team with ID {options['team_id']} does not exist!") - return + existing_team: Optional[Team] = None + if existing_team_id is not None and existing_team_id != 0: + try: + existing_team = Team.objects.get(pk=existing_team_id) + except Team.DoesNotExist: + print(f"Team with ID {options['team_id']} does not exist!") + return print("Instantiating the Matrix...") matrix = HedgeboxMatrix( seed, @@ -81,8 +82,8 @@ def handle(self, *args, **options): days_past=options["days_past"], days_future=options["days_future"], n_clusters=options["n_clusters"], - group_type_index_offset=GroupTypeMapping.objects.filter(team_id=existing_team_id).count() - if existing_team_id + group_type_index_offset=GroupTypeMapping.objects.filter(project_id=existing_team.project_id).count() + if existing_team else 0, ) print("Running simulation...") diff --git a/posthog/models/cohort/cohort.py b/posthog/models/cohort/cohort.py index 1ab980bfc6796..f658bbd07e6de 100644 --- a/posthog/models/cohort/cohort.py +++ b/posthog/models/cohort/cohort.py @@ -250,11 +250,16 @@ def calculate_people_ch(self, pending_version: int, *, initiating_user_id: Optio clear_stale_cohort.delay(self.pk, before_version=pending_version) - def insert_users_by_list(self, items: list[str]) -> None: - """ - Items is a list of distinct_ids + def insert_users_by_list(self, items: list[str], *, team_id: Optional[int] = None) -> None: """ + Insert a list of users identified by their distinct ID into the cohort, for the given team. + Args: + items: List of distinct IDs of users to be inserted into the cohort. + team_id: ID of the team for which to insert the users. Defaults to `self.team`, because of a lot of existing usage in tests. + """ + if team_id is None: + team_id = self.team_id batchsize = 1000 from posthog.models.cohort.util import ( insert_static_cohort, @@ -272,10 +277,10 @@ def insert_users_by_list(self, items: list[str]) -> None: for i in range(0, len(items), batchsize): batch = items[i : i + batchsize] persons_query = ( - Person.objects.filter(team_id=self.team_id) + Person.objects.filter(team_id=team_id) .filter( Q( - persondistinctid__team_id=self.team_id, + persondistinctid__team_id=team_id, persondistinctid__distinct_id__in=batch, ) ) @@ -284,7 +289,7 @@ def insert_users_by_list(self, items: list[str]) -> None: insert_static_cohort( list(persons_query.values_list("uuid", flat=True)), self.pk, - self.team, + team_id=team_id, ) sql, params = persons_query.distinct("pk").only("pk").query.sql_with_params() query = UPDATE_QUERY.format( @@ -297,7 +302,7 @@ def insert_users_by_list(self, items: list[str]) -> None: ) cursor.execute(query, params) - count = get_static_cohort_size(self) + count = get_static_cohort_size(cohort_id=self.id, team_id=self.team_id) self.count = count self.is_calculating = False @@ -313,7 +318,18 @@ def insert_users_by_list(self, items: list[str]) -> None: self.save() capture_exception(err) - def insert_users_list_by_uuid(self, items: list[str], insert_in_clickhouse: bool = False, batchsize=1000) -> None: + def insert_users_list_by_uuid( + self, items: list[str], insert_in_clickhouse: bool = False, batchsize=1000, *, team_id: int + ) -> None: + """ + Insert a list of users identified by their UUID into the cohort, for the given team. + + Args: + items: List of user UUIDs to be inserted into the cohort. + insert_in_clickhouse: Whether the data should also be inserted into ClickHouse. + batchsize: Number of UUIDs to process in each batch. + team_id: The ID of the team to which the cohort belongs. + """ from posthog.models.cohort.util import get_static_cohort_size, insert_static_cohort try: @@ -321,13 +337,13 @@ def insert_users_list_by_uuid(self, items: list[str], insert_in_clickhouse: bool for i in range(0, len(items), batchsize): batch = items[i : i + batchsize] persons_query = ( - Person.objects.filter(team_id=self.team_id).filter(uuid__in=batch).exclude(cohort__id=self.id) + Person.objects.filter(team_id=team_id).filter(uuid__in=batch).exclude(cohort__id=self.id) ) if insert_in_clickhouse: insert_static_cohort( list(persons_query.values_list("uuid", flat=True)), self.pk, - self.team, + team_id=team_id, ) sql, params = persons_query.distinct("pk").only("pk").query.sql_with_params() query = UPDATE_QUERY.format( @@ -340,7 +356,7 @@ def insert_users_list_by_uuid(self, items: list[str], insert_in_clickhouse: bool ) cursor.execute(query, params) - count = get_static_cohort_size(self) + count = get_static_cohort_size(cohort_id=self.id, team_id=self.team_id) self.count = count self.is_calculating = False @@ -357,12 +373,6 @@ def insert_users_list_by_uuid(self, items: list[str], insert_in_clickhouse: bool self.save() capture_exception(err) - def _clickhouse_persons_query(self, batch_size=10000, offset=0): - from posthog.models.cohort.util import get_person_ids_by_cohort_id - - uuids = get_person_ids_by_cohort_id(team=self.team, cohort_id=self.pk, limit=batch_size, offset=offset) - return Person.objects.filter(uuid__in=uuids, team=self.team) - __repr__ = sane_repr("id", "name", "last_calculation") diff --git a/posthog/models/cohort/sql.py b/posthog/models/cohort/sql.py index a84394bae94a8..603f8addf08a2 100644 --- a/posthog/models/cohort/sql.py +++ b/posthog/models/cohort/sql.py @@ -91,6 +91,8 @@ """ STALE_COHORTPEOPLE = f""" -SELECT count() FROM cohortpeople -WHERE team_id = %(team_id)s AND cohort_id = %(cohort_id)s AND version < %(version)s +SELECT team_id, count() AS stale_people_count FROM cohortpeople +WHERE team_id IN %(team_ids)s AND cohort_id = %(cohort_id)s AND version < %(version)s +GROUP BY team_id +HAVING stale_people_count > 0 """ diff --git a/posthog/models/cohort/util.py b/posthog/models/cohort/util.py index fe589236fa62e..395085453c5e3 100644 --- a/posthog/models/cohort/util.py +++ b/posthog/models/cohort/util.py @@ -34,13 +34,10 @@ STALE_COHORTPEOPLE, ) from posthog.models.person.sql import ( - GET_LATEST_PERSON_SQL, - GET_PERSON_IDS_BY_FILTER, INSERT_PERSON_STATIC_COHORT, PERSON_STATIC_COHORT_TABLE, ) from posthog.models.property import Property, PropertyGroup -from posthog.queries.insight import insight_sync_execute from posthog.queries.person_distinct_id_query import get_team_distinct_ids_query # temporary marker to denote when cohortpeople table started being populated @@ -75,14 +72,14 @@ def format_person_query(cohort: Cohort, index: int, hogql_context: HogQLContext) return query, params -def print_cohort_hogql_query(cohort: Cohort, hogql_context: HogQLContext) -> str: +def print_cohort_hogql_query(cohort: Cohort, hogql_context: HogQLContext, *, team: Team) -> str: from posthog.hogql_queries.query_runner import get_query_runner if not cohort.query: raise ValueError("Cohort has no query") query = get_query_runner( - cast(dict, cohort.query), team=cast(Team, cohort.team), limit_context=LimitContext.COHORT_CALCULATION + cast(dict, cohort.query), team=team, limit_context=LimitContext.COHORT_CALCULATION ).to_query() for select_query in extract_select_queries(query): @@ -109,7 +106,7 @@ def print_cohort_hogql_query(cohort: Cohort, hogql_context: HogQLContext) -> str hogql_context.enable_select_queries = True hogql_context.limit_top_select = False - create_default_modifiers_for_team(cohort.team, hogql_context.modifiers) + create_default_modifiers_for_team(team, hogql_context.modifiers) return print_ast(query, context=hogql_context, dialect="clickhouse") @@ -262,10 +259,7 @@ def format_filter_query( def format_cohort_subquery( - cohort: Cohort, - index: int, - hogql_context: HogQLContext, - custom_match_field="person_id", + cohort: Cohort, index: int, hogql_context: HogQLContext, custom_match_field="person_id" ) -> tuple[str, dict[str, Any]]: is_precalculated = is_precalculated_query(cohort) if is_precalculated: @@ -277,46 +271,13 @@ def format_cohort_subquery( return person_query, params -def get_person_ids_by_cohort_id( - team: Team, - cohort_id: int, - limit: Optional[int] = None, - offset: Optional[int] = None, -): - from posthog.models.property.util import parse_prop_grouped_clauses - - filter = Filter(data={"properties": [{"key": "id", "value": cohort_id, "type": "cohort"}]}) - filter_query, filter_params = parse_prop_grouped_clauses( - team_id=team.pk, - property_group=filter.property_groups, - table_name="pdi", - hogql_context=filter.hogql_context, - ) - - results = insight_sync_execute( - GET_PERSON_IDS_BY_FILTER.format( - person_query=GET_LATEST_PERSON_SQL, - distinct_query=filter_query, - query="", - GET_TEAM_PERSON_DISTINCT_IDS=get_team_distinct_ids_query(team.pk), - offset="OFFSET %(offset)s" if offset else "", - limit="ORDER BY _timestamp ASC LIMIT %(limit)s" if limit else "", - ), - {**filter_params, "team_id": team.pk, "offset": offset, "limit": limit}, - query_type="get_person_ids_by_cohort_id", - team_id=team.pk, - ) - - return [str(row[0]) for row in results] - - -def insert_static_cohort(person_uuids: list[Optional[uuid.UUID]], cohort_id: int, team: Team): +def insert_static_cohort(person_uuids: list[Optional[uuid.UUID]], cohort_id: int, *, team_id: int): persons = [ { "id": str(uuid.uuid4()), "person_id": str(person_uuid), "cohort_id": cohort_id, - "team_id": team.pk, + "team_id": team_id, "_timestamp": datetime.now(), } for person_uuid in person_uuids @@ -324,12 +285,12 @@ def insert_static_cohort(person_uuids: list[Optional[uuid.UUID]], cohort_id: int sync_execute(INSERT_PERSON_STATIC_COHORT, persons) -def get_static_cohort_size(cohort: Cohort) -> Optional[int]: +def get_static_cohort_size(*, cohort_id: int, team_id: int) -> Optional[int]: count_result = sync_execute( GET_STATIC_COHORT_SIZE_SQL, { - "cohort_id": cohort.pk, - "team_id": cohort.team_id, + "cohort_id": cohort_id, + "team_id": team_id, }, ) @@ -342,22 +303,39 @@ def get_static_cohort_size(cohort: Cohort) -> Optional[int]: def recalculate_cohortpeople( cohort: Cohort, pending_version: int, *, initiating_user_id: Optional[int] ) -> Optional[int]: - hogql_context = HogQLContext(within_non_hogql_query=True, team_id=cohort.team_id) + """ + Recalculate cohort people for all environments of the project. + NOTE: Currently this only returns the count for the team where the cohort was created. Instead it should return for all teams. + """ + relevant_teams = Team.objects.order_by("id").filter(project_id=cohort.team.project_id) + count_by_team_id: dict[int, int] = {} + for team in relevant_teams: + count_for_team = _recalculate_cohortpeople_for_team( + cohort, pending_version, team, initiating_user_id=initiating_user_id + ) + count_by_team_id[team.id] = count_for_team or 0 + return count_by_team_id[cohort.team_id] + + +def _recalculate_cohortpeople_for_team( + cohort: Cohort, pending_version: int, team: Team, *, initiating_user_id: Optional[int] +) -> Optional[int]: + hogql_context = HogQLContext(within_non_hogql_query=True, team_id=team.id) cohort_query, cohort_params = format_person_query(cohort, 0, hogql_context) - before_count = get_cohort_size(cohort) + before_count = get_cohort_size(cohort, team_id=team.id) if before_count: logger.warn( "Recalculating cohortpeople starting", - team_id=cohort.team_id, + team_id=team.id, cohort_id=cohort.pk, size_before=before_count, ) recalcluate_cohortpeople_sql = RECALCULATE_COHORT_BY_ID.format(cohort_filter=cohort_query) - tag_queries(kind="cohort_calculation", team_id=cohort.team_id, query_type="CohortsQuery") + tag_queries(kind="cohort_calculation", team_id=team.id, query_type="CohortsQuery") if initiating_user_id: tag_queries(user_id=initiating_user_id) @@ -367,7 +345,7 @@ def recalculate_cohortpeople( **cohort_params, **hogql_context.values, "cohort_id": cohort.pk, - "team_id": cohort.team_id, + "team_id": team.id, "new_version": pending_version, }, settings={ @@ -379,12 +357,12 @@ def recalculate_cohortpeople( workload=Workload.OFFLINE, ) - count = get_cohort_size(cohort, override_version=pending_version) + count = get_cohort_size(cohort, override_version=pending_version, team_id=team.id) if count is not None and before_count is not None: logger.warn( "Recalculating cohortpeople done", - team_id=cohort.team_id, + team_id=team.id, cohort_id=cohort.pk, size_before=before_count, size=count, @@ -395,33 +373,40 @@ def recalculate_cohortpeople( def clear_stale_cohortpeople(cohort: Cohort, before_version: int) -> None: if cohort.version and cohort.version > 0: + relevant_team_ids = list(Team.objects.filter(project_id=cohort.team.project_id).values_list("pk", flat=True)) stale_count_result = sync_execute( STALE_COHORTPEOPLE, { "cohort_id": cohort.pk, - "team_id": cohort.team_id, + "team_ids": relevant_team_ids, "version": before_version, }, ) - if stale_count_result and len(stale_count_result) and len(stale_count_result[0]): - stale_count = stale_count_result[0][0] - if stale_count > 0: - # Don't do anything if it already exists - AsyncDeletion.objects.get_or_create( - deletion_type=DeletionType.Cohort_stale, - team_id=cohort.team.pk, - key=f"{cohort.pk}_{before_version}", - ) + team_ids_with_stale_cohortpeople = [row[0] for row in stale_count_result] + if team_ids_with_stale_cohortpeople: + AsyncDeletion.objects.bulk_create( + [ + AsyncDeletion( + deletion_type=DeletionType.Cohort_stale, + team_id=team_id, + # Only appending `team_id` if it's not the same as the cohort's `team_id``, so that + # the migration to environments does not accidentally cause duplicate `AsyncDeletion`s + key=f"{cohort.pk}_{before_version}{('_'+team_id) if team_id != cohort.team_id else ''}", + ) + for team_id in team_ids_with_stale_cohortpeople + ], + ignore_conflicts=True, + ) -def get_cohort_size(cohort: Cohort, override_version: Optional[int] = None) -> Optional[int]: +def get_cohort_size(cohort: Cohort, override_version: Optional[int] = None, *, team_id: int) -> Optional[int]: count_result = sync_execute( GET_COHORT_SIZE_SQL, { "cohort_id": cohort.pk, "version": override_version if override_version is not None else cohort.version, - "team_id": cohort.team_id, + "team_id": team_id, }, workload=Workload.OFFLINE, ) @@ -545,7 +530,7 @@ def get_dependent_cohorts( continue else: current_cohort = Cohort.objects.db_manager(using_database).get( - pk=cohort_id, team_id=cohort.team_id, deleted=False + pk=cohort_id, team__project_id=cohort.team.project_id, deleted=False ) seen_cohorts_cache[cohort_id] = current_cohort if current_cohort.id not in seen_cohort_ids: diff --git a/posthog/models/feature_flag/feature_flag.py b/posthog/models/feature_flag/feature_flag.py index c21af6a397117..beca926b7fbac 100644 --- a/posthog/models/feature_flag/feature_flag.py +++ b/posthog/models/feature_flag/feature_flag.py @@ -187,7 +187,7 @@ def transform_cohort_filters_for_easy_evaluation( return self.conditions else: cohort = Cohort.objects.db_manager(using_database).get( - pk=cohort_id, team_id=self.team_id, deleted=False + pk=cohort_id, team__project_id=self.team.project_id, deleted=False ) seen_cohorts_cache[cohort_id] = cohort except Cohort.DoesNotExist: @@ -291,7 +291,7 @@ def get_cohort_ids( continue else: cohort = Cohort.objects.db_manager(using_database).get( - pk=cohort_id, team_id=self.team_id, deleted=False + pk=cohort_id, team__project_id=self.team.project_id, deleted=False ) seen_cohorts_cache[cohort_id] = cohort diff --git a/posthog/models/feature_flag/user_blast_radius.py b/posthog/models/feature_flag/user_blast_radius.py index 712df09ed5002..bf08e8eed950a 100644 --- a/posthog/models/feature_flag/user_blast_radius.py +++ b/posthog/models/feature_flag/user_blast_radius.py @@ -77,7 +77,7 @@ def get_user_blast_radius( if len(cohort_filters) == 1: try: - target_cohort = Cohort.objects.get(id=cohort_filters[0].value, team=team) + target_cohort = Cohort.objects.get(id=cohort_filters[0].value, team__project_id=team.project_id) except Cohort.DoesNotExist: pass finally: diff --git a/posthog/models/filters/mixins/simplify.py b/posthog/models/filters/mixins/simplify.py index b152e07113f11..01f3d2c4d4745 100644 --- a/posthog/models/filters/mixins/simplify.py +++ b/posthog/models/filters/mixins/simplify.py @@ -108,7 +108,7 @@ def _simplify_property(self, team: "Team", property: "Property", **kwargs) -> "P from posthog.models.cohort.util import simplified_cohort_filter_properties try: - cohort = Cohort.objects.get(pk=property.value, team_id=team.pk) + cohort = Cohort.objects.get(pk=property.value, team__project_id=team.project_id) except Cohort.DoesNotExist: # :TODO: Handle non-existing resource in-query instead return PropertyGroup(type=PropertyOperatorType.AND, values=[property]) diff --git a/posthog/models/property/util.py b/posthog/models/property/util.py index 90651b6cd1e5f..ef63b2f69c670 100644 --- a/posthog/models/property/util.py +++ b/posthog/models/property/util.py @@ -14,10 +14,7 @@ from rest_framework import exceptions from posthog.clickhouse.kafka_engine import trim_quotes_expr -from posthog.clickhouse.materialized_columns import ( - TableWithProperties, - get_enabled_materialized_columns, -) +from posthog.clickhouse.materialized_columns import TableWithProperties, get_materialized_column_for_property from posthog.constants import PropertyOperatorType from posthog.hogql import ast from posthog.hogql.hogql import HogQLContext @@ -711,17 +708,18 @@ def get_property_string_expr( (optional) alias of the table being queried :return: """ - materialized_columns = get_enabled_materialized_columns(table) if allow_denormalized_props else {} - table_string = f"{table_alias}." if table_alias is not None and table_alias != "" else "" if ( allow_denormalized_props - and (property_name, materialised_table_column) in materialized_columns + and ( + materialized_column := get_materialized_column_for_property(table, materialised_table_column, property_name) + ) + and not materialized_column.is_nullable and "group" not in materialised_table_column ): return ( - f'{table_string}"{materialized_columns[(property_name, materialised_table_column)]}"', + f'{table_string}"{materialized_column.name}"', True, ) diff --git a/posthog/models/test/test_async_deletion_model.py b/posthog/models/test/test_async_deletion_model.py index 8f4125be67a3c..e5649d6e812d9 100644 --- a/posthog/models/test/test_async_deletion_model.py +++ b/posthog/models/test/test_async_deletion_model.py @@ -365,7 +365,7 @@ def test_delete_auxilary_models_via_team(self): group_key="org:5", properties={}, ) - insert_static_cohort([uuid4()], 0, self.teams[0]) + insert_static_cohort([uuid4()], 0, team_id=self.teams[0].pk) self._insert_cohortpeople_row(self.teams[0], uuid4(), 3) create_plugin_log_entry( team_id=self.teams[0].pk, @@ -403,7 +403,7 @@ def test_delete_auxilary_models_via_team_unrelated(self): group_key="org:5", properties={}, ) - insert_static_cohort([uuid4()], 0, self.teams[1]) + insert_static_cohort([uuid4()], 0, team_id=self.teams[1].pk) self._insert_cohortpeople_row(self.teams[1], uuid4(), 3) create_plugin_log_entry( team_id=self.teams[1].pk, diff --git a/posthog/queries/column_optimizer/foss_column_optimizer.py b/posthog/queries/column_optimizer/foss_column_optimizer.py index 4fffbd1faa350..c998d92480b5a 100644 --- a/posthog/queries/column_optimizer/foss_column_optimizer.py +++ b/posthog/queries/column_optimizer/foss_column_optimizer.py @@ -3,7 +3,7 @@ from typing import Union, cast from collections.abc import Generator -from posthog.clickhouse.materialized_columns import ColumnName, get_enabled_materialized_columns +from posthog.clickhouse.materialized_columns import ColumnName, get_materialized_column_for_property from posthog.constants import TREND_FILTER_TYPE_ACTIONS, FunnelCorrelationType from posthog.models.action.util import ( get_action_tables_and_properties, @@ -72,12 +72,14 @@ def columns_to_query( table_column: str = "properties", ) -> set[ColumnName]: "Transforms a list of property names to what columns are needed for that query" - - materialized_columns = get_enabled_materialized_columns(table) - return { - materialized_columns.get((property_name, table_column), table_column) - for property_name, _, _ in used_properties - } + column_names = set() + for property_name, _, _ in used_properties: + column = get_materialized_column_for_property(table, table_column, property_name) + if column is not None and not column.is_nullable: + column_names.add(column.name) + else: + column_names.add(table_column) + return column_names @cached_property def is_using_person_properties(self) -> bool: diff --git a/posthog/queries/test/__snapshots__/test_trends.ambr b/posthog/queries/test/__snapshots__/test_trends.ambr index 879da96b30821..01ab1c2e0e23e 100644 --- a/posthog/queries/test/__snapshots__/test_trends.ambr +++ b/posthog/queries/test/__snapshots__/test_trends.ambr @@ -22,11 +22,14 @@ # name: TestTrends.test_action_filtering_with_cohort.2 ''' /* celery:posthog.tasks.calculate_cohort.clear_stale_cohort */ - SELECT count() + SELECT team_id, + count() AS stale_people_count FROM cohortpeople - WHERE team_id = 99999 + WHERE team_id IN [1, 2, 3, 4, 5 /* ... */] AND cohort_id = 99999 AND version < 2 + GROUP BY team_id + HAVING stale_people_count > 0 ''' # --- # name: TestTrends.test_action_filtering_with_cohort.3 @@ -110,11 +113,14 @@ # name: TestTrends.test_action_filtering_with_cohort_poe_v2.2 ''' /* celery:posthog.tasks.calculate_cohort.clear_stale_cohort */ - SELECT count() + SELECT team_id, + count() AS stale_people_count FROM cohortpeople - WHERE team_id = 99999 + WHERE team_id IN [1, 2, 3, 4, 5 /* ... */] AND cohort_id = 99999 AND version < 2 + GROUP BY team_id + HAVING stale_people_count > 0 ''' # --- # name: TestTrends.test_action_filtering_with_cohort_poe_v2.3 diff --git a/posthog/session_recordings/test/__snapshots__/test_session_recordings.ambr b/posthog/session_recordings/test/__snapshots__/test_session_recordings.ambr index 88a534a569646..c25bdb4d587b4 100644 --- a/posthog/session_recordings/test/__snapshots__/test_session_recordings.ambr +++ b/posthog/session_recordings/test/__snapshots__/test_session_recordings.ambr @@ -738,7 +738,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_get_session_recordings.24 @@ -865,7 +865,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_get_session_recordings.28 @@ -1637,7 +1637,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.100 @@ -1788,7 +1788,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.105 @@ -1915,7 +1915,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.109 @@ -2543,7 +2543,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.125 @@ -2670,7 +2670,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.129 @@ -3234,7 +3234,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.145 @@ -3361,7 +3361,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.149 @@ -3988,7 +3988,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.165 @@ -4115,7 +4115,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.169 @@ -4706,7 +4706,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.185 @@ -4833,7 +4833,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.189 @@ -5506,7 +5506,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.205 @@ -5633,7 +5633,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.209 @@ -5940,7 +5940,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.25 @@ -6067,7 +6067,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.29 @@ -6671,7 +6671,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.45 @@ -6798,7 +6798,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.49 @@ -7214,7 +7214,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.60 @@ -7365,7 +7365,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.65 @@ -7492,7 +7492,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.69 @@ -8116,7 +8116,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.85 @@ -8243,7 +8243,7 @@ "posthog_grouptypemapping"."name_singular", "posthog_grouptypemapping"."name_plural" FROM "posthog_grouptypemapping" - WHERE "posthog_grouptypemapping"."team_id" = 99999 + WHERE "posthog_grouptypemapping"."project_id" = 99999 ''' # --- # name: TestSessionRecordings.test_listing_recordings_is_not_nplus1_for_persons.89 diff --git a/posthog/tasks/calculate_cohort.py b/posthog/tasks/calculate_cohort.py index 53bc65d9abef6..be28173b5000f 100644 --- a/posthog/tasks/calculate_cohort.py +++ b/posthog/tasks/calculate_cohort.py @@ -1,20 +1,23 @@ import time from typing import Any, Optional +from django.conf import settings + +from posthog.models.team.team import Team import structlog from celery import shared_task from dateutil.relativedelta import relativedelta from django.db.models import F, ExpressionWrapper, DurationField, Q from django.utils import timezone from prometheus_client import Gauge -from sentry_sdk import set_tag +from sentry_sdk import capture_exception, set_tag from datetime import timedelta from posthog.api.monitoring import Feature from posthog.models import Cohort from posthog.models.cohort import get_and_update_pending_version -from posthog.models.cohort.util import clear_stale_cohortpeople +from posthog.models.cohort.util import clear_stale_cohortpeople, get_static_cohort_size from posthog.models.user import User COHORT_RECALCULATIONS_BACKLOG_GAUGE = Gauge( @@ -109,37 +112,65 @@ def calculate_cohort_ch(cohort_id: int, pending_version: int, initiating_user_id @shared_task(ignore_result=True, max_retries=1) -def calculate_cohort_from_list(cohort_id: int, items: list[str]) -> None: +def calculate_cohort_from_list(cohort_id: int, items: list[str], team_id: Optional[int] = None) -> None: + """ + team_id is only optional for backwards compatibility with the old celery task signature. + All new tasks should pass team_id explicitly. + """ start_time = time.time() cohort = Cohort.objects.get(pk=cohort_id) + if team_id is None: + team_id = cohort.team_id - cohort.insert_users_by_list(items) + cohort.insert_users_by_list(items, team_id=team_id) logger.warn("Calculating cohort {} from CSV took {:.2f} seconds".format(cohort.pk, (time.time() - start_time))) @shared_task(ignore_result=True, max_retries=1) -def insert_cohort_from_insight_filter(cohort_id: int, filter_data: dict[str, Any]) -> None: - from posthog.api.cohort import ( - insert_cohort_actors_into_ch, - insert_cohort_people_into_pg, - ) +def insert_cohort_from_insight_filter( + cohort_id: int, filter_data: dict[str, Any], team_id: Optional[int] = None +) -> None: + """ + team_id is only optional for backwards compatibility with the old celery task signature. + All new tasks should pass team_id explicitly. + """ + from posthog.api.cohort import insert_cohort_actors_into_ch, insert_cohort_people_into_pg cohort = Cohort.objects.get(pk=cohort_id) + if team_id is None: + team_id = cohort.team_id - insert_cohort_actors_into_ch(cohort, filter_data) - insert_cohort_people_into_pg(cohort=cohort) + insert_cohort_actors_into_ch(cohort, filter_data, team_id=team_id) + insert_cohort_people_into_pg(cohort, team_id=team_id) @shared_task(ignore_result=True, max_retries=1) -def insert_cohort_from_query(cohort_id: int) -> None: - from posthog.api.cohort import ( - insert_cohort_people_into_pg, - insert_cohort_query_actors_into_ch, - ) +def insert_cohort_from_query(cohort_id: int, team_id: Optional[int] = None) -> None: + """ + team_id is only optional for backwards compatibility with the old celery task signature. + All new tasks should pass team_id explicitly. + """ + from posthog.api.cohort import insert_cohort_people_into_pg, insert_cohort_query_actors_into_ch cohort = Cohort.objects.get(pk=cohort_id) - insert_cohort_query_actors_into_ch(cohort) - insert_cohort_people_into_pg(cohort=cohort) + if team_id is None: + team_id = cohort.team_id + team = Team.objects.get(pk=team_id) + try: + insert_cohort_query_actors_into_ch(cohort, team=team) + insert_cohort_people_into_pg(cohort, team_id=team_id) + cohort.count = get_static_cohort_size(cohort_id=cohort.id, team_id=cohort.team_id) + cohort.errors_calculating = 0 + cohort.last_calculation = timezone.now() + except: + cohort.errors_calculating = F("errors_calculating") + 1 + cohort.last_error_at = timezone.now() + capture_exception() + if settings.DEBUG: + raise + finally: + cohort.is_calculating = False + cohort.save() @shared_task(ignore_result=True, max_retries=1) diff --git a/posthog/tasks/exports/csv_exporter.py b/posthog/tasks/exports/csv_exporter.py index 751b8f5db70cc..7657db26c203f 100644 --- a/posthog/tasks/exports/csv_exporter.py +++ b/posthog/tasks/exports/csv_exporter.py @@ -170,19 +170,21 @@ def _convert_response_to_csv_data(data: Any) -> Generator[Any, None, None]: yield line return elif isinstance(first_result.get("data"), list): + is_comparison = first_result.get("compare_label") + + # take date labels from current results, when comparing against previous + # as previous results will be indexed with offset + date_labels_item = next((x for x in results if x.get("compare_label") == "current"), None) + # TRENDS LIKE for index, item in enumerate(results): label = item.get("label", f"Series #{index + 1}") compare_label = item.get("compare_label", "") series_name = f"{label} - {compare_label}" if compare_label else label - line = {"series": series_name} - # take labels from current results, when comparing against previous - if item.get("compare_label") == "previous": - label_item = results[index - 1] - else: - label_item = item + line = {"series": series_name} + label_item = date_labels_item if is_comparison else item action = item.get("action") if isinstance(action, dict) and action.get("custom_name"): diff --git a/posthog/tasks/exports/test/test_csv_exporter.py b/posthog/tasks/exports/test/test_csv_exporter.py index 29b57da6d7a0b..4d53742ed65bb 100644 --- a/posthog/tasks/exports/test/test_csv_exporter.py +++ b/posthog/tasks/exports/test/test_csv_exporter.py @@ -2,13 +2,13 @@ from typing import Any, Optional from unittest import mock from unittest.mock import MagicMock, Mock, patch, ANY +from dateutil.relativedelta import relativedelta from openpyxl import load_workbook from io import BytesIO import pytest from boto3 import resource from botocore.client import Config -from dateutil.relativedelta import relativedelta from django.test import override_settings from django.utils.timezone import now from requests.exceptions import HTTPError @@ -703,23 +703,75 @@ def test_csv_exporter_trends_query_with_compare_previous_option( self, ) -> None: _create_person(distinct_ids=[f"user_1"], team=self.team) - events_by_person = { - "user_1": [ - { - "event": "$pageview", - "timestamp": datetime(2023, 3, 21, 13, 46), - }, - { - "event": "$pageview", - "timestamp": datetime(2023, 3, 21, 13, 46), - }, - { - "event": "$pageview", - "timestamp": datetime(2023, 3, 22, 13, 47), - }, - ], - } - journeys_for(events_by_person, self.team) + + date = datetime(2023, 3, 21, 13, 46) + date_next_week = date + relativedelta(days=7) + + _create_event( + event="$pageview", + distinct_id="1", + team=self.team, + timestamp=date, + properties={"$browser": "Safari"}, + ) + _create_event( + event="$pageview", + distinct_id="1", + team=self.team, + timestamp=date, + properties={"$browser": "Chrome"}, + ) + _create_event( + event="$pageview", + distinct_id="1", + team=self.team, + timestamp=date, + properties={"$browser": "Chrome"}, + ) + _create_event( + event="$pageview", + distinct_id="1", + team=self.team, + timestamp=date, + properties={"$browser": "Firefox"}, + ) + + _create_event( + event="$pageview", + distinct_id="1", + team=self.team, + timestamp=date_next_week, + properties={"$browser": "Chrome"}, + ) + _create_event( + event="$pageview", + distinct_id="1", + team=self.team, + timestamp=date_next_week, + properties={"$browser": "Chrome"}, + ) + _create_event( + event="$pageview", + distinct_id="1", + team=self.team, + timestamp=date_next_week, + properties={"$browser": "Chrome"}, + ) + _create_event( + event="$pageview", + distinct_id="1", + team=self.team, + timestamp=date_next_week, + properties={"$browser": "Firefox"}, + ) + _create_event( + event="$pageview", + distinct_id="1", + team=self.team, + timestamp=date_next_week, + properties={"$browser": "Firefox"}, + ) + flush_persons_and_events() exported_asset = ExportedAsset( @@ -728,7 +780,10 @@ def test_csv_exporter_trends_query_with_compare_previous_option( export_context={ "source": { "kind": "TrendsQuery", - "dateRange": {"date_to": "2023-03-22", "date_from": "2023-03-22"}, + "dateRange": { + "date_from": date.strftime("%Y-%m-%d"), + "date_to": date_next_week.strftime("%Y-%m-%d"), + }, "series": [ { "kind": "EventsNode", @@ -738,7 +793,8 @@ def test_csv_exporter_trends_query_with_compare_previous_option( }, ], "interval": "day", - "compareFilter": {"compare": True}, + "compareFilter": {"compare": True, "compare_to": "-1w"}, + "breakdownFilter": {"breakdown": "$browser", "breakdown_type": "event"}, } }, ) @@ -747,5 +803,17 @@ def test_csv_exporter_trends_query_with_compare_previous_option( with self.settings(OBJECT_STORAGE_ENABLED=True, OBJECT_STORAGE_EXPORTS_FOLDER="Test-Exports"): csv_exporter.export_tabular(exported_asset) content = object_storage.read(exported_asset.content_location) # type: ignore - lines = (content or "").strip().split("\r\n") - self.assertEqual(lines, ["series,22-Mar-2023", "$pageview - current,1", "$pageview - previous,2"]) + + lines = (content or "").strip().splitlines() + + expected_lines = [ + "series,21-Mar-2023,22-Mar-2023,23-Mar-2023,24-Mar-2023,25-Mar-2023,26-Mar-2023,27-Mar-2023,28-Mar-2023", + "Chrome - current,2.0,0.0,0.0,0.0,0.0,0.0,0.0,3.0", + "Firefox - current,1.0,0.0,0.0,0.0,0.0,0.0,0.0,2.0", + "Safari - current,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0", + "Chrome - previous,0.0,0.0,0.0,0.0,0.0,0.0,0.0,2.0", + "Firefox - previous,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0", + "Safari - previous,0.0,0.0,0.0,0.0,0.0,0.0,0.0,1.0", + ] + + self.assertEqual(lines, expected_lines) diff --git a/posthog/tasks/test/__snapshots__/test_usage_report.ambr b/posthog/tasks/test/__snapshots__/test_usage_report.ambr index 2230c532da5ca..36733da586c57 100644 --- a/posthog/tasks/test/__snapshots__/test_usage_report.ambr +++ b/posthog/tasks/test/__snapshots__/test_usage_report.ambr @@ -3,7 +3,7 @@ ''' SELECT team_id, - multiIf(event LIKE 'helicone%', 'helicone_events', event LIKE 'langfuse%', 'langfuse_events', event LIKE 'keywords_ai%', 'keywords_ai_events', event LIKE 'traceloop%', 'traceloop_events', JSONExtractString(properties, '$lib') = 'web', 'web_events', JSONExtractString(properties, '$lib') = 'js', 'web_lite_events', JSONExtractString(properties, '$lib') = 'posthog-node', 'node_events', JSONExtractString(properties, '$lib') = 'posthog-android', 'android_events', JSONExtractString(properties, '$lib') = 'posthog-flutter', 'flutter_events', JSONExtractString(properties, '$lib') = 'posthog-ios', 'ios_events', JSONExtractString(properties, '$lib') = 'posthog-go', 'go_events', JSONExtractString(properties, '$lib') = 'posthog-java', 'java_events', JSONExtractString(properties, '$lib') = 'posthog-react-native', 'react_native_events', JSONExtractString(properties, '$lib') = 'posthog-ruby', 'ruby_events', JSONExtractString(properties, '$lib') = 'posthog-python', 'python_events', JSONExtractString(properties, '$lib') = 'posthog-php', 'php_events', 'other') AS metric, + multiIf(event LIKE 'helicone%', 'helicone_events', event LIKE 'langfuse%', 'langfuse_events', event LIKE 'keywords_ai%', 'keywords_ai_events', event LIKE 'traceloop%', 'traceloop_events', replaceRegexpAll(JSONExtractRaw(properties, '$lib'), '^"|"$', '') = 'web', 'web_events', replaceRegexpAll(JSONExtractRaw(properties, '$lib'), '^"|"$', '') = 'js', 'web_lite_events', replaceRegexpAll(JSONExtractRaw(properties, '$lib'), '^"|"$', '') = 'posthog-node', 'node_events', replaceRegexpAll(JSONExtractRaw(properties, '$lib'), '^"|"$', '') = 'posthog-android', 'android_events', replaceRegexpAll(JSONExtractRaw(properties, '$lib'), '^"|"$', '') = 'posthog-flutter', 'flutter_events', replaceRegexpAll(JSONExtractRaw(properties, '$lib'), '^"|"$', '') = 'posthog-ios', 'ios_events', replaceRegexpAll(JSONExtractRaw(properties, '$lib'), '^"|"$', '') = 'posthog-go', 'go_events', replaceRegexpAll(JSONExtractRaw(properties, '$lib'), '^"|"$', '') = 'posthog-java', 'java_events', replaceRegexpAll(JSONExtractRaw(properties, '$lib'), '^"|"$', '') = 'posthog-react-native', 'react_native_events', replaceRegexpAll(JSONExtractRaw(properties, '$lib'), '^"|"$', '') = 'posthog-ruby', 'ruby_events', replaceRegexpAll(JSONExtractRaw(properties, '$lib'), '^"|"$', '') = 'posthog-python', 'python_events', replaceRegexpAll(JSONExtractRaw(properties, '$lib'), '^"|"$', '') = 'posthog-php', 'php_events', 'other') AS metric, count(1) as count FROM events WHERE timestamp BETWEEN '2022-01-10 00:00:00' AND '2022-01-10 23:59:59' diff --git a/posthog/tasks/usage_report.py b/posthog/tasks/usage_report.py index 968354fff3032..6964668c5fffc 100644 --- a/posthog/tasks/usage_report.py +++ b/posthog/tasks/usage_report.py @@ -19,7 +19,6 @@ from posthog import version_requirement from posthog.clickhouse.client.connection import Workload -from posthog.clickhouse.materialized_columns import get_enabled_materialized_columns from posthog.client import sync_execute from posthog.cloud_utils import get_cached_instance_license, is_cloud from posthog.constants import FlagRequestType @@ -29,6 +28,7 @@ from posthog.models.feature_flag import FeatureFlag from posthog.models.organization import Organization from posthog.models.plugin import PluginConfig +from posthog.models.property.util import get_property_string_expr from posthog.models.team.team import Team from posthog.models.utils import namedtuplefetchall from posthog.settings import CLICKHOUSE_CLUSTER, INSTANCE_TAG @@ -460,10 +460,8 @@ def get_teams_with_event_count_with_groups_in_period(begin: datetime, end: datet @timed_log() @retry(tries=QUERY_RETRIES, delay=QUERY_RETRY_DELAY, backoff=QUERY_RETRY_BACKOFF) def get_all_event_metrics_in_period(begin: datetime, end: datetime) -> dict[str, list[tuple[int, int]]]: - materialized_columns = get_enabled_materialized_columns("events") - # Check if $lib is materialized - lib_expression = materialized_columns.get(("$lib", "properties"), "JSONExtractString(properties, '$lib')") + lib_expression, _ = get_property_string_expr("events", "$lib", "'$lib'", "properties") results = sync_execute( f""" diff --git a/posthog/temporal/data_imports/pipelines/sql_database/__init__.py b/posthog/temporal/data_imports/pipelines/sql_database/__init__.py index 41b8ceec1ef41..ae81f9fa61fe6 100644 --- a/posthog/temporal/data_imports/pipelines/sql_database/__init__.py +++ b/posthog/temporal/data_imports/pipelines/sql_database/__init__.py @@ -21,6 +21,9 @@ from posthog.warehouse.models.external_data_source import ExternalDataSource from sqlalchemy.sql import text +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization + from .helpers import ( table_rows, engine_from_credentials, @@ -111,8 +114,11 @@ def sql_source_for_type( def snowflake_source( account_id: str, - user: str, - password: str, + user: Optional[str], + password: Optional[str], + passphrase: Optional[str], + private_key: Optional[str], + auth_type: str, database: str, warehouse: str, schema: str, @@ -122,13 +128,6 @@ def snowflake_source( incremental_field: Optional[str] = None, incremental_field_type: Optional[IncrementalFieldType] = None, ) -> DltSource: - account_id = quote(account_id) - user = quote(user) - password = quote(password) - database = quote(database) - warehouse = quote(warehouse) - role = quote(role) if role else None - if incremental_field is not None and incremental_field_type is not None: incremental: dlt.sources.incremental | None = dlt.sources.incremental( cursor_path=incremental_field, initial_value=incremental_type_to_initial_value(incremental_field_type) @@ -136,9 +135,46 @@ def snowflake_source( else: incremental = None - credentials = ConnectionStringCredentials( - f"snowflake://{user}:{password}@{account_id}/{database}/{schema}?warehouse={warehouse}{f'&role={role}' if role else ''}" - ) + if auth_type == "password" and user is not None and password is not None: + account_id = quote(account_id) + user = quote(user) + password = quote(password) + database = quote(database) + warehouse = quote(warehouse) + role = quote(role) if role else None + + credentials = create_engine( + f"snowflake://{user}:{password}@{account_id}/{database}/{schema}?warehouse={warehouse}{f'&role={role}' if role else ''}" + ) + else: + assert private_key is not None + assert user is not None + + account_id = quote(account_id) + user = quote(user) + database = quote(database) + warehouse = quote(warehouse) + role = quote(role) if role else None + + p_key = serialization.load_pem_private_key( + private_key.encode("utf-8"), + password=passphrase.encode() if passphrase is not None else None, + backend=default_backend(), + ) + + pkb = p_key.private_bytes( + encoding=serialization.Encoding.DER, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + + credentials = create_engine( + f"snowflake://{user}@{account_id}/{database}/{schema}?warehouse={warehouse}{f'&role={role}' if role else ''}", + connect_args={ + "private_key": pkb, + }, + ) + db_source = sql_database( credentials=credentials, schema=schema, diff --git a/posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py b/posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py index 92f4d1f87cec4..a3fc1c6b2838b 100644 --- a/posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py +++ b/posthog/temporal/data_imports/pipelines/sql_database_v2/__init__.py @@ -20,6 +20,9 @@ from posthog.warehouse.models import ExternalDataSource from posthog.warehouse.types import IncrementalFieldType +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization + from .helpers import ( SelectAny, table_rows, @@ -127,8 +130,11 @@ def sql_source_for_type( def snowflake_source( account_id: str, - user: str, - password: str, + user: Optional[str], + password: Optional[str], + passphrase: Optional[str], + private_key: Optional[str], + auth_type: str, database: str, warehouse: str, schema: str, @@ -138,13 +144,6 @@ def snowflake_source( incremental_field: Optional[str] = None, incremental_field_type: Optional[IncrementalFieldType] = None, ) -> DltSource: - account_id = quote(account_id) - user = quote(user) - password = quote(password) - database = quote(database) - warehouse = quote(warehouse) - role = quote(role) if role else None - if incremental_field is not None and incremental_field_type is not None: incremental: dlt.sources.incremental | None = dlt.sources.incremental( cursor_path=incremental_field, initial_value=incremental_type_to_initial_value(incremental_field_type) @@ -152,9 +151,46 @@ def snowflake_source( else: incremental = None - credentials = ConnectionStringCredentials( - f"snowflake://{user}:{password}@{account_id}/{database}/{schema}?warehouse={warehouse}{f'&role={role}' if role else ''}" - ) + if auth_type == "password" and user is not None and password is not None: + account_id = quote(account_id) + user = quote(user) + password = quote(password) + database = quote(database) + warehouse = quote(warehouse) + role = quote(role) if role else None + + credentials = create_engine( + f"snowflake://{user}:{password}@{account_id}/{database}/{schema}?warehouse={warehouse}{f'&role={role}' if role else ''}" + ) + else: + assert private_key is not None + assert user is not None + + account_id = quote(account_id) + user = quote(user) + database = quote(database) + warehouse = quote(warehouse) + role = quote(role) if role else None + + p_key = serialization.load_pem_private_key( + private_key.encode("utf-8"), + password=passphrase.encode() if passphrase is not None else None, + backend=default_backend(), + ) + + pkb = p_key.private_bytes( + encoding=serialization.Encoding.DER, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + + credentials = create_engine( + f"snowflake://{user}@{account_id}/{database}/{schema}?warehouse={warehouse}{f'&role={role}' if role else ''}", + connect_args={ + "private_key": pkb, + }, + ) + db_source = sql_database( credentials=credentials, schema=schema, diff --git a/posthog/temporal/data_imports/workflow_activities/import_data_sync.py b/posthog/temporal/data_imports/workflow_activities/import_data_sync.py index e308e19965bdd..37b65c4c68fad 100644 --- a/posthog/temporal/data_imports/workflow_activities/import_data_sync.py +++ b/posthog/temporal/data_imports/workflow_activities/import_data_sync.py @@ -270,17 +270,24 @@ def import_data_activity_sync(inputs: ImportDataActivityInputs): ) account_id = model.pipeline.job_inputs.get("account_id") - user = model.pipeline.job_inputs.get("user") - password = model.pipeline.job_inputs.get("password") database = model.pipeline.job_inputs.get("database") warehouse = model.pipeline.job_inputs.get("warehouse") sf_schema = model.pipeline.job_inputs.get("schema") role = model.pipeline.job_inputs.get("role") + auth_type = model.pipeline.job_inputs.get("auth_type", "password") + auth_type_username = model.pipeline.job_inputs.get("user") + auth_type_password = model.pipeline.job_inputs.get("password") + auth_type_passphrase = model.pipeline.job_inputs.get("passphrase") + auth_type_private_key = model.pipeline.job_inputs.get("private_key") + source = snowflake_source( account_id=account_id, - user=user, - password=password, + auth_type=auth_type, + user=auth_type_username, + password=auth_type_password, + private_key=auth_type_private_key, + passphrase=auth_type_passphrase, database=database, schema=sf_schema, warehouse=warehouse, diff --git a/posthog/temporal/data_imports/workflow_activities/sync_new_schemas.py b/posthog/temporal/data_imports/workflow_activities/sync_new_schemas.py index 67f8c820e2837..b63d7ea869e16 100644 --- a/posthog/temporal/data_imports/workflow_activities/sync_new_schemas.py +++ b/posthog/temporal/data_imports/workflow_activities/sync_new_schemas.py @@ -86,14 +86,29 @@ def sync_new_schemas_activity(inputs: SyncNewSchemasActivityInputs) -> None: return account_id = source.job_inputs.get("account_id") - user = source.job_inputs.get("user") - password = source.job_inputs.get("password") database = source.job_inputs.get("database") warehouse = source.job_inputs.get("warehouse") sf_schema = source.job_inputs.get("schema") role = source.job_inputs.get("role") - sql_schemas = get_snowflake_schemas(account_id, database, warehouse, user, password, sf_schema, role) + auth_type = source.job_inputs.get("auth_type", "password") + auth_type_username = source.job_inputs.get("user") + auth_type_password = source.job_inputs.get("password") + auth_type_passphrase = source.job_inputs.get("passphrase") + auth_type_private_key = source.job_inputs.get("private_key") + + sql_schemas = get_snowflake_schemas( + account_id=account_id, + database=database, + warehouse=warehouse, + user=auth_type_username, + password=auth_type_password, + schema=sf_schema, + role=role, + auth_type=auth_type, + passphrase=auth_type_passphrase, + private_key=auth_type_private_key, + ) schemas_to_sync = list(sql_schemas.keys()) else: diff --git a/posthog/test/base.py b/posthog/test/base.py index 43dcc0e130964..53f4932f2898f 100644 --- a/posthog/test/base.py +++ b/posthog/test/base.py @@ -30,7 +30,6 @@ from posthog import rate_limit, redis from posthog.clickhouse.client import sync_execute from posthog.clickhouse.client.connection import ch_pool -from posthog.clickhouse.materialized_columns import get_materialized_columns from posthog.clickhouse.plugin_log_entries import TRUNCATE_PLUGIN_LOG_ENTRIES_TABLE_SQL from posthog.cloud_utils import TEST_clear_instance_license_cache from posthog.models import Dashboard, DashboardTile, Insight, Organization, Team, User @@ -121,6 +120,8 @@ def clean_varying_query_parts(query, replace_all_numbers): else: query = re.sub(r"(team|cohort)_id(\"?) = \d+", r"\1_id\2 = 99999", query) + query = re.sub(r"(team|cohort)_id(\"?) IN \(\d+(, ?\d+)*\)", r"\1_id\2 IN (1, 2, 3, 4, 5 /* ... */)", query) + query = re.sub(r"(team|cohort)_id(\"?) IN \[\d+(, ?\d+)*\]", r"\1_id\2 IN [1, 2, 3, 4, 5 /* ... */]", query) query = re.sub(r"\d+ as (team|cohort)_id(\"?)", r"99999 as \1_id\2", query) # feature flag conditions use primary keys as columns in queries, so replace those always query = re.sub(r"flag_\d+_condition", r"flag_X_condition", query) @@ -575,35 +576,31 @@ def stripResponse(response, remove=("action", "label", "persons_urls", "filter") return response -def default_materialised_columns(): +def cleanup_materialized_columns(): try: + from ee.clickhouse.materialized_columns.columns import get_materialized_columns from ee.clickhouse.materialized_columns.test.test_columns import EVENTS_TABLE_DEFAULT_MATERIALIZED_COLUMNS except: # EE not available? Skip - return [] - - default_columns = [] - for prop in EVENTS_TABLE_DEFAULT_MATERIALIZED_COLUMNS: - column_name = get_materialized_columns("events")[(prop, "properties")] - default_columns.append(column_name) - - return default_columns - + return -def cleanup_materialized_columns(): def optionally_drop(table, filter=None): drops = ",".join( [ - f"DROP COLUMN {column_name}" - for column_name in get_materialized_columns(table).values() - if filter is None or filter(column_name) + f"DROP COLUMN {column.name}" + for column in get_materialized_columns(table).values() + if filter is None or filter(column.name) ] ) if drops: sync_execute(f"ALTER TABLE {table} {drops} SETTINGS mutations_sync = 2") - default_columns = default_materialised_columns() - optionally_drop("events", lambda name: name not in default_columns) + default_column_names = { + get_materialized_columns("events")[(prop, "properties")].name + for prop in EVENTS_TABLE_DEFAULT_MATERIALIZED_COLUMNS + } + + optionally_drop("events", lambda name: name not in default_column_names) optionally_drop("person") optionally_drop("groups") diff --git a/posthog/urls.py b/posthog/urls.py index 078a66c5af8b3..e0f79123a10f4 100644 --- a/posthog/urls.py +++ b/posthog/urls.py @@ -238,6 +238,8 @@ def opt_slash_path(route: str, view: Callable, name: Optional[str] = None) -> UR path("year_in_posthog/2022//", year_in_posthog.render_2022), path("year_in_posthog/2023/", year_in_posthog.render_2023), path("year_in_posthog/2023//", year_in_posthog.render_2023), + path("year_in_posthog/2024/", year_in_posthog.render_2024), + path("year_in_posthog/2024//", year_in_posthog.render_2024), ] if settings.DEBUG: diff --git a/posthog/utils.py b/posthog/utils.py index d8ea9315fec7c..5a22bfcdde9ff 100644 --- a/posthog/utils.py +++ b/posthog/utils.py @@ -348,7 +348,7 @@ def render_template( context["js_url"] = get_js_url(request) try: - year_in_hog_url = f"/year_in_posthog/2023/{str(request.user.uuid)}" # type: ignore + year_in_hog_url = f"/year_in_posthog/2024/{str(request.user.uuid)}" # type: ignore except: year_in_hog_url = None diff --git a/posthog/warehouse/api/external_data_schema.py b/posthog/warehouse/api/external_data_schema.py index 55a4447907021..9391268bb69d0 100644 --- a/posthog/warehouse/api/external_data_schema.py +++ b/posthog/warehouse/api/external_data_schema.py @@ -350,14 +350,23 @@ def incremental_fields(self, request: Request, *args: Any, **kwargs: Any): sf_schema = source.job_inputs.get("schema") role = source.job_inputs.get("role") + auth_type = source.job_inputs.get("auth_type", "password") + auth_type_username = source.job_inputs.get("user") + auth_type_password = source.job_inputs.get("password") + auth_type_passphrase = source.job_inputs.get("passphrase") + auth_type_private_key = source.job_inputs.get("private_key") + sf_schemas = get_snowflake_schemas( account_id=account_id, database=database, warehouse=warehouse, - user=user, - password=password, + user=auth_type_username, + password=auth_type_password, schema=sf_schema, role=role, + auth_type=auth_type, + passphrase=auth_type_passphrase, + private_key=auth_type_private_key, ) columns = sf_schemas.get(instance.name, []) diff --git a/posthog/warehouse/api/external_data_source.py b/posthog/warehouse/api/external_data_source.py index 02e3c68a77422..28b1ebda1bf2e 100644 --- a/posthog/warehouse/api/external_data_source.py +++ b/posthog/warehouse/api/external_data_source.py @@ -640,10 +640,15 @@ def _handle_snowflake_source( database = payload.get("database") warehouse = payload.get("warehouse") role = payload.get("role") - user = payload.get("user") - password = payload.get("password") schema = payload.get("schema") + auth_type_obj = payload.get("auth_type", {}) + auth_type = auth_type_obj.get("selection", None) + auth_type_username = auth_type_obj.get("username", None) + auth_type_password = auth_type_obj.get("password", None) + auth_type_passphrase = auth_type_obj.get("passphrase", None) + auth_type_private_key = auth_type_obj.get("private_key", None) + new_source_model = ExternalDataSource.objects.create( source_id=str(uuid.uuid4()), connection_id=str(uuid.uuid4()), @@ -656,14 +661,28 @@ def _handle_snowflake_source( "database": database, "warehouse": warehouse, "role": role, - "user": user, - "password": password, "schema": schema, + "auth_type": auth_type, + "user": auth_type_username, + "password": auth_type_password, + "passphrase": auth_type_passphrase, + "private_key": auth_type_private_key, }, prefix=prefix, ) - schemas = get_snowflake_schemas(account_id, database, warehouse, user, password, schema, role) + schemas = get_snowflake_schemas( + account_id=account_id, + database=database, + warehouse=warehouse, + user=auth_type_username, + password=auth_type_password, + schema=schema, + role=role, + passphrase=auth_type_passphrase, + private_key=auth_type_private_key, + auth_type=auth_type, + ) return new_source_model, list(schemas.keys()) @@ -1068,20 +1087,48 @@ def database_schema(self, request: Request, *arg: Any, **kwargs: Any): database = request.data.get("database") warehouse = request.data.get("warehouse") role = request.data.get("role") - user = request.data.get("user") - password = request.data.get("password") schema = request.data.get("schema") - if not account_id or not warehouse or not database or not user or not password or not schema: + auth_type_obj = request.data.get("auth_type", {}) + auth_type = auth_type_obj.get("selection", None) + auth_type_username = auth_type_obj.get("username", None) + auth_type_password = auth_type_obj.get("password", None) + auth_type_passphrase = auth_type_obj.get("passphrase", None) + auth_type_private_key = auth_type_obj.get("private_key", None) + + if not account_id or not warehouse or not database or not schema: + return Response( + status=status.HTTP_400_BAD_REQUEST, + data={"message": "Missing required parameters: account id, warehouse, database, schema"}, + ) + + if auth_type == "password" and (not auth_type_username or not auth_type_password): + return Response( + status=status.HTTP_400_BAD_REQUEST, + data={"message": "Missing required parameters: username, password"}, + ) + + if auth_type == "keypair" and ( + not auth_type_passphrase or not auth_type_private_key or not auth_type_username + ): return Response( status=status.HTTP_400_BAD_REQUEST, - data={ - "message": "Missing required parameters: account id, warehouse, database, user, password, schema" - }, + data={"message": "Missing required parameters: passphrase, private key"}, ) try: - result = get_snowflake_schemas(account_id, database, warehouse, user, password, schema, role) + result = get_snowflake_schemas( + account_id=account_id, + database=database, + warehouse=warehouse, + user=auth_type_username, + password=auth_type_password, + schema=schema, + role=role, + passphrase=auth_type_passphrase, + private_key=auth_type_private_key, + auth_type=auth_type, + ) if len(result.keys()) == 0: return Response( status=status.HTTP_400_BAD_REQUEST, diff --git a/posthog/warehouse/models/external_data_schema.py b/posthog/warehouse/models/external_data_schema.py index fae744be0795c..b9629c6410672 100644 --- a/posthog/warehouse/models/external_data_schema.py +++ b/posthog/warehouse/models/external_data_schema.py @@ -1,5 +1,7 @@ from collections import defaultdict from datetime import datetime, timedelta +import tempfile +import os from typing import Any, Optional from django.db import models from django_deprecate_fields import deprecate_field @@ -97,6 +99,22 @@ def soft_delete(self): self.deleted_at = datetime.now() self.save() + def update_incremental_field_last_value(self, last_value: Any) -> None: + incremental_field_type = self.sync_type_config.get("incremental_field_type") + + last_value_py = last_value.item() if isinstance(last_value, numpy.generic) else last_value + + if ( + incremental_field_type == IncrementalFieldType.Integer + or incremental_field_type == IncrementalFieldType.Numeric + ): + last_value_json = last_value_py + else: + last_value_json = str(last_value_py) + + self.sync_type_config["incremental_field_last_value"] = last_value_json + self.save() + @database_sync_to_async def asave_external_data_schema(schema: ExternalDataSchema) -> None: @@ -218,16 +236,43 @@ def filter_snowflake_incremental_fields(columns: list[tuple[str, str]]) -> list[ def get_snowflake_schemas( - account_id: str, database: str, warehouse: str, user: str, password: str, schema: str, role: Optional[str] = None + account_id: str, + database: str, + warehouse: str, + user: Optional[str], + password: Optional[str], + passphrase: Optional[str], + private_key: Optional[str], + auth_type: str, + schema: str, + role: Optional[str] = None, ) -> dict[str, list[tuple[str, str]]]: + auth_connect_args: dict[str, str | None] = {} + file_name: str | None = None + + if auth_type == "keypair" and private_key is not None: + with tempfile.NamedTemporaryFile(delete=False) as tf: + tf.write(private_key.encode("utf-8")) + file_name = tf.name + + auth_connect_args = { + "user": user, + "private_key_file": file_name, + "private_key_file_pwd": passphrase, + } + else: + auth_connect_args = { + "password": password, + "user": user, + } + with snowflake.connector.connect( - user=user, - password=password, account=account_id, warehouse=warehouse, database=database, schema="information_schema", role=role, + **auth_connect_args, ) as connection: with connection.cursor() as cursor: if cursor is None: @@ -243,7 +288,10 @@ def get_snowflake_schemas( for row in result: schema_list[row[0]].append((row[1], row[2])) - return schema_list + if file_name is not None: + os.unlink(file_name) + + return schema_list def filter_postgres_incremental_fields(columns: list[tuple[str, str]]) -> list[tuple[str, IncrementalFieldType]]: diff --git a/posthog/year_in_posthog/2023.html b/posthog/year_in_posthog/2024.html similarity index 99% rename from posthog/year_in_posthog/2023.html rename to posthog/year_in_posthog/2024.html index 5604fb0c8fbcb..cb52db77ad278 100644 --- a/posthog/year_in_posthog/2023.html +++ b/posthog/year_in_posthog/2024.html @@ -24,7 +24,7 @@ posthog.init('{{api_token}}', { api_host: window.location.origin, loaded: function (posthog) { posthog.capture('year in posthog viewed', { - 'yearInPostHog': 2023, 'badge': '{{badge}}', {% for stat in stats %} + 'yearInPostHog': 2024, 'badge': '{{badge}}', {% for stat in stats %} '{{ stat.description }}': {{ stat.count }}, {% endfor %} }); diff --git a/posthog/year_in_posthog/calculate_2023.py b/posthog/year_in_posthog/calculate_2024.py similarity index 92% rename from posthog/year_in_posthog/calculate_2023.py rename to posthog/year_in_posthog/calculate_2024.py index 03428d2711d30..a260318cccb9c 100644 --- a/posthog/year_in_posthog/calculate_2023.py +++ b/posthog/year_in_posthog/calculate_2024.py @@ -18,7 +18,7 @@ posthog_dashboarditem WHERE NOT created_by_id IS NULL - AND date_part('year', created_at) = 2023 + AND date_part('year', created_at) = 2024 AND ( name IS NOT NULL OR derived_name IS NOT NULL @@ -41,7 +41,7 @@ key not ilike 'survey-targeting%%' AND key not ilike 'prompt-%%' AND key not ilike 'interview-%%' - AND date_part('year', created_at) = 2023 + AND date_part('year', created_at) = 2024 AND created_by_id = (select id from posthog_user where uuid = %(user_uuid)s) GROUP BY created_by_id @@ -56,7 +56,7 @@ FROM posthog_sessionrecordingviewed WHERE - date_part('year', created_at) = 2023 + date_part('year', created_at) = 2024 AND user_id = (select id from posthog_user where uuid = %(user_uuid)s) GROUP BY user_id @@ -71,7 +71,7 @@ FROM posthog_experiment WHERE - date_part('year', created_at) = 2023 + date_part('year', created_at) = 2024 AND created_by_id = (select id from posthog_user where uuid = %(user_uuid)s) GROUP BY created_by_id @@ -86,7 +86,7 @@ FROM posthog_dashboard WHERE - date_part('year', created_at) = 2023 + date_part('year', created_at) = 2024 AND created_by_id = (select id from posthog_user where uuid = %(user_uuid)s) GROUP BY created_by_id @@ -102,7 +102,7 @@ posthog_survey WHERE NOT created_by_id IS NULL - AND date_part('year', created_at) = 2023 + AND date_part('year', created_at) = 2024 AND created_by_id = (select id from posthog_user where uuid = %(user_uuid)s) group by created_by_id @@ -111,7 +111,7 @@ id, array_remove( ARRAY [ - case when posthog_user.last_login >= '1-Jan-2023' then 'astronaut' end, + case when posthog_user.last_login >= '1-Jan-2024' then 'astronaut' end, insight_stats.badge, flag_stats.badge, recording_viewed_stats.badge, @@ -147,7 +147,7 @@ def dictfetchall(cursor): @cache_for(timedelta(seconds=0 if settings.DEBUG else 30)) -def calculate_year_in_posthog_2023(user_uuid: str) -> Optional[dict]: +def calculate_year_in_posthog_2024(user_uuid: str) -> Optional[dict]: with connection.cursor() as cursor: cursor.execute(query, {"user_uuid": user_uuid}) rows = dictfetchall(cursor) diff --git a/posthog/year_in_posthog/hibernating.html b/posthog/year_in_posthog/hibernating.html index 873f10528be13..c7e57048087a9 100644 --- a/posthog/year_in_posthog/hibernating.html +++ b/posthog/year_in_posthog/hibernating.html @@ -24,7 +24,7 @@ posthog.init('{{api_token}}', { api_host: window.location.origin, loaded: function (posthog) { posthog.capture('year in posthog hibernating viewed', { - 'yearInPostHog': 2022 + 'yearInPostHog': 2023 }); } }) diff --git a/posthog/year_in_posthog/sharing-buttons.html b/posthog/year_in_posthog/sharing-buttons.html index 269f25cb456b3..8cabf76a97c19 100644 --- a/posthog/year_in_posthog/sharing-buttons.html +++ b/posthog/year_in_posthog/sharing-buttons.html @@ -1,8 +1,8 @@

    Share to
    + diff --git a/posthog/year_in_posthog/year_in_posthog.py b/posthog/year_in_posthog/year_in_posthog.py index a6ac65fa2fdaa..7e622f5bab5b5 100644 --- a/posthog/year_in_posthog/year_in_posthog.py +++ b/posthog/year_in_posthog/year_in_posthog.py @@ -9,7 +9,7 @@ from sentry_sdk import capture_exception from posthog import settings -from posthog.year_in_posthog.calculate_2023 import calculate_year_in_posthog_2023 +from posthog.year_in_posthog.calculate_2024 import calculate_year_in_posthog_2024 logger = structlog.get_logger(__name__) @@ -85,11 +85,11 @@ def sort_list_based_on_preference(badges: list[str]) -> str: @cache_control(public=True, max_age=300) # cache for 5 minutes -def render_2023(request, user_uuid: str) -> HttpResponse: +def render_2024(request, user_uuid: str) -> HttpResponse: data = None try: - data = calculate_year_in_posthog_2023(user_uuid) + data = calculate_year_in_posthog_2024(user_uuid) badge = sort_list_based_on_preference(data.get("badges") or ["astronaut"]) stats = stats_for_user(data) @@ -124,12 +124,12 @@ def render_2023(request, user_uuid: str) -> HttpResponse: "page_url": f"{request.scheme}://{request.META['HTTP_HOST']}{request.get_full_path()}", } - template = get_template("2023.html") + template = get_template("2024.html") html = template.render(context, request=request) return HttpResponse(html) except Exception as e: capture_exception(e) - logger.error("year_in_posthog_2023_error_rendering_2023_page", exc_info=True, exc=e, data=data or "no data") + logger.error("year_in_posthog_2024_error_rendering_2024_page", exc_info=True, exc=e, data=data or "no data") template = get_template("hibernating.html") html = template.render({"message": "Something went wrong 🫠"}, request=request) return HttpResponse(html, status=500) @@ -140,3 +140,10 @@ def render_2022(request, user_uuid: str) -> HttpResponse: template = get_template("hibernating.html") html = template.render({"message": "This is the 2022 Year in PostHog. That's too long ago 🙃"}, request=request) return HttpResponse(html) + + +@cache_control(public=True, max_age=300) # cache for 5 minutes +def render_2023(request, user_uuid: str) -> HttpResponse: + template = get_template("hibernating.html") + html = template.render({"message": "This is the 2023 Year in PostHog. That's too long ago 🙃"}, request=request) + return HttpResponse(html)