diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml index 09f3bb8d07a54..e0a5e2694fa14 100644 --- a/.github/workflows/ci-backend.yml +++ b/.github/workflows/ci-backend.yml @@ -130,7 +130,7 @@ jobs: - name: Check formatting run: | - ruff format --exclude posthog/hogql/grammar --check --diff . + ruff format --check --diff . - name: Add Problem Matcher run: echo "::add-matcher::.github/mypy-problem-matcher.json" @@ -197,22 +197,33 @@ jobs: sudo apt-get update sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl - - name: Install python dependencies - run: | - uv pip install --system -r requirements.txt -r requirements-dev.txt + # First running migrations from master, to simulate the real-world scenario - - uses: actions/checkout@v3 + - name: Checkout master + uses: actions/checkout@v3 with: ref: master - - name: Run migrations up to master + - name: Install python dependencies for master run: | - # We need to ensure we have requirements for the master branch - # now also, so we can run migrations up to master. uv pip install --system -r requirements.txt -r requirements-dev.txt + + - name: Run migrations up to master + run: | python manage.py migrate - - uses: actions/checkout@v3 + # Now we can consider this PR's migrations + + - name: Checkout this PR + uses: actions/checkout@v3 + + - name: Install python dependencies for this PR + run: | + uv pip install --system -r requirements.txt -r requirements-dev.txt + + - name: Run migrations for this PR + run: | + python manage.py migrate - name: Check migrations run: | diff --git a/bin/deploy-hobby b/bin/deploy-hobby index 5e7f2f8bfb7e6..d57c81bc996d4 100755 --- a/bin/deploy-hobby +++ b/bin/deploy-hobby @@ -67,7 +67,7 @@ curl -o /dev/null -L --header "Content-Type: application/json" -d "{ \"properties\": {\"domain\": \"${DOMAIN}\"}, \"type\": \"capture\", \"event\": \"magic_curl_install_start\" -}" https://app.posthog.com/batch/ &> /dev/null +}" https://us.i.posthog.com/batch/ &> /dev/null # update apt cache echo "Grabbing latest apt caches" @@ -223,7 +223,7 @@ curl -o /dev/null -L --header "Content-Type: application/json" -d "{ \"properties\": {\"domain\": \"${DOMAIN}\"}, \"type\": \"capture\", \"event\": \"magic_curl_install_complete\" -}" https://app.posthog.com/batch/ &> /dev/null +}" https://us.i.posthog.com/batch/ &> /dev/null echo "" echo "To stop the stack run 'docker-compose stop'" echo "To start the stack again run 'docker-compose start'" diff --git a/cypress/e2e/auto-redirect.cy.ts b/cypress/e2e/auto-redirect.cy.ts index 748a41037cc4c..4cacef126c381 100644 --- a/cypress/e2e/auto-redirect.cy.ts +++ b/cypress/e2e/auto-redirect.cy.ts @@ -29,7 +29,7 @@ describe('Redirect to other subdomain if logged in', () => { cy.visit(`/login?next=${redirect_path}`) - cy.setCookie('ph_current_instance', `"app.posthog.com"`) + cy.setCookie('ph_current_instance', `"us.posthog.com"`) cy.setCookie('is-logged-in', '1') cy.reload() diff --git a/ee/clickhouse/queries/event_query.py b/ee/clickhouse/queries/event_query.py index 259b4c4894786..b1b4dbb695e63 100644 --- a/ee/clickhouse/queries/event_query.py +++ b/ee/clickhouse/queries/event_query.py @@ -33,13 +33,19 @@ def __init__( should_join_distinct_ids=False, should_join_persons=False, # Extra events/person table columns to fetch since parent query needs them - extra_fields: List[ColumnName] = [], - extra_event_properties: List[PropertyName] = [], - extra_person_fields: List[ColumnName] = [], + extra_fields: Optional[List[ColumnName]] = None, + extra_event_properties: Optional[List[PropertyName]] = None, + extra_person_fields: Optional[List[ColumnName]] = None, override_aggregate_users_by_distinct_id: Optional[bool] = None, person_on_events_mode: PersonsOnEventsMode = PersonsOnEventsMode.disabled, **kwargs, ) -> None: + if extra_person_fields is None: + extra_person_fields = [] + if extra_event_properties is None: + extra_event_properties = [] + if extra_fields is None: + extra_fields = [] super().__init__( filter=filter, team=team, diff --git a/ee/clickhouse/queries/funnels/funnel_correlation.py b/ee/clickhouse/queries/funnels/funnel_correlation.py index 3ca6801ee6af6..ed3995968a001 100644 --- a/ee/clickhouse/queries/funnels/funnel_correlation.py +++ b/ee/clickhouse/queries/funnels/funnel_correlation.py @@ -868,9 +868,9 @@ def get_partial_event_contingency_tables(self) -> Tuple[List[EventContingencyTab # Get the total success/failure counts from the results results = [result for result in results_with_total if result[0] != self.TOTAL_IDENTIFIER] - _, success_total, failure_total = [ + _, success_total, failure_total = next( result for result in results_with_total if result[0] == self.TOTAL_IDENTIFIER - ][0] + ) # Add a little structure, and keep it close to the query definition so it's # obvious what's going on with result indices. diff --git a/ee/clickhouse/queries/test/test_cohort_query.py b/ee/clickhouse/queries/test/test_cohort_query.py index 25d0b92ed866f..95c1e6837b37d 100644 --- a/ee/clickhouse/queries/test/test_cohort_query.py +++ b/ee/clickhouse/queries/test/test_cohort_query.py @@ -27,8 +27,10 @@ def _make_event_sequence( interval_days, period_event_counts, event="$pageview", - properties={}, + properties=None, ): + if properties is None: + properties = {} for period_index, event_count in enumerate(period_event_counts): for i in range(event_count): _create_event( diff --git a/ee/clickhouse/views/test/test_clickhouse_retention.py b/ee/clickhouse/views/test/test_clickhouse_retention.py index f64aa17ca5834..0e5a8ad0fafdf 100644 --- a/ee/clickhouse/views/test/test_clickhouse_retention.py +++ b/ee/clickhouse/views/test/test_clickhouse_retention.py @@ -592,7 +592,7 @@ def test_can_specify_breakdown_event_property_and_retrieve_people(self): ), ) - chrome_cohort = [cohort for cohort in retention["result"] if cohort["label"] == "Chrome"][0] + chrome_cohort = next(cohort for cohort in retention["result"] if cohort["label"] == "Chrome") people_url = chrome_cohort["values"][0]["people_url"] people_response = self.client.get(people_url) assert people_response.status_code == 200 diff --git a/ee/models/license.py b/ee/models/license.py index d1b575ec801c2..f0e12d3d2f440 100644 --- a/ee/models/license.py +++ b/ee/models/license.py @@ -72,7 +72,8 @@ class License(models.Model): ] ENTERPRISE_PLAN = "enterprise" - ENTERPRISE_FEATURES = SCALE_FEATURES + [ + ENTERPRISE_FEATURES = [ + *SCALE_FEATURES, AvailableFeature.ADVANCED_PERMISSIONS, AvailableFeature.PROJECT_BASED_PERMISSIONING, AvailableFeature.SAML, diff --git a/ee/settings.py b/ee/settings.py index 448c9ef67aa22..7342bdf98f987 100644 --- a/ee/settings.py +++ b/ee/settings.py @@ -1,6 +1,7 @@ """ Django settings for PostHog Enterprise Edition. """ + import os from typing import Dict, List @@ -15,7 +16,8 @@ } # SSO -AUTHENTICATION_BACKENDS = AUTHENTICATION_BACKENDS + [ +AUTHENTICATION_BACKENDS = [ + *AUTHENTICATION_BACKENDS, "ee.api.authentication.MultitenantSAMLAuth", "social_core.backends.google.GoogleOAuth2", ] diff --git a/frontend/__snapshots__/replay-components-propertyicons--android-recording--dark.png b/frontend/__snapshots__/replay-components-propertyicons--android-recording--dark.png index 1e89f87950222..330d3bd5de11d 100644 Binary files a/frontend/__snapshots__/replay-components-propertyicons--android-recording--dark.png and b/frontend/__snapshots__/replay-components-propertyicons--android-recording--dark.png differ diff --git a/frontend/__snapshots__/replay-components-propertyicons--android-recording--light.png b/frontend/__snapshots__/replay-components-propertyicons--android-recording--light.png index e714c6f371d7f..d964d3cf036f8 100644 Binary files a/frontend/__snapshots__/replay-components-propertyicons--android-recording--light.png and b/frontend/__snapshots__/replay-components-propertyicons--android-recording--light.png differ diff --git a/frontend/__snapshots__/replay-components-propertyicons--loading--dark.png b/frontend/__snapshots__/replay-components-propertyicons--loading--dark.png index 8a404e2a4678f..2e57ae5e23fcb 100644 Binary files a/frontend/__snapshots__/replay-components-propertyicons--loading--dark.png and b/frontend/__snapshots__/replay-components-propertyicons--loading--dark.png differ diff --git a/frontend/__snapshots__/replay-components-propertyicons--loading--light.png b/frontend/__snapshots__/replay-components-propertyicons--loading--light.png index 272b11050ae7b..47664e292068d 100644 Binary files a/frontend/__snapshots__/replay-components-propertyicons--loading--light.png and b/frontend/__snapshots__/replay-components-propertyicons--loading--light.png differ diff --git a/frontend/__snapshots__/replay-components-propertyicons--web-recording--dark.png b/frontend/__snapshots__/replay-components-propertyicons--web-recording--dark.png index 16805c539b8be..8c5c98645af78 100644 Binary files a/frontend/__snapshots__/replay-components-propertyicons--web-recording--dark.png and b/frontend/__snapshots__/replay-components-propertyicons--web-recording--dark.png differ diff --git a/frontend/__snapshots__/replay-components-propertyicons--web-recording--light.png b/frontend/__snapshots__/replay-components-propertyicons--web-recording--light.png index f84bfb91ffc82..2b58e688291d9 100644 Binary files a/frontend/__snapshots__/replay-components-propertyicons--web-recording--light.png and b/frontend/__snapshots__/replay-components-propertyicons--web-recording--light.png differ diff --git a/frontend/__snapshots__/replay-player-failure--recent-recordings-404--dark.png b/frontend/__snapshots__/replay-player-failure--recent-recordings-404--dark.png index 47bf7ab05f463..4338f4712a54a 100644 Binary files a/frontend/__snapshots__/replay-player-failure--recent-recordings-404--dark.png and b/frontend/__snapshots__/replay-player-failure--recent-recordings-404--dark.png differ diff --git a/frontend/__snapshots__/replay-player-failure--recent-recordings-404--light.png b/frontend/__snapshots__/replay-player-failure--recent-recordings-404--light.png index b8636e803a367..2b3cade561e6f 100644 Binary files a/frontend/__snapshots__/replay-player-failure--recent-recordings-404--light.png and b/frontend/__snapshots__/replay-player-failure--recent-recordings-404--light.png differ diff --git a/frontend/__snapshots__/scenes-app-feature-flags--edit-feature-flag--dark.png b/frontend/__snapshots__/scenes-app-feature-flags--edit-feature-flag--dark.png index 7e63dcc4450bd..2f638baa35bac 100644 Binary files a/frontend/__snapshots__/scenes-app-feature-flags--edit-feature-flag--dark.png and b/frontend/__snapshots__/scenes-app-feature-flags--edit-feature-flag--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-feature-flags--edit-feature-flag--light.png b/frontend/__snapshots__/scenes-app-feature-flags--edit-feature-flag--light.png index ad59b116fccc7..46e10c41687f5 100644 Binary files a/frontend/__snapshots__/scenes-app-feature-flags--edit-feature-flag--light.png and b/frontend/__snapshots__/scenes-app-feature-flags--edit-feature-flag--light.png differ diff --git a/frontend/__snapshots__/scenes-app-feature-flags--edit-multi-variate-feature-flag--dark.png b/frontend/__snapshots__/scenes-app-feature-flags--edit-multi-variate-feature-flag--dark.png index 97fb2a7698f4c..413bb10e14a0d 100644 Binary files a/frontend/__snapshots__/scenes-app-feature-flags--edit-multi-variate-feature-flag--dark.png and b/frontend/__snapshots__/scenes-app-feature-flags--edit-multi-variate-feature-flag--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-feature-flags--edit-multi-variate-feature-flag--light.png b/frontend/__snapshots__/scenes-app-feature-flags--edit-multi-variate-feature-flag--light.png index f005ed0bf07c5..b6fe07b5c0eab 100644 Binary files a/frontend/__snapshots__/scenes-app-feature-flags--edit-multi-variate-feature-flag--light.png and b/frontend/__snapshots__/scenes-app-feature-flags--edit-multi-variate-feature-flag--light.png differ diff --git a/frontend/__snapshots__/scenes-app-notebooks--recordings-playlist--dark.png b/frontend/__snapshots__/scenes-app-notebooks--recordings-playlist--dark.png index be16d2121c557..592abd071d8f9 100644 Binary files a/frontend/__snapshots__/scenes-app-notebooks--recordings-playlist--dark.png and b/frontend/__snapshots__/scenes-app-notebooks--recordings-playlist--dark.png differ diff --git a/frontend/__snapshots__/scenes-app-notebooks--recordings-playlist--light.png b/frontend/__snapshots__/scenes-app-notebooks--recordings-playlist--light.png index 95c52c2f024ca..44912f5aa119f 100644 Binary files a/frontend/__snapshots__/scenes-app-notebooks--recordings-playlist--light.png and b/frontend/__snapshots__/scenes-app-notebooks--recordings-playlist--light.png differ diff --git a/frontend/src/layout.html b/frontend/src/layout.html index f3468333c3d5a..f82afa51791d6 100644 --- a/frontend/src/layout.html +++ b/frontend/src/layout.html @@ -7,7 +7,7 @@ {% include "head.html" %} ([ } if (event.data.type === 'external-navigation') { - // This should only be triggered for app|eu.posthog.com links + // This should only be triggered for us|eu.posthog.com links actions.handleExternalUrl(event.data.url) return } diff --git a/frontend/src/lib/components/Cards/InsightCard/TopHeading.tsx b/frontend/src/lib/components/Cards/InsightCard/TopHeading.tsx index 3d059c2a11236..19f81f0dc8309 100644 --- a/frontend/src/lib/components/Cards/InsightCard/TopHeading.tsx +++ b/frontend/src/lib/components/Cards/InsightCard/TopHeading.tsx @@ -32,8 +32,11 @@ export function TopHeading({ insight }: { insight: InsightModel }): JSX.Element } } - const defaultDateRange = query == undefined || isInsightQueryNode(query) ? 'Last 7 days' : null - const dateText = dateFilterToText(date_from, date_to, defaultDateRange) + let dateText: string | null = null + if (insightType?.name !== 'Retention') { + const defaultDateRange = query == undefined || isInsightQueryNode(query) ? 'Last 7 days' : null + dateText = dateFilterToText(date_from, date_to, defaultDateRange) + } return ( <> {insightType?.name} diff --git a/frontend/src/lib/components/JSSnippet.tsx b/frontend/src/lib/components/JSSnippet.tsx index 119d84622b9af..f3248a53fb628 100644 --- a/frontend/src/lib/components/JSSnippet.tsx +++ b/frontend/src/lib/components/JSSnippet.tsx @@ -3,13 +3,21 @@ import { CodeSnippet, Language } from 'lib/components/CodeSnippet' import { apiHostOrigin } from 'lib/utils/apiHost' import { teamLogic } from 'scenes/teamLogic' -export function JSSnippet(): JSX.Element { +export function useJsSnippet(indent = 0): string { const { currentTeam } = useValues(teamLogic) - return ( - {``} - ) + return [ + '', + ] + .map((x) => ' '.repeat(indent) + x) + .join('\n') +} + +export function JSSnippet(): JSX.Element { + const snippet = useJsSnippet() + + return {snippet} } diff --git a/frontend/src/lib/components/PropertyIcon.tsx b/frontend/src/lib/components/PropertyIcon.tsx index 8d4d9577fe1c0..881380953d64f 100644 --- a/frontend/src/lib/components/PropertyIcon.tsx +++ b/frontend/src/lib/components/PropertyIcon.tsx @@ -18,8 +18,7 @@ import { IconWeb, IconWindows, } from 'lib/lemon-ui/icons' -import { Tooltip } from 'lib/lemon-ui/Tooltip' -import { HTMLAttributes, ReactNode } from 'react' +import { forwardRef, HTMLAttributes, Ref } from 'react' import { countryCodeToFlag } from 'scenes/insights/views/WorldMap' const osIcons = { @@ -66,12 +65,13 @@ interface PropertyIconProps { property: string value?: string className?: string - noTooltip?: boolean onClick?: HTMLAttributes['onClick'] - tooltipTitle?: (property: string, value?: string) => ReactNode // Tooltip title will default to `value` } -export function PropertyIcon({ property, value, className, noTooltip, tooltipTitle }: PropertyIconProps): JSX.Element { +export const PropertyIcon = forwardRef(function PropertyIcon( + { property, value, className }: PropertyIconProps, + ref: Ref +): JSX.Element { if (!property || !(property in PROPERTIES_ICON_MAP)) { return <> } @@ -86,7 +86,9 @@ export function PropertyIcon({ property, value, className, noTooltip, tooltipTit icon = countryCodeToFlag(value) } - const content =
{icon}
- - return noTooltip ? content : {content} -} + return ( +
+ {icon} +
+ ) +}) diff --git a/frontend/src/lib/utils/apiHost.ts b/frontend/src/lib/utils/apiHost.ts index 6b47d8c9c8aea..d61f473573cef 100644 --- a/frontend/src/lib/utils/apiHost.ts +++ b/frontend/src/lib/utils/apiHost.ts @@ -1,8 +1,11 @@ export function apiHostOrigin(): string { let apiHost = window.location.origin - // similar to https://github.com/PostHog/posthog-js/blob/b79315b7a4fa0caded7026bda2fec01defb0ba73/src/posthog-core.ts#L1742 + if (apiHost === 'https://us.posthog.com') { - apiHost = 'https://app.posthog.com' + apiHost = 'https://us.i.posthog.com' + } else if (apiHost === 'https://eu.posthog.com') { + apiHost = 'https://eu.i.posthog.com' } + return apiHost } diff --git a/frontend/src/scenes/data-warehouse/new/sourceWizardLogic.tsx b/frontend/src/scenes/data-warehouse/new/sourceWizardLogic.tsx index a99f597d5a43e..2f400d9e14872 100644 --- a/frontend/src/scenes/data-warehouse/new/sourceWizardLogic.tsx +++ b/frontend/src/scenes/data-warehouse/new/sourceWizardLogic.tsx @@ -380,7 +380,7 @@ export const sourceWizardLogic = kea([ }), listeners(({ actions, values }) => ({ onBack: () => { - if (values.currentStep <= 2) { + if (values.currentStep <= 1) { actions.selectConnector(null) } }, @@ -447,6 +447,7 @@ export const sourceWizardLogic = kea([ }) lemonToast.success('New Data Resource Created') actions.setSourceId(id) + actions.resetSourceConnectionDetails() actions.onNext() } catch (e: any) { lemonToast.error(e.data?.message ?? e.message) diff --git a/frontend/src/scenes/experiments/ExperimentView/components.tsx b/frontend/src/scenes/experiments/ExperimentView/components.tsx index 78d50619bfead..deef44945819d 100644 --- a/frontend/src/scenes/experiments/ExperimentView/components.tsx +++ b/frontend/src/scenes/experiments/ExperimentView/components.tsx @@ -273,7 +273,7 @@ export function ExperimentLoadingAnimation(): JSX.Element { } export function PageHeaderCustom(): JSX.Element { - const { experiment, isExperimentRunning } = useValues(experimentLogic) + const { experiment, isExperimentRunning, isExperimentStopped } = useValues(experimentLogic) const { launchExperiment, resetRunningExperiment, @@ -307,38 +307,44 @@ export function PageHeaderCustom(): JSX.Element { )} {experiment && isExperimentRunning && (
- <> - - (exposureCohortId ? undefined : createExposureCohort())} - fullWidth - data-attr={`${exposureCohortId ? 'view' : 'create'}-exposure-cohort`} - to={exposureCohortId ? urls.cohort(exposureCohortId) : undefined} - targetBlank={!!exposureCohortId} - > - {exposureCohortId ? 'View' : 'Create'} exposure cohort - - loadExperimentResults(true)} - fullWidth - data-attr="refresh-experiment" - > - Refresh experiment results - - loadSecondaryMetricResults(true)} - fullWidth - data-attr="refresh-secondary-metrics" - > - Refresh secondary metrics - - - } - /> - - + {!isExperimentStopped && !experiment.archived && ( + <> + + + exposureCohortId ? undefined : createExposureCohort() + } + fullWidth + data-attr={`${ + exposureCohortId ? 'view' : 'create' + }-exposure-cohort`} + to={exposureCohortId ? urls.cohort(exposureCohortId) : undefined} + targetBlank={!!exposureCohortId} + > + {exposureCohortId ? 'View' : 'Create'} exposure cohort + + loadExperimentResults(true)} + fullWidth + data-attr="refresh-experiment" + > + Refresh experiment results + + loadSecondaryMetricResults(true)} + fullWidth + data-attr="refresh-secondary-metrics" + > + Refresh secondary metrics + + + } + /> + + + )} {!experiment.end_date && ( )} - {experiment?.end_date && - dayjs().isSameOrAfter(dayjs(experiment.end_date), 'day') && - !experiment.archived && ( - archiveExperiment()}> - Archive - - )} + {isExperimentStopped && ( + archiveExperiment()}> + Archive + + )}
)} diff --git a/frontend/src/scenes/experiments/experimentLogic.tsx b/frontend/src/scenes/experiments/experimentLogic.tsx index 527d7dd142076..f234fa6ead56a 100644 --- a/frontend/src/scenes/experiments/experimentLogic.tsx +++ b/frontend/src/scenes/experiments/experimentLogic.tsx @@ -396,7 +396,7 @@ export const experimentLogic = kea([ // the new query with any existing query and that causes validation problems when there are // unsupported properties in the now merged query. const newQuery = filtersToQueryNode(newInsightFilters) - if (filters?.insight === InsightType.FUNNELS) { + if (newInsightFilters?.insight === InsightType.FUNNELS) { ;(newQuery as TrendsQuery).trendsFilter = undefined } else { ;(newQuery as FunnelsQuery).funnelsFilter = undefined diff --git a/frontend/src/scenes/feature-flags/FeatureFlag.tsx b/frontend/src/scenes/feature-flags/FeatureFlag.tsx index 61ebd51c745b1..c6affd75911ee 100644 --- a/frontend/src/scenes/feature-flags/FeatureFlag.tsx +++ b/frontend/src/scenes/feature-flags/FeatureFlag.tsx @@ -101,7 +101,7 @@ export function FeatureFlag({ id }: { id?: string } = {}): JSX.Element { deleteFeatureFlag, editFeatureFlag, loadFeatureFlag, - triggerFeatureFlagUpdate, + saveFeatureFlag, createStaticCohort, setFeatureFlagFilters, setActiveTab, @@ -384,7 +384,7 @@ export function FeatureFlag({ id }: { id?: string } = {}): JSX.Element { person is identified. This ensures the experience for the anonymous person is carried forward to the authenticated person.{' '} Learn more @@ -485,28 +485,45 @@ export function FeatureFlag({ id }: { id?: string } = {}): JSX.Element { href: urls.featureFlag(id), }} caption={ - <> - {featureFlag.name || Description (optional)} - {featureFlag?.tags && ( - <> - {featureFlag.can_edit ? ( - { - // TODO: Use an existing function instead of this new one for updates - triggerFeatureFlagUpdate({ tags }) - }} - tagsAvailable={tags.filter( - (tag) => !featureFlag.tags?.includes(tag) - )} - className="mt-2" - /> - ) : featureFlag.tags.length ? ( - - ) : null} - - )} - +
+
+
+
+ Key:{' '} + + {featureFlag.key} + +
+
+
+ {featureFlag?.tags && ( + <> + {featureFlag.tags.length > 0 ? ( + Tags: + ) : null}{' '} + {featureFlag.can_edit ? ( + { + saveFeatureFlag({ tags }) + }} + tagsAvailable={tags.filter( + (tag) => !featureFlag.tags?.includes(tag) + )} + /> + ) : featureFlag.tags.length > 0 ? ( + + ) : null} + + )} +
+
+
{featureFlag.name || Description (optional)}
+
} buttons={ <> diff --git a/frontend/src/scenes/feature-flags/FeatureFlagSnippets.tsx b/frontend/src/scenes/feature-flags/FeatureFlagSnippets.tsx index 06037bb23c2ea..6130aa984820f 100644 --- a/frontend/src/scenes/feature-flags/FeatureFlagSnippets.tsx +++ b/frontend/src/scenes/feature-flags/FeatureFlagSnippets.tsx @@ -534,7 +534,7 @@ export function JSBootstrappingSnippet(): JSX.Element { // This avoids the delay between the library loading and feature flags becoming available to use. posthog.init('{project_api_key}', { - api_host: 'https://app.posthog.com', + api_host: '${apiHostOrigin()}' bootstrap: { distinctID: 'your-anonymous-id', diff --git a/frontend/src/scenes/feature-flags/featureFlagLogic.ts b/frontend/src/scenes/feature-flags/featureFlagLogic.ts index c4e6842aff7e5..32c029e760e43 100644 --- a/frontend/src/scenes/feature-flags/featureFlagLogic.ts +++ b/frontend/src/scenes/feature-flags/featureFlagLogic.ts @@ -3,7 +3,6 @@ import { DeepPartialMap, forms, ValidationErrorType } from 'kea-forms' import { loaders } from 'kea-loaders' import { router, urlToAction } from 'kea-router' import api from 'lib/api' -import { convertPropertyGroupToProperties } from 'lib/components/PropertyFilters/utils' import { dayjs } from 'lib/dayjs' import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' import { featureFlagLogic as enabledFeaturesLogic } from 'lib/logic/featureFlagLogic' @@ -23,7 +22,6 @@ import { userLogic } from 'scenes/userLogic' import { groupsModel } from '~/models/groupsModel' import { - AnyPropertyFilter, AvailableFeature, Breadcrumb, CohortType, @@ -196,6 +194,8 @@ export const featureFlagLogic = kea([ actions: [ newDashboardLogic({ featureFlagId: typeof props.id === 'number' ? props.id : undefined }), ['submitNewDashboardSuccessWithResult'], + featureFlagsLogic, + ['updateFlag', 'deleteFlag'], ], })), actions({ @@ -216,7 +216,6 @@ export const featureFlagLogic = kea([ loadInsightAtIndex: (index: number, filters: Partial) => ({ index, filters }), setInsightResultAtIndex: (index: number, average: number) => ({ index, average }), loadAllInsightsForFlag: true, - triggerFeatureFlagUpdate: (payload: Partial) => ({ payload }), generateUsageDashboard: true, enrichUsageDashboard: true, setCopyDestinationProject: (id: number | null) => ({ id }), @@ -259,21 +258,6 @@ export const featureFlagLogic = kea([ { ...NEW_FLAG } as FeatureFlagType, { setFeatureFlag: (_, { featureFlag }) => { - if (featureFlag.filters.groups) { - // TODO: This propertygroup conversion is non-sensical, don't need it here. - const groups = featureFlag.filters.groups.map((group) => { - if (group.properties) { - return { - ...group, - properties: convertPropertyGroupToProperties( - group.properties - ) as AnyPropertyFilter[], - } - } - return group - }) - return { ...featureFlag, filters: { ...featureFlag?.filters, groups } } - } return featureFlag }, setFeatureFlagFilters: (state, { filters }) => { @@ -673,7 +657,7 @@ export const featureFlagLogic = kea([ }, saveFeatureFlagSuccess: ({ featureFlag }) => { lemonToast.success('Feature flag saved') - featureFlagsLogic.findMounted()?.actions.updateFlag(featureFlag) + actions.updateFlag(featureFlag) featureFlag.id && router.actions.replace(urls.featureFlag(featureFlag.id)) actions.editFeatureFlag(false) }, @@ -682,8 +666,7 @@ export const featureFlagLogic = kea([ endpoint: `projects/${values.currentTeamId}/feature_flags`, object: { name: featureFlag.key, id: featureFlag.id }, callback: () => { - featureFlag.id && featureFlagsLogic.findMounted()?.actions.deleteFlag(featureFlag.id) - featureFlagsLogic.findMounted()?.actions.loadFeatureFlags() + featureFlag.id && actions.deleteFlag(featureFlag.id) router.actions.push(urls.featureFlags()) }, }) @@ -725,16 +708,6 @@ export const featureFlagLogic = kea([ values.featureFlag.rollback_conditions[index].threshold_metric as FilterType ) }, - triggerFeatureFlagUpdate: async ({ payload }) => { - if (values.featureFlag) { - const updatedFlag = await api.update( - `api/projects/${values.currentTeamId}/feature_flags/${values.featureFlag.id}`, - payload - ) - actions.setFeatureFlag(updatedFlag) - featureFlagsLogic.findMounted()?.actions.updateFlag(updatedFlag) - } - }, copyFlagSuccess: ({ featureFlagCopy }) => { if (featureFlagCopy?.success.length) { const operation = values.projectsWithCurrentFlag.find( @@ -953,13 +926,12 @@ export const featureFlagLogic = kea([ afterMount(({ props, actions }) => { const foundFlag = featureFlagsLogic.findMounted()?.values.featureFlags.find((flag) => flag.id === props.id) if (foundFlag) { - const formatPayloads = variantKeyToIndexFeatureFlagPayloads(foundFlag) - actions.setFeatureFlag(formatPayloads) + const formatPayloadsWithFlag = variantKeyToIndexFeatureFlagPayloads(foundFlag) + actions.setFeatureFlag(formatPayloadsWithFlag) actions.loadRelatedInsights() actions.loadAllInsightsForFlag() } else if (props.id !== 'new') { actions.loadFeatureFlag() } - actions.loadSentryStats() }), ]) diff --git a/frontend/src/scenes/notebooks/Nodes/NotebookNodePerson.tsx b/frontend/src/scenes/notebooks/Nodes/NotebookNodePerson.tsx index df2ac18295f26..1c249164b197a 100644 --- a/frontend/src/scenes/notebooks/Nodes/NotebookNodePerson.tsx +++ b/frontend/src/scenes/notebooks/Nodes/NotebookNodePerson.tsx @@ -1,7 +1,7 @@ import { createPostHogWidgetNode } from 'scenes/notebooks/Nodes/NodeWrapper' import { NotebookNodeType, PropertyDefinitionType } from '~/types' import { useActions, useValues } from 'kea' -import { LemonDivider } from '@posthog/lemon-ui' +import { LemonDivider, Tooltip } from '@posthog/lemon-ui' import { urls } from 'scenes/urls' import { PersonIcon, TZLabel } from '@posthog/apps-common' import { personLogic } from 'scenes/persons/personLogic' @@ -78,17 +78,16 @@ const Component = ({ attributes }: NotebookNodeProps ( + title={
{tooltipValue ?? 'N/A'}
- )} - /> + } + > + + ) }) ) : ( diff --git a/frontend/src/scenes/notebooks/NotebookTemplates/notebookTemplates.ts b/frontend/src/scenes/notebooks/NotebookTemplates/notebookTemplates.ts index 3ff279c9b060f..f1c63fe133674 100644 --- a/frontend/src/scenes/notebooks/NotebookTemplates/notebookTemplates.ts +++ b/frontend/src/scenes/notebooks/NotebookTemplates/notebookTemplates.ts @@ -313,7 +313,7 @@ export const LOCAL_NOTEBOOK_TEMPLATES: NotebookType[] = [ __init: null, children: null, file: null, - src: 'https://app.posthog.com/uploaded_media/018c494d-132b-0000-2004-8861f35c13b5', + src: 'https://us.posthog.com/uploaded_media/018c494d-132b-0000-2004-8861f35c13b5', }, }, { @@ -643,7 +643,7 @@ export const LOCAL_NOTEBOOK_TEMPLATES: NotebookType[] = [ __init: null, children: null, file: null, - src: 'https://app.posthog.com/uploaded_media/018c496c-d79a-0000-bbc8-fdb0c77ec46f', + src: 'https://us.posthog.com/uploaded_media/018c496c-d79a-0000-bbc8-fdb0c77ec46f', }, }, { diff --git a/frontend/src/scenes/onboarding/sdks/sdk-install-instructions/flutter.tsx b/frontend/src/scenes/onboarding/sdks/sdk-install-instructions/flutter.tsx index 67e3a4a7799d3..e5164e93d378e 100644 --- a/frontend/src/scenes/onboarding/sdks/sdk-install-instructions/flutter.tsx +++ b/frontend/src/scenes/onboarding/sdks/sdk-install-instructions/flutter.tsx @@ -1,5 +1,6 @@ import { useValues } from 'kea' import { CodeSnippet, Language } from 'lib/components/CodeSnippet' +import { useJsSnippet } from 'lib/components/JSSnippet' import { apiHostOrigin } from 'lib/utils/apiHost' import { teamLogic } from 'scenes/teamLogic' @@ -38,8 +39,7 @@ function FlutterIOSSetupSnippet(): JSX.Element { } function FlutterWebSetupSnippet(): JSX.Element { - const { currentTeam } = useValues(teamLogic) - const url = apiHostOrigin() + const jsSnippet = useJsSnippet(4) return ( @@ -47,10 +47,7 @@ function FlutterWebSetupSnippet(): JSX.Element { ... - +${jsSnippet} diff --git a/frontend/src/scenes/onboarding/sdks/sdk-install-instructions/next-js.tsx b/frontend/src/scenes/onboarding/sdks/sdk-install-instructions/next-js.tsx index 2aa0271b2dee5..8b6a77fbdc3de 100644 --- a/frontend/src/scenes/onboarding/sdks/sdk-install-instructions/next-js.tsx +++ b/frontend/src/scenes/onboarding/sdks/sdk-install-instructions/next-js.tsx @@ -27,7 +27,7 @@ import { PostHogProvider } from 'posthog-js/react' if (typeof window !== 'undefined') { // checks that we are client-side posthog.init(process.env.NEXT_PUBLIC_POSTHOG_KEY, { - api_host: process.env.NEXT_PUBLIC_POSTHOG_HOST || 'https://app.posthog.com', + api_host: process.env.NEXT_PUBLIC_POSTHOG_HOST || '${apiHostOrigin()}', loaded: (posthog) => { if (process.env.NODE_ENV === 'development') posthog.debug() // debug mode in development }, diff --git a/frontend/src/scenes/persons/RelatedFeatureFlags.tsx b/frontend/src/scenes/persons/RelatedFeatureFlags.tsx index 150ed59840cc7..90b9233ce6603 100644 --- a/frontend/src/scenes/persons/RelatedFeatureFlags.tsx +++ b/frontend/src/scenes/persons/RelatedFeatureFlags.tsx @@ -154,7 +154,7 @@ export function RelatedFeatureFlags({ distinctId, groups }: Props): JSX.Element } } }} - value="all" + value={filters.type || 'all'} dropdownMaxContentWidth /> @@ -180,7 +180,7 @@ export function RelatedFeatureFlags({ distinctId, groups }: Props): JSX.Element } } }} - value="all" + value={filters.reason || 'all'} dropdownMaxContentWidth /> @@ -206,7 +206,7 @@ export function RelatedFeatureFlags({ distinctId, groups }: Props): JSX.Element { label: 'Disabled', value: 'false' }, ] as { label: string; value: string }[] } - value="all" + value={filters.active || 'all'} dropdownMaxContentWidth /> diff --git a/frontend/src/scenes/persons/relatedFeatureFlagsLogic.ts b/frontend/src/scenes/persons/relatedFeatureFlagsLogic.ts index 7f60dfda38c67..08e27edffb5c5 100644 --- a/frontend/src/scenes/persons/relatedFeatureFlagsLogic.ts +++ b/frontend/src/scenes/persons/relatedFeatureFlagsLogic.ts @@ -88,7 +88,9 @@ export const relatedFeatureFlagsLogic = kea([ (selectors) => [selectors.relatedFeatureFlags, selectors.featureFlags], (relatedFlags, featureFlags): RelatedFeatureFlag[] => { if (relatedFlags && featureFlags) { - return featureFlags.map((flag) => ({ ...relatedFlags[flag.key], ...flag })) + return featureFlags + .map((flag) => ({ ...relatedFlags[flag.key], ...flag })) + .filter((flag) => flag.evaluation !== undefined) } return [] }, diff --git a/frontend/src/scenes/session-recordings/player/PlayerFrameOverlay.tsx b/frontend/src/scenes/session-recordings/player/PlayerFrameOverlay.tsx index 570a245c377e6..f591ce327a150 100644 --- a/frontend/src/scenes/session-recordings/player/PlayerFrameOverlay.tsx +++ b/frontend/src/scenes/session-recordings/player/PlayerFrameOverlay.tsx @@ -4,7 +4,7 @@ import { IconPlay } from '@posthog/icons' import clsx from 'clsx' import { useActions, useValues } from 'kea' import { useFeatureFlag } from 'lib/hooks/useFeatureFlag' -import { IconErrorOutline } from 'lib/lemon-ui/icons' +import { IconErrorOutline, IconSync } from 'lib/lemon-ui/icons' import { LemonButton } from 'lib/lemon-ui/LemonButton' import { useState } from 'react' import { sessionRecordingPlayerLogic } from 'scenes/session-recordings/player/sessionRecordingPlayerLogic' @@ -16,7 +16,7 @@ import { PlayerUpNext } from './PlayerUpNext' import { SimilarRecordings } from './SimilarRecordings' const PlayerFrameOverlayContent = (): JSX.Element | null => { - const { currentPlayerState } = useValues(sessionRecordingPlayerLogic) + const { currentPlayerState, endReached } = useValues(sessionRecordingPlayerLogic) let content = null const pausedState = currentPlayerState === SessionPlayerState.PAUSE || currentPlayerState === SessionPlayerState.READY @@ -59,7 +59,11 @@ const PlayerFrameOverlayContent = (): JSX.Element | null => { ) } if (pausedState) { - content = + content = endReached ? ( + + ) : ( + + ) } if (currentPlayerState === SessionPlayerState.SKIP) { content =
Skipping inactivity
diff --git a/frontend/src/scenes/session-recordings/player/controller/PlayerController.tsx b/frontend/src/scenes/session-recordings/player/controller/PlayerController.tsx index 77a0374e70904..98b391ec7d780 100644 --- a/frontend/src/scenes/session-recordings/player/controller/PlayerController.tsx +++ b/frontend/src/scenes/session-recordings/player/controller/PlayerController.tsx @@ -2,7 +2,7 @@ import { IconFastForward, IconPause, IconPlay } from '@posthog/icons' import { LemonMenu, LemonSwitch } from '@posthog/lemon-ui' import clsx from 'clsx' import { useActions, useValues } from 'kea' -import { IconFullScreen } from 'lib/lemon-ui/icons' +import { IconFullScreen, IconSync } from 'lib/lemon-ui/icons' import { LemonButton } from 'lib/lemon-ui/LemonButton' import { Tooltip } from 'lib/lemon-ui/Tooltip' import { @@ -18,7 +18,7 @@ import { SeekSkip, Timestamp } from './PlayerControllerTime' import { Seekbar } from './Seekbar' export function PlayerController(): JSX.Element { - const { playingState, isFullScreen } = useValues(sessionRecordingPlayerLogic) + const { playingState, isFullScreen, endReached } = useValues(sessionRecordingPlayerLogic) const { togglePlayPause, setIsFullScreen } = useActions(sessionRecordingPlayerLogic) const { speed, skipInactivitySetting } = useValues(playerSettingsLogic) @@ -37,13 +37,19 @@ export function PlayerController(): JSX.Element { size="small" onClick={togglePlayPause} tooltip={ - <> - {showPause ? 'Pause' : 'Play'} +
+ {showPause ? 'Pause' : endReached ? 'Restart' : 'Play'} - +
} > - {showPause ? : } + {showPause ? ( + + ) : endReached ? ( + + ) : ( + + )} diff --git a/frontend/src/scenes/session-recordings/player/controller/PlayerControllerTime.tsx b/frontend/src/scenes/session-recordings/player/controller/PlayerControllerTime.tsx index 6ca22131ef1dc..cdfc3474a0823 100644 --- a/frontend/src/scenes/session-recordings/player/controller/PlayerControllerTime.tsx +++ b/frontend/src/scenes/session-recordings/player/controller/PlayerControllerTime.tsx @@ -30,19 +30,26 @@ export function Timestamp(): JSX.Element { } active > - {timestampFormat === TimestampFormat.Relative ? ( - <> - {colonDelimitedDuration(startTimeSeconds, fixedUnits)} /{' '} - {colonDelimitedDuration(endTimeSeconds, fixedUnits)} - - ) : ( - <> - {currentTimestamp - ? dayjs(currentTimestamp).tz('UTC').format('DD/MM/YYYY, HH:mm:ss') - : '--/--/----, 00:00:00'}{' '} - UTC - - )} + + {timestampFormat === TimestampFormat.Relative ? ( + <> + {colonDelimitedDuration(startTimeSeconds, fixedUnits)} /{' '} + {colonDelimitedDuration(endTimeSeconds, fixedUnits)} + + ) : ( + <> + {currentTimestamp + ? dayjs(currentTimestamp).tz('UTC').format('DD/MM/YYYY, HH:mm:ss') + : '--/--/----, 00:00:00'}{' '} + UTC + + )} + ) } diff --git a/frontend/src/scenes/session-recordings/player/inspector/PlayerInspectorControls.tsx b/frontend/src/scenes/session-recordings/player/inspector/PlayerInspectorControls.tsx index 4724c1f58548e..9f09164e785cb 100644 --- a/frontend/src/scenes/session-recordings/player/inspector/PlayerInspectorControls.tsx +++ b/frontend/src/scenes/session-recordings/player/inspector/PlayerInspectorControls.tsx @@ -1,8 +1,8 @@ -import { IconBug, IconDashboard, IconInfo, IconPause, IconTerminal, IconX } from '@posthog/icons' +import { IconBug, IconDashboard, IconInfo, IconTerminal, IconX } from '@posthog/icons' import { LemonButton, LemonCheckbox, LemonInput, LemonSelect, LemonTabs, Tooltip } from '@posthog/lemon-ui' import { useActions, useValues } from 'kea' import { FEATURE_FLAGS } from 'lib/constants' -import { IconPlayCircle, IconUnverifiedEvent } from 'lib/lemon-ui/icons' +import { IconUnverifiedEvent } from 'lib/lemon-ui/icons' import { Spinner } from 'lib/lemon-ui/Spinner/Spinner' import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { capitalizeFirstLetter } from 'lib/utils' @@ -69,10 +69,10 @@ function TabButtons({ export function PlayerInspectorControls({ onClose }: { onClose: () => void }): JSX.Element { const { logicProps } = useValues(sessionRecordingPlayerLogic) const inspectorLogic = playerInspectorLogic(logicProps) - const { tab, windowIdFilter, syncScrollingPaused, windowIds, showMatchingEventsFilter } = useValues(inspectorLogic) - const { setWindowIdFilter, setSyncScrollPaused, setTab } = useActions(inspectorLogic) - const { showOnlyMatching, miniFilters, syncScroll, searchQuery } = useValues(playerSettingsLogic) - const { setShowOnlyMatching, setMiniFilter, setSyncScroll, setSearchQuery } = useActions(playerSettingsLogic) + const { tab, windowIdFilter, windowIds, showMatchingEventsFilter } = useValues(inspectorLogic) + const { setWindowIdFilter, setTab } = useActions(inspectorLogic) + const { showOnlyMatching, miniFilters, searchQuery } = useValues(playerSettingsLogic) + const { setShowOnlyMatching, setMiniFilter, setSearchQuery } = useActions(playerSettingsLogic) const mode = logicProps.mode ?? SessionRecordingPlayerMode.Standard @@ -179,36 +179,6 @@ export function PlayerInspectorControls({ onClose }: { onClose: () => void }): J ) : null} - -
- { - // If the user has syncScrolling on, but it is paused due to interacting with the Inspector, we want to resume it - if (syncScroll && syncScrollingPaused) { - setSyncScrollPaused(false) - } else { - // Otherwise we are just toggling the setting - setSyncScroll(!syncScroll) - } - }} - tooltipPlacement="left" - tooltip={ - syncScroll && syncScrollingPaused - ? 'Synced scrolling is paused - click to resume' - : 'Scroll the list in sync with the recording playback' - } - > - {syncScroll && syncScrollingPaused ? ( - - ) : ( - - )} - -
{showMatchingEventsFilter ? (
diff --git a/frontend/src/scenes/session-recordings/player/inspector/PlayerInspectorList.tsx b/frontend/src/scenes/session-recordings/player/inspector/PlayerInspectorList.tsx index fbd58fa6c67bd..7ba6d4ab16af3 100644 --- a/frontend/src/scenes/session-recordings/player/inspector/PlayerInspectorList.tsx +++ b/frontend/src/scenes/session-recordings/player/inspector/PlayerInspectorList.tsx @@ -15,7 +15,6 @@ import { userLogic } from 'scenes/userLogic' import { sidePanelSettingsLogic } from '~/layout/navigation-3000/sidepanel/panels/sidePanelSettingsLogic' import { AvailableFeature, SessionRecordingPlayerTab } from '~/types' -import { playerSettingsLogic } from '../playerSettingsLogic' import { sessionRecordingPlayerLogic } from '../sessionRecordingPlayerLogic' import { PlayerInspectorListItem } from './components/PlayerInspectorListItem' import { playerInspectorLogic } from './playerInspectorLogic' @@ -114,10 +113,9 @@ export function PlayerInspectorList(): JSX.Element { const { logicProps, snapshotsLoaded, sessionPlayerMetaData } = useValues(sessionRecordingPlayerLogic) const inspectorLogic = playerInspectorLogic(logicProps) - const { items, tabsState, playbackIndicatorIndex, playbackIndicatorIndexStop, syncScrollingPaused, tab } = + const { items, tabsState, playbackIndicatorIndex, playbackIndicatorIndexStop, syncScrollPaused, tab } = useValues(inspectorLogic) const { setSyncScrollPaused } = useActions(inspectorLogic) - const { syncScroll } = useValues(playerSettingsLogic) const { currentTeam } = useValues(teamLogic) const { hasAvailableFeature } = useValues(userLogic) const performanceAvailable: boolean = hasAvailableFeature(AvailableFeature.RECORDINGS_PERFORMANCE) @@ -161,12 +159,12 @@ export function PlayerInspectorList(): JSX.Element { .getElementById('PlayerInspectorListMarker') ?.setAttribute('style', `transform: translateY(${offset}px)`) - if (!syncScrollingPaused && syncScroll) { + if (!syncScrollPaused) { scrolledByJsFlag.current = true listRef.current.scrollToRow(playbackIndicatorIndex) } } - }, [playbackIndicatorIndex, syncScroll]) + }, [playbackIndicatorIndex]) const renderRow: ListRowRenderer = ({ index, key, parent, style }) => { return ( @@ -226,6 +224,22 @@ export function PlayerInspectorList(): JSX.Element { /> )} + {syncScrollPaused && ( +
+ { + if (listRef.current) { + listRef.current.scrollToRow(playbackIndicatorIndex) + } + // Tricky: Need to dely to make sure the row scrolled has finished + setTimeout(() => setSyncScrollPaused(false), 100) + }} + > + Sync scrolling + +
+ )}
) : tabsState[tab] === 'loading' ? (
diff --git a/frontend/src/scenes/session-recordings/player/inspector/playerInspectorLogic.ts b/frontend/src/scenes/session-recordings/player/inspector/playerInspectorLogic.ts index 91bdd827c65b6..5428d7d78d976 100644 --- a/frontend/src/scenes/session-recordings/player/inspector/playerInspectorLogic.ts +++ b/frontend/src/scenes/session-recordings/player/inspector/playerInspectorLogic.ts @@ -159,7 +159,7 @@ export const playerInspectorLogic = kea([ connect((props: PlayerInspectorLogicProps) => ({ actions: [ playerSettingsLogic, - ['setTab', 'setMiniFilter', 'setSyncScroll', 'setSearchQuery'], + ['setTab', 'setMiniFilter', 'setSearchQuery'], eventUsageLogic, ['reportRecordingInspectorItemExpanded'], sessionRecordingDataLogic(props), @@ -210,13 +210,12 @@ export const playerInspectorLogic = kea([ }, ], - syncScrollingPaused: [ + syncScrollPaused: [ false, { setTab: () => false, setSyncScrollPaused: (_, { paused }) => paused, setItemExpanded: () => true, - setSyncScroll: () => false, }, ], })), diff --git a/frontend/src/scenes/session-recordings/player/playerSettingsLogic.ts b/frontend/src/scenes/session-recordings/player/playerSettingsLogic.ts index 75255d5014801..381ab7d09a6e2 100644 --- a/frontend/src/scenes/session-recordings/player/playerSettingsLogic.ts +++ b/frontend/src/scenes/session-recordings/player/playerSettingsLogic.ts @@ -184,12 +184,10 @@ export const playerSettingsLogic = kea([ setTab: (tab: SessionRecordingPlayerTab) => ({ tab }), setMiniFilter: (key: string, enabled: boolean) => ({ key, enabled }), setSearchQuery: (search: string) => ({ search }), - setSyncScroll: (enabled: boolean) => ({ enabled }), setDurationTypeToShow: (type: DurationType) => ({ type }), setShowFilters: (showFilters: boolean) => ({ showFilters }), setPrefersAdvancedFilters: (prefersAdvancedFilters: boolean) => ({ prefersAdvancedFilters }), setQuickFilterProperties: (properties: string[]) => ({ properties }), - setShowRecordingListProperties: (enabled: boolean) => ({ enabled }), setTimestampFormat: (format: TimestampFormat) => ({ format }), }), reducers(() => ({ @@ -234,13 +232,6 @@ export const playerSettingsLogic = kea([ setSpeed: (_, { speed }) => speed, }, ], - showRecordingListProperties: [ - false, - { persist: true }, - { - setShowRecordingListProperties: (_, { enabled }) => enabled, - }, - ], timestampFormat: [ TimestampFormat.Relative as TimestampFormat, { persist: true }, @@ -338,14 +329,6 @@ export const playerSettingsLogic = kea([ setSearchQuery: (_, { search }) => search || '', }, ], - - syncScroll: [ - true, - { persist: true }, - { - setSyncScroll: (_, { enabled }) => enabled, - }, - ], })), selectors({ diff --git a/frontend/src/scenes/session-recordings/playlist/SessionRecordingPreview.tsx b/frontend/src/scenes/session-recordings/playlist/SessionRecordingPreview.tsx index 19350d0ab4b8d..88e470cda6f64 100644 --- a/frontend/src/scenes/session-recordings/playlist/SessionRecordingPreview.tsx +++ b/frontend/src/scenes/session-recordings/playlist/SessionRecordingPreview.tsx @@ -1,27 +1,15 @@ -import { - IconBug, - IconCalendar, - IconCursorClick, - IconKeyboard, - IconMagicWand, - IconPinFilled, - IconTerminal, -} from '@posthog/icons' -import { LemonDivider, LemonDropdown, Link } from '@posthog/lemon-ui' +import { IconBug, IconCursorClick, IconKeyboard, IconPinFilled } from '@posthog/icons' import clsx from 'clsx' import { useValues } from 'kea' -import { FlaggedFeature } from 'lib/components/FlaggedFeature' import { PropertyIcon } from 'lib/components/PropertyIcon' import { TZLabel } from 'lib/components/TZLabel' import { FEATURE_FLAGS } from 'lib/constants' -import { IconLink } from 'lib/lemon-ui/icons' -import { LemonButton } from 'lib/lemon-ui/LemonButton' import { LemonSkeleton } from 'lib/lemon-ui/LemonSkeleton' import { Tooltip } from 'lib/lemon-ui/Tooltip' import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { colonDelimitedDuration } from 'lib/utils' +import { countryCodeToName } from 'scenes/insights/views/WorldMap' import { DraggableToNotebook } from 'scenes/notebooks/AddToNotebook/DraggableToNotebook' -import { useNotebookNode } from 'scenes/notebooks/Nodes/NotebookNodeContext' import { asDisplay } from 'scenes/persons/person-utils' import { playerSettingsLogic } from 'scenes/session-recordings/player/playerSettingsLogic' import { urls } from 'scenes/urls' @@ -121,20 +109,17 @@ export interface PropertyIconsProps { export function PropertyIcons({ recordingProperties, loading, iconClassNames }: PropertyIconsProps): JSX.Element { return ( -
+
{loading ? ( -
- - -
+ ) : ( recordingProperties.map(({ property, value, label }) => ( -
- - - {!value ? 'Not captured' : label || value} - -
+ + + )) )}
@@ -144,13 +129,11 @@ export function PropertyIcons({ recordingProperties, loading, iconClassNames }: function FirstURL(props: { startUrl: string | undefined }): JSX.Element { const firstPath = props.startUrl?.replace(/https?:\/\//g, '').split(/[?|#]/)[0] return ( -
- - - {firstPath} - + + + {firstPath} -
+ ) } @@ -186,190 +169,86 @@ export function SessionRecordingPreview({ isActive, onClick, pinned, - summariseFn, - sessionSummaryLoading, }: SessionRecordingPreviewProps): JSX.Element { const { orderBy } = useValues(sessionRecordingsPlaylistLogic) - const { durationTypeToShow, showRecordingListProperties } = useValues(playerSettingsLogic) - - const nodeLogic = useNotebookNode() - const inNotebook = !!nodeLogic - - const iconClassnames = 'text-base text-muted-alt' - - const innerContent = ( -
onClick?.()} - > -
-
-
-
- {asDisplay(recording.person)} -
-
- -
- - -
- -
- - - {orderBy === 'console_error_count' ? ( - - ) : ( - - )} -
-
- -
- {!recording.viewed ? : null} - {pinned ? : null} -
-
- ) - - return ( - - {showRecordingListProperties && !inNotebook ? ( - - } - closeOnClickInside={false} - > - {innerContent} - - ) : ( - innerContent - )} - - ) -} + const { durationTypeToShow } = useValues(playerSettingsLogic) -function SessionRecordingPreviewPopover({ - recording, - summariseFn, - sessionSummaryLoading, -}: { - recording: SessionRecordingType - summariseFn?: (recording: SessionRecordingType) => void - sessionSummaryLoading?: boolean -}): JSX.Element { const { recordingPropertiesById, recordingPropertiesLoading } = useValues(sessionRecordingsListPropertiesLogic) const recordingProperties = recordingPropertiesById[recording.id] const loading = !recordingProperties && recordingPropertiesLoading const iconProperties = gatherIconProperties(recordingProperties, recording) - const iconClassNames = 'text-muted-alt mr-2 shrink-0' + const iconClassNames = 'text-muted-alt shrink-0' return ( -
-
-

Session data

- -
- - -
- - - {recording.start_url} - -
+ +
onClick?.()} + > +
+
+
+ {asDisplay(recording.person)} +
-
-
-
-
- - -
-

Activity

+
+
+ + +
+ + + + {recording.click_count} + + + + + + {recording.keypress_count} + + +
+
-
-
- - {recording.click_count} clicks -
-
- - {recording.keypress_count} key presses -
-
- - {recording.console_error_count} console errors + {orderBy === 'console_error_count' ? ( + + ) : ( + + )}
+ +
-
- - {summariseFn && ( - <> - -
- {recording.summary ? ( - {recording.summary} - ) : ( -
- } - onClick={(e) => { - e.preventDefault() - e.stopPropagation() - if (!recording.summary) { - summariseFn(recording) - } - }} - loading={sessionSummaryLoading} - > - Generate AI summary - -
- )} -
- - )} - -
+
+ {!recording.viewed ? : null} + {pinned ? : null} +
+
+ ) } diff --git a/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx b/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx index 21eae025a4b42..8b8e1c151d630 100644 --- a/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx +++ b/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx @@ -8,7 +8,6 @@ import { BindLogic, useActions, useValues } from 'kea' import { EmptyMessage } from 'lib/components/EmptyMessage/EmptyMessage' import { PropertyKeyInfo } from 'lib/components/PropertyKeyInfo' import { FEATURE_FLAGS } from 'lib/constants' -import { useKeyboardHotkeys } from 'lib/hooks/useKeyboardHotkeys' import { useResizeBreakpoints } from 'lib/hooks/useResizeObserver' import { IconWithCount } from 'lib/lemon-ui/icons' import { LemonBanner } from 'lib/lemon-ui/LemonBanner' @@ -16,16 +15,14 @@ import { LemonTableLoader } from 'lib/lemon-ui/LemonTable/LemonTableLoader' import { Spinner } from 'lib/lemon-ui/Spinner' import { Tooltip } from 'lib/lemon-ui/Tooltip' import { featureFlagLogic } from 'lib/logic/featureFlagLogic' -import React, { useEffect, useRef, useState } from 'react' +import React, { useEffect, useRef } from 'react' import { DraggableToNotebook } from 'scenes/notebooks/AddToNotebook/DraggableToNotebook' import { useNotebookNode } from 'scenes/notebooks/Nodes/NotebookNodeContext' import { urls } from 'scenes/urls' -import { KeyboardShortcut } from '~/layout/navigation-3000/components/KeyboardShortcut' import { ReplayTabs, SessionRecordingType } from '~/types' import { SessionRecordingsFilters } from '../filters/SessionRecordingsFilters' -import { playerSettingsLogic } from '../player/playerSettingsLogic' import { SessionRecordingPlayer } from '../player/SessionRecordingPlayer' import { SessionRecordingPreview, SessionRecordingPreviewSkeleton } from './SessionRecordingPreview' import { @@ -118,8 +115,6 @@ function RecordingsLists(): JSX.Element { logicProps, showOtherRecordings, recordingsCount, - sessionSummaryLoading, - sessionBeingSummarized, } = useValues(sessionRecordingsPlaylistLogic) const { setSelectedRecordingId, @@ -130,32 +125,14 @@ function RecordingsLists(): JSX.Element { setShowSettings, resetFilters, toggleShowOtherRecordings, - summarizeSession, } = useActions(sessionRecordingsPlaylistLogic) - const { showRecordingListProperties } = useValues(playerSettingsLogic) - const { setShowRecordingListProperties } = useActions(playerSettingsLogic) const onRecordingClick = (recording: SessionRecordingType): void => { setSelectedRecordingId(recording.id) } - const onSummarizeClick = (recording: SessionRecordingType): void => { - summarizeSession(recording.id) - } - const lastScrollPositionRef = useRef(0) const contentRef = useRef(null) - const [isHovering, setIsHovering] = useState(null) - - useKeyboardHotkeys( - { - p: { - action: () => setShowRecordingListProperties(!showRecordingListProperties), - disabled: !isHovering, - }, - }, - [isHovering] - ) const handleScroll = (e: React.UIEvent): void => { // If we are scrolling down then check if we are at the bottom of the list @@ -258,11 +235,7 @@ function RecordingsLists(): JSX.Element { ) : null} {pinnedRecordings.length || otherRecordings.length ? ( -
    setIsHovering(true)} onMouseLeave={() => setIsHovering(false)}> -
    - Hint: Hover list and press to preview -
    - +
      {pinnedRecordings.length ? ( @@ -283,10 +256,6 @@ function RecordingsLists(): JSX.Element { onClick={() => onRecordingClick(rec)} isActive={activeSessionRecordingId === rec.id} pinned={false} - summariseFn={onSummarizeClick} - sessionSummaryLoading={ - sessionSummaryLoading && sessionBeingSummarized === rec.id - } />
)) diff --git a/frontend/src/scenes/toolbar-launch/ToolbarLaunch.tsx b/frontend/src/scenes/toolbar-launch/ToolbarLaunch.tsx index b5f1602ec42ff..53dfd8769762e 100644 --- a/frontend/src/scenes/toolbar-launch/ToolbarLaunch.tsx +++ b/frontend/src/scenes/toolbar-launch/ToolbarLaunch.tsx @@ -48,7 +48,7 @@ function ToolbarLaunch(): JSX.Element {

Click on the URL to launch the toolbar.{' '} - {window.location.host === 'app.posthog.com' && 'Remember to disable your adblocker.'} + {window.location.host.includes('.posthog.com') && 'Remember to disable your adblocker.'}

diff --git a/latest_migrations.manifest b/latest_migrations.manifest index 3909765119557..dac9ed4ce4539 100644 --- a/latest_migrations.manifest +++ b/latest_migrations.manifest @@ -5,7 +5,7 @@ contenttypes: 0002_remove_content_type_name ee: 0016_rolemembership_organization_member otp_static: 0002_throttling otp_totp: 0002_auto_20190420_0723 -posthog: 0402_externaldatajob_schema +posthog: 0403_plugin_has_private_access sessions: 0001_initial social_django: 0010_uid_db_index two_factor: 0007_auto_20201201_1019 diff --git a/manage.py b/manage.py index 80de73776159b..09efd7a625ad4 100755 --- a/manage.py +++ b/manage.py @@ -1,5 +1,6 @@ #!/usr/bin/env python """Django's command-line utility for administrative tasks.""" + import os import sys diff --git a/mypy-baseline.txt b/mypy-baseline.txt index 58a2acbea7c86..5a2ab24ae125a 100644 --- a/mypy-baseline.txt +++ b/mypy-baseline.txt @@ -2,10 +2,6 @@ posthog/temporal/common/utils.py:0: error: Argument 1 to "abstractclassmethod" h posthog/temporal/common/utils.py:0: note: This is likely because "from_activity" has named arguments: "cls". Consider marking them positional-only posthog/temporal/common/utils.py:0: error: Argument 2 to "__get__" of "classmethod" has incompatible type "type[HeartbeatType]"; expected "type[Never]" [arg-type] posthog/temporal/data_imports/pipelines/zendesk/talk_api.py:0: error: Incompatible types in assignment (expression has type "None", variable has type "str") [assignment] -posthog/hogql/database/argmax.py:0: error: Argument "chain" to "Field" has incompatible type "list[str]"; expected "list[str | int]" [arg-type] -posthog/hogql/database/argmax.py:0: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance -posthog/hogql/database/argmax.py:0: note: Consider using "Sequence" instead, which is covariant -posthog/hogql/database/argmax.py:0: error: Unsupported operand types for + ("list[str]" and "list[str | int]") [operator] posthog/hogql/database/schema/numbers.py:0: error: Incompatible types in assignment (expression has type "dict[str, IntegerDatabaseField]", variable has type "dict[str, FieldOrTable]") [assignment] posthog/hogql/database/schema/numbers.py:0: note: "Dict" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance posthog/hogql/database/schema/numbers.py:0: note: Consider using "Mapping" instead, which is covariant in the value type @@ -51,14 +47,6 @@ posthog/hogql/visitor.py:0: error: Argument 1 to "visit" of "Visitor" has incomp posthog/hogql/visitor.py:0: error: Argument 1 to "visit" of "Visitor" has incompatible type "Expr | None"; expected "AST" [arg-type] posthog/hogql/visitor.py:0: error: Argument 1 to "visit" of "Visitor" has incompatible type "WindowFrameExpr | None"; expected "AST" [arg-type] posthog/hogql/visitor.py:0: error: Argument 1 to "visit" of "Visitor" has incompatible type "WindowFrameExpr | None"; expected "AST" [arg-type] -posthog/hogql/database/schema/log_entries.py:0: error: Argument "chain" to "Field" has incompatible type "list[str]"; expected "list[str | int]" [arg-type] -posthog/hogql/database/schema/log_entries.py:0: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance -posthog/hogql/database/schema/log_entries.py:0: note: Consider using "Sequence" instead, which is covariant -posthog/hogql/database/schema/log_entries.py:0: error: Unsupported operand types for + ("list[str]" and "list[str | int]") [operator] -posthog/hogql/database/schema/log_entries.py:0: error: Argument "chain" to "Field" has incompatible type "list[str]"; expected "list[str | int]" [arg-type] -posthog/hogql/database/schema/log_entries.py:0: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance -posthog/hogql/database/schema/log_entries.py:0: note: Consider using "Sequence" instead, which is covariant -posthog/hogql/database/schema/log_entries.py:0: error: Unsupported operand types for + ("list[str]" and "list[str | int]") [operator] posthog/hogql/database/schema/groups.py:0: error: Incompatible types in assignment (expression has type "dict[str, DatabaseField]", variable has type "dict[str, FieldOrTable]") [assignment] posthog/hogql/database/schema/groups.py:0: note: "Dict" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance posthog/hogql/database/schema/groups.py:0: note: Consider using "Mapping" instead, which is covariant in the value type @@ -76,18 +64,6 @@ posthog/hogql/parser.py:0: error: "None" has no attribute "text" [attr-defined] posthog/hogql/parser.py:0: error: Statement is unreachable [unreachable] posthog/hogql/database/schema/person_distinct_ids.py:0: error: Argument 1 to "select_from_person_distinct_ids_table" has incompatible type "dict[str, list[str]]"; expected "dict[str, list[str | int]]" [arg-type] posthog/hogql/database/schema/person_distinct_id_overrides.py:0: error: Argument 1 to "select_from_person_distinct_id_overrides_table" has incompatible type "dict[str, list[str]]"; expected "dict[str, list[str | int]]" [arg-type] -posthog/hogql/database/schema/cohort_people.py:0: error: Argument "chain" to "Field" has incompatible type "list[str]"; expected "list[str | int]" [arg-type] -posthog/hogql/database/schema/cohort_people.py:0: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance -posthog/hogql/database/schema/cohort_people.py:0: note: Consider using "Sequence" instead, which is covariant -posthog/hogql/database/schema/cohort_people.py:0: error: Unsupported operand types for + ("list[str]" and "list[str | int]") [operator] -posthog/hogql/database/schema/session_replay_events.py:0: error: Argument "chain" to "Field" has incompatible type "list[str]"; expected "list[str | int]" [arg-type] -posthog/hogql/database/schema/session_replay_events.py:0: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance -posthog/hogql/database/schema/session_replay_events.py:0: note: Consider using "Sequence" instead, which is covariant -posthog/hogql/database/schema/session_replay_events.py:0: error: Unsupported operand types for + ("list[str]" and "list[str | int]") [operator] -posthog/hogql/database/schema/session_replay_events.py:0: error: Argument "chain" to "Field" has incompatible type "list[str]"; expected "list[str | int]" [arg-type] -posthog/hogql/database/schema/session_replay_events.py:0: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance -posthog/hogql/database/schema/session_replay_events.py:0: note: Consider using "Sequence" instead, which is covariant -posthog/hogql/database/schema/session_replay_events.py:0: error: Unsupported operand types for + ("list[str]" and "list[str | int]") [operator] posthog/plugins/utils.py:0: error: Subclass of "str" and "bytes" cannot exist: would have incompatible method signatures [unreachable] posthog/plugins/utils.py:0: error: Statement is unreachable [unreachable] posthog/models/filters/base_filter.py:0: error: "HogQLContext" has no attribute "person_on_events_mode" [attr-defined] @@ -292,9 +268,6 @@ posthog/queries/trends/util.py:0: error: Argument 1 to "translate_hogql" has inc posthog/hogql/property.py:0: error: Argument "chain" to "Field" has incompatible type "list[str]"; expected "list[str | int]" [arg-type] posthog/hogql/property.py:0: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance posthog/hogql/property.py:0: note: Consider using "Sequence" instead, which is covariant -posthog/hogql/property.py:0: error: Argument "chain" to "Field" has incompatible type "list[str]"; expected "list[str | int]" [arg-type] -posthog/hogql/property.py:0: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance -posthog/hogql/property.py:0: note: Consider using "Sequence" instead, which is covariant posthog/hogql/property.py:0: error: Incompatible type for lookup 'pk': (got "str | float", expected "str | int") [misc] posthog/hogql/filters.py:0: error: Incompatible default for argument "team" (default has type "None", argument has type "Team") [assignment] posthog/hogql/filters.py:0: note: PEP 484 prohibits implicit Optional. Accordingly, mypy has changed its default to no_implicit_optional=True @@ -329,9 +302,11 @@ posthog/queries/funnels/base.py:0: error: "HogQLContext" has no attribute "perso posthog/queries/funnels/base.py:0: error: Argument 1 to "translate_hogql" has incompatible type "str | int"; expected "str" [arg-type] ee/clickhouse/queries/funnels/funnel_correlation.py:0: error: Statement is unreachable [unreachable] posthog/caching/calculate_results.py:0: error: Argument 3 to "process_query" has incompatible type "bool"; expected "LimitContext | None" [arg-type] +posthog/api/person.py:0: error: Argument 1 to has incompatible type "*tuple[str, ...]"; expected "type[BaseRenderer]" [arg-type] posthog/api/person.py:0: error: Argument 1 to "loads" has incompatible type "str | None"; expected "str | bytes | bytearray" [arg-type] posthog/api/person.py:0: error: Argument "user" to "log_activity" has incompatible type "User | AnonymousUser"; expected "User | None" [arg-type] posthog/api/person.py:0: error: Argument "user" to "log_activity" has incompatible type "User | AnonymousUser"; expected "User | None" [arg-type] +posthog/api/person.py:0: error: Cannot determine type of "group_properties_filter_group" [has-type] posthog/hogql_queries/web_analytics/web_analytics_query_runner.py:0: error: Argument 1 to "append" of "list" has incompatible type "EventPropertyFilter"; expected "Expr" [arg-type] posthog/hogql_queries/insights/trends/trends_query_runner.py:0: error: Signature of "to_actors_query" incompatible with supertype "QueryRunner" [override] posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: Superclass: @@ -373,6 +348,7 @@ posthog/hogql_queries/legacy_compatibility/process_insight.py:0: error: Incompat posthog/hogql_queries/legacy_compatibility/process_insight.py:0: error: Incompatible types in assignment (expression has type "Filter", variable has type "RetentionFilter") [assignment] posthog/api/insight.py:0: error: Argument 1 to "is_insight_with_hogql_support" has incompatible type "Insight | DashboardTile"; expected "Insight" [arg-type] posthog/api/insight.py:0: error: Argument 1 to "process_insight" has incompatible type "Insight | DashboardTile"; expected "Insight" [arg-type] +posthog/api/insight.py:0: error: Argument 1 to has incompatible type "*tuple[str, ...]"; expected "type[BaseRenderer]" [arg-type] posthog/api/dashboards/dashboard.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] posthog/api/feature_flag.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] posthog/api/feature_flag.py:0: error: Item "Sequence[Any]" of "Any | Sequence[Any] | None" has no attribute "filters" [union-attr] @@ -504,9 +480,6 @@ posthog/hogql/test/test_resolver.py:0: error: "FieldOrTable" has no attribute "f posthog/hogql/test/test_resolver.py:0: error: Item "None" of "JoinExpr | None" has no attribute "table" [union-attr] posthog/hogql/test/test_resolver.py:0: error: Argument 1 to "clone_expr" has incompatible type "SelectQuery | SelectUnionQuery | Field | Any | None"; expected "Expr" [arg-type] posthog/hogql/test/test_resolver.py:0: error: Item "None" of "JoinExpr | None" has no attribute "alias" [union-attr] -posthog/hogql/test/test_property.py:0: error: Incompatible default for argument "placeholders" (default has type "None", argument has type "dict[str, Any]") [assignment] -posthog/hogql/test/test_property.py:0: note: PEP 484 prohibits implicit Optional. Accordingly, mypy has changed its default to no_implicit_optional=True -posthog/hogql/test/test_property.py:0: note: Use https://github.com/hauntsaninja/no_implicit_optional to automatically upgrade your codebase posthog/hogql/test/test_property.py:0: error: Argument 1 to "_property_to_expr" of "TestProperty" has incompatible type "HogQLPropertyFilter"; expected "PropertyGroup | Property | dict[Any, Any] | list[Any]" [arg-type] posthog/hogql/test/test_printer.py:0: error: Argument 2 to "Database" has incompatible type "int"; expected "WeekStartDay | None" [arg-type] posthog/hogql/test/test_printer.py:0: error: Argument 2 to "Database" has incompatible type "int"; expected "WeekStartDay | None" [arg-type] @@ -526,12 +499,6 @@ posthog/hogql/test/test_modifiers.py:0: error: Unsupported right operand type fo posthog/hogql/test/test_modifiers.py:0: error: Unsupported right operand type for in ("str | None") [operator] posthog/hogql/test/test_modifiers.py:0: error: Unsupported right operand type for in ("str | None") [operator] posthog/hogql/test/test_modifiers.py:0: error: Unsupported right operand type for in ("str | None") [operator] -posthog/hogql/test/test_filters.py:0: error: Incompatible default for argument "placeholders" (default has type "None", argument has type "dict[str, Any]") [assignment] -posthog/hogql/test/test_filters.py:0: note: PEP 484 prohibits implicit Optional. Accordingly, mypy has changed its default to no_implicit_optional=True -posthog/hogql/test/test_filters.py:0: note: Use https://github.com/hauntsaninja/no_implicit_optional to automatically upgrade your codebase -posthog/hogql/test/test_filters.py:0: error: Incompatible default for argument "placeholders" (default has type "None", argument has type "dict[str, Any]") [assignment] -posthog/hogql/test/test_filters.py:0: note: PEP 484 prohibits implicit Optional. Accordingly, mypy has changed its default to no_implicit_optional=True -posthog/hogql/test/test_filters.py:0: note: Use https://github.com/hauntsaninja/no_implicit_optional to automatically upgrade your codebase posthog/hogql/test/_test_parser.py:0: error: Invalid base class [misc] posthog/hogql/test/_test_parser.py:0: error: Argument "table" to "JoinExpr" has incompatible type "Placeholder"; expected "SelectQuery | SelectUnionQuery | Field | None" [arg-type] posthog/hogql/test/_test_parser.py:0: error: Item "None" of "JoinExpr | None" has no attribute "table" [union-attr] @@ -551,6 +518,7 @@ posthog/hogql/database/schema/test/test_channel_type.py:0: error: Value of type posthog/hogql/database/schema/test/test_channel_type.py:0: error: Value of type "list[Any] | None" is not indexable [index] posthog/hogql/database/schema/event_sessions.py:0: error: Statement is unreachable [unreachable] posthog/api/organization_member.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] +posthog/api/action.py:0: error: Argument 1 to has incompatible type "*tuple[str, ...]"; expected "type[BaseRenderer]" [arg-type] ee/api/role.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] ee/clickhouse/views/insights.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] posthog/temporal/data_imports/workflow_activities/create_job_model.py:0: error: Argument 6 has incompatible type "ExternalDataSchema"; expected "str" [arg-type] @@ -663,6 +631,7 @@ posthog/api/property_definition.py:0: error: Item "None" of "Organization | Any posthog/api/property_definition.py:0: error: Incompatible types in assignment (expression has type "type[EnterprisePropertyDefinitionSerializer]", variable has type "type[PropertyDefinitionSerializer]") [assignment] posthog/api/property_definition.py:0: error: Item "AnonymousUser" of "User | AnonymousUser" has no attribute "organization" [union-attr] posthog/api/property_definition.py:0: error: Item "None" of "Organization | Any | None" has no attribute "is_feature_available" [union-attr] +posthog/api/event.py:0: error: Argument 1 to has incompatible type "*tuple[str, ...]"; expected "type[BaseRenderer]" [arg-type] posthog/api/dashboards/dashboard_templates.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] ee/api/feature_flag_role_access.py:0: error: Metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases [misc] posthog/temporal/tests/batch_exports/test_run_updates.py:0: error: Unused "type: ignore" comment [unused-ignore] @@ -722,6 +691,7 @@ posthog/management/commands/test/test_create_batch_export_from_app.py:0: error: posthog/management/commands/test/test_create_batch_export_from_app.py:0: note: Possible overload variants: posthog/management/commands/test/test_create_batch_export_from_app.py:0: note: def __getitem__(self, SupportsIndex, /) -> str posthog/management/commands/test/test_create_batch_export_from_app.py:0: note: def __getitem__(self, slice, /) -> list[str] +posthog/api/test/test_capture.py:0: error: Statement is unreachable [unreachable] posthog/api/test/test_capture.py:0: error: Dict entry 0 has incompatible type "Any": "float"; expected "str": "int" [dict-item] posthog/api/test/test_capture.py:0: error: Dict entry 0 has incompatible type "Any": "float"; expected "str": "int" [dict-item] posthog/api/test/test_capture.py:0: error: Dict entry 0 has incompatible type "Any": "float"; expected "str": "int" [dict-item] diff --git a/package.json b/package.json index 5c160594e4a43..e08bbf6ade657 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,7 @@ "build:esbuild": "node frontend/build.mjs", "schema:build": "pnpm run schema:build:json && pnpm run schema:build:python", "schema:build:json": "ts-node bin/build-schema.mjs && prettier --write frontend/src/queries/schema.json", - "schema:build:python": "datamodel-codegen --class-name='SchemaRoot' --collapse-root-models --target-python-version 3.10 --disable-timestamp --use-one-literal-as-default --use-default --use-default-kwarg --use-subclass-enum --input frontend/src/queries/schema.json --input-file-type jsonschema --output posthog/schema.py --output-model-type pydantic_v2.BaseModel && ruff format posthog/schema.py", + "schema:build:python": "datamodel-codegen --class-name='SchemaRoot' --collapse-root-models --target-python-version 3.10 --disable-timestamp --use-one-literal-as-default --use-default --use-default-kwarg --use-subclass-enum --input frontend/src/queries/schema.json --input-file-type jsonschema --output posthog/schema.py --output-model-type pydantic_v2.BaseModel && ruff format posthog/schema.py && ruff check --fix posthog/schema.py", "grammar:build": "npm run grammar:build:python && npm run grammar:build:cpp", "grammar:build:python": "cd posthog/hogql/grammar && antlr -Dlanguage=Python3 HogQLLexer.g4 && antlr -visitor -no-listener -Dlanguage=Python3 HogQLParser.g4", "grammar:build:cpp": "cd posthog/hogql/grammar && antlr -o ../../../hogql_parser -Dlanguage=Cpp HogQLLexer.g4 && antlr -o ../../../hogql_parser -visitor -no-listener -Dlanguage=Cpp HogQLParser.g4", @@ -47,7 +47,7 @@ "typescript:check": "tsc --noEmit && echo \"No errors reported by tsc.\"", "lint:js": "eslint frontend/src", "lint:css": "stylelint \"frontend/**/*.{css,scss}\"", - "format:backend": "ruff --exclude posthog/hogql/grammar .", + "format:backend": "ruff .", "format:frontend": "pnpm lint:js --fix && pnpm lint:css --fix && pnpm prettier", "format": "pnpm format:backend && pnpm format:frontend", "typegen:write": "kea-typegen write --delete --show-ts-errors", @@ -145,7 +145,7 @@ "pmtiles": "^2.11.0", "postcss": "^8.4.31", "postcss-preset-env": "^9.3.0", - "posthog-js": "1.128.1", + "posthog-js": "1.128.3", "posthog-js-lite": "2.5.0", "prettier": "^2.8.8", "prop-types": "^15.7.2", @@ -337,8 +337,8 @@ "pnpm --dir plugin-server exec prettier --write" ], "!(posthog/hogql/grammar/*)*.{py,pyi}": [ - "ruff format", - "ruff check --fix" + "ruff check --fix", + "ruff format" ] }, "browserslist": { diff --git a/plugin-server/src/utils/posthog.ts b/plugin-server/src/utils/posthog.ts index 2f6ada2300fb5..b63604628eb2f 100644 --- a/plugin-server/src/utils/posthog.ts +++ b/plugin-server/src/utils/posthog.ts @@ -1,7 +1,7 @@ import { PostHog } from 'posthog-node' export const posthog = new PostHog('sTMFPsFhdP1Ssg', { - host: 'https://app.posthog.com', + host: 'https://us.i.posthog.com', }) if (process.env.NODE_ENV === 'test') { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 84f402083f4a8..8497ec648a88f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -254,8 +254,8 @@ dependencies: specifier: ^9.3.0 version: 9.3.0(postcss@8.4.31) posthog-js: - specifier: 1.128.1 - version: 1.128.1 + specifier: 1.128.3 + version: 1.128.3 posthog-js-lite: specifier: 2.5.0 version: 2.5.0 @@ -17457,8 +17457,8 @@ packages: resolution: {integrity: sha512-Urvlp0Vu9h3td0BVFWt0QXFJDoOZcaAD83XM9d91NKMKTVPZtfU0ysoxstIf5mw/ce9ZfuMgpWPaagrZI4rmSg==} dev: false - /posthog-js@1.128.1: - resolution: {integrity: sha512-+CIiZf+ijhgAF8g6K+PfaDbSBiADfRaXzrlYKmu5IEN8ghunFd06EV5QM68cwLUEkti4FXn7AAM3k9/KxJgvcA==} + /posthog-js@1.128.3: + resolution: {integrity: sha512-ES5FLTw/u2JTHocJZJtJibVkbk+xc4u9XTxWQPGE1ZVbUOH4lVjSXbEtI56fJvSJaaAuGSQ43kB5crJZ2gNG+g==} dependencies: fflate: 0.4.8 preact: 10.20.2 diff --git a/posthog/api/action.py b/posthog/api/action.py index 8c3caf435e343..437f0227c817f 100644 --- a/posthog/api/action.py +++ b/posthog/api/action.py @@ -165,7 +165,7 @@ class ActionViewSet( viewsets.ModelViewSet, ): scope_object = "action" - renderer_classes = tuple(api_settings.DEFAULT_RENDERER_CLASSES) + (csvrenderers.PaginatedCSVRenderer,) + renderer_classes = (*tuple(api_settings.DEFAULT_RENDERER_CLASSES), csvrenderers.PaginatedCSVRenderer) queryset = Action.objects.all() serializer_class = ActionSerializer authentication_classes = [TemporaryTokenAuthentication] diff --git a/posthog/api/capture.py b/posthog/api/capture.py index 6b921dd27ea48..31592e90e790d 100644 --- a/posthog/api/capture.py +++ b/posthog/api/capture.py @@ -59,10 +59,7 @@ # events that are ingested via a separate path than analytics events. They have # fewer restrictions on e.g. the order they need to be processed in. SESSION_RECORDING_DEDICATED_KAFKA_EVENTS = ("$snapshot_items",) -SESSION_RECORDING_EVENT_NAMES = ( - "$snapshot", - "$performance_event", -) + SESSION_RECORDING_DEDICATED_KAFKA_EVENTS +SESSION_RECORDING_EVENT_NAMES = ("$snapshot", "$performance_event", *SESSION_RECORDING_DEDICATED_KAFKA_EVENTS) EVENTS_RECEIVED_COUNTER = Counter( "capture_events_received_total", @@ -604,9 +601,7 @@ def capture_internal( if event["event"] in SESSION_RECORDING_EVENT_NAMES: session_id = event["properties"]["$session_id"] - headers = [ - ("token", token), - ] + extra_headers + headers = [("token", token), *extra_headers] overflowing = False if token in settings.REPLAY_OVERFLOW_FORCED_TOKENS: diff --git a/posthog/api/dashboards/dashboard.py b/posthog/api/dashboards/dashboard.py index 100e8745b8db1..a89d41814d616 100644 --- a/posthog/api/dashboards/dashboard.py +++ b/posthog/api/dashboards/dashboard.py @@ -398,23 +398,25 @@ class DashboardsViewSet( viewsets.ModelViewSet, ): scope_object = "dashboard" - queryset = Dashboard.objects.order_by("name") + queryset = Dashboard.objects_including_soft_deleted.order_by("name") permission_classes = [CanEditDashboard] def get_serializer_class(self) -> Type[BaseSerializer]: return DashboardBasicSerializer if self.action == "list" else DashboardSerializer def get_queryset(self) -> QuerySet: - if ( + queryset = super().get_queryset() + + include_deleted = ( self.action == "partial_update" and "deleted" in self.request.data and not self.request.data.get("deleted") and len(self.request.data) == 1 - ): + ) + + if not include_deleted: # a dashboard can be un-deleted by patching {"deleted": False} - queryset = Dashboard.objects_including_soft_deleted - else: - queryset = super().get_queryset() + queryset = queryset.filter(deleted=False) queryset = queryset.prefetch_related("sharingconfiguration_set").select_related( "team__organization", diff --git a/posthog/api/event.py b/posthog/api/event.py index 1d251572be874..6366ee866f657 100644 --- a/posthog/api/event.py +++ b/posthog/api/event.py @@ -85,7 +85,7 @@ class EventViewSet( viewsets.GenericViewSet, ): scope_object = "query" - renderer_classes = tuple(api_settings.DEFAULT_RENDERER_CLASSES) + (csvrenderers.PaginatedCSVRenderer,) + renderer_classes = (*tuple(api_settings.DEFAULT_RENDERER_CLASSES), csvrenderers.PaginatedCSVRenderer) serializer_class = ClickhouseEventSerializer throttle_classes = [ClickHouseBurstRateThrottle, ClickHouseSustainedRateThrottle] pagination_class = UncountedLimitOffsetPagination diff --git a/posthog/api/feature_flag.py b/posthog/api/feature_flag.py index e09e70c01b6f1..8bf1dbb5d3cf4 100644 --- a/posthog/api/feature_flag.py +++ b/posthog/api/feature_flag.py @@ -241,6 +241,14 @@ def properties_all_match(predicate): detail=f"Invalid date value: {prop.value}", code="invalid_date" ) + # make sure regex and icontains properties have string values + if prop.operator in ["regex", "icontains", "not_regex", "not_icontains"] and not isinstance( + prop.value, str + ): + raise serializers.ValidationError( + detail=f"Invalid value for operator {prop.operator}: {prop.value}", code="invalid_value" + ) + payloads = filters.get("payloads", {}) if not isinstance(payloads, dict): diff --git a/posthog/api/insight.py b/posthog/api/insight.py index 9e4e7c3af6466..528dc53767934 100644 --- a/posthog/api/insight.py +++ b/posthog/api/insight.py @@ -572,7 +572,7 @@ class InsightViewSet( ClickHouseBurstRateThrottle, ClickHouseSustainedRateThrottle, ] - renderer_classes = tuple(api_settings.DEFAULT_RENDERER_CLASSES) + (csvrenderers.CSVRenderer,) + renderer_classes = (*tuple(api_settings.DEFAULT_RENDERER_CLASSES), csvrenderers.CSVRenderer) filter_backends = [DjangoFilterBackend] filterset_fields = ["short_id", "created_by"] sharing_enabled_actions = ["retrieve", "list"] @@ -838,12 +838,12 @@ def trend(self, request: request.Request, *args: Any, **kwargs: Any): export = "{}/insights/{}/\n".format(SITE_URL, request.GET["export_insight_id"]).encode() + export response = HttpResponse(export) - response[ - "Content-Disposition" - ] = 'attachment; filename="{name} ({date_from} {date_to}) from PostHog.csv"'.format( - name=slugify(request.GET.get("export_name", "export")), - date_from=filter.date_from.strftime("%Y-%m-%d -") if filter.date_from else "up until", - date_to=filter.date_to.strftime("%Y-%m-%d"), + response["Content-Disposition"] = ( + 'attachment; filename="{name} ({date_from} {date_to}) from PostHog.csv"'.format( + name=slugify(request.GET.get("export_name", "export")), + date_from=filter.date_from.strftime("%Y-%m-%d -") if filter.date_from else "up until", + date_to=filter.date_to.strftime("%Y-%m-%d"), + ) ) return response diff --git a/posthog/api/person.py b/posthog/api/person.py index 585fcc33cb86d..942f07e9a9ef8 100644 --- a/posthog/api/person.py +++ b/posthog/api/person.py @@ -224,7 +224,7 @@ class PersonViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet): """ scope_object = "person" - renderer_classes = tuple(api_settings.DEFAULT_RENDERER_CLASSES) + (csvrenderers.PaginatedCSVRenderer,) + renderer_classes = (*tuple(api_settings.DEFAULT_RENDERER_CLASSES), csvrenderers.PaginatedCSVRenderer) queryset = Person.objects.all() serializer_class = PersonSerializer pagination_class = PersonLimitOffsetPagination @@ -932,21 +932,11 @@ def prepare_actor_query_filter(filter: T) -> T: new_group = { "type": "OR", "values": [ - { - "key": "email", - "type": "person", - "value": search, - "operator": "icontains", - }, + {"key": "email", "type": "person", "value": search, "operator": "icontains"}, {"key": "name", "type": "person", "value": search, "operator": "icontains"}, - { - "key": "distinct_id", - "type": "event", - "value": search, - "operator": "icontains", - }, - ] - + group_properties_filter_group, + {"key": "distinct_id", "type": "event", "value": search, "operator": "icontains"}, + *group_properties_filter_group, + ], } prop_group = ( {"type": "AND", "values": [new_group, filter.property_groups.to_dict()]} diff --git a/posthog/api/plugin.py b/posthog/api/plugin.py index 468da9d5ccfd7..2a6e00f325451 100644 --- a/posthog/api/plugin.py +++ b/posthog/api/plugin.py @@ -63,7 +63,11 @@ def _update_plugin_attachments(request: request.Request, plugin_config: PluginCo _update_plugin_attachment(request, plugin_config, match.group(1), None, user) -def get_plugin_config_changes(old_config: Dict[str, Any], new_config: Dict[str, Any], secret_fields=[]) -> List[Change]: +def get_plugin_config_changes( + old_config: Dict[str, Any], new_config: Dict[str, Any], secret_fields=None +) -> List[Change]: + if secret_fields is None: + secret_fields = [] config_changes = dict_changes_between("Plugin", old_config, new_config) for i, change in enumerate(config_changes): @@ -79,8 +83,10 @@ def get_plugin_config_changes(old_config: Dict[str, Any], new_config: Dict[str, def log_enabled_change_activity( - new_plugin_config: PluginConfig, old_enabled: bool, user: User, was_impersonated: bool, changes=[] + new_plugin_config: PluginConfig, old_enabled: bool, user: User, was_impersonated: bool, changes=None ): + if changes is None: + changes = [] if old_enabled != new_plugin_config.enabled: log_activity( organization_id=new_plugin_config.team.organization.id, @@ -864,7 +870,7 @@ def frontend(self, request: request.Request, **kwargs): def _get_secret_fields_for_plugin(plugin: Plugin) -> Set[str]: # A set of keys for config fields that have secret = true - secret_fields = {field["key"] for field in plugin.config_schema if "secret" in field and field["secret"]} + secret_fields = {field["key"] for field in plugin.config_schema if isinstance(field, dict) and field.get("secret")} return secret_fields diff --git a/posthog/api/signup.py b/posthog/api/signup.py index b8c3db86c3341..c31f37b891eb3 100644 --- a/posthog/api/signup.py +++ b/posthog/api/signup.py @@ -503,9 +503,7 @@ def social_create_user( user=user.id if user else None, ) if user: - backend_processor = ( - "domain_whitelist" - ) # This is actually `jit_provisioning` (name kept for backwards-compatibility purposes) + backend_processor = "domain_whitelist" # This is actually `jit_provisioning` (name kept for backwards-compatibility purposes) from_invite = True # jit_provisioning means they're definitely not organization_first_user if not user: diff --git a/posthog/api/team.py b/posthog/api/team.py index 1b615bd692643..c8b2513b6798c 100644 --- a/posthog/api/team.py +++ b/posthog/api/team.py @@ -421,7 +421,8 @@ def get_permissions(self) -> List: IsAuthenticated, APIScopePermission, PremiumMultiProjectPermissions, - ] + self.permission_classes + *self.permission_classes, + ] base_permissions = [permission() for permission in common_permissions] diff --git a/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr b/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr index 3a13d80bf85a7..a120ce5c58068 100644 --- a/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr +++ b/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr @@ -390,8 +390,8 @@ INNER JOIN "posthog_team" ON ("posthog_dashboard"."team_id" = "posthog_team"."id") INNER JOIN "posthog_organization" ON ("posthog_team"."organization_id" = "posthog_organization"."id") LEFT OUTER JOIN "posthog_user" ON ("posthog_dashboard"."created_by_id" = "posthog_user"."id") - WHERE (NOT ("posthog_dashboard"."deleted") - AND "posthog_dashboard"."team_id" = 2 + WHERE ("posthog_dashboard"."team_id" = 2 + AND NOT "posthog_dashboard"."deleted" AND "posthog_dashboard"."id" = 2) LIMIT 21 ''' @@ -2615,8 +2615,8 @@ ''' SELECT COUNT(*) AS "__count" FROM "posthog_dashboard" - WHERE (NOT ("posthog_dashboard"."deleted") - AND "posthog_dashboard"."team_id" = 2) + WHERE ("posthog_dashboard"."team_id" = 2 + AND NOT "posthog_dashboard"."deleted") ''' # --- # name: TestDashboard.test_listing_dashboards_is_not_nplus1.55 @@ -2738,8 +2738,8 @@ INNER JOIN "posthog_team" ON ("posthog_dashboard"."team_id" = "posthog_team"."id") INNER JOIN "posthog_organization" ON ("posthog_team"."organization_id" = "posthog_organization"."id") LEFT OUTER JOIN "posthog_user" ON ("posthog_dashboard"."created_by_id" = "posthog_user"."id") - WHERE (NOT ("posthog_dashboard"."deleted") - AND "posthog_dashboard"."team_id" = 2) + WHERE ("posthog_dashboard"."team_id" = 2 + AND NOT "posthog_dashboard"."deleted") ORDER BY "posthog_dashboard"."name" ASC LIMIT 300 ''' @@ -2777,7 +2777,7 @@ 2, 3, 4, - 5 /* ... */) /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/ + 5 /* ... */) ''' # --- # name: TestDashboard.test_listing_dashboards_is_not_nplus1.6 @@ -6929,8 +6929,8 @@ INNER JOIN "posthog_team" ON ("posthog_dashboard"."team_id" = "posthog_team"."id") INNER JOIN "posthog_organization" ON ("posthog_team"."organization_id" = "posthog_organization"."id") LEFT OUTER JOIN "posthog_user" ON ("posthog_dashboard"."created_by_id" = "posthog_user"."id") - WHERE (NOT ("posthog_dashboard"."deleted") - AND "posthog_dashboard"."team_id" = 2 + WHERE ("posthog_dashboard"."team_id" = 2 + AND NOT "posthog_dashboard"."deleted" AND "posthog_dashboard"."id" = 2) LIMIT 21 ''' @@ -10687,8 +10687,8 @@ INNER JOIN "posthog_team" ON ("posthog_dashboard"."team_id" = "posthog_team"."id") INNER JOIN "posthog_organization" ON ("posthog_team"."organization_id" = "posthog_organization"."id") LEFT OUTER JOIN "posthog_user" ON ("posthog_dashboard"."created_by_id" = "posthog_user"."id") - WHERE (NOT ("posthog_dashboard"."deleted") - AND "posthog_dashboard"."team_id" = 2 + WHERE ("posthog_dashboard"."team_id" = 2 + AND NOT "posthog_dashboard"."deleted" AND "posthog_dashboard"."id" = 2) LIMIT 21 ''' @@ -11797,8 +11797,8 @@ ''' SELECT COUNT(*) AS "__count" FROM "posthog_dashboard" - WHERE (NOT ("posthog_dashboard"."deleted") - AND "posthog_dashboard"."team_id" = 2) + WHERE ("posthog_dashboard"."team_id" = 2 + AND NOT "posthog_dashboard"."deleted") ''' # --- # name: TestDashboard.test_retrieve_dashboard_list.3 @@ -11931,8 +11931,8 @@ INNER JOIN "posthog_team" ON ("posthog_dashboard"."team_id" = "posthog_team"."id") INNER JOIN "posthog_organization" ON ("posthog_team"."organization_id" = "posthog_organization"."id") LEFT OUTER JOIN "posthog_user" ON ("posthog_dashboard"."created_by_id" = "posthog_user"."id") - WHERE (NOT ("posthog_dashboard"."deleted") - AND "posthog_dashboard"."team_id" = 2) + WHERE ("posthog_dashboard"."team_id" = 2 + AND NOT "posthog_dashboard"."deleted") ORDER BY "posthog_dashboard"."name" ASC LIMIT 100 ''' diff --git a/posthog/api/test/test_capture.py b/posthog/api/test/test_capture.py index a0fc8826c95c6..f771aca99b39d 100644 --- a/posthog/api/test/test_capture.py +++ b/posthog/api/test/test_capture.py @@ -63,7 +63,7 @@ def mocked_get_ingest_context_from_token(_: Any) -> None: openapi_spec = cast(Dict[str, Any], parser.specification) large_data_array = [ - {"key": random.choice(string.ascii_letters) for _ in range(512 * 1024)} + {"key": "".join(random.choice(string.ascii_letters) for _ in range(512 * 1024))} ] # 512 * 1024 is the max size of a single message and random letters shouldn't be compressible, so this should be at least 2 messages android_json = { @@ -188,7 +188,7 @@ def _to_arguments(self, patch_process_event_with_plugins: Any) -> dict: def _send_original_version_session_recording_event( self, number_of_events: int = 1, - event_data: Dict | None = {}, + event_data: Dict | None = None, snapshot_source=3, snapshot_type=1, session_id="abc123", @@ -198,6 +198,8 @@ def _send_original_version_session_recording_event( ) -> dict: if event_data is None: event_data = {} + if event_data is None: + event_data = {} event = { "event": "$snapshot", @@ -1525,8 +1527,8 @@ def test_handle_invalid_snapshot(self): ] ) def test_cors_allows_tracing_headers(self, _: str, path: str, headers: List[str]) -> None: - expected_headers = ",".join(["X-Requested-With", "Content-Type"] + headers) - presented_headers = ",".join(headers + ["someotherrandomheader"]) + expected_headers = ",".join(["X-Requested-With", "Content-Type", *headers]) + presented_headers = ",".join([*headers, "someotherrandomheader"]) response = self.client.options( path, HTTP_ORIGIN="https://localhost", diff --git a/posthog/api/test/test_comments.py b/posthog/api/test/test_comments.py index 42ede7a56587b..6807c924cbbf1 100644 --- a/posthog/api/test/test_comments.py +++ b/posthog/api/test/test_comments.py @@ -7,7 +7,9 @@ class TestComments(APIBaseTest, QueryMatchingTest): - def _create_comment(self, data={}) -> Any: + def _create_comment(self, data=None) -> Any: + if data is None: + data = {} payload = { "content": "my content", "scope": "Notebook", diff --git a/posthog/api/test/test_decide.py b/posthog/api/test/test_decide.py index 05b8f11d78dd6..e89fb0b3c1270 100644 --- a/posthog/api/test/test_decide.py +++ b/posthog/api/test/test_decide.py @@ -73,12 +73,14 @@ def _post_decide( origin="http://127.0.0.1:8000", api_version=1, distinct_id="example_id", - groups={}, + groups=None, geoip_disable=False, ip="127.0.0.1", disable_flags=False, user_agent: Optional[str] = None, ): + if groups is None: + groups = {} return self.client.post( f"/decide/?v={api_version}", { @@ -3336,10 +3338,12 @@ def _post_decide( origin="http://127.0.0.1:8000", api_version=1, distinct_id="example_id", - groups={}, + groups=None, geoip_disable=False, ip="127.0.0.1", ): + if groups is None: + groups = {} return self.client.post( f"/decide/?v={api_version}", { @@ -3571,11 +3575,15 @@ def _post_decide( origin="http://127.0.0.1:8000", api_version=3, distinct_id="example_id", - groups={}, - person_props={}, + groups=None, + person_props=None, geoip_disable=False, ip="127.0.0.1", ): + if person_props is None: + person_props = {} + if groups is None: + groups = {} return self.client.post( f"/decide/?v={api_version}", { diff --git a/posthog/api/test/test_feature_flag.py b/posthog/api/test/test_feature_flag.py index 770883a191490..18236c8332f00 100644 --- a/posthog/api/test/test_feature_flag.py +++ b/posthog/api/test/test_feature_flag.py @@ -83,6 +83,166 @@ def test_cant_create_flag_with_duplicate_key(self): ) self.assertEqual(FeatureFlag.objects.count(), count) + def test_cant_create_flag_with_invalid_filters(self): + count = FeatureFlag.objects.count() + + response = self.client.post( + f"/api/projects/{self.team.id}/feature_flags", + { + "name": "Beta feature", + "key": "beta-x", + "filters": { + "groups": [ + { + "rollout_percentage": 65, + "properties": [ + { + "key": "email", + "type": "person", + "value": ["@posthog.com"], + "operator": "icontains", + } + ], + } + ] + }, + }, + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual( + response.json(), + { + "type": "validation_error", + "code": "invalid_value", + "detail": "Invalid value for operator icontains: ['@posthog.com']", + "attr": "filters", + }, + ) + + response = self.client.post( + f"/api/projects/{self.team.id}/feature_flags", + { + "name": "Beta feature", + "key": "beta-x", + "filters": { + "groups": [ + { + "rollout_percentage": 65, + "properties": [ + { + "key": "email", + "type": "person", + "value": ["@posthog.com"], + "operator": "regex", + } + ], + } + ] + }, + }, + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual( + response.json(), + { + "type": "validation_error", + "code": "invalid_value", + "detail": "Invalid value for operator regex: ['@posthog.com']", + "attr": "filters", + }, + ) + + response = self.client.post( + f"/api/projects/{self.team.id}/feature_flags", + { + "name": "Beta feature", + "key": "beta-x", + "filters": { + "groups": [ + { + "rollout_percentage": 65, + "properties": [ + { + "key": "email", + "type": "person", + "value": ["@posthog.com"], + "operator": "not_icontains", + } + ], + } + ] + }, + }, + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual( + response.json(), + { + "type": "validation_error", + "code": "invalid_value", + "detail": "Invalid value for operator not_icontains: ['@posthog.com']", + "attr": "filters", + }, + ) + + response = self.client.post( + f"/api/projects/{self.team.id}/feature_flags", + { + "name": "Beta feature", + "key": "beta-x", + "filters": { + "groups": [ + { + "rollout_percentage": 65, + "properties": [ + { + "key": "email", + "type": "person", + "value": ["@posthog.com"], + "operator": "not_regex", + } + ], + } + ] + }, + }, + ) + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual( + response.json(), + { + "type": "validation_error", + "code": "invalid_value", + "detail": "Invalid value for operator not_regex: ['@posthog.com']", + "attr": "filters", + }, + ) + self.assertEqual(FeatureFlag.objects.count(), count) + + response = self.client.post( + f"/api/projects/{self.team.id}/feature_flags", + { + "name": "Beta feature", + "key": "beta-x", + "filters": { + "groups": [ + { + "rollout_percentage": 65, + "properties": [ + { + "key": "email", + "type": "person", + "value": '["@posthog.com"]', # fine as long as a string + "operator": "not_regex", + } + ], + } + ] + }, + }, + ) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + def test_cant_update_flag_with_duplicate_key(self): another_feature_flag = FeatureFlag.objects.create( team=self.team, diff --git a/posthog/api/test/test_preflight.py b/posthog/api/test/test_preflight.py index 9d82e512814aa..9d8b59d09a61e 100644 --- a/posthog/api/test/test_preflight.py +++ b/posthog/api/test/test_preflight.py @@ -19,7 +19,9 @@ class TestPreflight(APIBaseTest, QueryMatchingTest): def instance_preferences(self, **kwargs): return {"debug_queries": False, "disable_paid_fs": False, **kwargs} - def preflight_dict(self, options={}): + def preflight_dict(self, options=None): + if options is None: + options = {} return { "django": True, "redis": True, @@ -47,7 +49,9 @@ def preflight_dict(self, options={}): **options, } - def preflight_authenticated_dict(self, options={}): + def preflight_authenticated_dict(self, options=None): + if options is None: + options = {} preflight = { "opt_out_capture": False, "licensed_users_available": None, diff --git a/posthog/api/utils.py b/posthog/api/utils.py index 75373856ccd56..d34530cda14cc 100644 --- a/posthog/api/utils.py +++ b/posthog/api/utils.py @@ -251,8 +251,10 @@ def create_event_definitions_sql( event_type: EventDefinitionType, is_enterprise: bool = False, conditions: str = "", - order_expressions: List[Tuple[str, Literal["ASC", "DESC"]]] = [], + order_expressions: Optional[List[Tuple[str, Literal["ASC", "DESC"]]]] = None, ) -> str: + if order_expressions is None: + order_expressions = [] if is_enterprise: from ee.models import EnterpriseEventDefinition diff --git a/posthog/batch_exports/http.py b/posthog/batch_exports/http.py index e39a01c9cbeec..eaca9da1218a3 100644 --- a/posthog/batch_exports/http.py +++ b/posthog/batch_exports/http.py @@ -95,35 +95,21 @@ class BatchExportRunViewSet(TeamAndOrgViewSetMixin, viewsets.ReadOnlyModelViewSe queryset = BatchExportRun.objects.all() serializer_class = BatchExportRunSerializer pagination_class = RunsCursorPagination + filter_rewrite_rules = {"team_id": "batch_export__team_id"} - def get_queryset(self, date_range: tuple[dt.datetime, dt.datetime] | None = None): - if not isinstance(self.request.user, User) or self.request.user.current_team is None: - raise NotAuthenticated() - - if date_range: - return self.queryset.filter( - batch_export_id=self.kwargs["parent_lookup_batch_export_id"], - created_at__range=date_range, - ).order_by("-created_at") - else: - return self.queryset.filter(batch_export_id=self.kwargs["parent_lookup_batch_export_id"]).order_by( - "-created_at" - ) - - def list(self, request: request.Request, *args, **kwargs) -> response.Response: - """Get all BatchExportRuns for a BatchExport.""" - if not isinstance(request.user, User) or request.user.team is None: - raise NotAuthenticated() + def get_queryset(self): + queryset = super().get_queryset() - after = self.request.query_params.get("after", "-7d") - before = self.request.query_params.get("before", None) - after_datetime = relative_date_parse(after, request.user.team.timezone_info) - before_datetime = relative_date_parse(before, request.user.team.timezone_info) if before else now() + after = self.request.GET.get("after", "-7d") + before = self.request.GET.get("before", None) + after_datetime = relative_date_parse(after, self.team.timezone_info) + before_datetime = relative_date_parse(before, self.team.timezone_info) if before else now() date_range = (after_datetime, before_datetime) - page = self.paginate_queryset(self.get_queryset(date_range=date_range)) - serializer = self.get_serializer(page, many=True) - return self.get_paginated_response(serializer.data) + queryset = queryset.filter(batch_export_id=self.kwargs["parent_lookup_batch_export_id"]) + queryset = queryset.filter(created_at__range=date_range) + + return queryset.order_by("-created_at") class BatchExportDestinationSerializer(serializers.ModelSerializer): @@ -342,9 +328,6 @@ class BatchExportViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet): serializer_class = BatchExportSerializer def get_queryset(self): - if not isinstance(self.request.user, User): - raise NotAuthenticated() - return super().get_queryset().exclude(deleted=True).order_by("-created_at").prefetch_related("destination") @action(methods=["POST"], detail=True) diff --git a/posthog/batch_exports/models.py b/posthog/batch_exports/models.py index db51865560a33..615b087079625 100644 --- a/posthog/batch_exports/models.py +++ b/posthog/batch_exports/models.py @@ -230,9 +230,11 @@ def fetch_batch_export_log_entries( before: dt.datetime | None = None, search: str | None = None, limit: int | None = None, - level_filter: list[BatchExportLogEntryLevel] = [], + level_filter: typing.Optional[list[BatchExportLogEntryLevel]] = None, ) -> list[BatchExportLogEntry]: """Fetch a list of batch export log entries from ClickHouse.""" + if level_filter is None: + level_filter = [] clickhouse_where_parts: list[str] = [] clickhouse_kwargs: dict[str, typing.Any] = {} diff --git a/posthog/batch_exports/service.py b/posthog/batch_exports/service.py index 1accbad9791bc..0661da7a709cd 100644 --- a/posthog/batch_exports/service.py +++ b/posthog/batch_exports/service.py @@ -465,6 +465,21 @@ def update_batch_export_run( return model.get() +def count_failed_batch_export_runs(batch_export_id: UUID, last_n: int) -> int: + """Count failed batch export runs in the 'last_n' runs.""" + count_of_failures = ( + BatchExportRun.objects.filter( + id__in=BatchExportRun.objects.filter(batch_export_id=batch_export_id) + .order_by("-last_updated_at") + .values("id")[:last_n] + ) + .filter(status=BatchExportRun.Status.FAILED) + .count() + ) + + return count_of_failures + + def sync_batch_export(batch_export: BatchExport, created: bool): workflow, workflow_inputs = DESTINATION_WORKFLOWS[batch_export.destination.type] state = ScheduleState( diff --git a/posthog/clickhouse/client/escape.py b/posthog/clickhouse/client/escape.py index 49e7b1047f372..c1a2ae1cf4197 100644 --- a/posthog/clickhouse/client/escape.py +++ b/posthog/clickhouse/client/escape.py @@ -89,6 +89,7 @@ def escape_param_for_clickhouse(param: Any) -> str: version_patch="placeholder server_info value", revision="placeholder server_info value", display_name="placeholder server_info value", + used_revision="placeholder server_info value", timezone="UTC", ) return escape_param(param, context=context) diff --git a/posthog/clickhouse/client/migration_tools.py b/posthog/clickhouse/client/migration_tools.py index 0d105b0423972..f71abd489fd64 100644 --- a/posthog/clickhouse/client/migration_tools.py +++ b/posthog/clickhouse/client/migration_tools.py @@ -5,11 +5,14 @@ from posthog.clickhouse.client.execute import sync_execute -def run_sql_with_exceptions(sql: Union[str, Callable[[], str]], settings={}): +def run_sql_with_exceptions(sql: Union[str, Callable[[], str]], settings=None): """ migrations.RunSQL does not raise exceptions, so we need to wrap it in a function that does. """ + if settings is None: + settings = {} + def run_sql(database): nonlocal sql if callable(sql): diff --git a/posthog/email.py b/posthog/email.py index 99edbddc717ff..61edb7ae593d2 100644 --- a/posthog/email.py +++ b/posthog/email.py @@ -135,10 +135,12 @@ def __init__( campaign_key: str, subject: str, template_name: str, - template_context: Dict = {}, + template_context: Optional[Dict] = None, headers: Optional[Dict] = None, reply_to: Optional[str] = None, ): + if template_context is None: + template_context = {} if not is_email_available(): raise exceptions.ImproperlyConfigured("Email is not enabled in this instance.") diff --git a/posthog/event_usage.py b/posthog/event_usage.py index e1f7f48dcb421..ae8432c6b2731 100644 --- a/posthog/event_usage.py +++ b/posthog/event_usage.py @@ -217,7 +217,9 @@ def report_user_organization_membership_level_changed( ) -def report_user_action(user: User, event: str, properties: Dict = {}, team: Optional[Team] = None): +def report_user_action(user: User, event: str, properties: Optional[Dict] = None, team: Optional[Team] = None): + if properties is None: + properties = {} posthoganalytics.capture( user.distinct_id, event, @@ -252,12 +254,14 @@ def groups(organization: Optional[Organization] = None, team: Optional[Team] = N def report_team_action( team: Team, event: str, - properties: Dict = {}, + properties: Optional[Dict] = None, group_properties: Optional[Dict] = None, ): """ For capturing events where it is unclear which user was the core actor we can use the team instead """ + if properties is None: + properties = {} posthoganalytics.capture(str(team.uuid), event, properties=properties, groups=groups(team=team)) if group_properties: @@ -267,12 +271,14 @@ def report_team_action( def report_organization_action( organization: Organization, event: str, - properties: Dict = {}, + properties: Optional[Dict] = None, group_properties: Optional[Dict] = None, ): """ For capturing events where it is unclear which user was the core actor we can use the organization instead """ + if properties is None: + properties = {} posthoganalytics.capture( str(organization.id), event, diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index d5369dd30d40c..ccb3f9f34576d 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -408,7 +408,7 @@ class PropertyType(Type): joined_subquery_field_name: Optional[str] = field(default=None, init=False) def get_child(self, name: str | int, context: HogQLContext) -> "Type": - return PropertyType(chain=self.chain + [name], field_type=self.field_type) + return PropertyType(chain=[*self.chain, name], field_type=self.field_type) def has_child(self, name: str | int, context: HogQLContext) -> bool: return True diff --git a/posthog/hogql/constants.py b/posthog/hogql/constants.py index 46d3f36a04249..45e362c8f8e72 100644 --- a/posthog/hogql/constants.py +++ b/posthog/hogql/constants.py @@ -25,7 +25,7 @@ KEYWORDS = ["true", "false", "null"] # Keywords you can't alias to -RESERVED_KEYWORDS = KEYWORDS + ["team_id"] +RESERVED_KEYWORDS = [*KEYWORDS, "team_id"] # Limit applied to SELECT statements without LIMIT clause when queried via the API DEFAULT_RETURNED_ROWS = 100 diff --git a/posthog/hogql/database/argmax.py b/posthog/hogql/database/argmax.py index c6e479db07951..5872dc77d8b44 100644 --- a/posthog/hogql/database/argmax.py +++ b/posthog/hogql/database/argmax.py @@ -21,7 +21,7 @@ def argmax_select( fields_to_select.append( ast.Alias( alias=name, - expr=argmax_version(ast.Field(chain=[table_name] + chain)), + expr=argmax_version(ast.Field(chain=[table_name, *chain])), ) ) for key in group_fields: diff --git a/posthog/hogql/database/models.py b/posthog/hogql/database/models.py index 9752fc5f061ff..f6e985d92b4d7 100644 --- a/posthog/hogql/database/models.py +++ b/posthog/hogql/database/models.py @@ -91,7 +91,7 @@ def avoid_asterisk_fields(self) -> List[str]: return [] def get_asterisk(self): - fields_to_avoid = self.avoid_asterisk_fields() + ["team_id"] + fields_to_avoid = [*self.avoid_asterisk_fields(), "team_id"] asterisk: Dict[str, FieldOrTable] = {} for key, field in self.fields.items(): if key in fields_to_avoid: diff --git a/posthog/hogql/database/schema/channel_type.py b/posthog/hogql/database/schema/channel_type.py index 24e4d32bab05b..39c9b31d36918 100644 --- a/posthog/hogql/database/schema/channel_type.py +++ b/posthog/hogql/database/schema/channel_type.py @@ -98,7 +98,7 @@ def wrap_with_null_if_empty(expr: ast.Expr) -> ast.Expr: match({campaign}, '^(.*video.*)$'), 'Paid Video', - 'Paid Other' + 'Paid Unknown' ) ), @@ -125,7 +125,7 @@ def wrap_with_null_if_empty(expr: ast.Expr) -> ast.Expr: match({medium}, 'push$'), 'Push', - 'Other' + 'Unknown' ) ) )""", diff --git a/posthog/hogql/database/schema/cohort_people.py b/posthog/hogql/database/schema/cohort_people.py index f98b522672602..c556903d40cdf 100644 --- a/posthog/hogql/database/schema/cohort_people.py +++ b/posthog/hogql/database/schema/cohort_people.py @@ -40,7 +40,7 @@ def select_from_cohort_people_table(requested_fields: Dict[str, List[str | int]] requested_fields = {**requested_fields, "cohort_id": ["cohort_id"]} fields: List[ast.Expr] = [ - ast.Alias(alias=name, expr=ast.Field(chain=[table_name] + chain)) for name, chain in requested_fields.items() + ast.Alias(alias=name, expr=ast.Field(chain=[table_name, *chain])) for name, chain in requested_fields.items() ] return ast.SelectQuery( diff --git a/posthog/hogql/database/schema/log_entries.py b/posthog/hogql/database/schema/log_entries.py index 9f5dc816ac4b0..14efaff09ce1f 100644 --- a/posthog/hogql/database/schema/log_entries.py +++ b/posthog/hogql/database/schema/log_entries.py @@ -35,7 +35,7 @@ class ReplayConsoleLogsLogEntriesTable(LazyTable): fields: Dict[str, FieldOrTable] = LOG_ENTRIES_FIELDS def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node): - fields: List[ast.Expr] = [ast.Field(chain=["log_entries"] + chain) for name, chain in requested_fields.items()] + fields: List[ast.Expr] = [ast.Field(chain=["log_entries", *chain]) for name, chain in requested_fields.items()] return ast.SelectQuery( select=fields, @@ -58,7 +58,7 @@ class BatchExportLogEntriesTable(LazyTable): fields: Dict[str, FieldOrTable] = LOG_ENTRIES_FIELDS def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node): - fields: List[ast.Expr] = [ast.Field(chain=["log_entries"] + chain) for name, chain in requested_fields.items()] + fields: List[ast.Expr] = [ast.Field(chain=["log_entries", *chain]) for name, chain in requested_fields.items()] return ast.SelectQuery( select=fields, diff --git a/posthog/hogql/database/schema/session_replay_events.py b/posthog/hogql/database/schema/session_replay_events.py index baaecef89e049..a6f0fbed3bcf5 100644 --- a/posthog/hogql/database/schema/session_replay_events.py +++ b/posthog/hogql/database/schema/session_replay_events.py @@ -96,8 +96,8 @@ def select_from_session_replay_events_table(requested_fields: Dict[str, List[str if name in aggregate_fields: select_fields.append(ast.Alias(alias=name, expr=aggregate_fields[name])) else: - select_fields.append(ast.Alias(alias=name, expr=ast.Field(chain=[table_name] + chain))) - group_by_fields.append(ast.Field(chain=[table_name] + chain)) + select_fields.append(ast.Alias(alias=name, expr=ast.Field(chain=[table_name, *chain]))) + group_by_fields.append(ast.Field(chain=[table_name, *chain])) return ast.SelectQuery( select=select_fields, diff --git a/posthog/hogql/database/schema/test/test_channel_type.py b/posthog/hogql/database/schema/test/test_channel_type.py index 97dba3e13ba38..363e262944770 100644 --- a/posthog/hogql/database/schema/test/test_channel_type.py +++ b/posthog/hogql/database/schema/test/test_channel_type.py @@ -234,15 +234,15 @@ def test_organic_video(self): ), ) - def test_no_info_is_other(self): + def test_no_info_is_unknown(self): self.assertEqual( - "Other", + "Unknown", self._get_initial_channel_type({}), ) - def test_unknown_domain_is_other(self): + def test_unknown_domain_is_unknown(self): self.assertEqual( - "Other", + "Unknown", self._get_initial_channel_type( { "$initial_referring_domain": "some-unknown-domain.example.com", @@ -252,7 +252,7 @@ def test_unknown_domain_is_other(self): def test_doesnt_fail_on_numbers(self): self.assertEqual( - "Other", + "Unknown", self._get_initial_channel_type( { "$initial_referring_domain": "example.com", @@ -318,7 +318,7 @@ def test_firefox_google_search_for_shoes(self): def test_daily_mail_ad_click(self): # go to daily mail -> click ad self.assertEqual( - "Paid Other", + "Paid Unknown", self._get_initial_channel_type_from_wild_clicks( "https://www.vivaia.com/item/square-toe-v-cut-flats-p_10003645.html?gid=10011676¤cy=GBP&shipping_country_code=GB&gclid=EAIaIQobChMIxvGy5rr_ggMVYi0GAB0KSAumEAEYASABEgLZ2PD_BwE", "https://2bb5cd7f10ba63d8b55ecfac1a3948db.safeframe.googlesyndication.com/", diff --git a/posthog/hogql/database/schema/util/session_where_clause_extractor.py b/posthog/hogql/database/schema/util/session_where_clause_extractor.py index d1552ffa75f2f..3d94a4a0f691f 100644 --- a/posthog/hogql/database/schema/util/session_where_clause_extractor.py +++ b/posthog/hogql/database/schema/util/session_where_clause_extractor.py @@ -379,6 +379,8 @@ def visit_alias(self, node: ast.Alias) -> bool: table_type = node.type.resolve_table_type(self.context) if not table_type: return False + if isinstance(table_type, ast.TableAliasType): + table_type = table_type.table_type return ( isinstance(table_type, ast.TableType) and isinstance(table_type.table, EventsTable) @@ -409,7 +411,10 @@ def visit_field(self, node: ast.Field) -> ast.Field: if node.type and isinstance(node.type, ast.FieldType): resolved_field = node.type.resolve_database_field(self.context) - table = node.type.resolve_table_type(self.context).table + table_type = node.type.resolve_table_type(self.context) + if isinstance(table_type, ast.TableAliasType): + table_type = table_type.table_type + table = table_type.table if resolved_field and isinstance(resolved_field, DatabaseField): if (isinstance(table, EventsTable) and resolved_field.name == "timestamp") or ( isinstance(table, SessionsTable) and resolved_field.name == "$start_timestamp" diff --git a/posthog/hogql/database/schema/util/test/test_session_where_clause_extractor.py b/posthog/hogql/database/schema/util/test/test_session_where_clause_extractor.py index 3fa9df4e8a815..1e3464c1b9bd6 100644 --- a/posthog/hogql/database/schema/util/test/test_session_where_clause_extractor.py +++ b/posthog/hogql/database/schema/util/test/test_session_where_clause_extractor.py @@ -23,9 +23,8 @@ def f(s: Union[str, ast.Expr, None], placeholders: Optional[dict[str, ast.Expr]] def parse( s: str, placeholders: Optional[Dict[str, ast.Expr]] = None, -) -> ast.SelectQuery: +) -> ast.SelectQuery | ast.SelectUnionQuery: parsed = parse_select(s, placeholders=placeholders) - assert isinstance(parsed, ast.SelectQuery) return parsed @@ -245,6 +244,36 @@ def test_select_query(self): ) assert actual is None + def test_breakdown_subquery(self): + actual = f( + self.inliner.get_inner_where( + parse( + f""" +SELECT + count(DISTINCT e.$session_id) AS total, + toStartOfDay(timestamp) AS day_start, + multiIf(and(greaterOrEquals(session.$session_duration, 2.0), less(session.$session_duration, 4.5)), '[2.0,4.5]', and(greaterOrEquals(session.$session_duration, 4.5), less(session.$session_duration, 27.0)), '[4.5,27.0]', and(greaterOrEquals(session.$session_duration, 27.0), less(session.$session_duration, 44.0)), '[27.0,44.0]', and(greaterOrEquals(session.$session_duration, 44.0), less(session.$session_duration, 48.0)), '[44.0,48.0]', and(greaterOrEquals(session.$session_duration, 48.0), less(session.$session_duration, 57.5)), '[48.0,57.5]', and(greaterOrEquals(session.$session_duration, 57.5), less(session.$session_duration, 61.0)), '[57.5,61.0]', and(greaterOrEquals(session.$session_duration, 61.0), less(session.$session_duration, 74.0)), '[61.0,74.0]', and(greaterOrEquals(session.$session_duration, 74.0), less(session.$session_duration, 90.0)), '[74.0,90.0]', and(greaterOrEquals(session.$session_duration, 90.0), less(session.$session_duration, 98.5)), '[90.0,98.5]', and(greaterOrEquals(session.$session_duration, 98.5), less(session.$session_duration, 167.01)), '[98.5,167.01]', '["",""]') AS breakdown_value + FROM + events AS e SAMPLE 1 + WHERE + and(greaterOrEquals(timestamp, toStartOfDay(assumeNotNull(toDateTime('2024-04-13 00:00:00')))), lessOrEquals(timestamp, assumeNotNull(toDateTime('2024-04-20 23:59:59'))), equals(event, '$pageview'), in(person_id, (SELECT + person_id + FROM + raw_cohort_people + WHERE + and(equals(cohort_id, 2), equals(version, 0))))) + GROUP BY + day_start, + breakdown_value + """ + ) + ) + ) + expected = f( + "((raw_sessions.min_timestamp + toIntervalDay(3)) >= toStartOfDay(assumeNotNull(toDateTime('2024-04-13 00:00:00'))) AND (raw_sessions.min_timestamp - toIntervalDay(3)) <= assumeNotNull(toDateTime('2024-04-20 23:59:59')))" + ) + assert expected == actual + class TestSessionsQueriesHogQLToClickhouse(ClickhouseTestMixin, APIBaseTest): def print_query(self, query: str) -> str: @@ -311,5 +340,120 @@ def test_join_with_events(self): and(equals(events.team_id, {self.team.id}), greater(toTimeZone(events.timestamp, %(hogql_val_2)s), %(hogql_val_3)s)) GROUP BY sessions.session_id +LIMIT 10000""" + assert expected == actual + + def test_union(self): + actual = self.print_query( + """ +SELECT 0 as duration +UNION ALL +SELECT events.session.$session_duration as duration +FROM events +WHERE events.timestamp < today() + """ + ) + expected = f"""SELECT + 0 AS duration +LIMIT 10000 +UNION ALL +SELECT + events__session.`$session_duration` AS duration +FROM + events + LEFT JOIN (SELECT + dateDiff(%(hogql_val_0)s, min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, + sessions.session_id AS session_id + FROM + sessions + WHERE + and(equals(sessions.team_id, {self.team.id}), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, %(hogql_val_1)s), toIntervalDay(3)), today()), 0)) + GROUP BY + sessions.session_id, + sessions.session_id) AS events__session ON equals(events.`$session_id`, events__session.session_id) +WHERE + and(equals(events.team_id, {self.team.id}), less(toTimeZone(events.timestamp, %(hogql_val_2)s), today())) +LIMIT 10000""" + assert expected == actual + + def test_session_breakdown(self): + actual = self.print_query( + """SELECT count(DISTINCT e."$session_id") AS total, + toStartOfDay(timestamp) AS day_start, + multiIf(and(greaterOrEquals(session."$session_duration", 2.0), + less(session."$session_duration", 4.5)), + '[2.0,4.5]', + and(greaterOrEquals(session."$session_duration", 4.5), + less(session."$session_duration", 27.0)), + '[4.5,27.0]', + and(greaterOrEquals(session."$session_duration", 27.0), + less(session."$session_duration", 44.0)), + '[27.0,44.0]', + and(greaterOrEquals(session."$session_duration", 44.0), + less(session."$session_duration", 48.0)), + '[44.0,48.0]', + and(greaterOrEquals(session."$session_duration", 48.0), + less(session."$session_duration", 57.5)), + '[48.0,57.5]', + and(greaterOrEquals(session."$session_duration", 57.5), + less(session."$session_duration", 61.0)), + '[57.5,61.0]', + and(greaterOrEquals(session."$session_duration", 61.0), + less(session."$session_duration", 74.0)), + '[61.0,74.0]', + and(greaterOrEquals(session."$session_duration", 74.0), + less(session."$session_duration", 90.0)), + '[74.0,90.0]', + and(greaterOrEquals(session."$session_duration", 90.0), + less(session."$session_duration", 98.5)), + '[90.0,98.5]', and(greaterOrEquals(session."$session_duration", 98.5), + less(session."$session_duration", 167.01)), '[98.5,167.01]', + '["",""]') AS breakdown_value +FROM events AS e SAMPLE 1 +WHERE and(greaterOrEquals(timestamp, toStartOfDay(assumeNotNull(toDateTime('2024-04-13 00:00:00')))), + lessOrEquals(timestamp, assumeNotNull(toDateTime('2024-04-20 23:59:59'))), + equals(event, '$pageview'), in(person_id, (SELECT person_id + FROM raw_cohort_people + WHERE and(equals(cohort_id, 2), equals(version, 0))))) +GROUP BY day_start, + breakdown_value""" + ) + expected = f"""SELECT + count(DISTINCT e.`$session_id`) AS total, + toStartOfDay(toTimeZone(e.timestamp, %(hogql_val_7)s)) AS day_start, + multiIf(and(ifNull(greaterOrEquals(e__session.`$session_duration`, 2.0), 0), ifNull(less(e__session.`$session_duration`, 4.5), 0)), %(hogql_val_8)s, and(ifNull(greaterOrEquals(e__session.`$session_duration`, 4.5), 0), ifNull(less(e__session.`$session_duration`, 27.0), 0)), %(hogql_val_9)s, and(ifNull(greaterOrEquals(e__session.`$session_duration`, 27.0), 0), ifNull(less(e__session.`$session_duration`, 44.0), 0)), %(hogql_val_10)s, and(ifNull(greaterOrEquals(e__session.`$session_duration`, 44.0), 0), ifNull(less(e__session.`$session_duration`, 48.0), 0)), %(hogql_val_11)s, and(ifNull(greaterOrEquals(e__session.`$session_duration`, 48.0), 0), ifNull(less(e__session.`$session_duration`, 57.5), 0)), %(hogql_val_12)s, and(ifNull(greaterOrEquals(e__session.`$session_duration`, 57.5), 0), ifNull(less(e__session.`$session_duration`, 61.0), 0)), %(hogql_val_13)s, and(ifNull(greaterOrEquals(e__session.`$session_duration`, 61.0), 0), ifNull(less(e__session.`$session_duration`, 74.0), 0)), %(hogql_val_14)s, and(ifNull(greaterOrEquals(e__session.`$session_duration`, 74.0), 0), ifNull(less(e__session.`$session_duration`, 90.0), 0)), %(hogql_val_15)s, and(ifNull(greaterOrEquals(e__session.`$session_duration`, 90.0), 0), ifNull(less(e__session.`$session_duration`, 98.5), 0)), %(hogql_val_16)s, and(ifNull(greaterOrEquals(e__session.`$session_duration`, 98.5), 0), ifNull(less(e__session.`$session_duration`, 167.01), 0)), %(hogql_val_17)s, %(hogql_val_18)s) AS breakdown_value +FROM + events AS e SAMPLE 1 + INNER JOIN (SELECT + argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, + person_distinct_id2.distinct_id AS distinct_id + FROM + person_distinct_id2 + WHERE + equals(person_distinct_id2.team_id, {self.team.id}) + GROUP BY + person_distinct_id2.distinct_id + HAVING + ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) + LEFT JOIN (SELECT + dateDiff(%(hogql_val_0)s, min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, + sessions.session_id AS session_id + FROM + sessions + WHERE + and(equals(sessions.team_id, {self.team.id}), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, %(hogql_val_1)s), toIntervalDay(3)), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull(%(hogql_val_2)s, 6, %(hogql_val_3)s)))), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, %(hogql_val_4)s), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull(%(hogql_val_5)s, 6, %(hogql_val_6)s))), 0)) + GROUP BY + sessions.session_id, + sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) +WHERE + and(equals(e.team_id, {self.team.id}), and(greaterOrEquals(toTimeZone(e.timestamp, %(hogql_val_19)s), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull(%(hogql_val_20)s, 6, %(hogql_val_21)s)))), lessOrEquals(toTimeZone(e.timestamp, %(hogql_val_22)s), assumeNotNull(parseDateTime64BestEffortOrNull(%(hogql_val_23)s, 6, %(hogql_val_24)s))), equals(e.event, %(hogql_val_25)s), ifNull(in(e__pdi.person_id, (SELECT + cohortpeople.person_id AS person_id + FROM + cohortpeople + WHERE + and(equals(cohortpeople.team_id, {self.team.id}), and(equals(cohortpeople.cohort_id, 2), equals(cohortpeople.version, 0))))), 0))) +GROUP BY + day_start, + breakdown_value LIMIT 10000""" assert expected == actual diff --git a/posthog/hogql/filters.py b/posthog/hogql/filters.py index dcec66efad6e0..496cadf8da417 100644 --- a/posthog/hogql/filters.py +++ b/posthog/hogql/filters.py @@ -4,7 +4,6 @@ from posthog.hogql import ast from posthog.hogql.errors import QueryError -from posthog.hogql.parser import parse_expr from posthog.hogql.property import property_to_expr from posthog.hogql.visitor import CloningVisitor from posthog.models import Team @@ -59,14 +58,14 @@ def visit_placeholder(self, node): dateTo = self.filters.dateRange.date_to if self.filters.dateRange else None if dateTo is not None: try: - parsed_date = isoparse(dateTo) + parsed_date = isoparse(dateTo).replace(tzinfo=self.team.timezone_info) except ValueError: parsed_date = relative_date_parse(dateTo, self.team.timezone_info) exprs.append( - parse_expr( - "timestamp < {timestamp}", - {"timestamp": ast.Constant(value=parsed_date)}, - start=None, # do not add location information for "timestamp" to the metadata + ast.CompareOperation( + op=ast.CompareOperationOp.Lt, + left=ast.Field(chain=["timestamp"]), + right=ast.Constant(value=parsed_date), ) ) @@ -74,14 +73,14 @@ def visit_placeholder(self, node): dateFrom = self.filters.dateRange.date_from if self.filters.dateRange else None if dateFrom is not None and dateFrom != "all": try: - parsed_date = isoparse(dateFrom) + parsed_date = isoparse(dateFrom).replace(tzinfo=self.team.timezone_info) except ValueError: parsed_date = relative_date_parse(dateFrom, self.team.timezone_info) exprs.append( - parse_expr( - "timestamp >= {timestamp}", - {"timestamp": ast.Constant(value=parsed_date)}, - start=None, # do not add location information for "timestamp" to the metadata + ast.CompareOperation( + op=ast.CompareOperationOp.GtEq, + left=ast.Field(chain=["timestamp"]), + right=ast.Constant(value=parsed_date), ) ) diff --git a/posthog/hogql/functions/action.py b/posthog/hogql/functions/action.py new file mode 100644 index 0000000000000..02888081632f3 --- /dev/null +++ b/posthog/hogql/functions/action.py @@ -0,0 +1,45 @@ +from typing import List + +from posthog.hogql import ast +from posthog.hogql.context import HogQLContext +from posthog.hogql.errors import QueryError +from posthog.hogql.escape_sql import escape_clickhouse_string + + +def matches_action(node: ast.Expr, args: List[ast.Expr], context: HogQLContext) -> ast.Expr: + arg = args[0] + if not isinstance(arg, ast.Constant): + raise QueryError("action() takes only constant arguments", node=arg) + if context.team_id is None: + raise QueryError("action() can only be used in a query with a team_id", node=arg) + + from posthog.models import Action + from posthog.hogql.property import action_to_expr + + if (isinstance(arg.value, int) or isinstance(arg.value, float)) and not isinstance(arg.value, bool): + actions = Action.objects.filter(id=int(arg.value), team_id=context.team_id).all() + if len(actions) == 1: + context.add_notice( + start=arg.start, + end=arg.end, + message=f"Action #{actions[0].pk} can also be specified as {escape_clickhouse_string(actions[0].name)}", + fix=escape_clickhouse_string(actions[0].name), + ) + return action_to_expr(actions[0]) + raise QueryError(f"Could not find cohort with ID {arg.value}", node=arg) + + if isinstance(arg.value, str): + actions = Action.objects.filter(name=arg.value, team_id=context.team_id).all() + if len(actions) == 1: + context.add_notice( + start=arg.start, + end=arg.end, + message=f"Searching for action by name. Replace with numeric ID {actions[0].pk} to protect against renaming.", + fix=str(actions[0].pk), + ) + return action_to_expr(actions[0]) + elif len(actions) > 1: + raise QueryError(f"Found multiple actions with name '{arg.value}'", node=arg) + raise QueryError(f"Could not find an action with the name '{arg.value}'", node=arg) + + raise QueryError("action() takes exactly one string or integer argument", node=arg) diff --git a/posthog/hogql/functions/mapping.py b/posthog/hogql/functions/mapping.py index 9aff4371135ae..652e1711ff0bb 100644 --- a/posthog/hogql/functions/mapping.py +++ b/posthog/hogql/functions/mapping.py @@ -748,6 +748,7 @@ class HogQLFunctionMeta: "maxIntersectionsPositionIf": HogQLFunctionMeta("maxIntersectionsPositionIf", 3, 3, aggregate=True), } HOGQL_POSTHOG_FUNCTIONS: Dict[str, HogQLFunctionMeta] = { + "matchesAction": HogQLFunctionMeta("matchesAction", 1, 1), "sparkline": HogQLFunctionMeta("sparkline", 1, 1), "hogql_lookupDomainType": HogQLFunctionMeta("hogql_lookupDomainType", 1, 1), "hogql_lookupPaidDomainType": HogQLFunctionMeta("hogql_lookupPaidDomainType", 1, 1), diff --git a/posthog/hogql/functions/test/__snapshots__/test_action.ambr b/posthog/hogql/functions/test/__snapshots__/test_action.ambr new file mode 100644 index 0000000000000..97cd09fe4c9de --- /dev/null +++ b/posthog/hogql/functions/test/__snapshots__/test_action.ambr @@ -0,0 +1,37 @@ +# serializer version: 1 +# name: TestAction.test_matches_action_id + ''' + -- ClickHouse + + SELECT events.event AS event + FROM events + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_0)s)) + LIMIT 100 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event + FROM events + WHERE equals(event, 'RANDOM_TEST_ID::UUID') + LIMIT 100 + ''' +# --- +# name: TestAction.test_matches_action_name + ''' + -- ClickHouse + + SELECT events.event AS event + FROM events + WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_0)s)) + LIMIT 100 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1 + + -- HogQL + + SELECT event + FROM events + WHERE equals(event, 'RANDOM_TEST_ID::UUID') + LIMIT 100 + ''' +# --- diff --git a/posthog/hogql/functions/test/test_action.py b/posthog/hogql/functions/test/test_action.py new file mode 100644 index 0000000000000..a25ec57c21c4b --- /dev/null +++ b/posthog/hogql/functions/test/test_action.py @@ -0,0 +1,56 @@ +from posthog.hogql.query import execute_hogql_query +from posthog.models import Action, ActionStep +from posthog.models.utils import UUIDT +from posthog.test.base import ( + BaseTest, + _create_person, + _create_event, + flush_persons_and_events, +) + + +def _create_action(**kwargs): + team = kwargs.pop("team") + name = kwargs.pop("name") + action = Action.objects.create(team=team, name=name) + ActionStep.objects.create(action=action, event=name) + return action + + +class TestAction(BaseTest): + maxDiff = None + + def _create_random_events(self) -> str: + random_uuid = f"RANDOM_TEST_ID::{UUIDT()}" + _create_person( + properties={"$os": "Chrome", "random_uuid": random_uuid}, + team=self.team, + distinct_ids=["bla"], + is_identified=True, + ) + _create_event(distinct_id="bla", event=random_uuid, team=self.team) + _create_event(distinct_id="bla", event=random_uuid + "::extra", team=self.team) + flush_persons_and_events() + return random_uuid + + def test_matches_action_name(self): + random_uuid = self._create_random_events() + _create_action(team=self.team, name=random_uuid) + response = execute_hogql_query( + f"SELECT event FROM events WHERE matchesAction('{random_uuid}')", + self.team, + ) + assert response.results is not None + assert len(response.results) == 1 + assert response.results[0][0] == random_uuid + + def test_matches_action_id(self): + random_uuid = self._create_random_events() + action = _create_action(team=self.team, name=random_uuid) + response = execute_hogql_query( + f"SELECT event FROM events WHERE matchesAction({action.pk})", + self.team, + ) + assert response.results is not None + assert len(response.results) == 1 + assert response.results[0][0] == random_uuid diff --git a/posthog/hogql/parser.py b/posthog/hogql/parser.py index f374d70c8cfcf..0ec619f338909 100644 --- a/posthog/hogql/parser.py +++ b/posthog/hogql/parser.py @@ -752,7 +752,7 @@ def visitColumnExprFunction(self, ctx: HogQLParser.ColumnExprFunctionContext): def visitColumnExprAsterisk(self, ctx: HogQLParser.ColumnExprAsteriskContext): if ctx.tableIdentifier(): table = self.visit(ctx.tableIdentifier()) - return ast.Field(chain=table + ["*"]) + return ast.Field(chain=[*table, "*"]) return ast.Field(chain=["*"]) def visitColumnExprTagElement(self, ctx: HogQLParser.ColumnExprTagElementContext): diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 3f5be7cc42b83..ff4766f86074a 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -235,7 +235,7 @@ def visit_select_query(self, node: ast.SelectQuery): if where is None: where = extra_where elif isinstance(where, ast.And): - where = ast.And(exprs=[extra_where] + where.exprs) + where = ast.And(exprs=[extra_where, *where.exprs]) else: where = ast.And(exprs=[extra_where, where]) else: @@ -1169,7 +1169,7 @@ def _print_escaped_string(self, name: float | int | str | list | tuple | datetim return escape_hogql_string(name, timezone=self._get_timezone()) def _unsafe_json_extract_trim_quotes(self, unsafe_field: str, unsafe_args: List[str]) -> str: - return f"replaceRegexpAll(nullIf(nullIf(JSONExtractRaw({', '.join([unsafe_field] + unsafe_args)}), ''), 'null'), '^\"|\"$', '')" + return f"replaceRegexpAll(nullIf(nullIf(JSONExtractRaw({', '.join([unsafe_field, *unsafe_args])}), ''), 'null'), '^\"|\"$', '')" def _get_materialized_column( self, table_name: str, property_name: PropertyName, field_name: TableColumn diff --git a/posthog/hogql/property.py b/posthog/hogql/property.py index 821a8db5a23fd..501bc613bd539 100644 --- a/posthog/hogql/property.py +++ b/posthog/hogql/property.py @@ -163,7 +163,7 @@ def property_to_expr( chain = ["properties"] properties_field = ast.Field(chain=chain) - field = ast.Field(chain=chain + [property.key]) + field = ast.Field(chain=[*chain, property.key]) if isinstance(value, list): if len(value) == 0: diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index 2c578c85e4913..fce251dc8a08d 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -13,6 +13,7 @@ SavedQuery, ) from posthog.hogql.errors import ImpossibleASTError, QueryError, ResolutionError +from posthog.hogql.functions.action import matches_action from posthog.hogql.functions.cohort import cohort_query_node from posthog.hogql.functions.mapping import validate_function_args from posthog.hogql.functions.sparkline import sparkline @@ -388,6 +389,8 @@ def visit_call(self, node: ast.Call): validate_function_args(node.args, func_meta.min_args, func_meta.max_args, node.name) if node.name == "sparkline": return self.visit(sparkline(node=node, args=node.args)) + if node.name == "matchesAction": + return self.visit(matches_action(node=node, args=node.args, context=self.context)) node = super().visit_call(node) arg_types: List[ast.ConstantType] = [] @@ -461,7 +464,7 @@ def visit_field(self, node: ast.Field): if table_count > 1: raise QueryError("Cannot use '*' without table name when there are multiple tables in the query") table_type = ( - scope.anonymous_tables[0] if len(scope.anonymous_tables) > 0 else list(scope.tables.values())[0] + scope.anonymous_tables[0] if len(scope.anonymous_tables) > 0 else next(iter(scope.tables.values())) ) type = ast.AsteriskType(table_type=table_type) diff --git a/posthog/hogql/test/test_filters.py b/posthog/hogql/test/test_filters.py index 951d5814f213a..5aba11a3b28c6 100644 --- a/posthog/hogql/test/test_filters.py +++ b/posthog/hogql/test/test_filters.py @@ -1,4 +1,4 @@ -from typing import Dict, Any +from typing import Dict, Any, Optional from posthog.hogql import ast from posthog.hogql.context import HogQLContext @@ -18,10 +18,10 @@ class TestFilters(BaseTest): maxDiff = None - def _parse_expr(self, expr: str, placeholders: Dict[str, Any] = None): + def _parse_expr(self, expr: str, placeholders: Optional[Dict[str, Any]] = None): return clear_locations(parse_expr(expr, placeholders=placeholders)) - def _parse_select(self, select: str, placeholders: Dict[str, Any] = None): + def _parse_select(self, select: str, placeholders: Optional[Dict[str, Any]] = None): return clear_locations(parse_select(select, placeholders=placeholders)) def _print_ast(self, node: ast.Expr): @@ -63,6 +63,34 @@ def test_replace_filters_date_range(self): "SELECT event FROM events WHERE less(timestamp, toDateTime('2020-02-02 00:00:00.000000')) LIMIT 10000", ) + select = replace_filters( + self._parse_select("SELECT event FROM events where {filters}"), + HogQLFilters(dateRange=DateRange(date_from="2020-02-02", date_to="2020-02-03 23:59:59")), + self.team, + ) + self.assertEqual( + self._print_ast(select), + "SELECT event FROM events WHERE " + "and(less(timestamp, toDateTime('2020-02-03 23:59:59.000000')), " + "greaterOrEquals(timestamp, toDateTime('2020-02-02 00:00:00.000000'))) LIMIT 10000", + ) + + # now with different team timezone + self.team.timezone = "America/New_York" + self.team.save() + + select = replace_filters( + self._parse_select("SELECT event FROM events where {filters}"), + HogQLFilters(dateRange=DateRange(date_from="2020-02-02", date_to="2020-02-03 23:59:59")), + self.team, + ) + self.assertEqual( + self._print_ast(select), + "SELECT event FROM events WHERE " + "and(less(timestamp, toDateTime('2020-02-03 23:59:59.000000')), " + "greaterOrEquals(timestamp, toDateTime('2020-02-02 00:00:00.000000'))) LIMIT 10000", + ) + def test_replace_filters_event_property(self): select = replace_filters( self._parse_select("SELECT event FROM events where {filters}"), diff --git a/posthog/hogql/test/test_property.py b/posthog/hogql/test/test_property.py index 44cbf6a5b09bc..44b740552d8f0 100644 --- a/posthog/hogql/test/test_property.py +++ b/posthog/hogql/test/test_property.py @@ -46,7 +46,7 @@ def _property_to_expr( def _selector_to_expr(self, selector: str): return clear_locations(selector_to_expr(selector)) - def _parse_expr(self, expr: str, placeholders: Dict[str, Any] = None): + def _parse_expr(self, expr: str, placeholders: Optional[Dict[str, Any]] = None): return clear_locations(parse_expr(expr, placeholders=placeholders)) def test_has_aggregation(self): diff --git a/posthog/hogql_queries/insights/funnels/base.py b/posthog/hogql_queries/insights/funnels/base.py index 82b0161b8c833..1dade0de4b052 100644 --- a/posthog/hogql_queries/insights/funnels/base.py +++ b/posthog/hogql_queries/insights/funnels/base.py @@ -729,7 +729,7 @@ def _get_matching_events(self, max_steps: int) -> List[ast.Expr]: ): events = [] for i in range(0, max_steps): - event_fields = ["latest"] + self.extra_event_fields_and_properties + event_fields = ["latest", *self.extra_event_fields_and_properties] event_fields_with_step = ", ".join([f"{field}_{i}" for field in event_fields]) event_clause = f"({event_fields_with_step}) as step_{i}_matching_event" events.append(parse_expr(event_clause)) diff --git a/posthog/hogql_queries/insights/funnels/funnel_correlation_query_runner.py b/posthog/hogql_queries/insights/funnels/funnel_correlation_query_runner.py index 72dcf1993e1f3..04b1115fd38d2 100644 --- a/posthog/hogql_queries/insights/funnels/funnel_correlation_query_runner.py +++ b/posthog/hogql_queries/insights/funnels/funnel_correlation_query_runner.py @@ -245,9 +245,9 @@ def _calculate(self) -> tuple[List[EventOddsRatio], bool, str, HogQLQueryRespons # Get the total success/failure counts from the results results = [result for result in response.results if result[0] != self.TOTAL_IDENTIFIER] - _, success_total, failure_total = [result for result in response.results if result[0] == self.TOTAL_IDENTIFIER][ - 0 - ] + _, success_total, failure_total = next( + result for result in response.results if result[0] == self.TOTAL_IDENTIFIER + ) # Add a little structure, and keep it close to the query definition so it's # obvious what's going on with result indices. diff --git a/posthog/hogql_queries/insights/funnels/funnel_event_query.py b/posthog/hogql_queries/insights/funnels/funnel_event_query.py index f2d0e115e2d0b..b2fd19083ed75 100644 --- a/posthog/hogql_queries/insights/funnels/funnel_event_query.py +++ b/posthog/hogql_queries/insights/funnels/funnel_event_query.py @@ -1,4 +1,4 @@ -from typing import List, Set, Union +from typing import List, Set, Union, Optional from posthog.clickhouse.materialized_columns.column import ColumnName from posthog.hogql import ast from posthog.hogql.parser import parse_expr @@ -21,9 +21,13 @@ class FunnelEventQuery: def __init__( self, context: FunnelQueryContext, - extra_fields: List[ColumnName] = [], - extra_event_properties: List[PropertyName] = [], + extra_fields: Optional[List[ColumnName]] = None, + extra_event_properties: Optional[List[PropertyName]] = None, ): + if extra_event_properties is None: + extra_event_properties = [] + if extra_fields is None: + extra_fields = [] self.context = context self._extra_fields = extra_fields diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_breakdowns_by_current_url.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_breakdowns_by_current_url.py index b745ea87761eb..859f3e627aab7 100644 --- a/posthog/hogql_queries/insights/funnels/test/test_funnel_breakdowns_by_current_url.py +++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_breakdowns_by_current_url.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Dict, cast +from typing import Dict, cast, Optional from posthog.hogql_queries.insights.funnels.funnels_query_runner import FunnelsQueryRunner from posthog.hogql_queries.legacy_compatibility.filter_to_query import filter_to_query @@ -116,7 +116,11 @@ def setUp(self): journeys_for(journey, team=self.team, create_people=True) - def _run(self, extra: Dict = {}, events_extra: Dict = {}): + def _run(self, extra: Optional[Dict] = None, events_extra: Optional[Dict] = None): + if events_extra is None: + events_extra = {} + if extra is None: + extra = {} filters = { "events": [ { diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_trends_persons.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_trends_persons.py index 54a8b4cf063ea..9aac61f1d0564 100644 --- a/posthog/hogql_queries/insights/funnels/test/test_funnel_trends_persons.py +++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_trends_persons.py @@ -74,7 +74,7 @@ def test_funnel_trend_persons_returns_recordings(self): self.assertEqual(results[0][0], persons["user_one"].uuid) self.assertEqual( # [person["matched_recordings"][0]["session_id"] for person in results], - [list(results[0][2])[0]["session_id"]], + [next(iter(results[0][2]))["session_id"]], ["s1b"], ) @@ -124,7 +124,7 @@ def test_funnel_trend_persons_with_no_to_step(self): self.assertEqual(results[0][0], persons["user_one"].uuid) self.assertEqual( # [person["matched_recordings"][0]["session_id"] for person in results], - [list(results[0][2])[0]["session_id"]], + [next(iter(results[0][2]))["session_id"]], ["s1c"], ) @@ -163,6 +163,6 @@ def test_funnel_trend_persons_with_drop_off(self): self.assertEqual(results[0][0], persons["user_one"].uuid) self.assertEqual( # [person["matched_recordings"][0].get("session_id") for person in results], - [list(results[0][2])[0]["session_id"]], + [next(iter(results[0][2]))["session_id"]], ["s1a"], ) diff --git a/posthog/hogql_queries/insights/test/test_insight_actors_query_runner.py b/posthog/hogql_queries/insights/test/test_insight_actors_query_runner.py index 1dad592a2449e..bb963cf1f8b62 100644 --- a/posthog/hogql_queries/insights/test/test_insight_actors_query_runner.py +++ b/posthog/hogql_queries/insights/test/test_insight_actors_query_runner.py @@ -1,4 +1,4 @@ -from typing import Dict, Any +from typing import Dict, Any, Optional from freezegun import freeze_time @@ -69,7 +69,9 @@ def _create_test_events(self): ] ) - def select(self, query: str, placeholders: Dict[str, Any] = {}): + def select(self, query: str, placeholders: Optional[Dict[str, Any]] = None): + if placeholders is None: + placeholders = {} return execute_hogql_query( query=query, team=self.team, diff --git a/posthog/hogql_queries/insights/trends/breakdown_values.py b/posthog/hogql_queries/insights/trends/breakdown_values.py index fb349f279d19a..6a9b9a24a22f0 100644 --- a/posthog/hogql_queries/insights/trends/breakdown_values.py +++ b/posthog/hogql_queries/insights/trends/breakdown_values.py @@ -228,7 +228,7 @@ def get_breakdown_values(self) -> List[str | int]: if self.hide_other_aggregation is not True and self.histogram_bin_count is None: values = [BREAKDOWN_NULL_STRING_LABEL if value in (None, "") else value for value in values] if needs_other: - values = [BREAKDOWN_OTHER_STRING_LABEL] + values + values = [BREAKDOWN_OTHER_STRING_LABEL, *values] if len(values) == 0: values.insert(0, None) diff --git a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr index 5885c57710928..7902fbb4b5674 100644 --- a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr +++ b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr @@ -3819,7 +3819,7 @@ (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, sessions.session_id AS session_id FROM sessions - WHERE equals(sessions.team_id, 2) + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) GROUP BY sessions.session_id, sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'))) @@ -3842,7 +3842,7 @@ (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, sessions.session_id AS session_id FROM sessions - WHERE equals(sessions.team_id, 2) + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) GROUP BY sessions.session_id, sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) @@ -3863,7 +3863,7 @@ (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, sessions.session_id AS session_id FROM sessions - WHERE equals(sessions.team_id, 2) + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) GROUP BY sessions.session_id, sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'))) @@ -3886,7 +3886,7 @@ (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, sessions.session_id AS session_id FROM sessions - WHERE equals(sessions.team_id, 2) + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) GROUP BY sessions.session_id, sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) @@ -4275,7 +4275,7 @@ (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, sessions.session_id AS session_id FROM sessions - WHERE equals(sessions.team_id, 2) + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) GROUP BY sessions.session_id, sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) INNER JOIN @@ -4316,7 +4316,7 @@ (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, sessions.session_id AS session_id FROM sessions - WHERE equals(sessions.team_id, 2) + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) GROUP BY sessions.session_id, sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) INNER JOIN @@ -4382,7 +4382,7 @@ (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, sessions.session_id AS session_id FROM sessions - WHERE equals(sessions.team_id, 2) + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) GROUP BY sessions.session_id, sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')) @@ -4402,7 +4402,7 @@ (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, sessions.session_id AS session_id FROM sessions - WHERE equals(sessions.team_id, 2) + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) GROUP BY sessions.session_id, sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')) @@ -4435,7 +4435,7 @@ (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, sessions.session_id AS session_id FROM sessions - WHERE equals(sessions.team_id, 2) + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) GROUP BY sessions.session_id, sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')) @@ -4474,7 +4474,7 @@ (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, sessions.session_id AS session_id FROM sessions - WHERE equals(sessions.team_id, 2) + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) GROUP BY sessions.session_id, sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')) @@ -4499,7 +4499,7 @@ (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, sessions.session_id AS session_id FROM sessions - WHERE equals(sessions.team_id, 2) + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) GROUP BY sessions.session_id, sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'))) @@ -4545,7 +4545,7 @@ (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, sessions.session_id AS session_id FROM sessions - WHERE equals(sessions.team_id, 2) + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) GROUP BY sessions.session_id, sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) @@ -4575,7 +4575,7 @@ (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, sessions.session_id AS session_id FROM sessions - WHERE equals(sessions.team_id, 2) + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) GROUP BY sessions.session_id, sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'))) @@ -4621,7 +4621,7 @@ (SELECT dateDiff('second', min(sessions.min_timestamp), max(sessions.max_timestamp)) AS `$session_duration`, sessions.session_id AS session_id FROM sessions - WHERE equals(sessions.team_id, 2) + WHERE and(equals(sessions.team_id, 2), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(minus(toTimeZone(sessions.min_timestamp, 'UTC'), toIntervalDay(3)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0)) GROUP BY sessions.session_id, sessions.session_id) AS e__session ON equals(e.`$session_id`, e__session.session_id) WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), true) diff --git a/posthog/hogql_queries/legacy_compatibility/clean_properties.py b/posthog/hogql_queries/legacy_compatibility/clean_properties.py index a6e8e8663bb44..e77cf1ee1f944 100644 --- a/posthog/hogql_queries/legacy_compatibility/clean_properties.py +++ b/posthog/hogql_queries/legacy_compatibility/clean_properties.py @@ -121,8 +121,8 @@ def is_old_style_properties(properties): def transform_old_style_properties(properties): - key = list(properties.keys())[0] - value = list(properties.values())[0] + key = next(iter(properties.keys())) + value = next(iter(properties.values())) key_split = key.split("__") return [ { diff --git a/posthog/hogql_queries/legacy_compatibility/filter_to_query.py b/posthog/hogql_queries/legacy_compatibility/filter_to_query.py index a16991a2f1ab3..382b37fa56db0 100644 --- a/posthog/hogql_queries/legacy_compatibility/filter_to_query.py +++ b/posthog/hogql_queries/legacy_compatibility/filter_to_query.py @@ -381,7 +381,7 @@ def _insight_filter(filter: Dict): else: raise Exception(f"Invalid insight type {filter.get('insight')}.") - if len(list(insight_filter.values())[0].model_dump(exclude_defaults=True)) == 0: + if len(next(iter(insight_filter.values())).model_dump(exclude_defaults=True)) == 0: return {} return insight_filter diff --git a/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py b/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py index 12ef703271c51..ffb758858d151 100644 --- a/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py +++ b/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py @@ -55,21 +55,19 @@ def property_filters_without_pathname(self) -> List[Union[EventPropertyFilter, P return [p for p in self.query.properties if p.key != "$pathname"] def session_where(self, include_previous_period: Optional[bool] = None): - properties = ( - [ - parse_expr( - "events.timestamp < {date_to} AND events.timestamp >= minus({date_from}, toIntervalHour(1))", - placeholders={ - "date_from": self.query_date_range.previous_period_date_from_as_hogql() - if include_previous_period - else self.query_date_range.date_from_as_hogql(), - "date_to": self.query_date_range.date_to_as_hogql(), - }, - ) - ] - + self.property_filters_without_pathname - + self._test_account_filters - ) + properties = [ + parse_expr( + "events.timestamp < {date_to} AND events.timestamp >= minus({date_from}, toIntervalHour(1))", + placeholders={ + "date_from": self.query_date_range.previous_period_date_from_as_hogql() + if include_previous_period + else self.query_date_range.date_from_as_hogql(), + "date_to": self.query_date_range.date_to_as_hogql(), + }, + ), + *self.property_filters_without_pathname, + *self._test_account_filters, + ] return property_to_expr( properties, self.team, diff --git a/posthog/management/commands/backfill_persons_and_groups_on_events.py b/posthog/management/commands/backfill_persons_and_groups_on_events.py index b7fb2fcbc46e9..0e90461a701d5 100644 --- a/posthog/management/commands/backfill_persons_and_groups_on_events.py +++ b/posthog/management/commands/backfill_persons_and_groups_on_events.py @@ -120,7 +120,9 @@ query_number = 0 -def print_and_execute_query(sql: str, name: str, dry_run: bool, timeout=180, query_args={}) -> Any: +def print_and_execute_query(sql: str, name: str, dry_run: bool, timeout=180, query_args=None) -> Any: + if query_args is None: + query_args = {} global query_number if not settings.TEST: diff --git a/posthog/management/commands/create_channel_definitions_file.py b/posthog/management/commands/create_channel_definitions_file.py index 5ff198a7334d3..859bbe3c631ce 100644 --- a/posthog/management/commands/create_channel_definitions_file.py +++ b/posthog/management/commands/create_channel_definitions_file.py @@ -62,8 +62,8 @@ def handle_entry(entry): entries: OrderedDict[Tuple[str, str], SourceEntry] = OrderedDict(map(handle_entry, split_items)) # add google domains to this, from https://www.google.com/supported_domains - for google_domain in ( - ".google.com .google.ad .google.ae .google.com.af .google.com.ag .google.al .google.am .google.co.ao " + for google_domain in [ + *".google.com .google.ad .google.ae .google.com.af .google.com.ag .google.al .google.am .google.co.ao " ".google.com.ar .google.as .google.at .google.com.au .google.az .google.ba .google.com.bd .google.be " ".google.bf .google.bg .google.com.bh .google.bi .google.bj .google.com.bn .google.com.bo " ".google.com.br .google.bs .google.bt .google.co.bw .google.by .google.com.bz .google.ca .google.cd " @@ -87,8 +87,9 @@ def handle_entry(entry): ".google.co.th .google.com.tj .google.tl .google.tm .google.tn .google.to .google.com.tr .google.tt " ".google.com.tw .google.co.tz .google.com.ua .google.co.ug .google.co.uk .google.com.uy .google.co.uz " ".google.com.vc .google.co.ve .google.co.vi .google.com.vn .google.vu .google.ws .google.rs " - ".google.co.za .google.co.zm .google.co.zw .google.cat" - ).split(" ") + ["google"]: + ".google.co.za .google.co.zm .google.co.zw .google.cat".split(" "), + "google", + ]: google_domain = google_domain.strip() if google_domain[0] == ".": google_domain = google_domain[1:] diff --git a/posthog/management/commands/execute_temporal_workflow.py b/posthog/management/commands/execute_temporal_workflow.py index e59574969072c..61c257cecc5b9 100644 --- a/posthog/management/commands/execute_temporal_workflow.py +++ b/posthog/management/commands/execute_temporal_workflow.py @@ -99,7 +99,7 @@ def handle(self, *args, **options): retry_policy = RetryPolicy(maximum_attempts=int(options["max_attempts"])) try: - workflow = [workflow for workflow in WORKFLOWS if workflow.is_named(workflow_name)][0] + workflow = next(workflow for workflow in WORKFLOWS if workflow.is_named(workflow_name)) except IndexError: raise ValueError(f"No workflow with name '{workflow_name}'") except AttributeError: diff --git a/posthog/middleware.py b/posthog/middleware.py index 281723f460fea..e43ef3a620f18 100644 --- a/posthog/middleware.py +++ b/posthog/middleware.py @@ -94,7 +94,7 @@ def extract_client_ip(self, request: HttpRequest): client_ip = forwarded_for.pop(0) if settings.TRUST_ALL_PROXIES: return client_ip - proxies = [closest_proxy] + forwarded_for + proxies = [closest_proxy, *forwarded_for] for proxy in proxies: if proxy not in self.trusted_proxies: return None @@ -486,7 +486,7 @@ def __call__(self, request: HttpRequest): def per_request_logging_context_middleware( - get_response: Callable[[HttpRequest], HttpResponse] + get_response: Callable[[HttpRequest], HttpResponse], ) -> Callable[[HttpRequest], HttpResponse]: """ We get some default logging context from the django-structlog middleware, @@ -517,7 +517,7 @@ def middleware(request: HttpRequest) -> HttpResponse: def user_logging_context_middleware( - get_response: Callable[[HttpRequest], HttpResponse] + get_response: Callable[[HttpRequest], HttpResponse], ) -> Callable[[HttpRequest], HttpResponse]: """ This middleware adds the team_id to the logging context if it exists. Note diff --git a/posthog/migrations/0403_plugin_has_private_access.py b/posthog/migrations/0403_plugin_has_private_access.py new file mode 100644 index 0000000000000..fdcb3adabe7d3 --- /dev/null +++ b/posthog/migrations/0403_plugin_has_private_access.py @@ -0,0 +1,17 @@ +# Generated by Django 4.1.13 on 2024-04-19 14:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("posthog", "0402_externaldatajob_schema"), + ] + + operations = [ + migrations.AddField( + model_name="plugin", + name="has_private_access", + field=models.ManyToManyField(to="posthog.organization"), + ), + ] diff --git a/posthog/models/dashboard.py b/posthog/models/dashboard.py index 86af344be038e..9be7e0de14e93 100644 --- a/posthog/models/dashboard.py +++ b/posthog/models/dashboard.py @@ -81,7 +81,7 @@ class PrivilegeLevel(models.IntegerChoices): __repr__ = sane_repr("team_id", "id", "name") def __str__(self): - return self.name or self.id + return self.name or str(self.id) @property def is_sharing_enabled(self): diff --git a/posthog/models/event/util.py b/posthog/models/event/util.py index 1d8357b855b71..c55094898016d 100644 --- a/posthog/models/event/util.py +++ b/posthog/models/event/util.py @@ -31,7 +31,7 @@ def create_event( team: Team, distinct_id: str, timestamp: Optional[Union[timezone.datetime, str]] = None, - properties: Optional[Dict] = {}, + properties: Optional[Dict] = None, elements: Optional[List[Element]] = None, person_id: Optional[uuid.UUID] = None, person_properties: Optional[Dict] = None, @@ -48,6 +48,8 @@ def create_event( group4_created_at: Optional[Union[timezone.datetime, str]] = None, person_mode: Literal["full", "propertyless"] = "full", ) -> str: + if properties is None: + properties = {} if not timestamp: timestamp = timezone.now() assert timestamp is not None @@ -285,9 +287,11 @@ class Meta: ] -def parse_properties(properties: str, allow_list: Set[str] = set()) -> Dict: +def parse_properties(properties: str, allow_list: Optional[Set[str]] = None) -> Dict: # parse_constants gets called for any NaN, Infinity etc values # we just want those to be returned as None + if allow_list is None: + allow_list = set() props = json.loads(properties or "{}", parse_constant=lambda x: None) return { key: value.strip('"') if isinstance(value, str) else value diff --git a/posthog/models/feature_flag/flag_matching.py b/posthog/models/feature_flag/flag_matching.py index e9faf83effa4f..134af65dfdad7 100644 --- a/posthog/models/feature_flag/flag_matching.py +++ b/posthog/models/feature_flag/flag_matching.py @@ -135,14 +135,22 @@ def __init__( self, feature_flags: List[FeatureFlag], distinct_id: str, - groups: Dict[GroupTypeName, str] = {}, + groups: Optional[Dict[GroupTypeName, str]] = None, cache: Optional[FlagsMatcherCache] = None, - hash_key_overrides: Dict[str, str] = {}, - property_value_overrides: Dict[str, Union[str, int]] = {}, - group_property_value_overrides: Dict[str, Dict[str, Union[str, int]]] = {}, + hash_key_overrides: Optional[Dict[str, str]] = None, + property_value_overrides: Optional[Dict[str, Union[str, int]]] = None, + group_property_value_overrides: Optional[Dict[str, Dict[str, Union[str, int]]]] = None, skip_database_flags: bool = False, cohorts_cache: Optional[Dict[int, CohortOrEmpty]] = None, ): + if group_property_value_overrides is None: + group_property_value_overrides = {} + if property_value_overrides is None: + property_value_overrides = {} + if hash_key_overrides is None: + hash_key_overrides = {} + if groups is None: + groups = {} self.feature_flags = feature_flags self.distinct_id = distinct_id self.groups = groups @@ -712,11 +720,17 @@ def _get_all_feature_flags( team_id: int, distinct_id: str, person_overrides: Optional[Dict[str, str]] = None, - groups: Dict[GroupTypeName, str] = {}, - property_value_overrides: Dict[str, Union[str, int]] = {}, - group_property_value_overrides: Dict[str, Dict[str, Union[str, int]]] = {}, + groups: Optional[Dict[GroupTypeName, str]] = None, + property_value_overrides: Optional[Dict[str, Union[str, int]]] = None, + group_property_value_overrides: Optional[Dict[str, Dict[str, Union[str, int]]]] = None, skip_database_flags: bool = False, ) -> Tuple[Dict[str, Union[str, bool]], Dict[str, dict], Dict[str, object], bool]: + if group_property_value_overrides is None: + group_property_value_overrides = {} + if property_value_overrides is None: + property_value_overrides = {} + if groups is None: + groups = {} cache = FlagsMatcherCache(team_id) if feature_flags: @@ -738,11 +752,17 @@ def _get_all_feature_flags( def get_all_feature_flags( team_id: int, distinct_id: str, - groups: Dict[GroupTypeName, str] = {}, + groups: Optional[Dict[GroupTypeName, str]] = None, hash_key_override: Optional[str] = None, - property_value_overrides: Dict[str, Union[str, int]] = {}, - group_property_value_overrides: Dict[str, Dict[str, Union[str, int]]] = {}, + property_value_overrides: Optional[Dict[str, Union[str, int]]] = None, + group_property_value_overrides: Optional[Dict[str, Dict[str, Union[str, int]]]] = None, ) -> Tuple[Dict[str, Union[str, bool]], Dict[str, dict], Dict[str, object], bool]: + if group_property_value_overrides is None: + group_property_value_overrides = {} + if property_value_overrides is None: + property_value_overrides = {} + if groups is None: + groups = {} property_value_overrides, group_property_value_overrides = add_local_person_and_group_properties( distinct_id, groups, property_value_overrides, group_property_value_overrides ) diff --git a/posthog/models/filters/retention_filter.py b/posthog/models/filters/retention_filter.py index 9cc3e8d0c7a08..338d3d87e3e64 100644 --- a/posthog/models/filters/retention_filter.py +++ b/posthog/models/filters/retention_filter.py @@ -48,7 +48,9 @@ class RetentionFilter( SampleMixin, BaseFilter, ): - def __init__(self, data: Dict[str, Any] = {}, request: Optional[Request] = None, **kwargs) -> None: + def __init__(self, data: Optional[Dict[str, Any]] = None, request: Optional[Request] = None, **kwargs) -> None: + if data is None: + data = {} if data: data["insight"] = INSIGHT_RETENTION else: diff --git a/posthog/models/filters/test/test_filter.py b/posthog/models/filters/test/test_filter.py index a584bbd415916..63a947bca6770 100644 --- a/posthog/models/filters/test/test_filter.py +++ b/posthog/models/filters/test/test_filter.py @@ -993,8 +993,10 @@ def filter_persons_with_annotation(filter: Filter, team: Team): def filter_persons_with_property_group( - filter: Filter, team: Team, property_overrides: Dict[str, Any] = {} + filter: Filter, team: Team, property_overrides: Optional[Dict[str, Any]] = None ) -> List[str]: + if property_overrides is None: + property_overrides = {} flush_persons_and_events() persons = Person.objects.filter(property_group_to_Q(team.pk, filter.property_groups, property_overrides)) persons = persons.filter(team_id=team.pk) diff --git a/posthog/models/filters/utils.py b/posthog/models/filters/utils.py index 0b31b209afa69..d91b49b3e05bf 100644 --- a/posthog/models/filters/utils.py +++ b/posthog/models/filters/utils.py @@ -21,12 +21,14 @@ def earliest_timestamp_func(team_id: int): return get_earliest_timestamp(team_id) -def get_filter(team, data: dict = {}, request: Optional[Request] = None): +def get_filter(team, data: Optional[dict] = None, request: Optional[Request] = None): from .filter import Filter from .path_filter import PathFilter from .retention_filter import RetentionFilter from .stickiness_filter import StickinessFilter + if data is None: + data = {} insight = data.get("insight") if not insight and request: insight = request.GET.get("insight") or request.data.get("insight") diff --git a/posthog/models/performance/sql.py b/posthog/models/performance/sql.py index 4c6a97f34a615..26f184f6cbac4 100644 --- a/posthog/models/performance/sql.py +++ b/posthog/models/performance/sql.py @@ -1,4 +1,5 @@ """https://developer.mozilla.org/en-US/docs/Web/API/PerformanceEntry""" + from posthog import settings from posthog.clickhouse.kafka_engine import ( KAFKA_COLUMNS_WITH_PARTITION, diff --git a/posthog/models/person/util.py b/posthog/models/person/util.py index 7e8afc3db5e78..f6bcc60ebc333 100644 --- a/posthog/models/person/util.py +++ b/posthog/models/person/util.py @@ -127,13 +127,15 @@ def create_person( team_id: int, version: int, uuid: Optional[str] = None, - properties: Optional[Dict] = {}, + properties: Optional[Dict] = None, sync: bool = False, is_identified: bool = False, is_deleted: bool = False, timestamp: Optional[Union[datetime.datetime, str]] = None, created_at: Optional[datetime.datetime] = None, ) -> str: + if properties is None: + properties = {} if uuid: uuid = str(uuid) else: diff --git a/posthog/models/plugin.py b/posthog/models/plugin.py index bdd1a5f8f496e..900b1abec7741 100644 --- a/posthog/models/plugin.py +++ b/posthog/models/plugin.py @@ -197,6 +197,11 @@ class PluginType(models.TextChoices): updated_at: models.DateTimeField = models.DateTimeField(null=True, blank=True) log_level: models.IntegerField = models.IntegerField(null=True, blank=True) + # Some plugins are private, only certain organizations should be able to access them + # Sometimes we want to deprecate plugins, where the first step is limiting access to organizations using them + # Sometimes we want to test out new plugins by only enabling them for certain organizations at first + has_private_access = models.ManyToManyField(Organization) + objects: PluginManager = PluginManager() def get_default_config(self) -> Dict[str, Any]: @@ -421,8 +426,10 @@ def fetch_plugin_log_entries( before: Optional[timezone.datetime] = None, search: Optional[str] = None, limit: Optional[int] = None, - type_filter: List[PluginLogEntryType] = [], + type_filter: Optional[List[PluginLogEntryType]] = None, ) -> List[PluginLogEntry]: + if type_filter is None: + type_filter = [] clickhouse_where_parts: List[str] = [] clickhouse_kwargs: Dict[str, Any] = {} if team_id is not None: diff --git a/posthog/models/property_definition.py b/posthog/models/property_definition.py index 2efc8f203192d..0a6f89354a639 100644 --- a/posthog/models/property_definition.py +++ b/posthog/models/property_definition.py @@ -80,12 +80,11 @@ class Meta: # creates an index pganalyze identified as missing # https://app.pganalyze.com/servers/i35ydkosi5cy5n7tly45vkjcqa/checks/index_advisor/missing_index/15282978 models.Index(fields=["team_id", "type", "is_numerical"]), - ] + [ GinIndex( name="index_property_definition_name", fields=["name"], opclasses=["gin_trgm_ops"], - ) # To speed up DB-based fuzzy searching + ), # To speed up DB-based fuzzy searching ] constraints = [ models.CheckConstraint( diff --git a/posthog/models/tagged_item.py b/posthog/models/tagged_item.py index 4c55c4a663791..612f2f39399c3 100644 --- a/posthog/models/tagged_item.py +++ b/posthog/models/tagged_item.py @@ -102,7 +102,7 @@ class TaggedItem(UUIDModel): ) class Meta: - unique_together = ("tag",) + RELATED_OBJECTS + unique_together = ("tag", *RELATED_OBJECTS) # Make sure to add new key to uniqueness constraint when extending tag functionality to new model constraints = [ *[build_partial_uniqueness_constraint(field=field) for field in RELATED_OBJECTS], diff --git a/posthog/models/team/team.py b/posthog/models/team/team.py index 19cb99cf67762..6f5f927fe000a 100644 --- a/posthog/models/team/team.py +++ b/posthog/models/team/team.py @@ -81,13 +81,9 @@ def set_test_account_filters(self, organization: Optional[Any]) -> List: example_email = re.search(r"@[\w.]+", example_emails[0]) if example_email: return [ - { - "key": "email", - "operator": "not_icontains", - "value": example_email.group(), - "type": "person", - } - ] + filters + {"key": "email", "operator": "not_icontains", "value": example_email.group(), "type": "person"}, + *filters, + ] return filters def create_with_data(self, user: Any = None, default_dashboards: bool = True, **kwargs) -> "Team": diff --git a/posthog/models/user.py b/posthog/models/user.py index 8968c4c17675e..cb4b1063cc961 100644 --- a/posthog/models/user.py +++ b/posthog/models/user.py @@ -21,9 +21,10 @@ class Notifications(TypedDict, total=False): plugin_disabled: bool + batch_export_run_failure: bool -NOTIFICATION_DEFAULTS: Notifications = {"plugin_disabled": True} +NOTIFICATION_DEFAULTS: Notifications = {"plugin_disabled": True, "batch_export_run_failure": True} # We don't ned the following attributes in most cases, so we defer them by default DEFERED_ATTRS = ["requested_password_reset_at"] diff --git a/posthog/models/utils.py b/posthog/models/utils.py index b00a87eb881c5..a093cf1e4ebde 100644 --- a/posthog/models/utils.py +++ b/posthog/models/utils.py @@ -122,7 +122,7 @@ class Meta: def sane_repr(*attrs: str, include_id=True) -> Callable[[object], str]: if "id" not in attrs and "pk" not in attrs and include_id: - attrs = ("id",) + attrs + attrs = ("id", *attrs) def _repr(self): pairs = (f"{attr}={repr(getattr(self, attr))}" for attr in attrs) @@ -206,7 +206,7 @@ def create_with_slug(create_func: Callable[..., T], default_slug: str = "", *arg def get_deferred_field_set_for_model( model: Type[models.Model], - fields_not_deferred: Set[str] = set(), + fields_not_deferred: Optional[Set[str]] = None, field_prefix: str = "", ) -> Set[str]: """Return a set of field names to be deferred for a given model. Used with `.defer()` after `select_related` @@ -225,6 +225,8 @@ def get_deferred_field_set_for_model( fields_not_deferred: the models fields to exclude from the deferred field set field_prefix: a prefix to add to the field names e.g. ("team__organization__") to work in the query set """ + if fields_not_deferred is None: + fields_not_deferred = set() return {f"{field_prefix}{x.name}" for x in model._meta.fields if x.name not in fields_not_deferred} diff --git a/posthog/ph_client.py b/posthog/ph_client.py index e81161a59d470..9775ebd9a0334 100644 --- a/posthog/ph_client.py +++ b/posthog/ph_client.py @@ -14,10 +14,10 @@ def get_ph_client(): region = get_instance_region() if region == "EU": api_key = "phc_dZ4GK1LRjhB97XozMSkEwPXx7OVANaJEwLErkY1phUF" - host = "https://eu.posthog.com" + host = "https://eu.i.posthog.com" elif region == "US": api_key = "sTMFPsFhdP1Ssg" - host = "https://app.posthog.com" + host = "https://us.i.posthog.com" if not api_key: return diff --git a/posthog/queries/base.py b/posthog/queries/base.py index 393c14e3042d7..7dff88f602099 100644 --- a/posthog/queries/base.py +++ b/posthog/queries/base.py @@ -276,10 +276,12 @@ def lookup_q(key: str, value: Any) -> Q: def property_to_Q( team_id: int, property: Property, - override_property_values: Dict[str, Any] = {}, + override_property_values: Optional[Dict[str, Any]] = None, cohorts_cache: Optional[Dict[int, CohortOrEmpty]] = None, using_database: str = "default", ) -> Q: + if override_property_values is None: + override_property_values = {} if property.type not in ["person", "group", "cohort", "event"]: # We need to support event type for backwards compatibility, even though it's treated as a person property type raise ValueError(f"property_to_Q: type is not supported: {repr(property.type)}") @@ -380,10 +382,12 @@ def property_to_Q( def property_group_to_Q( team_id: int, property_group: PropertyGroup, - override_property_values: Dict[str, Any] = {}, + override_property_values: Optional[Dict[str, Any]] = None, cohorts_cache: Optional[Dict[int, CohortOrEmpty]] = None, using_database: str = "default", ) -> Q: + if override_property_values is None: + override_property_values = {} filters = Q() if not property_group or len(property_group.values) == 0: @@ -423,7 +427,7 @@ def property_group_to_Q( def properties_to_Q( team_id: int, properties: List[Property], - override_property_values: Dict[str, Any] = {}, + override_property_values: Optional[Dict[str, Any]] = None, cohorts_cache: Optional[Dict[int, CohortOrEmpty]] = None, using_database: str = "default", ) -> Q: @@ -431,6 +435,8 @@ def properties_to_Q( Converts a filter to Q, for use in Django ORM .filter() If you're filtering a Person/Group QuerySet, use is_direct_query to avoid doing an unnecessary nested loop """ + if override_property_values is None: + override_property_values = {} filters = Q() if len(properties) == 0: diff --git a/posthog/queries/breakdown_props.py b/posthog/queries/breakdown_props.py index 397ee061332e6..fffb0aef0f2f0 100644 --- a/posthog/queries/breakdown_props.py +++ b/posthog/queries/breakdown_props.py @@ -46,7 +46,7 @@ def get_breakdown_prop_values( entity: Entity, aggregate_operation: str, team: Team, - extra_params={}, + extra_params=None, column_optimizer: Optional[ColumnOptimizer] = None, person_properties_mode: PersonPropertiesMode = PersonPropertiesMode.USING_PERSON_PROPERTIES_COLUMN, use_all_funnel_entities: bool = False, @@ -58,6 +58,8 @@ def get_breakdown_prop_values( When dealing with a histogram though, buckets are returned instead of values. """ + if extra_params is None: + extra_params = {} column_optimizer = column_optimizer or ColumnOptimizer(filter, team.id) date_params = {} diff --git a/posthog/queries/event_query/event_query.py b/posthog/queries/event_query/event_query.py index bcd7002e66f47..8737876d00116 100644 --- a/posthog/queries/event_query/event_query.py +++ b/posthog/queries/event_query/event_query.py @@ -60,13 +60,19 @@ def __init__( should_join_persons=False, should_join_sessions=False, # Extra events/person table columns to fetch since parent query needs them - extra_fields: List[ColumnName] = [], - extra_event_properties: List[PropertyName] = [], - extra_person_fields: List[ColumnName] = [], + extra_fields: Optional[List[ColumnName]] = None, + extra_event_properties: Optional[List[PropertyName]] = None, + extra_person_fields: Optional[List[ColumnName]] = None, override_aggregate_users_by_distinct_id: Optional[bool] = None, person_on_events_mode: PersonsOnEventsMode = PersonsOnEventsMode.disabled, **kwargs, ) -> None: + if extra_person_fields is None: + extra_person_fields = [] + if extra_event_properties is None: + extra_event_properties = [] + if extra_fields is None: + extra_fields = [] self._filter = filter self._team_id = team.pk self._team = team diff --git a/posthog/queries/foss_cohort_query.py b/posthog/queries/foss_cohort_query.py index 91d16ec3ec5a4..352fc19ee13cf 100644 --- a/posthog/queries/foss_cohort_query.py +++ b/posthog/queries/foss_cohort_query.py @@ -139,12 +139,18 @@ def __init__( should_join_distinct_ids=False, should_join_persons=False, # Extra events/person table columns to fetch since parent query needs them - extra_fields: List[ColumnName] = [], - extra_event_properties: List[PropertyName] = [], - extra_person_fields: List[ColumnName] = [], + extra_fields: Optional[List[ColumnName]] = None, + extra_event_properties: Optional[List[PropertyName]] = None, + extra_person_fields: Optional[List[ColumnName]] = None, override_aggregate_users_by_distinct_id: Optional[bool] = None, **kwargs, ) -> None: + if extra_person_fields is None: + extra_person_fields = [] + if extra_event_properties is None: + extra_event_properties = [] + if extra_fields is None: + extra_fields = [] self._fields = [] self._events = [] self._earliest_time_for_event_query = None diff --git a/posthog/queries/funnels/base.py b/posthog/queries/funnels/base.py index a96ba9b9f7f7c..c4258c6f6eb9f 100644 --- a/posthog/queries/funnels/base.py +++ b/posthog/queries/funnels/base.py @@ -667,7 +667,7 @@ def _get_matching_events(self, max_steps: int): if self._filter.include_recordings: events = [] for i in range(0, max_steps): - event_fields = ["latest"] + self.extra_event_fields_and_properties + event_fields = ["latest", *self.extra_event_fields_and_properties] event_fields_with_step = ", ".join([f'"{field}_{i}"' for field in event_fields]) event_clause = f"({event_fields_with_step}) as step_{i}_matching_event" events.append(event_clause) diff --git a/posthog/queries/funnels/test/test_breakdowns_by_current_url.py b/posthog/queries/funnels/test/test_breakdowns_by_current_url.py index bb6673387b64d..7994b195fca94 100644 --- a/posthog/queries/funnels/test/test_breakdowns_by_current_url.py +++ b/posthog/queries/funnels/test/test_breakdowns_by_current_url.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Dict +from typing import Dict, Optional from posthog.models import Filter from posthog.queries.funnels import ClickhouseFunnel @@ -115,7 +115,11 @@ def setUp(self): journeys_for(journey, team=self.team, create_people=True) - def _run(self, extra: Dict = {}, events_extra: Dict = {}): + def _run(self, extra: Optional[Dict] = None, events_extra: Optional[Dict] = None): + if events_extra is None: + events_extra = {} + if extra is None: + extra = {} response = ClickhouseFunnel( Filter( data={ diff --git a/posthog/queries/trends/test/test_breakdowns.py b/posthog/queries/trends/test/test_breakdowns.py index 48ed9033c0458..78b5a01e45aaa 100644 --- a/posthog/queries/trends/test/test_breakdowns.py +++ b/posthog/queries/trends/test/test_breakdowns.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Dict +from typing import Dict, Optional from posthog.constants import TRENDS_TABLE from posthog.models import Filter @@ -104,7 +104,11 @@ def setUp(self): journeys_for(journey, team=self.team, create_people=True) - def _run(self, extra: Dict = {}, events_extra: Dict = {}): + def _run(self, extra: Optional[Dict] = None, events_extra: Optional[Dict] = None): + if events_extra is None: + events_extra = {} + if extra is None: + extra = {} response = Trends().run( Filter( data={ diff --git a/posthog/queries/trends/test/test_breakdowns_by_current_url.py b/posthog/queries/trends/test/test_breakdowns_by_current_url.py index bc7a81595843b..26e0c40ae6404 100644 --- a/posthog/queries/trends/test/test_breakdowns_by_current_url.py +++ b/posthog/queries/trends/test/test_breakdowns_by_current_url.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Dict +from typing import Dict, Optional from posthog.models import Filter from posthog.queries.trends.trends import Trends @@ -99,7 +99,11 @@ def setUp(self): journeys_for(journey, team=self.team, create_people=True) - def _run(self, extra: Dict = {}, events_extra: Dict = {}): + def _run(self, extra: Optional[Dict] = None, events_extra: Optional[Dict] = None): + if events_extra is None: + events_extra = {} + if extra is None: + extra = {} response = Trends().run( Filter( data={ diff --git a/posthog/queries/trends/test/test_formula.py b/posthog/queries/trends/test/test_formula.py index adbf54fa05f79..01e838336e5c8 100644 --- a/posthog/queries/trends/test/test_formula.py +++ b/posthog/queries/trends/test/test_formula.py @@ -129,7 +129,9 @@ def setUp(self): }, ) - def _run(self, extra: Dict = {}, run_at: Optional[str] = None): + def _run(self, extra: Optional[Dict] = None, run_at: Optional[str] = None): + if extra is None: + extra = {} with freeze_time(run_at or "2020-01-04T13:01:01Z"): action_response = Trends().run( Filter( diff --git a/posthog/queries/trends/test/test_paging_breakdowns.py b/posthog/queries/trends/test/test_paging_breakdowns.py index 31db69f75b529..b4040fee61897 100644 --- a/posthog/queries/trends/test/test_paging_breakdowns.py +++ b/posthog/queries/trends/test/test_paging_breakdowns.py @@ -38,7 +38,9 @@ def setUp(self): create_people=True, ) - def _run(self, extra: Dict = {}, run_at: Optional[str] = None): + def _run(self, extra: Optional[Dict] = None, run_at: Optional[str] = None): + if extra is None: + extra = {} with freeze_time(run_at or "2020-01-04T13:01:01Z"): action_response = Trends().run( Filter( diff --git a/posthog/queries/trends/util.py b/posthog/queries/trends/util.py index bb11f0c38293d..e002145de9957 100644 --- a/posthog/queries/trends/util.py +++ b/posthog/queries/trends/util.py @@ -102,9 +102,11 @@ def process_math( def parse_response( stats: Dict, filter: Filter, - additional_values: Dict = {}, + additional_values: Optional[Dict] = None, entity: Optional[Entity] = None, ) -> Dict[str, Any]: + if additional_values is None: + additional_values = {} counts = stats[1] labels = [item.strftime("%-d-%b-%Y{}".format(" %H:%M" if filter.interval == "hour" else "")) for item in stats[0]] days = [item.strftime("%Y-%m-%d{}".format(" %H:%M:%S" if filter.interval == "hour" else "")) for item in stats[0]] diff --git a/posthog/session_recordings/queries/test/test_session_recording_properties.py b/posthog/session_recordings/queries/test/test_session_recording_properties.py index aa152b0b2fa16..7972eb742abb0 100644 --- a/posthog/session_recordings/queries/test/test_session_recording_properties.py +++ b/posthog/session_recordings/queries/test/test_session_recording_properties.py @@ -25,8 +25,10 @@ def create_event( timestamp, team=None, event_name="$pageview", - properties={"$os": "Windows 95", "$current_url": "aloha.com/2"}, + properties=None, ): + if properties is None: + properties = {"$os": "Windows 95", "$current_url": "aloha.com/2"} if team is None: team = self.team _create_event( diff --git a/posthog/session_recordings/test/test_session_recording_helpers.py b/posthog/session_recordings/test/test_session_recording_helpers.py index 1fd6bb3191948..b6b83e02c28d9 100644 --- a/posthog/session_recordings/test/test_session_recording_helpers.py +++ b/posthog/session_recordings/test/test_session_recording_helpers.py @@ -280,7 +280,6 @@ def test_new_ingestion_large_full_snapshot_is_separated(raw_snapshot_events, moc "distinct_id": "abc123", }, }, - ] + [ { "event": "$snapshot", "properties": { diff --git a/posthog/session_recordings/test/test_session_recordings.py b/posthog/session_recordings/test/test_session_recordings.py index 03e73aabe054f..12085f55925eb 100644 --- a/posthog/session_recordings/test/test_session_recordings.py +++ b/posthog/session_recordings/test/test_session_recordings.py @@ -780,7 +780,7 @@ def test_can_get_session_recording_realtime_utf16_data( # by default a session recording is deleted, so we have to explicitly mark the mock as not deleted mock_get_session_recording.return_value = SessionRecording(session_id=session_id, team=self.team, deleted=False) - annoying_data_from_javascript = "\uD801\uDC37 probably from console logs" + annoying_data_from_javascript = "\ud801\udc37 probably from console logs" mock_realtime_snapshots.return_value = [ {"some": annoying_data_from_javascript}, diff --git a/posthog/settings/feature_flags.py b/posthog/settings/feature_flags.py index 371f497376663..8b1f5b3e3f94e 100644 --- a/posthog/settings/feature_flags.py +++ b/posthog/settings/feature_flags.py @@ -4,7 +4,8 @@ # These flags will be force-enabled on the frontend # The features here are released, but the flags are just not yet removed from the code -PERSISTED_FEATURE_FLAGS = get_list(os.getenv("PERSISTED_FEATURE_FLAGS", "")) + [ +PERSISTED_FEATURE_FLAGS = [ + *get_list(os.getenv("PERSISTED_FEATURE_FLAGS", "")), "simplify-actions", "historical-exports-v2", "ingestion-warnings-enabled", diff --git a/posthog/settings/sentry.py b/posthog/settings/sentry.py index 0d3fcee485506..d279af33c6a94 100644 --- a/posthog/settings/sentry.py +++ b/posthog/settings/sentry.py @@ -9,6 +9,7 @@ from sentry_sdk.integrations.django import DjangoIntegration from sentry_sdk.integrations.logging import LoggingIntegration from sentry_sdk.integrations.redis import RedisIntegration +from sentry_sdk.integrations.clickhouse_driver import ClickhouseDriverIntegration from posthog.git import get_git_commit_full from posthog.settings import get_from_env @@ -141,8 +142,6 @@ def traces_sampler(sampling_context: dict) -> float: def sentry_init() -> None: if not TEST and os.getenv("SENTRY_DSN"): - sentry_sdk.utils.MAX_STRING_LENGTH = 10_000_000 - # Setting this on enables more visibility, at the risk of capturing personal information we should not: # - standard sentry "client IP" field, through send_default_pii # - django access logs (info level) @@ -151,7 +150,6 @@ def sentry_init() -> None: send_pii = get_from_env("SENTRY_SEND_PII", type_cast=bool, default=False) sentry_logging_level = logging.INFO if send_pii else logging.ERROR - sentry_logging = LoggingIntegration(level=sentry_logging_level, event_level=None) profiles_sample_rate = get_from_env("SENTRY_PROFILES_SAMPLE_RATE", type_cast=float, default=0.0) release = get_git_commit_full() @@ -164,9 +162,11 @@ def sentry_init() -> None: DjangoIntegration(), CeleryIntegration(), RedisIntegration(), - sentry_logging, + ClickhouseDriverIntegration(), + LoggingIntegration(level=sentry_logging_level, event_level=None), ], - request_bodies="always" if send_pii else "never", + max_request_body_size="always" if send_pii else "never", + max_value_length=8192, # Increased from the default of 1024 to capture SQL statements in full sample_rate=1.0, # Configures the sample rate for error events, in the range of 0.0 to 1.0 (default). # If set to 0.1 only 10% of error events will be sent. Events are picked randomly. diff --git a/posthog/settings/web.py b/posthog/settings/web.py index f54c2e32fc28c..ee6961de70e79 100644 --- a/posthog/settings/web.py +++ b/posthog/settings/web.py @@ -341,7 +341,7 @@ def add_recorder_js_headers(headers, path, url): # https://github.com/korfuri/django-prometheus for more details # We keep the number of buckets low to reduce resource usage on the Prometheus -PROMETHEUS_LATENCY_BUCKETS = [0.1, 0.3, 0.9, 2.7, 8.1] + [float("inf")] +PROMETHEUS_LATENCY_BUCKETS = [0.1, 0.3, 0.9, 2.7, 8.1, float("inf")] SALT_KEY = os.getenv("SALT_KEY", "0123456789abcdefghijklmnopqrstuvwxyz") diff --git a/posthog/tasks/email.py b/posthog/tasks/email.py index d27c00d9ae85a..d06d15ee12ace 100644 --- a/posthog/tasks/email.py +++ b/posthog/tasks/email.py @@ -4,10 +4,12 @@ import posthoganalytics import structlog +from asgiref.sync import sync_to_async from celery import shared_task from django.conf import settings from django.utils import timezone +from posthog.batch_exports.models import BatchExportRun from posthog.cloud_utils import is_cloud from posthog.email import EMAIL_TASK_KWARGS, EmailMessage, is_email_available from posthog.models import ( @@ -157,6 +159,62 @@ def send_fatal_plugin_error( message.send(send_async=False) +@shared_task(**EMAIL_TASK_KWARGS) +async def send_batch_export_run_failure( + batch_export_run_id: int, +) -> None: + is_email_available_result = await sync_to_async(is_email_available)(with_absolute_urls=True) + if not is_email_available_result: + return + + batch_export_run: BatchExportRun = await sync_to_async( + BatchExportRun.objects.select_related("batch_export__team").get + )(id=batch_export_run_id) + team: Team = batch_export_run.batch_export.team + # NOTE: We are taking only the date component to cap the number of emails at one per day per batch export. + last_updated_at_date = batch_export_run.last_updated_at.strftime("%Y-%m-%d") + + campaign_key: str = ( + f"batch_export_run_email_batch_export_{batch_export_run.batch_export.id}_last_updated_at_{last_updated_at_date}" + ) + + message = await sync_to_async(EmailMessage)( + campaign_key=campaign_key, + subject=f"PostHog: {batch_export_run.batch_export.name} batch export run failure", + template_name="batch_export_run_failure", + template_context={ + "time": batch_export_run.last_updated_at.strftime("%I:%M%p %Z on %B %d"), + "team": team, + "id": batch_export_run.batch_export.id, + "name": batch_export_run.batch_export.name, + }, + ) + memberships_to_email = [] + memberships = OrganizationMembership.objects.select_related("user", "organization").filter( + organization_id=team.organization_id + ) + all_memberships: list[OrganizationMembership] = await sync_to_async(list)(memberships) # type: ignore + for membership in all_memberships: + has_notification_settings_enabled = await sync_to_async(membership.user.notification_settings.get)( + "batch_export_run_failure", True + ) + if has_notification_settings_enabled is False: + continue + team_permissions = UserPermissions(membership.user).team(team) + # Only send the email to users who have access to the affected project + # Those without access have `effective_membership_level` of `None` + if ( + team_permissions.effective_membership_level_for_parent_membership(membership.organization, membership) + is not None + ): + memberships_to_email.append(membership) + + if memberships_to_email: + for membership in memberships_to_email: + message.add_recipient(email=membership.user.email, name=membership.user.first_name) + await sync_to_async(message.send)(send_async=True) + + @shared_task(**EMAIL_TASK_KWARGS) def send_canary_email(user_email: str) -> None: message = EmailMessage( diff --git a/posthog/tasks/test/test_email.py b/posthog/tasks/test/test_email.py index 571132fd1ca84..447d0d442bfc8 100644 --- a/posthog/tasks/test/test_email.py +++ b/posthog/tasks/test/test_email.py @@ -1,10 +1,14 @@ +import datetime as dt from typing import Tuple from unittest.mock import MagicMock, patch +import pytest +from asgiref.sync import sync_to_async from freezegun import freeze_time from posthog.api.authentication import password_reset_token_generator from posthog.api.email_verification import email_verification_token_generator +from posthog.batch_exports.models import BatchExport, BatchExportDestination, BatchExportRun from posthog.models import Organization, Team, User from posthog.models.instance_setting import set_instance_setting from posthog.models.organization import OrganizationInvite, OrganizationMembership @@ -12,6 +16,7 @@ from posthog.tasks.email import ( send_async_migration_complete_email, send_async_migration_errored_email, + send_batch_export_run_failure, send_canary_email, send_email_verification, send_fatal_plugin_error, @@ -144,6 +149,62 @@ def test_send_fatal_plugin_error_with_settings(self, MockEmailMessage: MagicMock # should be sent to both assert len(mocked_email_messages[1].to) == 2 + @pytest.mark.asyncio + async def test_send_batch_export_run_failure(self, MockEmailMessage: MagicMock) -> None: + mocked_email_messages = mock_email_messages(MockEmailMessage) + _, user = await sync_to_async(create_org_team_and_user)("2022-01-02 00:00:00", "admin@posthog.com") + batch_export_destination = await sync_to_async(BatchExportDestination.objects.create)( + type=BatchExportDestination.Destination.S3, config={"bucket_name": "my_production_s3_bucket"} + ) + batch_export = await sync_to_async(BatchExport.objects.create)( + team=user.team, name="A batch export", destination=batch_export_destination + ) + now = dt.datetime.now() + batch_export_run = await sync_to_async(BatchExportRun.objects.create)( + batch_export=batch_export, + status=BatchExportRun.Status.FAILED, + data_interval_start=now - dt.timedelta(hours=1), + data_interval_end=now, + ) + + await send_batch_export_run_failure(batch_export_run.id) + + assert len(mocked_email_messages) == 1 + assert mocked_email_messages[0].send.call_count == 1 + assert mocked_email_messages[0].html_body + + @pytest.mark.asyncio + async def test_send_batch_export_run_failure_with_settings(self, MockEmailMessage: MagicMock) -> None: + mocked_email_messages = mock_email_messages(MockEmailMessage) + batch_export_destination = await sync_to_async(BatchExportDestination.objects.create)( + type=BatchExportDestination.Destination.S3, config={"bucket_name": "my_production_s3_bucket"} + ) + batch_export = await sync_to_async(BatchExport.objects.create)( + team=self.user.team, name="A batch export", destination=batch_export_destination + ) + now = dt.datetime.now() + batch_export_run = await sync_to_async(BatchExportRun.objects.create)( + batch_export=batch_export, + status=BatchExportRun.Status.FAILED, + data_interval_start=now - dt.timedelta(hours=1), + data_interval_end=now, + ) + + await sync_to_async(self._create_user)("test2@posthog.com") + self.user.partial_notification_settings = {"batch_export_run_failure": False} + await sync_to_async(self.user.save)() + + await send_batch_export_run_failure(batch_export_run.id) + # Should only be sent to user2 + assert mocked_email_messages[0].to == [{"recipient": "test2@posthog.com", "raw_email": "test2@posthog.com"}] + + self.user.partial_notification_settings = {"batch_export_run_failure": True} + await sync_to_async(self.user.save)() + + await send_batch_export_run_failure(batch_export_run.id) + # should be sent to both + assert len(mocked_email_messages[1].to) == 2 + def test_send_canary_email(self, MockEmailMessage: MagicMock) -> None: mocked_email_messages = mock_email_messages(MockEmailMessage) send_canary_email("test@posthog.com") diff --git a/posthog/tasks/test/test_usage_report.py b/posthog/tasks/test/test_usage_report.py index 055629bf055ca..d977f27560b51 100644 --- a/posthog/tasks/test/test_usage_report.py +++ b/posthog/tasks/test/test_usage_report.py @@ -325,7 +325,7 @@ def _create_sample_usage_data(self) -> None: flush_persons_and_events() def _select_report_by_org_id(self, org_id: str, reports: List[Dict]) -> Dict: - return [report for report in reports if report["organization_id"] == org_id][0] + return next(report for report in reports if report["organization_id"] == org_id) def _create_plugin(self, name: str, enabled: bool) -> None: plugin = Plugin.objects.create(organization_id=self.team.organization.pk, name=name) diff --git a/posthog/templates/email/batch_export_run_failure.html b/posthog/templates/email/batch_export_run_failure.html new file mode 100644 index 0000000000000..04cf2021e342c --- /dev/null +++ b/posthog/templates/email/batch_export_run_failure.html @@ -0,0 +1,31 @@ +{% extends "email/base.html" %} {% load posthog_assets %} {% load posthog_filters %} +{% block preheader %}If failures keep occurring we will disable this batch export{% endblock %} +{% block heading %}PostHog batch export {{ name }} has failed{% endblock %} +{% block section %} +

+ There's been a fatal error with your batch export {{ name }} at {{ time }}. Due to the nature of the error, it cannot be retried automatically and requires manual intervention. + + We recommend reviewing the batch export logs for error details: +

+ +
+ Go to the batch export +
+ +

+ After reviewing the logs, and addressing any errors in them, you can retry the batch export run manually. If the batch export continues to fail we will disable it. +

+{% endblock %} + +{% block footer %} +Need help? +Visit support +or +read our documentation.

+ +Manage these notifications in PostHog + +{% endblock %} diff --git a/posthog/temporal/batch_exports/batch_exports.py b/posthog/temporal/batch_exports/batch_exports.py index 0e12fc14635b4..66279ccd7183e 100644 --- a/posthog/temporal/batch_exports/batch_exports.py +++ b/posthog/temporal/batch_exports/batch_exports.py @@ -14,8 +14,10 @@ from posthog.batch_exports.models import BatchExportBackfill, BatchExportRun from posthog.batch_exports.service import ( BatchExportField, + count_failed_batch_export_runs, create_batch_export_backfill, create_batch_export_run, + pause_batch_export, update_batch_export_backfill_status, update_batch_export_run, ) @@ -24,6 +26,7 @@ get_export_started_metric, ) from posthog.temporal.common.clickhouse import ClickHouseClient, get_client +from posthog.temporal.common.client import connect from posthog.temporal.common.logger import bind_temporal_worker_logger SELECT_QUERY_TEMPLATE = Template( @@ -48,7 +51,7 @@ -- These 'timestamp' checks are a heuristic to exploit the sort key. -- Ideally, we need a schema that serves our needs, i.e. with a sort key on the _timestamp field used for batch exports. -- As a side-effect, this heuristic will discard historical loads older than a day. -AND timestamp >= toDateTime64({data_interval_start}, 6, 'UTC') - INTERVAL 2 DAY +AND timestamp >= toDateTime64({data_interval_start}, 6, 'UTC') - INTERVAL 4 DAY AND timestamp < toDateTime64({data_interval_end}, 6, 'UTC') + INTERVAL 1 DAY """ @@ -370,33 +373,47 @@ class FinishBatchExportRunInputs: Attributes: id: The id of the batch export run. This should be a valid UUID string. + batch_export_id: The id of the batch export this run belongs to. team_id: The team id of the batch export. status: The status this batch export is finishing with. latest_error: The latest error message captured, if any. records_completed: Number of records successfully exported. records_total_count: Total count of records this run noted. + failure_threshold: Used when determining to pause a batch export that has failed. + See the docstring in 'pause_batch_export_if_over_failure_threshold'. + failure_check_window: Used when determining to pause a batch export that has failed. + See the docstring in 'pause_batch_export_if_over_failure_threshold'. """ id: str + batch_export_id: str team_id: int status: str latest_error: str | None = None records_completed: int | None = None records_total_count: int | None = None + failure_threshold: int = 10 + failure_check_window: int = 50 @activity.defn async def finish_batch_export_run(inputs: FinishBatchExportRunInputs) -> None: - """Activity that finishes a BatchExportRun. + """Activity that finishes a 'BatchExportRun'. + + Finishing means setting and handling the status of a 'BatchExportRun' model, as well + as setting any additional supported model attributes. - Finishing means a final update to the status of the BatchExportRun model. + The only status that requires handling is 'FAILED' as we also check if the number of failures in + 'failure_check_window' exceeds 'failure_threshold' and attempt to pause the batch export if + that's the case. Also, a notification is sent to users on every failure. """ logger = await bind_temporal_worker_logger(team_id=inputs.team_id) + not_model_params = ("id", "team_id", "batch_export_id", "failure_threshold", "failure_check_window") update_params = { key: value for key, value in dataclasses.asdict(inputs).items() - if key not in ("id", "team_id") and value is not None + if key not in not_model_params and value is not None } batch_export_run = await sync_to_async(update_batch_export_run)( run_id=uuid.UUID(inputs.id), @@ -404,11 +421,41 @@ async def finish_batch_export_run(inputs: FinishBatchExportRunInputs) -> None: **update_params, ) - if batch_export_run.status in (BatchExportRun.Status.FAILED, BatchExportRun.Status.FAILED_RETRYABLE): - logger.error("BatchExport failed with error: %s", batch_export_run.latest_error) + if batch_export_run.status == BatchExportRun.Status.FAILED_RETRYABLE: + logger.error("Batch export failed with error: %s", batch_export_run.latest_error) + + elif batch_export_run.status == BatchExportRun.Status.FAILED: + logger.error("Batch export failed with non-retryable error: %s", batch_export_run.latest_error) + + from posthog.tasks.email import send_batch_export_run_failure + + try: + await send_batch_export_run_failure(inputs.id) + except Exception: + logger.exception("Failure email notification could not be sent") + + try: + was_paused = await pause_batch_export_if_over_failure_threshold( + inputs.batch_export_id, + check_window=inputs.failure_check_window, + failure_threshold=inputs.failure_threshold, + ) + except Exception: + # Pausing could error if the underlying schedule is deleted. + # Our application logic should prevent that, but I want to log it in case it ever happens + # as that would indicate a bug. + logger.exception("Batch export could not be automatically paused") + was_paused = False + + if was_paused: + logger.warning( + "Batch export was automatically paused due to exceeding failure threshold and exhausting " + "all automated retries." + "The batch export can be manually unpaused after addressing any errors." + ) elif batch_export_run.status == BatchExportRun.Status.CANCELLED: - logger.warning("BatchExport was cancelled.") + logger.warning("Batch export was cancelled") else: logger.info( @@ -418,6 +465,59 @@ async def finish_batch_export_run(inputs: FinishBatchExportRunInputs) -> None: ) +async def pause_batch_export_if_over_failure_threshold( + batch_export_id: str, check_window: int, failure_threshold: int = 10 +) -> bool: + """Pause a batch export if it exceeds failure threshold. + + A 'check_window' was added to account for batch exports that have a history of failures but have some + occassional successes in the middle. This is relevant particularly for low-volume exports: + A batch export without rows to export always succeeds, even if it's not properly configured. So, the failures + could be scattered between these successes. + + Keep in mind that if 'check_window' is less than 'failure_threshold', there is no point in even counting, + so we raise an exception. + + We check if the count of failed runs in the last 'check_window' runs exceeds 'failure_threshold'. This means + that 'pause_batch_export_if_over_failure_threshold' should only be called when handling a failed run, + otherwise we could be pausing a batch export that is just now recovering (as old failed runs in 'check_window' + contribute to exceeding 'failure_threshold'). + + Arguments: + batch_export_id: The ID of the batch export to check and pause. + check_window: The window of runs to consider for computing a count of failures. + failure_threshold: The number of runs that must have failed for a batch export to be paused. + + Returns: + A bool indicating if the batch export is paused. + + Raises: + ValueError: If 'check_window' is smaller than 'failure_threshold' as that check would be redundant and, + likely, a bug. + """ + if check_window < failure_threshold: + raise ValueError("'failure_threshold' cannot be higher than 'check_window'") + + count = await sync_to_async(count_failed_batch_export_runs)(uuid.UUID(batch_export_id), last_n=check_window) + + if count < failure_threshold: + return False + + client = await connect( + settings.TEMPORAL_HOST, + settings.TEMPORAL_PORT, + settings.TEMPORAL_NAMESPACE, + settings.TEMPORAL_CLIENT_ROOT_CA, + settings.TEMPORAL_CLIENT_CERT, + settings.TEMPORAL_CLIENT_KEY, + ) + + await sync_to_async(pause_batch_export)( + client, batch_export_id=batch_export_id, note="Paused due to exceeding failure threshold" + ) + return True + + @dataclasses.dataclass class CreateBatchExportBackfillInputs: team_id: int diff --git a/posthog/temporal/batch_exports/bigquery_batch_export.py b/posthog/temporal/batch_exports/bigquery_batch_export.py index f9ddd29bd528f..93a2e522e1e7f 100644 --- a/posthog/temporal/batch_exports/bigquery_batch_export.py +++ b/posthog/temporal/batch_exports/bigquery_batch_export.py @@ -390,12 +390,9 @@ async def run(self, inputs: BigQueryBatchExportInputs): ), ) - finish_inputs = FinishBatchExportRunInputs( - id=run_id, status=BatchExportRun.Status.COMPLETED, team_id=inputs.team_id - ) - finish_inputs = FinishBatchExportRunInputs( id=run_id, + batch_export_id=inputs.batch_export_id, status=BatchExportRun.Status.COMPLETED, team_id=inputs.team_id, ) diff --git a/posthog/temporal/batch_exports/http_batch_export.py b/posthog/temporal/batch_exports/http_batch_export.py index 993806c004c5e..f86703f3cf792 100644 --- a/posthog/temporal/batch_exports/http_batch_export.py +++ b/posthog/temporal/batch_exports/http_batch_export.py @@ -339,6 +339,7 @@ async def run(self, inputs: HttpBatchExportInputs): finish_inputs = FinishBatchExportRunInputs( id=run_id, + batch_export_id=inputs.batch_export_id, status=BatchExportRun.Status.COMPLETED, team_id=inputs.team_id, ) diff --git a/posthog/temporal/batch_exports/postgres_batch_export.py b/posthog/temporal/batch_exports/postgres_batch_export.py index 54b3f316393c2..6281862a72f21 100644 --- a/posthog/temporal/batch_exports/postgres_batch_export.py +++ b/posthog/temporal/batch_exports/postgres_batch_export.py @@ -399,6 +399,7 @@ async def run(self, inputs: PostgresBatchExportInputs): finish_inputs = FinishBatchExportRunInputs( id=run_id, + batch_export_id=inputs.batch_export_id, status=BatchExportRun.Status.COMPLETED, team_id=inputs.team_id, ) diff --git a/posthog/temporal/batch_exports/redshift_batch_export.py b/posthog/temporal/batch_exports/redshift_batch_export.py index cd1f299751cc8..e98fa9106c15f 100644 --- a/posthog/temporal/batch_exports/redshift_batch_export.py +++ b/posthog/temporal/batch_exports/redshift_batch_export.py @@ -428,6 +428,7 @@ async def run(self, inputs: RedshiftBatchExportInputs): finish_inputs = FinishBatchExportRunInputs( id=run_id, + batch_export_id=inputs.batch_export_id, status=BatchExportRun.Status.COMPLETED, team_id=inputs.team_id, ) diff --git a/posthog/temporal/batch_exports/s3_batch_export.py b/posthog/temporal/batch_exports/s3_batch_export.py index e5ad6dd07144e..febdac88b45cd 100644 --- a/posthog/temporal/batch_exports/s3_batch_export.py +++ b/posthog/temporal/batch_exports/s3_batch_export.py @@ -645,6 +645,7 @@ async def run(self, inputs: S3BatchExportInputs): finish_inputs = FinishBatchExportRunInputs( id=run_id, + batch_export_id=inputs.batch_export_id, status=BatchExportRun.Status.COMPLETED, team_id=inputs.team_id, ) diff --git a/posthog/temporal/batch_exports/snowflake_batch_export.py b/posthog/temporal/batch_exports/snowflake_batch_export.py index 19b090340a9c9..2d782c1f94d5c 100644 --- a/posthog/temporal/batch_exports/snowflake_batch_export.py +++ b/posthog/temporal/batch_exports/snowflake_batch_export.py @@ -591,6 +591,7 @@ async def run(self, inputs: SnowflakeBatchExportInputs): finish_inputs = FinishBatchExportRunInputs( id=run_id, + batch_export_id=inputs.batch_export_id, status=BatchExportRun.Status.COMPLETED, team_id=inputs.team_id, ) diff --git a/posthog/temporal/batch_exports/utils.py b/posthog/temporal/batch_exports/utils.py index a097776389cac..f165ae070a83f 100644 --- a/posthog/temporal/batch_exports/utils.py +++ b/posthog/temporal/batch_exports/utils.py @@ -9,7 +9,7 @@ def peek_first_and_rewind( - gen: collections.abc.Generator[T, None, None] + gen: collections.abc.Generator[T, None, None], ) -> tuple[T, collections.abc.Generator[T, None, None]]: """Peek into the first element in a generator and rewind the advance. diff --git a/posthog/temporal/data_imports/pipelines/zendesk/credentials.py b/posthog/temporal/data_imports/pipelines/zendesk/credentials.py index e4dfda2013573..88a0659b7ce1a 100644 --- a/posthog/temporal/data_imports/pipelines/zendesk/credentials.py +++ b/posthog/temporal/data_imports/pipelines/zendesk/credentials.py @@ -1,6 +1,7 @@ """ This module handles how credentials are read in dlt sources """ + from typing import ClassVar, List, Union from dlt.common.configuration import configspec from dlt.common.configuration.specs import CredentialsConfiguration diff --git a/posthog/temporal/tests/batch_exports/test_logger.py b/posthog/temporal/tests/batch_exports/test_logger.py index 5c12cef1d034a..4ee3ca9a014aa 100644 --- a/posthog/temporal/tests/batch_exports/test_logger.py +++ b/posthog/temporal/tests/batch_exports/test_logger.py @@ -82,7 +82,7 @@ def __init__(self, *args, **kwargs): def producer(self) -> aiokafka.AIOKafkaProducer: if self._producer is None: self._producer = aiokafka.AIOKafkaProducer( - bootstrap_servers=settings.KAFKA_HOSTS + ["localhost:9092"], + bootstrap_servers=[*settings.KAFKA_HOSTS, "localhost:9092"], security_protocol=settings.KAFKA_SECURITY_PROTOCOL or "PLAINTEXT", acks="all", request_timeout_ms=1000000, diff --git a/posthog/temporal/tests/batch_exports/test_run_updates.py b/posthog/temporal/tests/batch_exports/test_run_updates.py index 7269b3455d8f1..c7838c4ebca8d 100644 --- a/posthog/temporal/tests/batch_exports/test_run_updates.py +++ b/posthog/temporal/tests/batch_exports/test_run_updates.py @@ -3,6 +3,7 @@ import pytest from asgiref.sync import sync_to_async +from posthog.batch_exports.service import disable_and_delete_export, sync_batch_export from posthog.models import ( BatchExport, BatchExportDestination, @@ -63,12 +64,17 @@ def destination(team): @pytest.fixture def batch_export(destination, team): """A test BatchExport.""" - batch_export = BatchExport.objects.create(name="test export", team=team, destination=destination, interval="hour") + batch_export = BatchExport.objects.create( + name="test export", team=team, destination=destination, interval="hour", paused=False + ) batch_export.save() + sync_batch_export(batch_export, created=True) + yield batch_export + disable_and_delete_export(batch_export) batch_export.delete() @@ -125,6 +131,7 @@ async def test_finish_batch_export_run(activity_environment, team, batch_export) finish_inputs = FinishBatchExportRunInputs( id=str(run_id), + batch_export_id=str(batch_export.id), status="Completed", team_id=inputs.team_id, ) @@ -135,3 +142,77 @@ async def test_finish_batch_export_run(activity_environment, team, batch_export) assert run is not None assert run.status == "Completed" assert run.records_total_count == records_total_count + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.asyncio +async def test_finish_batch_export_run_pauses_if_reaching_failure_threshold(activity_environment, team, batch_export): + """Test if 'finish_batch_export_run' will pause a batch export upon reaching failure_threshold.""" + start = dt.datetime(2023, 4, 24, tzinfo=dt.timezone.utc) + end = dt.datetime(2023, 4, 25, tzinfo=dt.timezone.utc) + + inputs = StartBatchExportRunInputs( + team_id=team.id, + batch_export_id=str(batch_export.id), + data_interval_start=start.isoformat(), + data_interval_end=end.isoformat(), + ) + + batch_export_id = str(batch_export.id) + failure_threshold = 10 + + for run_number in range(1, failure_threshold * 2): + run_id, _ = await activity_environment.run(start_batch_export_run, inputs) + + finish_inputs = FinishBatchExportRunInputs( + id=str(run_id), + batch_export_id=batch_export_id, + status=BatchExportRun.Status.FAILED, + team_id=inputs.team_id, + latest_error="Oh No!", + failure_threshold=failure_threshold, + ) + + await activity_environment.run(finish_batch_export_run, finish_inputs) + await sync_to_async(batch_export.refresh_from_db)() + + if run_number >= failure_threshold: + assert batch_export.paused is True + else: + assert batch_export.paused is False + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.asyncio +async def test_finish_batch_export_run_never_pauses_with_small_check_window(activity_environment, team, batch_export): + """Test if 'finish_batch_export_run' will never pause a batch export with a small check window.""" + start = dt.datetime(2023, 4, 24, tzinfo=dt.timezone.utc) + end = dt.datetime(2023, 4, 25, tzinfo=dt.timezone.utc) + + inputs = StartBatchExportRunInputs( + team_id=team.id, + batch_export_id=str(batch_export.id), + data_interval_start=start.isoformat(), + data_interval_end=end.isoformat(), + ) + + batch_export_id = str(batch_export.id) + failure_threshold = 10 + + for _ in range(1, failure_threshold * 2): + run_id, _ = await activity_environment.run(start_batch_export_run, inputs) + + finish_inputs = FinishBatchExportRunInputs( + id=str(run_id), + batch_export_id=batch_export_id, + status=BatchExportRun.Status.FAILED, + team_id=inputs.team_id, + latest_error="Oh No!", + failure_threshold=failure_threshold, + failure_check_window=failure_threshold - 1, + ) + + await activity_environment.run(finish_batch_export_run, finish_inputs) + await sync_to_async(batch_export.refresh_from_db)() + + assert batch_export.paused is False diff --git a/posthog/temporal/tests/utils/datetimes.py b/posthog/temporal/tests/utils/datetimes.py index ec0c10980bbdf..c168e885a3e8d 100644 --- a/posthog/temporal/tests/utils/datetimes.py +++ b/posthog/temporal/tests/utils/datetimes.py @@ -1,4 +1,5 @@ """Test utilities that operate with datetime.datetimes.""" + import datetime as dt diff --git a/posthog/temporal/tests/utils/events.py b/posthog/temporal/tests/utils/events.py index 884901ca9aa92..71ce7f7f61615 100644 --- a/posthog/temporal/tests/utils/events.py +++ b/posthog/temporal/tests/utils/events.py @@ -1,4 +1,5 @@ """Test utilities that deal with test event generation.""" + import datetime as dt import json import random diff --git a/posthog/temporal/tests/utils/models.py b/posthog/temporal/tests/utils/models.py index 04da6fe21b0fb..4ed75ad50aae8 100644 --- a/posthog/temporal/tests/utils/models.py +++ b/posthog/temporal/tests/utils/models.py @@ -1,4 +1,5 @@ """Test utilities to manipulate BatchExport* models.""" + import uuid import temporalio.client diff --git a/posthog/test/base.py b/posthog/test/base.py index 6d4735679a0f3..c96738aafa139 100644 --- a/posthog/test/base.py +++ b/posthog/test/base.py @@ -409,9 +409,9 @@ def cleanup_materialized_columns(): def also_test_with_materialized_columns( - event_properties=[], - person_properties=[], - group_properties=[], + event_properties=None, + person_properties=None, + group_properties=None, verify_no_jsonextract=True, # :TODO: Remove this when groups-on-events is released materialize_only_with_person_on_events=False, @@ -422,6 +422,12 @@ def also_test_with_materialized_columns( Requires a unittest class with ClickhouseTestMixin mixed in """ + if group_properties is None: + group_properties = [] + if person_properties is None: + person_properties = [] + if event_properties is None: + event_properties = [] try: from ee.clickhouse.materialized_columns.analyze import materialize except: diff --git a/posthog/test/test_latest_migrations.py b/posthog/test/test_latest_migrations.py index 36a047af8a66a..1d60179576f5c 100644 --- a/posthog/test/test_latest_migrations.py +++ b/posthog/test/test_latest_migrations.py @@ -33,6 +33,6 @@ def _get_newest_migration_file(path: str) -> str: def _get_latest_migration_from_manifest(django_app: str) -> str: root = pathlib.Path().resolve() manifest = pathlib.Path(f"{root}/latest_migrations.manifest").read_text() - posthog_latest_migration = [line for line in manifest.splitlines() if line.startswith(f"{django_app}: ")][0] + posthog_latest_migration = next(line for line in manifest.splitlines() if line.startswith(f"{django_app}: ")) return posthog_latest_migration.replace(f"{django_app}: ", "") diff --git a/posthog/utils.py b/posthog/utils.py index f186fdadb4adb..19e110507ab9b 100644 --- a/posthog/utils.py +++ b/posthog/utils.py @@ -275,7 +275,7 @@ def get_js_url(request: HttpRequest) -> str: def render_template( template_name: str, request: HttpRequest, - context: Dict = {}, + context: Optional[Dict] = None, *, team_for_public_context: Optional["Team"] = None, ) -> HttpResponse: @@ -284,6 +284,8 @@ def render_template( If team_for_public_context is provided, this means this is a public page such as a shared dashboard. """ + if context is None: + context = {} template = get_template(template_name) context["opt_out_capture"] = settings.OPT_OUT_CAPTURE @@ -471,7 +473,7 @@ def get_frontend_apps(team_id: int) -> Dict[int, Dict[str, Any]]: for p in plugin_configs: config = p["pluginconfig__config"] or {} config_schema = p["config_schema"] or {} - secret_fields = {field["key"] for field in config_schema if "secret" in field and field["secret"]} + secret_fields = {field["key"] for field in config_schema if field.get("secret")} for key in secret_fields: if key in config: config[key] = "** SECRET FIELD **" diff --git a/posthog/year_in_posthog/2023.html b/posthog/year_in_posthog/2023.html index 54ff75cc4cb06..113ec1730c381 100644 --- a/posthog/year_in_posthog/2023.html +++ b/posthog/year_in_posthog/2023.html @@ -20,27 +20,7 @@