+
)
}
export function ExperimentNext(): JSX.Element {
- const { experimentId, editingExistingExperiment, currentFormStep, props } = useValues(experimentLogic)
+ const { experimentId, editingExistingExperiment } = useValues(experimentLogic)
- const stepComponents = {
- 0:
,
- }
- const CurrentStepComponent = (currentFormStep && stepComponents[currentFormStep]) ||
-
- return (
- <>
- {experimentId === 'new' || editingExistingExperiment ? (
-
- )}
-
- )
+ return experimentId === 'new' || editingExistingExperiment ?
}
diff --git a/frontend/src/scenes/experiments/ExperimentPreview.tsx b/frontend/src/scenes/experiments/ExperimentPreview.tsx
index 3924eb67e5ace..6536294e95a58 100644
--- a/frontend/src/scenes/experiments/ExperimentPreview.tsx
+++ b/frontend/src/scenes/experiments/ExperimentPreview.tsx
@@ -444,7 +444,7 @@ export function MetricDisplay({ filters }: { filters?: FilterType }): JSX.Elemen
.map((event: ActionFilterType, idx: number) => (
-
+
{experimentInsightType === InsightType.FUNNELS ? (event.order || 0) + 1 : idx + 1}
diff --git a/frontend/src/scenes/experiments/ExperimentResult.tsx b/frontend/src/scenes/experiments/ExperimentResult.tsx
index 67633f8dd6f42..c978a5cd8c767 100644
--- a/frontend/src/scenes/experiments/ExperimentResult.tsx
+++ b/frontend/src/scenes/experiments/ExperimentResult.tsx
@@ -3,7 +3,6 @@ import './Experiment.scss'
import { IconInfo } from '@posthog/icons'
import { LemonTable, Tooltip } from '@posthog/lemon-ui'
import { useValues } from 'kea'
-import { getSeriesColor } from 'lib/colors'
import { EntityFilterInfo } from 'lib/components/EntityFilterInfo'
import { FunnelLayout } from 'lib/constants'
import { LemonProgress } from 'lib/lemon-ui/LemonProgress'
@@ -16,10 +15,8 @@ import { ChartDisplayType, FilterType, FunnelVizType, InsightShortId, InsightTyp
import { LoadingState } from './Experiment'
import { experimentLogic } from './experimentLogic'
+import { getExperimentInsightColour } from './utils'
-export function getExperimentInsightColour(variantIndex: number | null): string {
- return variantIndex !== null ? getSeriesColor(variantIndex) : 'var(--muted-3000)'
-}
interface ExperimentResultProps {
secondaryMetricId?: number
}
diff --git a/frontend/src/scenes/experiments/ExperimentView/DistributionTable.tsx b/frontend/src/scenes/experiments/ExperimentView/DistributionTable.tsx
new file mode 100644
index 0000000000000..43b3c50ed614b
--- /dev/null
+++ b/frontend/src/scenes/experiments/ExperimentView/DistributionTable.tsx
@@ -0,0 +1,66 @@
+import '../Experiment.scss'
+
+import { LemonTable, LemonTableColumns, Link } from '@posthog/lemon-ui'
+import { useValues } from 'kea'
+import { getSeriesColor } from 'lib/colors'
+import { capitalizeFirstLetter } from 'lib/utils'
+import { urls } from 'scenes/urls'
+
+import { MultivariateFlagVariant } from '~/types'
+
+import { experimentLogic } from '../experimentLogic'
+
+export function DistributionTable(): JSX.Element {
+ const { experiment } = useValues(experimentLogic)
+
+ const columns: LemonTableColumns = [
+ {
+ className: 'w-1/3',
+ key: 'key',
+ title: 'Variant',
+ render: function Key(_, item, index): JSX.Element {
+ return (
+
+
+
{capitalizeFirstLetter(item.key)}
+
{`${item.rollout_percentage}%`}
+ },
+ },
+ ]
+
+ return (
+
+
+
+
Distribution
+
+
+
+
+
+ Manage distribution
+
+
+
+
+
+
+
+
+ {experimentInsightType === InsightType.FUNNELS ? (event.order || 0) + 1 : idx + 1}
+
+
+
+
+
+
+ {event.properties?.map((prop: AnyPropertyFilter) => (
+
+ ))}
+
+
+ Exposure metric
+
+
+
+
+ {experiment.parameters?.custom_exposure_filter ? (
+
+ ) : (
+ Default via $feature_flag_called events
+ )}
+
+
+
+ Change exposure metric
+
+ {experiment.parameters?.custom_exposure_filter && (
+ updateExperimentExposure(null)}
+ >
+ Reset exposure
+
+ )}
+
+
+
+ )
+}
+
+export function ExperimentGoalModal({ experimentId }: { experimentId: Experiment['id'] }): JSX.Element {
+ const { experiment, isExperimentGoalModalOpen, experimentLoading } = useValues(experimentLogic({ experimentId }))
+ const { closeExperimentGoalModal, updateExperimentGoal, setNewExperimentInsight } = useActions(
+ experimentLogic({ experimentId })
+ )
+
+ return (
+
+
+ Cancel
+
+ {
+ updateExperimentGoal(experiment.filters)
+ }}
+ type="primary"
+ loading={experimentLoading}
+ data-attr="create-annotation-submit"
+ >
+ Save
+
+
+ }
+ >
+
+
+ )
+}
+
+export function ExperimentExposureModal({ experimentId }: { experimentId: Experiment['id'] }): JSX.Element {
+ const { experiment, isExperimentExposureModalOpen, experimentLoading } = useValues(
+ experimentLogic({ experimentId })
+ )
+ const { closeExperimentExposureModal, updateExperimentExposure, setExperimentExposureInsight } = useActions(
+ experimentLogic({ experimentId })
+ )
+
+ return (
+
+
+ Cancel
+
+ {
+ if (experiment.parameters.custom_exposure_filter) {
+ updateExperimentExposure(experiment.parameters.custom_exposure_filter)
+ }
+ }}
+ type="primary"
+ loading={experimentLoading}
+ data-attr="create-annotation-submit"
+ >
+ Save
+
+
+ }
+ >
+
+
+ )
+}
+
+export function Goal(): JSX.Element {
+ const { experiment, experimentId, experimentInsightType, experimentMathAggregationForTrends } =
+ useValues(experimentLogic)
+ const { openExperimentGoalModal } = useActions(experimentLogic({ experimentId }))
+
+ return (
+
+
Experiment goal
+
+ This {experimentInsightType === InsightType.FUNNELS ? 'funnel' : 'trend'}{' '}
+ {experimentInsightType === InsightType.FUNNELS
+ ? 'experiment measures conversion through each step of the user journey.'
+ : 'experiment tracks the performance of a single metric.'}
+
+
+
+
+ {experimentInsightType === InsightType.FUNNELS ? 'Conversion goal steps' : 'Trend goal'}
+
+
+
+ Change experiment goal
+
+
+ {experimentInsightType === InsightType.TRENDS &&
+ !experimentMathAggregationForTrends(experiment.filters) && (
+ <>
+
+
+
+ )}
+
+
+ )
+}
diff --git a/frontend/src/scenes/experiments/ExperimentView/Info.tsx b/frontend/src/scenes/experiments/ExperimentView/Info.tsx
new file mode 100644
index 0000000000000..b11b938860eac
--- /dev/null
+++ b/frontend/src/scenes/experiments/ExperimentView/Info.tsx
@@ -0,0 +1,87 @@
+import '../Experiment.scss'
+
+import { IconWarning } from '@posthog/icons'
+import { Link, ProfilePicture, Tooltip } from '@posthog/lemon-ui'
+import { useValues } from 'kea'
+import { CopyToClipboardInline } from 'lib/components/CopyToClipboard'
+import { TZLabel } from 'lib/components/TZLabel'
+import { IconOpenInNew } from 'lib/lemon-ui/icons'
+import { urls } from 'scenes/urls'
+
+import { ProgressStatus } from '~/types'
+
+import { StatusTag } from '../Experiment'
+import { experimentLogic } from '../experimentLogic'
+import { getExperimentStatus } from '../experimentsLogic'
+import { ResultsTag } from './components'
+
+export function Info(): JSX.Element {
+ const { experiment } = useValues(experimentLogic)
+ const { created_by, created_at } = experiment
+
+ if (!experiment.feature_flag) {
+ return <>
+ }
+
+ return (
+
+
+
+
+ {experiment.feature_flag && (
+
+
+ Feature flag
+
+ {getExperimentStatus(experiment) === ProgressStatus.Running &&
+ !experiment.feature_flag.active && (
+
+
+
+ )}
+
+ {experiment.feature_flag.key}
+
+
+
+
+
+ )}
+
+
+
+
+
+
Created at
+ {created_at &&
}
+
+
+
Created by
+ {created_by &&
}
+
+
+
+
+ )
+}
diff --git a/frontend/src/scenes/experiments/ExperimentView/NoResultsEmptyState.tsx b/frontend/src/scenes/experiments/ExperimentView/NoResultsEmptyState.tsx
new file mode 100644
index 0000000000000..c4c021a3c382e
--- /dev/null
+++ b/frontend/src/scenes/experiments/ExperimentView/NoResultsEmptyState.tsx
@@ -0,0 +1,33 @@
+import '../Experiment.scss'
+
+import { Empty } from 'antd'
+import { useValues } from 'kea'
+
+import { experimentLogic } from '../experimentLogic'
+
+export function NoResultsEmptyState(): JSX.Element {
+ const { experimentResultsLoading, experimentResultCalculationError } = useValues(experimentLogic)
+
+ if (experimentResultsLoading) {
+ return <>
+ }
+
+ return (
+
+
Results
+
+
+
+
There are no experiment results yet
+ {!!experimentResultCalculationError && (
+
{experimentResultCalculationError}
+ )}
+
+ Wait a bit longer for your users to be exposed to the experiment. Double check your feature flag
+ implementation if you're still not seeing results.
+
+
+
+
+ )
+}
diff --git a/frontend/src/scenes/experiments/ExperimentView/Overview.tsx b/frontend/src/scenes/experiments/ExperimentView/Overview.tsx
new file mode 100644
index 0000000000000..76cc2136116d4
--- /dev/null
+++ b/frontend/src/scenes/experiments/ExperimentView/Overview.tsx
@@ -0,0 +1,95 @@
+import '../Experiment.scss'
+
+import { LemonDivider } from '@posthog/lemon-ui'
+import { useValues } from 'kea'
+import { getSeriesColor } from 'lib/colors'
+import { capitalizeFirstLetter } from 'lib/utils'
+
+import { InsightType } from '~/types'
+
+import { experimentLogic } from '../experimentLogic'
+
+export function Overview(): JSX.Element {
+ const {
+ experimentResults,
+ getIndexForVariant,
+ experimentInsightType,
+ sortedConversionRates,
+ highestProbabilityVariant,
+ areResultsSignificant,
+ } = useValues(experimentLogic)
+
+ function SignificanceText(): JSX.Element {
+ return (
+ <>
+
Your results are
+
{`${areResultsSignificant ? 'significant' : 'not significant'}`}.
+
+ )
+ }
+
+ if (experimentInsightType === InsightType.FUNNELS) {
+ const winningVariant = sortedConversionRates[0]
+ const secondBestVariant = sortedConversionRates[1]
+ const difference = winningVariant.conversionRate - secondBestVariant.conversionRate
+
+ return (
+
+
Summary
+
+
+
{capitalizeFirstLetter(winningVariant.key)}
+
is winning with a conversion rate
+
+ increase of {`${difference.toFixed(2)}%`}
+
+
percentage points (vs
+
+
{capitalizeFirstLetter(secondBestVariant.key)}
+
).
+
+
+
+ )
+ }
+
+ const index = getIndexForVariant(experimentResults, highestProbabilityVariant || '')
+ if (highestProbabilityVariant && index !== null && experimentResults) {
+ const { probability } = experimentResults
+
+ return (
+
+
Overview
+
+
+
+
{capitalizeFirstLetter(highestProbabilityVariant)}
+
is winning with a
+
+ {`${(probability[highestProbabilityVariant] * 100).toFixed(2)}% probability`}
+
+
of being best.
+
+
+
+ )
+ }
+
+ return <>
+}
diff --git a/frontend/src/scenes/experiments/ExperimentView/ProgressBar.tsx b/frontend/src/scenes/experiments/ExperimentView/ProgressBar.tsx
new file mode 100644
index 0000000000000..1cedbcf500d6c
--- /dev/null
+++ b/frontend/src/scenes/experiments/ExperimentView/ProgressBar.tsx
@@ -0,0 +1,77 @@
+import '../Experiment.scss'
+
+import { useValues } from 'kea'
+import { dayjs } from 'lib/dayjs'
+import { LemonProgress } from 'lib/lemon-ui/LemonProgress'
+import { humanFriendlyNumber } from 'lib/utils'
+
+import { FunnelStep, InsightType } from '~/types'
+
+import { experimentLogic } from '../experimentLogic'
+
+export function ProgressBar(): JSX.Element {
+ const { experiment, experimentResults, experimentInsightType } = useValues(experimentLogic)
+
+ // Parameters for experiment results
+ // don't use creation variables in results
+ const funnelResultsPersonsTotal =
+ experimentInsightType === InsightType.FUNNELS && experimentResults?.insight
+ ? (experimentResults.insight as FunnelStep[][]).reduce(
+ (sum: number, variantResult: FunnelStep[]) => variantResult[0]?.count + sum,
+ 0
+ )
+ : 0
+
+ const experimentProgressPercent =
+ experimentInsightType === InsightType.FUNNELS
+ ? ((funnelResultsPersonsTotal || 0) / (experiment?.parameters?.recommended_sample_size || 1)) * 100
+ : (dayjs().diff(experiment?.start_date, 'day') / (experiment?.parameters?.recommended_running_time || 1)) *
+ 100
+
+ return (
+
+
{`${
+ experimentProgressPercent > 100 ? 100 : experimentProgressPercent.toFixed(2)
+ }% complete`}
+
+ {experimentInsightType === InsightType.TRENDS && experiment.start_date && (
+
+ {experiment.end_date ? (
+
+ Ran for {dayjs(experiment.end_date).diff(experiment.start_date, 'day')} days
+
+ ) : (
+
+ {dayjs().diff(experiment.start_date, 'day')} days running
+
+ )}
+
+ Goal: {experiment?.parameters?.recommended_running_time ?? 'Unknown'} days
+
+
+ )}
+ {experimentInsightType === InsightType.FUNNELS && (
+
+ {experiment.end_date ? (
+
+ Saw {humanFriendlyNumber(funnelResultsPersonsTotal)} participants
+
+ ) : (
+
+ {humanFriendlyNumber(funnelResultsPersonsTotal)} participants seen
+
+ )}
+
+ Goal: {humanFriendlyNumber(experiment?.parameters?.recommended_sample_size || 0)}{' '}
+ participants
+
+
+ )}
+
+ )
+}
diff --git a/frontend/src/scenes/experiments/ExperimentView/ReleaseConditionsTable.tsx b/frontend/src/scenes/experiments/ExperimentView/ReleaseConditionsTable.tsx
new file mode 100644
index 0000000000000..c0a4024e559f6
--- /dev/null
+++ b/frontend/src/scenes/experiments/ExperimentView/ReleaseConditionsTable.tsx
@@ -0,0 +1,77 @@
+import '../Experiment.scss'
+
+import { LemonTable, LemonTableColumns, LemonTag, Link } from '@posthog/lemon-ui'
+import { useValues } from 'kea'
+import { urls } from 'scenes/urls'
+
+import { groupsModel } from '~/models/groupsModel'
+import { FeatureFlagGroupType } from '~/types'
+
+import { experimentLogic } from '../experimentLogic'
+
+export function ReleaseConditionsTable(): JSX.Element {
+ const { experiment } = useValues(experimentLogic)
+ const { aggregationLabel } = useValues(groupsModel)
+
+ const columns: LemonTableColumns
= [
+ {
+ key: 'key',
+ title: '',
+ render: function Key(_, _item, index): JSX.Element {
+ return {`Set ${index + 1}`}
+ },
+ },
+ {
+ key: 'rollout_percentage',
+ title: 'Rollout',
+ render: function Key(_, item): JSX.Element {
+ const aggregationTargetName =
+ experiment.filters.aggregation_group_type_index != null
+ ? aggregationLabel(experiment.filters.aggregation_group_type_index).plural
+ : 'users'
+
+ const releaseText = `${item.rollout_percentage}% of ${aggregationTargetName}`
+
+ return (
+
+ {releaseText.startsWith('100% of') ? (
+ {releaseText}
+ ) : (
+ releaseText
+ )}
+
+ )
+ },
+ },
+ {
+ key: 'variant',
+ title: 'Override',
+ render: function Key(_, item): JSX.Element {
+ return {item.variant || '--'}
+ },
+ },
+ ]
+
+ return (
+
+
+
+
Release conditions
+
+
+
+
+
+ Manage release conditions
+
+
+
+
+
+
+
+
{capitalizeFirstLetter(item.variant)}
+
+ {item.results?.[idx].result ? (
+ item.results[idx].insightType === InsightType.FUNNELS ? (
+ <>{((item.results[idx].result as number) * 100).toFixed(1)}%
+ ) : (
+ <>{humanFriendlyNumber(item.results[idx].result as number)}
+ )
+ ) : (
+ <>--
+ )}
+
+ )
+ },
+ })
+ })
+
+ return (
+ <>
+
+ {existingModalSecondaryMetric && (
+ deleteMetric(metricIdx)}
+ >
+ Delete
+
+ )}
+
+
+ Cancel
+
+
+ {existingModalSecondaryMetric ? 'Save' : 'Create'}
+
+
+
+ }
+ >
+
+
+
+
+
+
Secondary metrics
+ {metrics.length > 0 && (
+
Click a metric name to compare variants on a graph.
+ )}
+
+
+
+
+ {metrics && metrics.length > 0 && metrics.length < 3 && isExperimentRunning && (
+
+
+ Add metric
+
+
+ )}
+
+
+
+ {metrics && metrics.length > 0 ? (
+
+ ) : (
+
+
+
+
+ Add up to 3 secondary metrics to gauge side effects of your experiment.
+
+
}
+ type="secondary"
+ size="small"
+ onClick={openModalToCreateSecondaryMetric}
+ >
+ Add metric
+
+
+
+ )}
+
+
+
{capitalizeFirstLetter(item.key)}
+
+ {experimentResults.insight?.[0] && 'action' in experimentResults.insight[0] && (
+
+ )}
+
+ {experimentMathAggregationForTrends(experimentResults?.filters) ? 'metric' : 'count'}
+
+
+ ),
+ render: function Key(_, item, index): JSX.Element {
+ return (
+
+ {countDataForVariant(experimentResults, item.key)}{' '}
+ {areTrendResultsConfusing && index === 0 && (
+
+
+
+ )}
+
+ )
+ },
+ })
+ columns.push({
+ key: 'exposure',
+ title: 'Exposure',
+ render: function Key(_, item): JSX.Element {
+ return {exposureCountDataForVariant(experimentResults, item.key)}
+ },
+ })
+ }
+
+ if (experimentInsightType === InsightType.FUNNELS) {
+ columns.push({
+ key: 'conversionRate',
+ title: 'Conversion rate',
+ render: function Key(_, item): JSX.Element {
+ const isWinning = item.key === sortedConversionRates[0].key
+ return (
+ {`${conversionRateForVariant(
+ experimentResults,
+ item.key
+ )}%`}
+ )
+ },
+ })
+ }
+
+ columns.push({
+ key: 'winProbability',
+ title: 'Win probability',
+ render: function Key(_, item): JSX.Element {
+ const percentage =
+ experimentResults?.probability?.[item.key] != undefined &&
+ experimentResults.probability?.[item.key] * 100
+
+ return (
+ <>
+ {percentage ? (
+
+
+ {percentage.toFixed(2)}%
+
+ ) : (
+ '--'
+ )}
+
+ )
+ },
+ })
+
+ return (
+
+
+
+ )
+}
diff --git a/frontend/src/scenes/experiments/ExperimentView/components.tsx b/frontend/src/scenes/experiments/ExperimentView/components.tsx
new file mode 100644
index 0000000000000..1a22957925e68
--- /dev/null
+++ b/frontend/src/scenes/experiments/ExperimentView/components.tsx
@@ -0,0 +1,148 @@
+import '../Experiment.scss'
+
+import { LemonButton, LemonDivider, LemonTable, LemonTag, LemonTagType } from '@posthog/lemon-ui'
+import { useActions, useValues } from 'kea'
+import { AnimationType } from 'lib/animations/animations'
+import { Animation } from 'lib/components/Animation/Animation'
+import { PageHeader } from 'lib/components/PageHeader'
+import { dayjs } from 'lib/dayjs'
+import { More } from 'lib/lemon-ui/LemonButton/More'
+import { useEffect, useState } from 'react'
+
+import { ResetButton } from '../Experiment'
+import { experimentLogic } from '../experimentLogic'
+
+export function ResultsTag(): JSX.Element {
+ const { areResultsSignificant } = useValues(experimentLogic)
+ const result: { color: LemonTagType; label: string } = areResultsSignificant
+ ? { color: 'success', label: 'Significant' }
+ : { color: 'primary', label: 'Not significant' }
+
+ return (
+
+ {result.label}
+
+ )
+}
+
+export function ExperimentLoader(): JSX.Element {
+ return (
+
+ )
+}
+
+export function ExperimentLoadingAnimation(): JSX.Element {
+ function EllipsisAnimation(): JSX.Element {
+ const [ellipsis, setEllipsis] = useState('.')
+
+ useEffect(() => {
+ let count = 1
+ let direction = 1
+
+ const interval = setInterval(() => {
+ setEllipsis('.'.repeat(count))
+ count += direction
+
+ if (count === 3 || count === 1) {
+ direction *= -1
+ }
+ }, 300)
+
+ return () => clearInterval(interval)
+ }, [])
+
+ return {ellipsis}
+ }
+
+ return (
+
+
+
+ Fetching experiment results
+
+
+
+ setEditExperiment(true)}>
+ Edit
+
+ launchExperiment()}>
+ Launch
+
+
+ )}
+ {experiment && isExperimentRunning && (
+
+ <>
+
+ loadExperimentResults(true)}
+ fullWidth
+ data-attr="refresh-experiment"
+ >
+ Refresh experiment results
+
+ loadSecondaryMetricResults(true)}
+ fullWidth
+ data-attr="refresh-secondary-metrics"
+ >
+ Refresh secondary metrics
+
+
+ }
+ />
+
+
+
+ {!experiment.end_date && (
+ endExperiment()}>
+ Stop
+
+ )}
+ {experiment?.end_date &&
+ dayjs().isSameOrAfter(dayjs(experiment.end_date), 'day') &&
+ !experiment.archived && (
+ archiveExperiment()}>
+ Archive
+
+ )}
+
+ )}
+
+ }
+ />
+ )
+}
diff --git a/frontend/src/scenes/experiments/MetricSelector.tsx b/frontend/src/scenes/experiments/MetricSelector.tsx
index 4df25546fe8a9..fbfcd0617d61c 100644
--- a/frontend/src/scenes/experiments/MetricSelector.tsx
+++ b/frontend/src/scenes/experiments/MetricSelector.tsx
@@ -4,6 +4,7 @@ import { IconInfo } from '@posthog/icons'
import { LemonSelect } from '@posthog/lemon-ui'
import { BindLogic, useActions, useValues } from 'kea'
import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types'
+import { EXPERIMENT_DEFAULT_DURATION } from 'lib/constants'
import { LemonBanner } from 'lib/lemon-ui/LemonBanner'
import { Tooltip } from 'lib/lemon-ui/Tooltip'
import { Attribution } from 'scenes/insights/EditorFilters/AttributionFilter'
@@ -23,8 +24,6 @@ import { Query } from '~/queries/Query/Query'
import { FunnelsQuery, InsightQueryNode, TrendsQuery } from '~/queries/schema'
import { EditorFilterProps, FilterType, InsightLogicProps, InsightShortId, InsightType } from '~/types'
-import { DEFAULT_DURATION } from './experimentLogic'
-
export interface MetricSelectorProps {
dashboardItemId: InsightShortId
setPreviewInsight: (filters?: Partial) => void
@@ -75,8 +74,8 @@ export function MetricSelector({
{showDateRangeBanner && (
- Preview insights are generated based on {DEFAULT_DURATION} days of data. This can cause a mismatch
- between the preview and the actual results.
+ Preview insights are generated based on {EXPERIMENT_DEFAULT_DURATION} days of data. This can cause a
+ mismatch between the preview and the actual results.
)}
diff --git a/frontend/src/scenes/experiments/SecondaryMetricsTable.tsx b/frontend/src/scenes/experiments/SecondaryMetricsTable.tsx
index fbaae05233582..c15b4f8293a3a 100644
--- a/frontend/src/scenes/experiments/SecondaryMetricsTable.tsx
+++ b/frontend/src/scenes/experiments/SecondaryMetricsTable.tsx
@@ -16,9 +16,9 @@ import { InsightType } from '~/types'
import { SECONDARY_METRIC_INSIGHT_ID } from './constants'
import { experimentLogic, TabularSecondaryMetricResults } from './experimentLogic'
-import { getExperimentInsightColour } from './ExperimentResult'
import { MetricSelector } from './MetricSelector'
import { secondaryMetricsLogic, SecondaryMetricsProps } from './secondaryMetricsLogic'
+import { getExperimentInsightColour } from './utils'
export function SecondaryMetricsTable({
onMetricsChange,
diff --git a/frontend/src/scenes/experiments/experimentLogic.tsx b/frontend/src/scenes/experiments/experimentLogic.tsx
index 35e617d41470e..d2154c637a2a6 100644
--- a/frontend/src/scenes/experiments/experimentLogic.tsx
+++ b/frontend/src/scenes/experiments/experimentLogic.tsx
@@ -4,7 +4,7 @@ import { forms } from 'kea-forms'
import { loaders } from 'kea-loaders'
import { router, urlToAction } from 'kea-router'
import api from 'lib/api'
-import { FunnelLayout } from 'lib/constants'
+import { EXPERIMENT_DEFAULT_DURATION, FunnelLayout } from 'lib/constants'
import { dayjs } from 'lib/dayjs'
import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'
import { Tooltip } from 'lib/lemon-ui/Tooltip'
@@ -23,7 +23,7 @@ import { urls } from 'scenes/urls'
import { groupsModel } from '~/models/groupsModel'
import { filtersToQueryNode } from '~/queries/nodes/InsightQuery/utils/filtersToQueryNode'
import { queryNodeToFilter } from '~/queries/nodes/InsightQuery/utils/queryNodeToFilter'
-import { InsightVizNode } from '~/queries/schema'
+import { FunnelsQuery, InsightVizNode, TrendsQuery } from '~/queries/schema'
import {
ActionFilter as ActionFilterType,
Breadcrumb,
@@ -47,8 +47,6 @@ import { EXPERIMENT_EXPOSURE_INSIGHT_ID, EXPERIMENT_INSIGHT_ID } from './constan
import type { experimentLogicType } from './experimentLogicType'
import { experimentsLogic } from './experimentsLogic'
-export const DEFAULT_DURATION = 14 // days
-
const NEW_EXPERIMENT: Experiment = {
id: 'new',
name: '',
@@ -358,7 +356,7 @@ export const experimentLogic = kea([
newInsightFilters = cleanFilters({
insight: InsightType.FUNNELS,
funnel_viz_type: FunnelVizType.Steps,
- date_from: dayjs().subtract(DEFAULT_DURATION, 'day').format('YYYY-MM-DDTHH:mm'),
+ date_from: dayjs().subtract(EXPERIMENT_DEFAULT_DURATION, 'day').format('YYYY-MM-DDTHH:mm'),
date_to: dayjs().endOf('d').format('YYYY-MM-DDTHH:mm'),
layout: FunnelLayout.horizontal,
aggregation_group_type_index: aggregationGroupTypeIndex,
@@ -375,14 +373,23 @@ export const experimentLogic = kea([
: { events: [{ ...getDefaultEvent(), ...groupAggregation }] }
newInsightFilters = cleanFilters({
insight: InsightType.TRENDS,
- date_from: dayjs().subtract(DEFAULT_DURATION, 'day').format('YYYY-MM-DDTHH:mm'),
+ date_from: dayjs().subtract(EXPERIMENT_DEFAULT_DURATION, 'day').format('YYYY-MM-DDTHH:mm'),
date_to: dayjs().endOf('d').format('YYYY-MM-DDTHH:mm'),
...eventAddition,
...filters,
})
}
- actions.updateQuerySource(filtersToQueryNode(newInsightFilters))
+ // This allows switching between insight types. It's necessary as `updateQuerySource` merges
+ // the new query with any existing query and that causes validation problems when there are
+ // unsupported properties in the now merged query.
+ const newQuery = filtersToQueryNode(newInsightFilters)
+ if (filters?.insight === InsightType.FUNNELS) {
+ ;(newQuery as TrendsQuery).trendsFilter = undefined
+ } else {
+ ;(newQuery as FunnelsQuery).funnelsFilter = undefined
+ }
+ actions.updateQuerySource(newQuery)
},
// sync form value `filters` with query
setQuery: ({ query }) => {
@@ -391,7 +398,7 @@ export const experimentLogic = kea([
setExperimentExposureInsight: async ({ filters }) => {
const newInsightFilters = cleanFilters({
insight: InsightType.TRENDS,
- date_from: dayjs().subtract(DEFAULT_DURATION, 'day').format('YYYY-MM-DDTHH:mm'),
+ date_from: dayjs().subtract(EXPERIMENT_DEFAULT_DURATION, 'day').format('YYYY-MM-DDTHH:mm'),
date_to: dayjs().endOf('d').format('YYYY-MM-DDTHH:mm'),
...filters,
})
@@ -672,6 +679,16 @@ export const experimentLogic = kea([
return !!experiment?.start_date
},
],
+ isExperimentStopped: [
+ (s) => [s.experiment],
+ (experiment): boolean => {
+ return (
+ !!experiment?.end_date &&
+ dayjs().isSameOrAfter(dayjs(experiment.end_date), 'day') &&
+ !experiment.archived
+ )
+ },
+ ],
breadcrumbs: [
(s) => [s.experiment, s.experimentId],
(experiment, experimentId): Breadcrumb[] => [
@@ -801,7 +818,11 @@ export const experimentLogic = kea([
return parseFloat(
(
4 /
- Math.pow(Math.sqrt(lambda1 / DEFAULT_DURATION) - Math.sqrt(lambda2 / DEFAULT_DURATION), 2)
+ Math.pow(
+ Math.sqrt(lambda1 / EXPERIMENT_DEFAULT_DURATION) -
+ Math.sqrt(lambda2 / EXPERIMENT_DEFAULT_DURATION),
+ 2
+ )
).toFixed(1)
)
},
@@ -809,7 +830,7 @@ export const experimentLogic = kea([
expectedRunningTime: [
() => [],
() =>
- (entrants: number, sampleSize: number, duration: number = DEFAULT_DURATION): number => {
+ (entrants: number, sampleSize: number, duration: number = EXPERIMENT_DEFAULT_DURATION): number => {
// recommended people / (actual people / day) = expected days
return parseFloat((sampleSize / (entrants / duration)).toFixed(1))
},
@@ -1014,13 +1035,29 @@ export const experimentLogic = kea([
return variantsWithResults
},
],
+ sortedConversionRates: [
+ (s) => [s.experimentResults, s.variants, s.conversionRateForVariant],
+ (
+ experimentResults: any,
+ variants: any,
+ conversionRateForVariant: any
+ ): { key: string; conversionRate: number; index: number }[] => {
+ const conversionRates = []
+ for (let index = 0; index < variants.length; index++) {
+ const variant = variants[index].key
+ const conversionRate = parseFloat(conversionRateForVariant(experimentResults, variant))
+ conversionRates.push({ key: variant, conversionRate, index })
+ }
+ return conversionRates.sort((a, b) => b.conversionRate - a.conversionRate)
+ },
+ ],
}),
forms(({ actions, values }) => ({
experiment: {
options: { showErrorsOnTouch: true },
defaults: { ...NEW_EXPERIMENT } as Experiment,
errors: ({ name, feature_flag_key, parameters }) => ({
- name: !name && 'You have to enter a name.',
+ name: !name && 'Please enter a name',
feature_flag_key: validateFeatureFlagKey(feature_flag_key),
parameters: {
feature_flag_variants: parameters.feature_flag_variants?.map(({ key }) => ({
diff --git a/frontend/src/scenes/experiments/secondaryMetricsLogic.ts b/frontend/src/scenes/experiments/secondaryMetricsLogic.ts
index d3b04d4a29c38..a12bc0f4a7547 100644
--- a/frontend/src/scenes/experiments/secondaryMetricsLogic.ts
+++ b/frontend/src/scenes/experiments/secondaryMetricsLogic.ts
@@ -10,7 +10,7 @@ import { teamLogic } from 'scenes/teamLogic'
import { filtersToQueryNode } from '~/queries/nodes/InsightQuery/utils/filtersToQueryNode'
import { queryNodeToFilter } from '~/queries/nodes/InsightQuery/utils/queryNodeToFilter'
-import { InsightVizNode } from '~/queries/schema'
+import { FunnelsQuery, InsightVizNode, TrendsQuery } from '~/queries/schema'
import { Experiment, FilterType, FunnelVizType, InsightType, SecondaryExperimentMetric } from '~/types'
import { SECONDARY_METRIC_INSIGHT_ID } from './constants'
@@ -162,7 +162,16 @@ export const secondaryMetricsLogic = kea([
})
}
- actions.updateQuerySource(filtersToQueryNode(newInsightFilters))
+ // This allows switching between insight types. It's necessary as `updateQuerySource` merges
+ // the new query with any existing query and that causes validation problems when there are
+ // unsupported properties in the now merged query.
+ const newQuery = filtersToQueryNode(newInsightFilters)
+ if (filters?.insight === InsightType.FUNNELS) {
+ ;(newQuery as TrendsQuery).trendsFilter = undefined
+ } else {
+ ;(newQuery as FunnelsQuery).funnelsFilter = undefined
+ }
+ actions.updateQuerySource(newQuery)
},
// sync form value `filters` with query
setQuery: ({ query }) => {
diff --git a/frontend/src/scenes/experiments/utils.ts b/frontend/src/scenes/experiments/utils.ts
new file mode 100644
index 0000000000000..90d7b2c64f44b
--- /dev/null
+++ b/frontend/src/scenes/experiments/utils.ts
@@ -0,0 +1,19 @@
+import { getSeriesColor } from 'lib/colors'
+import { FunnelLayout } from 'lib/constants'
+
+import { ChartDisplayType, FilterType, FunnelVizType, InsightType } from '~/types'
+
+export function getExperimentInsightColour(variantIndex: number | null): string {
+ return variantIndex !== null ? getSeriesColor(variantIndex) : 'var(--muted-3000)'
+}
+
+export const transformResultFilters = (filters: Partial): Partial => ({
+ ...filters,
+ ...(filters.insight === InsightType.FUNNELS && {
+ layout: FunnelLayout.vertical,
+ funnel_viz_type: FunnelVizType.Steps,
+ }),
+ ...(filters.insight === InsightType.TRENDS && {
+ display: ChartDisplayType.ActionsLineGraphCumulative,
+ }),
+})
diff --git a/frontend/src/scenes/feature-flags/FeatureFlag.tsx b/frontend/src/scenes/feature-flags/FeatureFlag.tsx
index 0905db1420fa4..f99e578887799 100644
--- a/frontend/src/scenes/feature-flags/FeatureFlag.tsx
+++ b/frontend/src/scenes/feature-flags/FeatureFlag.tsx
@@ -174,17 +174,13 @@ export function FeatureFlag({ id }: { id?: string } = {}): JSX.Element {
key: FeatureFlagsTab.USAGE,
content: ,
})
- }
- if (featureFlags[FEATURE_FLAGS.MULTI_PROJECT_FEATURE_FLAGS]) {
tabs.push({
label: 'Projects',
key: FeatureFlagsTab.PROJECTS,
content: ,
})
- }
- if (featureFlags[FEATURE_FLAGS.SCHEDULED_CHANGES_FEATURE_FLAGS]) {
tabs.push({
label: 'Schedule',
key: FeatureFlagsTab.SCHEDULE,
@@ -220,7 +216,7 @@ export function FeatureFlag({ id }: { id?: string } = {}): JSX.Element {
})
}
- if (featureFlags[FEATURE_FLAGS.ROLE_BASED_ACCESS] && featureFlag.can_edit) {
+ if (featureFlag.can_edit) {
tabs.push({
label: 'Permissions',
key: FeatureFlagsTab.PERMISSIONS,
@@ -431,29 +427,27 @@ export function FeatureFlag({ id }: { id?: string } = {}): JSX.Element {
{featureFlags[FEATURE_FLAGS.AUTO_ROLLBACK_FEATURE_FLAGS] && (
)}
- {featureFlags[FEATURE_FLAGS.ROLE_BASED_ACCESS] && (
-
-
Permissions
-
-
-
- setRolesToAdd(roleIds)}
- rolesToAdd={rolesToAdd}
- addableRoles={addableRoles}
- addableRolesLoading={unfilteredAddableRolesLoading}
- onAdd={() => addAssociatedRoles()}
- roles={derivedRoles}
- deleteAssociatedRole={(id) =>
- deleteAssociatedRole({ roleId: id })
- }
- canEdit={featureFlag.can_edit}
- />
-
-
+
+
Permissions
+
+
+
+ setRolesToAdd(roleIds)}
+ rolesToAdd={rolesToAdd}
+ addableRoles={addableRoles}
+ addableRolesLoading={unfilteredAddableRolesLoading}
+ onAdd={() => addAssociatedRoles()}
+ roles={derivedRoles}
+ deleteAssociatedRole={(id) =>
+ deleteAssociatedRole({ roleId: id })
+ }
+ canEdit={featureFlag.can_edit}
+ />
+
- )}
+
)}
@@ -572,15 +566,13 @@ export function FeatureFlag({ id }: { id?: string } = {}): JSX.Element {
{
editFeatureFlag(true)
}}
- disabled={!featureFlag.can_edit}
>
Edit
diff --git a/frontend/src/scenes/feature-flags/FeatureFlagReleaseConditions.tsx b/frontend/src/scenes/feature-flags/FeatureFlagReleaseConditions.tsx
index 389e0e2e4f271..75e3b9a47a6e6 100644
--- a/frontend/src/scenes/feature-flags/FeatureFlagReleaseConditions.tsx
+++ b/frontend/src/scenes/feature-flags/FeatureFlagReleaseConditions.tsx
@@ -8,7 +8,7 @@ import { router } from 'kea-router'
import { allOperatorsToHumanName } from 'lib/components/DefinitionPopover/utils'
import { PropertyFilters } from 'lib/components/PropertyFilters/PropertyFilters'
import { isPropertyFilterWithOperator } from 'lib/components/PropertyFilters/utils'
-import { FEATURE_FLAGS, INSTANTLY_AVAILABLE_PROPERTIES } from 'lib/constants'
+import { INSTANTLY_AVAILABLE_PROPERTIES } from 'lib/constants'
import { groupsAccessLogic, GroupsAccessStatus } from 'lib/introductions/groupsAccessLogic'
import { GroupsIntroductionOption } from 'lib/introductions/GroupsIntroductionOption'
import { IconErrorOutline, IconOpenInNew, IconSubArrowRight } from 'lib/lemon-ui/icons'
@@ -60,7 +60,6 @@ export function FeatureFlagReleaseConditions({
affectedUsers,
totalUsers,
filtersTaxonomicOptions,
- enabledFeatures,
aggregationTargetName,
} = useValues(releaseConditionsLogic)
@@ -233,7 +232,7 @@ export function FeatureFlagReleaseConditions({
taxonomicFilterOptionsFromProp={filtersTaxonomicOptions}
hasRowOperator={false}
sendAllKeyUpdates
- allowRelativeDateOptions={!!enabledFeatures[FEATURE_FLAGS.NEW_FEATURE_FLAG_OPERATORS]}
+ allowRelativeDateOptions
errorMessages={
propertySelectErrors?.[index]?.properties?.some((message) => !!message.value)
? propertySelectErrors[index].properties?.map((message, index) => {
diff --git a/frontend/src/scenes/feature-flags/FeatureFlagReleaseConditionsLogic.ts b/frontend/src/scenes/feature-flags/FeatureFlagReleaseConditionsLogic.ts
index f0007c648cc28..dab0d6f408993 100644
--- a/frontend/src/scenes/feature-flags/FeatureFlagReleaseConditionsLogic.ts
+++ b/frontend/src/scenes/feature-flags/FeatureFlagReleaseConditionsLogic.ts
@@ -2,8 +2,6 @@ import { actions, afterMount, connect, kea, key, listeners, path, props, propsCh
import { subscriptions } from 'kea-subscriptions'
import api from 'lib/api'
import { TaxonomicFilterGroupType, TaxonomicFilterProps } from 'lib/components/TaxonomicFilter/types'
-import { FEATURE_FLAGS } from 'lib/constants'
-import { featureFlagLogic as enabledFeaturesLogic } from 'lib/logic/featureFlagLogic'
import { objectsEqual } from 'lib/utils'
import { groupsModel } from '~/models/groupsModel'
@@ -32,14 +30,7 @@ export const featureFlagReleaseConditionsLogic = kea
id ?? 'unknown'),
connect({
- values: [
- teamLogic,
- ['currentTeamId'],
- groupsModel,
- ['groupTypes', 'aggregationLabel'],
- enabledFeaturesLogic,
- ['featureFlags as enabledFeatures'],
- ],
+ values: [teamLogic, ['currentTeamId'], groupsModel, ['groupTypes', 'aggregationLabel']],
}),
actions({
setFilters: (filters: FeatureFlagFilters) => ({ filters }),
@@ -210,35 +201,28 @@ export const featureFlagReleaseConditionsLogic = kea [s.filters, s.groupTypes, s.enabledFeatures],
- (filters, groupTypes, enabledFeatures): TaxonomicFilterGroupType[] => {
- const baseGroupTypes = []
- const additionalGroupTypes = []
- const newFlagOperatorsEnabled = enabledFeatures[FEATURE_FLAGS.NEW_FEATURE_FLAG_OPERATORS]
+ (s) => [s.filters, s.groupTypes],
+ (filters, groupTypes): TaxonomicFilterGroupType[] => {
+ const targetGroupTypes = []
const targetGroup =
filters?.aggregation_group_type_index != null
? groupTypes.get(filters.aggregation_group_type_index as GroupTypeIndex)
: undefined
if (targetGroup) {
- baseGroupTypes.push(
+ targetGroupTypes.push(
`${TaxonomicFilterGroupType.GroupsPrefix}_${targetGroup?.group_type_index}` as unknown as TaxonomicFilterGroupType
)
- if (newFlagOperatorsEnabled) {
- additionalGroupTypes.push(
- `${TaxonomicFilterGroupType.GroupNamesPrefix}_${filters.aggregation_group_type_index}` as unknown as TaxonomicFilterGroupType
- )
- }
+ targetGroupTypes.push(
+ `${TaxonomicFilterGroupType.GroupNamesPrefix}_${filters.aggregation_group_type_index}` as unknown as TaxonomicFilterGroupType
+ )
} else {
- baseGroupTypes.push(TaxonomicFilterGroupType.PersonProperties)
- baseGroupTypes.push(TaxonomicFilterGroupType.Cohorts)
-
- if (newFlagOperatorsEnabled) {
- additionalGroupTypes.push(TaxonomicFilterGroupType.Metadata)
- }
+ targetGroupTypes.push(TaxonomicFilterGroupType.PersonProperties)
+ targetGroupTypes.push(TaxonomicFilterGroupType.Cohorts)
+ targetGroupTypes.push(TaxonomicFilterGroupType.Metadata)
}
- return [...baseGroupTypes, ...additionalGroupTypes]
+ return targetGroupTypes
},
],
aggregationTargetName: [
diff --git a/frontend/src/scenes/feature-flags/featureFlagLogic.ts b/frontend/src/scenes/feature-flags/featureFlagLogic.ts
index 7cca0a0924172..c4e6842aff7e5 100644
--- a/frontend/src/scenes/feature-flags/featureFlagLogic.ts
+++ b/frontend/src/scenes/feature-flags/featureFlagLogic.ts
@@ -110,7 +110,7 @@ const EMPTY_MULTIVARIATE_OPTIONS: MultivariateFlagOptions = {
/** Check whether a string is a valid feature flag key. If not, a reason string is returned - otherwise undefined. */
export function validateFeatureFlagKey(key: string): string | undefined {
return !key
- ? 'You need to set a key'
+ ? 'Please set a key'
: !key.match?.(/^([A-z]|[a-z]|[0-9]|-|_)+$/)
? 'Only letters, numbers, hyphens (-) & underscores (_) are allowed.'
: undefined
diff --git a/frontend/src/scenes/insights/EditorFilters/PathsEventTypes.tsx b/frontend/src/scenes/insights/EditorFilters/PathsEventTypes.tsx
index 71fe12ce8d68d..48bbd016595ee 100644
--- a/frontend/src/scenes/insights/EditorFilters/PathsEventTypes.tsx
+++ b/frontend/src/scenes/insights/EditorFilters/PathsEventTypes.tsx
@@ -60,7 +60,7 @@ export function PathsEventsTypes({ insightProps }: EditorFilterProps): JSX.Eleme
fullWidth
type="secondary"
dropdown={{
- sameWidth: true,
+ matchWidth: true,
closeOnClickInside: false,
overlay: options.map((option) => (
{
const groupType = taxonomicFilterGroupTypeToEntityType(taxonomicGroupType)
if (groupType === EntityTypes.DATA_WAREHOUSE) {
diff --git a/frontend/src/scenes/insights/views/BoldNumber/BoldNumber.tsx b/frontend/src/scenes/insights/views/BoldNumber/BoldNumber.tsx
index 3642648298642..f18fbeebfefb1 100644
--- a/frontend/src/scenes/insights/views/BoldNumber/BoldNumber.tsx
+++ b/frontend/src/scenes/insights/views/BoldNumber/BoldNumber.tsx
@@ -120,6 +120,10 @@ export function BoldNumber({ showPersonsModal = true }: ChartParams): JSX.Elemen
kind: NodeKind.InsightActorsQuery,
source: query.source,
},
+ additionalSelect: {
+ value_at_data_point: 'event_count',
+ matched_recordings: 'matched_recordings',
+ },
})
} else if (resultSeries.persons?.url) {
openPersonsModal({
@@ -213,6 +217,10 @@ function BoldNumberComparison({ showPersonsModal }: Pick
setVisible(false)}
diff --git a/frontend/src/scenes/insights/views/InsightsTable/InsightsTable.tsx b/frontend/src/scenes/insights/views/InsightsTable/InsightsTable.tsx
index d426ca87f525c..c6d037c3d1792 100644
--- a/frontend/src/scenes/insights/views/InsightsTable/InsightsTable.tsx
+++ b/frontend/src/scenes/insights/views/InsightsTable/InsightsTable.tsx
@@ -3,6 +3,7 @@ import './InsightsTable.scss'
import { useActions, useValues } from 'kea'
import { getSeriesColor } from 'lib/colors'
import { LemonTable, LemonTableColumn } from 'lib/lemon-ui/LemonTable'
+import { compare as compareFn } from 'natural-orderby'
import { insightLogic } from 'scenes/insights/insightLogic'
import { insightSceneLogic } from 'scenes/insights/insightSceneLogic'
import { isTrendsFilter } from 'scenes/insights/sharedUtils'
@@ -157,7 +158,7 @@ export function InsightsTable({
}
const labelA = formatItemBreakdownLabel(a)
const labelB = formatItemBreakdownLabel(b)
- return labelA.localeCompare(labelB)
+ return compareFn()(labelA, labelB)
},
})
if (isTrends && display === ChartDisplayType.WorldMap) {
diff --git a/frontend/src/scenes/notebooks/NotebookSelectButton/NotebookSelectButton.tsx b/frontend/src/scenes/notebooks/NotebookSelectButton/NotebookSelectButton.tsx
index 50358e1f18537..aab8e816e5db5 100644
--- a/frontend/src/scenes/notebooks/NotebookSelectButton/NotebookSelectButton.tsx
+++ b/frontend/src/scenes/notebooks/NotebookSelectButton/NotebookSelectButton.tsx
@@ -212,7 +212,7 @@ export function NotebookSelectPopover({
{displayTitle}
@@ -45,6 +43,17 @@ export function SessionRecordingErrors(): JSX.Element {
},
width: '50%',
},
+ {
+ title: '',
+ render: (_, cluster) => {
+ return (
+
+ )
+ },
+ },
{
title: 'Occurrences',
dataIndex: 'occurrences',
@@ -68,23 +77,41 @@ export function SessionRecordingErrors(): JSX.Element {
title: 'Actions',
render: function Render(_, cluster) {
return (
-
{
- e.preventDefault()
- openSessionPlayer({ id: cluster.sample.session_id })
- }}
- className="p-2 whitespace-nowrap"
- type="primary"
- >
- Watch example
-
+
+ {
+ e.preventDefault()
+ openSessionPlayer({ id: cluster.session_ids[0] })
+ }}
+ className="whitespace-nowrap"
+ type="primary"
+ >
+ Watch example
+
+ {
+ createPlaylist(
+ `Examples of '${parseTitle(cluster.sample)}'`,
+ cluster.session_ids
+ )
+ }}
+ className="whitespace-nowrap"
+ type="secondary"
+ tooltip="Create a playlist of recordings containing this issue"
+ >
+ Create playlist
+
+
)
},
},
]}
- dataSource={errors}
- expandable={{ expandedRowRender: (cluster) =>
}}
+ loading={errorsLoading}
+ dataSource={errors || []}
+ expandable={{
+ expandedRowRender: (cluster) =>
,
+ }}
/>
@@ -136,5 +163,20 @@ function parseTitle(error: string): string {
input = error
}
- return input.split('\n')[0].trim().substring(0, MAX_TITLE_LENGTH)
+ if (!input) {
+ return error
+ }
+
+ try {
+ // TRICKY - after json parsing we might not have a string,
+ // since the JSON parser will helpfully convert to other types too e.g. have seen objects here
+ if (typeof input !== 'string') {
+ input = JSON.stringify(input)
+ }
+
+ return input.split('\n')[0].trim().substring(0, MAX_TITLE_LENGTH) || error
+ } catch (e) {
+ captureException(e, { extra: { error }, tags: { feature: 'replay/error-clustering' } })
+ return error
+ }
}
diff --git a/frontend/src/scenes/session-recordings/errors/sessionRecordingErrorsLogic.ts b/frontend/src/scenes/session-recordings/errors/sessionRecordingErrorsLogic.ts
index 45b887fd33cbb..49de62c7bf5c4 100644
--- a/frontend/src/scenes/session-recordings/errors/sessionRecordingErrorsLogic.ts
+++ b/frontend/src/scenes/session-recordings/errors/sessionRecordingErrorsLogic.ts
@@ -1,13 +1,19 @@
-import { afterMount, kea, path } from 'kea'
+import { actions, afterMount, kea, listeners, path } from 'kea'
import { loaders } from 'kea-loaders'
+import { router } from 'kea-router'
import api from 'lib/api'
+import { urls } from 'scenes/urls'
import { ErrorClusterResponse } from '~/types'
+import { createPlaylist } from '../playlist/playlistUtils'
import type { sessionRecordingErrorsLogicType } from './sessionRecordingErrorsLogicType'
export const sessionRecordingErrorsLogic = kea
([
path(['scenes', 'session-recordings', 'detail', 'sessionRecordingErrorsLogic']),
+ actions({
+ createPlaylist: (name: string, sessionIds: string[]) => ({ name, sessionIds }),
+ }),
loaders(() => ({
errors: [
null as ErrorClusterResponse,
@@ -19,7 +25,19 @@ export const sessionRecordingErrorsLogic = kea(
},
],
})),
+ listeners(() => ({
+ createPlaylist: async ({ name, sessionIds }) => {
+ const playlist = await createPlaylist({ name: name })
+ if (playlist) {
+ const samples = sessionIds.slice(0, 10)
+ await Promise.all(
+ samples.map((sessionId) => api.recordings.addRecordingToPlaylist(playlist.short_id, sessionId))
+ )
+ router.actions.push(urls.replayPlaylist(playlist.short_id))
+ }
+ },
+ })),
afterMount(({ actions }) => {
actions.loadErrorClusters(false)
}),
diff --git a/frontend/src/scenes/session-recordings/file-playback/sessionRecordingFilePlaybackLogic.ts b/frontend/src/scenes/session-recordings/file-playback/sessionRecordingFilePlaybackLogic.ts
index fb01f15b9a0cc..b6f547603114b 100644
--- a/frontend/src/scenes/session-recordings/file-playback/sessionRecordingFilePlaybackLogic.ts
+++ b/frontend/src/scenes/session-recordings/file-playback/sessionRecordingFilePlaybackLogic.ts
@@ -3,7 +3,6 @@ import { eventWithTime } from '@rrweb/types'
import { BuiltLogic, connect, kea, listeners, path, reducers, selectors } from 'kea'
import { loaders } from 'kea-loaders'
import { beforeUnload } from 'kea-router'
-import { FEATURE_FLAGS } from 'lib/constants'
import { dayjs } from 'lib/dayjs'
import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { uuid } from 'lib/utils'
@@ -178,11 +177,7 @@ export const sessionRecordingFilePlaybackLogic = kea
diff --git a/frontend/src/scenes/session-recordings/player/inspector/PlayerInspectorControls.tsx b/frontend/src/scenes/session-recordings/player/inspector/PlayerInspectorControls.tsx
index 93b272ede828b..bf3fd911e604d 100644
--- a/frontend/src/scenes/session-recordings/player/inspector/PlayerInspectorControls.tsx
+++ b/frontend/src/scenes/session-recordings/player/inspector/PlayerInspectorControls.tsx
@@ -174,6 +174,7 @@ export function PlayerInspectorControls({ onClose }: { onClose: () => void }): J
setTimestampMode(timestampMode === 'absolute' ? 'relative' : 'absolute')}
tooltipPlacement="left"
@@ -191,14 +192,15 @@ export function PlayerInspectorControls({ onClose }: { onClose: () => void }): J
{
- // If the user has syncScrolling on but it is paused due to interacting with the Inspector, we want to resume it
+ // If the user has syncScrolling on, but it is paused due to interacting with the Inspector, we want to resume it
if (syncScroll && syncScrollingPaused) {
setSyncScrollPaused(false)
} else {
- // Otherwise we are just toggling the settting
+ // Otherwise we are just toggling the setting
setSyncScroll(!syncScroll)
}
}}
diff --git a/frontend/src/scenes/session-recordings/player/sessionRecordingDataLogic.ts b/frontend/src/scenes/session-recordings/player/sessionRecordingDataLogic.ts
index 8d630aa10acbb..0aff48444a7e9 100644
--- a/frontend/src/scenes/session-recordings/player/sessionRecordingDataLogic.ts
+++ b/frontend/src/scenes/session-recordings/player/sessionRecordingDataLogic.ts
@@ -67,9 +67,10 @@ function isRecordingSnapshot(x: unknown): x is RecordingSnapshot {
export const parseEncodedSnapshots = async (
items: (RecordingSnapshot | EncodedRecordingSnapshot | string)[],
sessionId: string,
- withMobileTransformer: boolean
+ // this is only kept so that we can export the untransformed data for debugging
+ withMobileTransformer: boolean = true
): Promise => {
- if (!postHogEEModule && withMobileTransformer) {
+ if (!postHogEEModule) {
postHogEEModule = await posthogEE()
}
const lineCount = items.length
@@ -239,11 +240,7 @@ async function processEncodedResponse(
let untransformed: RecordingSnapshot[] | null = null
const transformed = deduplicateSnapshots(
- await parseEncodedSnapshots(
- encodedResponse,
- props.sessionRecordingId,
- !!featureFlags[FEATURE_FLAGS.SESSION_REPLAY_MOBILE]
- ),
+ await parseEncodedSnapshots(encodedResponse, props.sessionRecordingId),
existingData?.snapshots ?? []
)
diff --git a/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx b/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx
index 22dd5881ae378..3c2a9842c0dbc 100644
--- a/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx
+++ b/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx
@@ -6,7 +6,6 @@ import clsx from 'clsx'
import { range } from 'd3'
import { BindLogic, useActions, useValues } from 'kea'
import { EmptyMessage } from 'lib/components/EmptyMessage/EmptyMessage'
-import { FlaggedFeature } from 'lib/components/FlaggedFeature'
import { PropertyKeyInfo } from 'lib/components/PropertyKeyInfo'
import { FEATURE_FLAGS } from 'lib/constants'
import { useResizeBreakpoints } from 'lib/hooks/useResizeObserver'
@@ -58,12 +57,10 @@ function UnusableEventsWarning(props: { unusableEventsInFilter: string[] }): JSX
the Web SDK
-
- ,{' '}
-
- the Android SDK
-
-
+ ,{' '}
+
+ the Android SDK
+
)
diff --git a/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylistTroubleshooting.tsx b/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylistTroubleshooting.tsx
index bda13153d9ccb..961b0b54fa246 100644
--- a/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylistTroubleshooting.tsx
+++ b/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylistTroubleshooting.tsx
@@ -1,7 +1,5 @@
import { LemonDivider, Link } from '@posthog/lemon-ui'
import { useActions, useValues } from 'kea'
-import { FlaggedFeature } from 'lib/components/FlaggedFeature'
-import { FEATURE_FLAGS } from 'lib/constants'
import { playerSettingsLogic } from '../player/playerSettingsLogic'
import { sessionRecordingsPlaylistLogic } from './sessionRecordingsPlaylistLogic'
@@ -21,9 +19,7 @@ export const SessionRecordingsPlaylistTroubleshooting = (): JSX.Element => {
-
- All recording sources:
-
+ All recording sources:
{otherRecordings.length > 0 && hideViewedRecordings && (
-
Viewed recordings hidden.{' '}
@@ -42,10 +38,8 @@ export const SessionRecordingsPlaylistTroubleshooting = (): JSX.Element => {
They are outside the retention period
-
-
- Web recordings
-
+
+ Web recordings
-
= {
export const permissionsLogic = kea([
path(['scenes', 'organization', 'Settings', 'Permissions', 'permissionsLogic']),
connect({
- values: [featureFlagLogic, ['featureFlags'], rolesLogic, ['roles']],
+ values: [rolesLogic, ['roles']],
actions: [rolesLogic, ['updateRole']],
}),
actions({
@@ -123,10 +121,6 @@ export const permissionsLogic = kea([
)
},
],
- shouldShowPermissionsTable: [
- (s) => [s.featureFlags],
- (featureFlags) => featureFlags[FEATURE_FLAGS.ROLE_BASED_ACCESS] === 'control',
- ],
resourceRolesAccess: [
(s) => [s.allPermissions, s.roles],
(permissions, roles) => {
diff --git a/frontend/src/scenes/surveys/SurveyView.tsx b/frontend/src/scenes/surveys/SurveyView.tsx
index f36568410fe1a..95c2ca6df47cf 100644
--- a/frontend/src/scenes/surveys/SurveyView.tsx
+++ b/frontend/src/scenes/surveys/SurveyView.tsx
@@ -6,12 +6,10 @@ import { LemonButton, LemonDivider, Link } from '@posthog/lemon-ui'
import { useActions, useValues } from 'kea'
import { EditableField } from 'lib/components/EditableField/EditableField'
import { PageHeader } from 'lib/components/PageHeader'
-import { FEATURE_FLAGS } from 'lib/constants'
import { dayjs } from 'lib/dayjs'
import { More } from 'lib/lemon-ui/LemonButton/More'
import { LemonSkeleton } from 'lib/lemon-ui/LemonSkeleton'
import { LemonTabs } from 'lib/lemon-ui/LemonTabs'
-import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { capitalizeFirstLetter, pluralize } from 'lib/utils'
import { useEffect, useState } from 'react'
import { urls } from 'scenes/urls'
@@ -260,7 +258,6 @@ export function SurveyResult({ disableEventsTable }: { disableEventsTable?: bool
surveyOpenTextResultsReady,
surveyNPSScore,
} = useValues(surveyLogic)
- const { featureFlags } = useValues(featureFlagLogic)
return (
<>
@@ -274,10 +271,8 @@ export function SurveyResult({ disableEventsTable }: { disableEventsTable?: bool
<>
{surveyNPSScore}
Total NPS Score
- {featureFlags[FEATURE_FLAGS.SURVEYS_RESULTS_VISUALIZATIONS] && (
- // TODO: rework this to show nps scores over time
-
- )}
+ {/* TODO: rework this to show nps scores over time */}
+
)}
([
path(['scenes', 'surveys', 'surveysLogic']),
connect(() => ({
- values: [
- userLogic,
- ['hasAvailableFeature'],
- teamLogic,
- ['currentTeam', 'currentTeamLoading'],
- featureFlagLogic,
- ['featureFlags'],
- ],
+ values: [userLogic, ['hasAvailableFeature'], teamLogic, ['currentTeam', 'currentTeamLoading']],
actions: [teamLogic, ['loadCurrentTeam']],
})),
actions({
@@ -151,21 +142,17 @@ export const surveysLogic = kea([
},
],
],
- payGateFlagOn: [(s) => [s.featureFlags], (featureFlags) => featureFlags[FEATURE_FLAGS.SURVEYS_PAYGATES]],
surveysStylingAvailable: [
- (s) => [s.hasAvailableFeature, s.payGateFlagOn],
- (hasAvailableFeature, payGateFlagOn) =>
- !payGateFlagOn || (payGateFlagOn && hasAvailableFeature(AvailableFeature.SURVEYS_STYLING)),
+ (s) => [s.hasAvailableFeature],
+ (hasAvailableFeature) => hasAvailableFeature(AvailableFeature.SURVEYS_STYLING),
],
surveysHTMLAvailable: [
- (s) => [s.hasAvailableFeature, s.payGateFlagOn],
- (hasAvailableFeature, payGateFlagOn) =>
- !payGateFlagOn || (payGateFlagOn && hasAvailableFeature(AvailableFeature.SURVEYS_TEXT_HTML)),
+ (s) => [s.hasAvailableFeature],
+ (hasAvailableFeature) => hasAvailableFeature(AvailableFeature.SURVEYS_TEXT_HTML),
],
surveysMultipleQuestionsAvailable: [
- (s) => [s.hasAvailableFeature, s.payGateFlagOn],
- (hasAvailableFeature, payGateFlagOn) =>
- !payGateFlagOn || (payGateFlagOn && hasAvailableFeature(AvailableFeature.SURVEYS_MULTIPLE_QUESTIONS)),
+ (s) => [s.hasAvailableFeature],
+ (hasAvailableFeature) => hasAvailableFeature(AvailableFeature.SURVEYS_MULTIPLE_QUESTIONS),
],
showSurveysDisabledBanner: [
(s) => [s.currentTeam, s.currentTeamLoading, s.surveys],
diff --git a/frontend/src/scenes/trends/persons-modal/PersonsModal.tsx b/frontend/src/scenes/trends/persons-modal/PersonsModal.tsx
index 5240c916f2218..0fd8a8b8cc931 100644
--- a/frontend/src/scenes/trends/persons-modal/PersonsModal.tsx
+++ b/frontend/src/scenes/trends/persons-modal/PersonsModal.tsx
@@ -369,6 +369,7 @@ export function ActorRow({ actor, onOpenRecording, propertiesTimelineFilter }: A
onClick={onOpenRecordingClick}
sideIcon={matchedRecordings.length === 1 ? : null}
type="secondary"
+ status={matchedRecordings.length > 1 ? 'alt' : undefined}
size="small"
>
{matchedRecordings.length > 1 ? `${matchedRecordings.length} recordings` : 'View recording'}
diff --git a/frontend/src/scenes/trends/viz/ActionsHorizontalBar.tsx b/frontend/src/scenes/trends/viz/ActionsHorizontalBar.tsx
index 03743f0c4dd29..ded94e7815490 100644
--- a/frontend/src/scenes/trends/viz/ActionsHorizontalBar.tsx
+++ b/frontend/src/scenes/trends/viz/ActionsHorizontalBar.tsx
@@ -115,6 +115,10 @@ export function ActionsHorizontalBar({ showPersonsModal = true }: ChartParams):
kind: NodeKind.InsightActorsQuery,
source: query.source,
},
+ additionalSelect: {
+ value_at_data_point: 'event_count',
+ matched_recordings: 'matched_recordings',
+ },
})
} else if (selectedUrl) {
openPersonsModal({
diff --git a/frontend/src/scenes/trends/viz/ActionsLineGraph.tsx b/frontend/src/scenes/trends/viz/ActionsLineGraph.tsx
index b930608587e22..b7f3ba8a46d0f 100644
--- a/frontend/src/scenes/trends/viz/ActionsLineGraph.tsx
+++ b/frontend/src/scenes/trends/viz/ActionsLineGraph.tsx
@@ -152,6 +152,10 @@ export function ActionsLineGraph({
breakdown: dataset.breakdown_value,
compare: dataset.compare_label,
},
+ additionalSelect: {
+ value_at_data_point: 'event_count',
+ matched_recordings: 'matched_recordings',
+ },
})
} else {
const datasetUrls = urlsForDatasets(
diff --git a/frontend/src/scenes/trends/viz/ActionsPie.tsx b/frontend/src/scenes/trends/viz/ActionsPie.tsx
index 86804b11e487f..839e06f0d71b6 100644
--- a/frontend/src/scenes/trends/viz/ActionsPie.tsx
+++ b/frontend/src/scenes/trends/viz/ActionsPie.tsx
@@ -118,6 +118,10 @@ export function ActionsPie({
kind: NodeKind.InsightActorsQuery,
source: query.source,
},
+ additionalSelect: {
+ value_at_data_point: 'event_count',
+ matched_recordings: 'matched_recordings',
+ },
})
} else if (selectedUrl) {
openPersonsModal({
diff --git a/frontend/src/toolbar/actions/ActionsListView.tsx b/frontend/src/toolbar/actions/ActionsListView.tsx
index 6054c0bc65241..fb999ed600949 100644
--- a/frontend/src/toolbar/actions/ActionsListView.tsx
+++ b/frontend/src/toolbar/actions/ActionsListView.tsx
@@ -1,7 +1,6 @@
import { Link } from '@posthog/lemon-ui'
import { useActions, useValues } from 'kea'
import { Spinner } from 'lib/lemon-ui/Spinner'
-import { useEffect } from 'react'
import { actionsLogic } from '~/toolbar/actions/actionsLogic'
import { actionsTabLogic } from '~/toolbar/actions/actionsTabLogic'
@@ -13,13 +12,8 @@ interface ActionsListViewProps {
export function ActionsListView({ actions }: ActionsListViewProps): JSX.Element {
const { allActionsLoading, searchTerm } = useValues(actionsLogic)
- const { getActions } = useActions(actionsLogic)
const { selectAction } = useActions(actionsTabLogic)
- useEffect(() => {
- getActions()
- }, [])
-
return (
{actions.length ? (
diff --git a/frontend/src/toolbar/actions/ActionsToolbarMenu.tsx b/frontend/src/toolbar/actions/ActionsToolbarMenu.tsx
index ecbb3800c9260..87ff37b719ee8 100644
--- a/frontend/src/toolbar/actions/ActionsToolbarMenu.tsx
+++ b/frontend/src/toolbar/actions/ActionsToolbarMenu.tsx
@@ -5,6 +5,7 @@ import { LemonButton } from 'lib/lemon-ui/LemonButton'
import { LemonInput } from 'lib/lemon-ui/LemonInput'
import { Link } from 'lib/lemon-ui/Link'
import { Spinner } from 'lib/lemon-ui/Spinner'
+import { useEffect } from 'react'
import { urls } from 'scenes/urls'
import { ActionsEditingToolbarMenu } from '~/toolbar/actions/ActionsEditingToolbarMenu'
@@ -16,13 +17,17 @@ import { toolbarConfigLogic } from '~/toolbar/toolbarConfigLogic'
const ActionsListToolbarMenu = (): JSX.Element => {
const { searchTerm } = useValues(actionsLogic)
- const { setSearchTerm } = useActions(actionsLogic)
+ const { setSearchTerm, getActions } = useActions(actionsLogic)
const { newAction } = useActions(actionsTabLogic)
const { allActions, sortedActions, allActionsLoading } = useValues(actionsLogic)
const { apiURL } = useValues(toolbarConfigLogic)
+ useEffect(() => {
+ getActions()
+ }, [])
+
return (
diff --git a/frontend/src/toolbar/bar/Toolbar.scss b/frontend/src/toolbar/bar/Toolbar.scss
index ff83b7c9d922a..ec98167332fbd 100644
--- a/frontend/src/toolbar/bar/Toolbar.scss
+++ b/frontend/src/toolbar/bar/Toolbar.scss
@@ -144,8 +144,4 @@
transform: var(--toolbar-translate) scale(0);
}
}
-
- &--unauthenticated {
- width: calc(5rem + 1px); // Account for border
- }
}
diff --git a/frontend/src/toolbar/bar/Toolbar.tsx b/frontend/src/toolbar/bar/Toolbar.tsx
index f255ccc26800d..8ed031f904d86 100644
--- a/frontend/src/toolbar/bar/Toolbar.tsx
+++ b/frontend/src/toolbar/bar/Toolbar.tsx
@@ -77,15 +77,18 @@ function MoreMenu(): JSX.Element {
}
maxContentWidth={true}
>
- } title="More options" />
+
+
+
)
}
-export function ToolbarInfoMenu(): JSX.Element {
+export function ToolbarInfoMenu(): JSX.Element | null {
const ref = useRef(null)
const { visibleMenu, isDragging, menuProperties, minimized, isBlurred } = useValues(toolbarLogic)
const { setMenu } = useActions(toolbarLogic)
+ const { isAuthenticated } = useValues(toolbarConfigLogic)
const content = minimized ? null : visibleMenu === 'flags' ? (
@@ -102,6 +105,10 @@ export function ToolbarInfoMenu(): JSX.Element {
return () => setMenu(null)
}, [ref.current])
+ if (!isAuthenticated) {
+ return null
+ }
+
return (
}
onClick={isAuthenticated ? toggleMinimized : authenticate}
title={isAuthenticated ? 'Minimize' : 'Authenticate the PostHog Toolbar'}
titleMinimized={isAuthenticated ? 'Expand the toolbar' : 'Authenticate the PostHog Toolbar'}
- />
+ >
+
+
{isAuthenticated ? (
<>
- } menuId="inspect" />
- } menuId="heatmap" />
- } menuId="actions" />
- } menuId="flags" title="Feature flags" />
+
+
+
+
+
+
+
+
+
+
+
+
- ) : null}
+ ) : (
+
+ Authenticate
+
+ )}
diff --git a/frontend/src/toolbar/bar/ToolbarButton.scss b/frontend/src/toolbar/bar/ToolbarButton.scss
index 0d0bb666fa540..ce480f3fbab35 100644
--- a/frontend/src/toolbar/bar/ToolbarButton.scss
+++ b/frontend/src/toolbar/bar/ToolbarButton.scss
@@ -15,6 +15,8 @@
width: 2rem;
height: 2rem;
min-height: var(--lemon-button-height);
+ margin: 0.25rem;
+ font-weight: 600;
color: var(--muted-alt);
appearance: none !important; // Important as this gets overridden by Ant styles...
cursor: pointer;
@@ -43,4 +45,13 @@
}
}
}
+
+ &--flex {
+ flex-grow: 1;
+ width: auto;
+
+ button {
+ width: 100%;
+ }
+ }
}
diff --git a/frontend/src/toolbar/bar/ToolbarButton.tsx b/frontend/src/toolbar/bar/ToolbarButton.tsx
index add0e5f2580ce..f5dfc755be469 100644
--- a/frontend/src/toolbar/bar/ToolbarButton.tsx
+++ b/frontend/src/toolbar/bar/ToolbarButton.tsx
@@ -10,17 +10,18 @@ import React from 'react'
import { MenuState, toolbarLogic } from './toolbarLogic'
export type ToolbarButtonProps = {
- icon: React.ReactElement | null
+ children: React.ReactNode
onClick?: () => void
title?: string
titleMinimized?: JSX.Element | string
menuId?: MenuState
+ flex?: boolean
}
export const ToolbarButton: FunctionComponent = React.forwardRef<
HTMLDivElement,
ToolbarButtonProps
->(({ icon, title, onClick, titleMinimized, menuId, ...props }, ref): JSX.Element => {
+>(({ children, title, onClick, titleMinimized, menuId, flex, ...props }, ref): JSX.Element => {
const { visibleMenu, minimized, isDragging } = useValues(toolbarLogic)
const { setVisibleMenu } = useActions(toolbarLogic)
@@ -54,9 +55,13 @@ export const ToolbarButton: FunctionComponent = React.forwar
}
const theButton = (
-
+
)
diff --git a/frontend/src/toolbar/elements/heatmapLogic.ts b/frontend/src/toolbar/elements/heatmapLogic.ts
index 8bb148b4f6adc..de7232bd70dfd 100644
--- a/frontend/src/toolbar/elements/heatmapLogic.ts
+++ b/frontend/src/toolbar/elements/heatmapLogic.ts
@@ -24,7 +24,7 @@ const emptyElementsStatsPages: PaginatedResponse
= {
export const heatmapLogic = kea([
path(['toolbar', 'elements', 'heatmapLogic']),
connect({
- values: [toolbarConfigLogic, ['apiURL'], currentPageLogic, ['href', 'wildcardHref']],
+ values: [currentPageLogic, ['href', 'wildcardHref']],
actions: [currentPageLogic, ['setHref', 'setWildcardHref']],
}),
actions({
@@ -113,7 +113,7 @@ export const heatmapLogic = kea([
...values.heatmapFilter,
}
const includeEventsParams = '&include=$autocapture&include=$rageclick'
- defaultUrl = `${values.apiURL}/api/element/stats/${encodeParams(
+ defaultUrl = `/api/element/stats/${encodeParams(
{ ...params, paginate_response: true },
'?'
)}${includeEventsParams}`
@@ -124,7 +124,7 @@ export const heatmapLogic = kea([
url || defaultUrl,
'GET',
undefined,
- url ? 'use-as-provided' : 'only-add-token'
+ url ? 'use-as-provided' : 'full'
)
if (response.status === 403) {
diff --git a/frontend/src/toolbar/flags/flagsToolbarLogic.ts b/frontend/src/toolbar/flags/flagsToolbarLogic.ts
index 60c1f568f45a6..e1f41cabca73c 100644
--- a/frontend/src/toolbar/flags/flagsToolbarLogic.ts
+++ b/frontend/src/toolbar/flags/flagsToolbarLogic.ts
@@ -40,11 +40,6 @@ export const flagsToolbarLogic = kea([
`/api/projects/@current/feature_flags/my_flags${encodeParams(params, '?')}`
)
- if (response.status >= 400) {
- toolbarConfigLogic.actions.tokenExpired()
- return []
- }
-
breakpoint()
if (!response.ok) {
return []
diff --git a/frontend/src/toolbar/toolbarConfigLogic.ts b/frontend/src/toolbar/toolbarConfigLogic.ts
index 1b4638b8f39f8..c0650d7552d55 100644
--- a/frontend/src/toolbar/toolbarConfigLogic.ts
+++ b/frontend/src/toolbar/toolbarConfigLogic.ts
@@ -6,7 +6,7 @@ import { posthog } from '~/toolbar/posthog'
import { ToolbarProps } from '~/types'
import type { toolbarConfigLogicType } from './toolbarConfigLogicType'
-import { clearSessionToolbarToken } from './utils'
+import { LOCALSTORAGE_KEY } from './utils'
export const toolbarConfigLogic = kea([
path(['toolbar', 'toolbarConfigLogic']),
@@ -19,48 +19,68 @@ export const toolbarConfigLogic = kea([
clearUserIntent: true,
showButton: true,
hideButton: true,
+ persistConfig: true,
}),
reducers(({ props }) => ({
- rawApiURL: [props.apiURL as string],
- rawJsURL: [(props.jsURL || props.apiURL) as string],
- temporaryToken: [props.temporaryToken || null, { logout: () => null, tokenExpired: () => null }],
+ // TRICKY: We cache a copy of the props. This allows us to connect the logic without passing the props in - only the top level caller has to do this.
+ props: [props],
+ temporaryToken: [
+ props.temporaryToken || null,
+ { logout: () => null, tokenExpired: () => null, authenticate: () => null },
+ ],
actionId: [props.actionId || null, { logout: () => null, clearUserIntent: () => null }],
userIntent: [props.userIntent || null, { logout: () => null, clearUserIntent: () => null }],
- source: [props.source || null, { logout: () => null }],
buttonVisible: [true, { showButton: () => true, hideButton: () => false, logout: () => false }],
- dataAttributes: [props.dataAttributes || []],
- posthog: [props.posthog ?? null],
})),
selectors({
- apiURL: [(s) => [s.rawApiURL], (apiURL) => `${apiURL.endsWith('/') ? apiURL.replace(/\/+$/, '') : apiURL}`],
+ posthog: [(s) => [s.props], (props) => props.posthog ?? null],
+ apiURL: [
+ (s) => [s.props],
+ (props: ToolbarProps) => `${props.apiURL?.endsWith('/') ? props.apiURL.replace(/\/+$/, '') : props.apiURL}`,
+ ],
jsURL: [
- (s) => [s.rawJsURL, s.apiURL],
- (rawJsURL, apiUrl) =>
- `${rawJsURL ? (rawJsURL.endsWith('/') ? rawJsURL.replace(/\/+$/, '') : rawJsURL) : apiUrl}`,
+ (s) => [s.props, s.apiURL],
+ (props: ToolbarProps, apiUrl) =>
+ `${props.jsURL ? (props.jsURL.endsWith('/') ? props.jsURL.replace(/\/+$/, '') : props.jsURL) : apiUrl}`,
],
+ dataAttributes: [(s) => [s.props], (props): string[] => props.dataAttributes ?? []],
isAuthenticated: [(s) => [s.temporaryToken], (temporaryToken) => !!temporaryToken],
}),
- listeners(({ values }) => ({
+ listeners(({ values, actions }) => ({
authenticate: () => {
posthog.capture('toolbar authenticate', { is_authenticated: values.isAuthenticated })
const encodedUrl = encodeURIComponent(window.location.href)
+ actions.persistConfig()
window.location.href = `${values.apiURL}/authorize_and_redirect/?redirect=${encodedUrl}`
- clearSessionToolbarToken()
},
logout: () => {
posthog.capture('toolbar logout')
- clearSessionToolbarToken()
+ localStorage.removeItem(LOCALSTORAGE_KEY)
},
tokenExpired: () => {
posthog.capture('toolbar token expired')
console.warn('PostHog Toolbar API token expired. Clearing session.')
- if (values.source !== 'localstorage') {
+ if (values.props.source !== 'localstorage') {
lemonToast.error('PostHog Toolbar API token expired.')
}
- clearSessionToolbarToken()
+ actions.persistConfig()
+ },
+
+ persistConfig: () => {
+ // Most params we don't change, only those that we may have modified during the session
+ const toolbarParams: ToolbarProps = {
+ ...values.props,
+ temporaryToken: values.temporaryToken ?? undefined,
+ actionId: values.actionId ?? undefined,
+ userIntent: values.userIntent ?? undefined,
+ posthog: undefined,
+ featureFlags: undefined,
+ }
+
+ localStorage.setItem(LOCALSTORAGE_KEY, JSON.stringify(toolbarParams))
},
})),
@@ -83,12 +103,10 @@ export async function toolbarFetch(
/*
allows caller to control how the provided URL is altered before use
if "full" then the payload and URL are taken apart and reconstructed
- if "only-add-token" the URL is unchanged, the payload is not used
- but the temporary token is added to the URL
if "use-as-provided" then the URL is used as-is, and the payload is not used
this is because the heatmapLogic needs more control over how the query parameters are constructed
*/
- urlConstruction: 'full' | 'only-add-token' | 'use-as-provided' = 'full'
+ urlConstruction: 'full' | 'use-as-provided' = 'full'
): Promise {
const temporaryToken = toolbarConfigLogic.findMounted()?.values.temporaryToken
const apiURL = toolbarConfigLogic.findMounted()?.values.apiURL
@@ -96,8 +114,6 @@ export async function toolbarFetch(
let fullUrl: string
if (urlConstruction === 'use-as-provided') {
fullUrl = url
- } else if (urlConstruction === 'only-add-token') {
- fullUrl = `${url}&temporary_token=${temporaryToken}`
} else {
const { pathname, searchParams } = combineUrl(url)
const params = { ...searchParams, temporary_token: temporaryToken }
@@ -119,10 +135,12 @@ export async function toolbarFetch(
})
if (response.status === 403) {
const responseData = await response.json()
- // Do not try to authenticate if the user has no project access altogether
- if (responseData.detail !== "You don't have access to the project.") {
+ if (responseData.detail === "You don't have access to the project.") {
toolbarConfigLogic.actions.authenticate()
}
}
+ if (response.status == 401) {
+ toolbarConfigLogic.actions.tokenExpired()
+ }
return response
}
diff --git a/frontend/src/toolbar/toolbarLogic.ts b/frontend/src/toolbar/toolbarLogic.ts
deleted file mode 100644
index d5183a6734f20..0000000000000
--- a/frontend/src/toolbar/toolbarLogic.ts
+++ /dev/null
@@ -1,87 +0,0 @@
-import { actions, afterMount, kea, listeners, path, props, reducers, selectors } from 'kea'
-import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'
-
-import { actionsTabLogic } from '~/toolbar/actions/actionsTabLogic'
-import { posthog } from '~/toolbar/posthog'
-import { clearSessionToolbarToken } from '~/toolbar/utils'
-import { ToolbarProps } from '~/types'
-
-import type { toolbarLogicType } from './toolbarLogicType'
-
-export const toolbarLogic = kea([
- path(['toolbar', 'toolbarLogic']),
- props({} as ToolbarProps),
-
- actions({
- authenticate: true,
- logout: true,
- tokenExpired: true,
- processUserIntent: true,
- clearUserIntent: true,
- showButton: true,
- hideButton: true,
- }),
-
- reducers(({ props }) => ({
- rawApiURL: [props.apiURL as string],
- rawJsURL: [(props.jsURL || props.apiURL) as string],
- temporaryToken: [props.temporaryToken || null, { logout: () => null, tokenExpired: () => null }],
- actionId: [props.actionId || null, { logout: () => null, clearUserIntent: () => null }],
- userIntent: [props.userIntent || null, { logout: () => null, clearUserIntent: () => null }],
- source: [props.source || null, { logout: () => null }],
- buttonVisible: [true, { showButton: () => true, hideButton: () => false, logout: () => false }],
- dataAttributes: [props.dataAttributes || []],
- posthog: [props.posthog ?? null],
- })),
-
- selectors({
- apiURL: [(s) => [s.rawApiURL], (apiURL) => `${apiURL.endsWith('/') ? apiURL.replace(/\/+$/, '') : apiURL}`],
- jsURL: [
- (s) => [s.rawJsURL, s.apiURL],
- (rawJsURL, apiUrl) =>
- `${rawJsURL ? (rawJsURL.endsWith('/') ? rawJsURL.replace(/\/+$/, '') : rawJsURL) : apiUrl}`,
- ],
- isAuthenticated: [(s) => [s.temporaryToken], (temporaryToken) => !!temporaryToken],
- }),
-
- listeners(({ values, props }) => ({
- authenticate: () => {
- posthog.capture('toolbar authenticate', { is_authenticated: values.isAuthenticated })
- const encodedUrl = encodeURIComponent(window.location.href)
- window.location.href = `${values.apiURL}/authorize_and_redirect/?redirect=${encodedUrl}`
- clearSessionToolbarToken()
- },
- logout: () => {
- posthog.capture('toolbar logout')
- clearSessionToolbarToken()
- },
- tokenExpired: () => {
- posthog.capture('toolbar token expired')
- console.warn('PostHog Toolbar API token expired. Clearing session.')
- if (values.source !== 'localstorage') {
- lemonToast.error('PostHog Toolbar API token expired.')
- }
- clearSessionToolbarToken()
- },
- processUserIntent: () => {
- if (props.userIntent === 'add-action' || props.userIntent === 'edit-action') {
- actionsTabLogic.actions.showButtonActions()
- // the right view will next be opened in `actionsTabLogic` on `getActionsSuccess`
- }
- },
- })),
-
- afterMount(({ props, actions, values }) => {
- if (props.instrument) {
- const distinctId = props.distinctId
- if (distinctId) {
- posthog.identify(distinctId, props.userEmail ? { email: props.userEmail } : {})
- }
- posthog.optIn()
- }
- if (props.userIntent) {
- actions.processUserIntent()
- }
- posthog.capture('toolbar loaded', { is_authenticated: values.isAuthenticated })
- }),
-])
diff --git a/frontend/src/toolbar/utils.ts b/frontend/src/toolbar/utils.ts
index 20441d2d385a9..9e2e67b56231f 100644
--- a/frontend/src/toolbar/utils.ts
+++ b/frontend/src/toolbar/utils.ts
@@ -7,6 +7,7 @@ import { ActionStepForm, BoxColor, ElementRect } from '~/toolbar/types'
import { ActionStepType, StringMatching } from '~/types'
export const TOOLBAR_ID = '__POSTHOG_TOOLBAR__'
+export const LOCALSTORAGE_KEY = '_postHogToolbarParams'
export function getSafeText(el: HTMLElement): string {
if (!el.childNodes || !el.childNodes.length) {
@@ -328,14 +329,6 @@ export function stepToDatabaseFormat(step: ActionStepForm): ActionStepType {
}
}
-export function clearSessionToolbarToken(): void {
- window.sessionStorage?.removeItem('_postHogToolbarParams')
- window.localStorage?.removeItem('_postHogToolbarParams')
- // keeping these around for compatibility, should be eventually removed
- window.sessionStorage?.removeItem('_postHogEditorParams')
- window.localStorage?.removeItem('_postHogEditorParams')
-}
-
export function getRectForElement(element: HTMLElement): ElementRect {
const elements = [elementToAreaRect(element)]
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index 2aaaba4809a87..131598f9a79d2 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -903,8 +903,10 @@ export interface SessionRecordingsResponse {
export type ErrorCluster = {
cluster: number
- sample: { session_id: string; error: string }
+ sample: string
occurrences: number
+ session_ids: string[]
+ sparkline: Record
unique_sessions: number
viewed: number
}
@@ -3040,7 +3042,7 @@ interface BreadcrumbBase {
/** Symbol, e.g. a lettermark or a profile picture. */
symbol?: React.ReactNode
/** Whether to show a custom popover */
- popover?: Pick
+ popover?: Pick
}
interface LinkBreadcrumb extends BreadcrumbBase {
/** Path to link to. */
@@ -3542,7 +3544,7 @@ export interface DataWarehouseViewLink {
created_at?: string | null
}
-export type ExternalDataSourceType = 'Stripe' | 'Hubspot' | 'Postgres'
+export type ExternalDataSourceType = 'Stripe' | 'Hubspot' | 'Postgres' | 'Zendesk'
export interface ExternalDataSourceCreatePayload {
source_type: ExternalDataSourceType
@@ -3595,6 +3597,7 @@ export type BatchExportDestinationS3 = {
encryption: string | null
kms_key_id: string | null
endpoint_url: string | null
+ file_format: string
}
}
diff --git a/latest_migrations.manifest b/latest_migrations.manifest
index f232dbc8c186c..f88359530eb78 100644
--- a/latest_migrations.manifest
+++ b/latest_migrations.manifest
@@ -5,7 +5,7 @@ contenttypes: 0002_remove_content_type_name
ee: 0016_rolemembership_organization_member
otp_static: 0002_throttling
otp_totp: 0002_auto_20190420_0723
-posthog: 0397_projects_backfill
+posthog: 0398_alter_externaldatasource_source_type
sessions: 0001_initial
social_django: 0010_uid_db_index
two_factor: 0007_auto_20201201_1019
diff --git a/mypy-baseline.txt b/mypy-baseline.txt
index 781ad2980830b..b8d2d1c94da64 100644
--- a/mypy-baseline.txt
+++ b/mypy-baseline.txt
@@ -1,6 +1,14 @@
posthog/temporal/common/utils.py:0: error: Argument 1 to "abstractclassmethod" has incompatible type "Callable[[HeartbeatDetails, Any], Any]"; expected "Callable[[type[Never], Any], Any]" [arg-type]
posthog/temporal/common/utils.py:0: note: This is likely because "from_activity" has named arguments: "cls". Consider marking them positional-only
posthog/temporal/common/utils.py:0: error: Argument 2 to "__get__" of "classmethod" has incompatible type "type[HeartbeatType]"; expected "type[Never]" [arg-type]
+posthog/temporal/data_imports/pipelines/zendesk/talk_api.py:0: error: Incompatible types in assignment (expression has type "None", variable has type "str") [assignment]
+posthog/temporal/data_imports/pipelines/zendesk/helpers.py:0: error: Argument 1 to "ensure_pendulum_datetime" has incompatible type "DateTime | Date | datetime | date | str | float | int | None"; expected "DateTime | Date | datetime | date | str | float | int" [arg-type]
+posthog/temporal/data_imports/pipelines/zendesk/helpers.py:0: error: Argument 1 to "ensure_pendulum_datetime" has incompatible type "str | None"; expected "DateTime | Date | datetime | date | str | float | int" [arg-type]
+posthog/temporal/data_imports/pipelines/zendesk/helpers.py:0: error: Argument 1 to "ensure_pendulum_datetime" has incompatible type "DateTime | Date | datetime | date | str | float | int | None"; expected "DateTime | Date | datetime | date | str | float | int" [arg-type]
+posthog/temporal/data_imports/pipelines/zendesk/helpers.py:0: error: Argument 1 to "ensure_pendulum_datetime" has incompatible type "str | None"; expected "DateTime | Date | datetime | date | str | float | int" [arg-type]
+posthog/temporal/data_imports/pipelines/zendesk/helpers.py:0: error: Argument 1 to "ensure_pendulum_datetime" has incompatible type "DateTime | Date | datetime | date | str | float | int | None"; expected "DateTime | Date | datetime | date | str | float | int" [arg-type]
+posthog/temporal/data_imports/pipelines/zendesk/helpers.py:0: error: Item "None" of "DateTime | None" has no attribute "int_timestamp" [union-attr]
+posthog/temporal/data_imports/pipelines/zendesk/helpers.py:0: error: Argument 1 to "ensure_pendulum_datetime" has incompatible type "str | None"; expected "DateTime | Date | datetime | date | str | float | int" [arg-type]
posthog/hogql/database/argmax.py:0: error: Argument "chain" to "Field" has incompatible type "list[str]"; expected "list[str | int]" [arg-type]
posthog/hogql/database/argmax.py:0: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance
posthog/hogql/database/argmax.py:0: note: Consider using "Sequence" instead, which is covariant
@@ -77,6 +85,7 @@ posthog/hogql/parser.py:0: error: "None" has no attribute "text" [attr-defined]
posthog/hogql/parser.py:0: error: "None" has no attribute "text" [attr-defined]
posthog/hogql/parser.py:0: error: Statement is unreachable [unreachable]
posthog/hogql/database/schema/person_distinct_ids.py:0: error: Argument 1 to "select_from_person_distinct_ids_table" has incompatible type "dict[str, list[str]]"; expected "dict[str, list[str | int]]" [arg-type]
+posthog/hogql/database/schema/person_distinct_id_overrides.py:0: error: Argument 1 to "select_from_person_distinct_id_overrides_table" has incompatible type "dict[str, list[str]]"; expected "dict[str, list[str | int]]" [arg-type]
posthog/hogql/database/schema/cohort_people.py:0: error: Argument "chain" to "Field" has incompatible type "list[str]"; expected "list[str | int]" [arg-type]
posthog/hogql/database/schema/cohort_people.py:0: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance
posthog/hogql/database/schema/cohort_people.py:0: note: Consider using "Sequence" instead, which is covariant
@@ -97,6 +106,7 @@ posthog/hogql/database/database.py:0: error: "FieldOrTable" has no attribute "fi
posthog/hogql/database/database.py:0: error: "FieldOrTable" has no attribute "fields" [attr-defined]
posthog/hogql/database/database.py:0: error: "FieldOrTable" has no attribute "fields" [attr-defined]
posthog/hogql/database/database.py:0: error: "FieldOrTable" has no attribute "fields" [attr-defined]
+posthog/hogql/database/database.py:0: error: "FieldOrTable" has no attribute "fields" [attr-defined]
posthog/hogql/database/database.py:0: error: Incompatible types (expression has type "Literal['view', 'lazy_table']", TypedDict item "type" has type "Literal['integer', 'float', 'string', 'datetime', 'date', 'boolean', 'array', 'json', 'lazy_table', 'virtual_table', 'field_traverser', 'expression']") [typeddict-item]
posthog/warehouse/models/datawarehouse_saved_query.py:0: error: Argument 1 to "create_hogql_database" has incompatible type "int | None"; expected "int" [arg-type]
posthog/warehouse/models/datawarehouse_saved_query.py:0: error: Incompatible types in assignment (expression has type "Expr", variable has type "SelectQuery | SelectUnionQuery") [assignment]
@@ -243,7 +253,6 @@ posthog/hogql/resolver.py:0: error: Argument 1 to "join" of "str" has incompatib
posthog/temporal/data_imports/external_data_job.py:0: error: Argument "team_id" has incompatible type "int"; expected "str" [arg-type]
posthog/temporal/data_imports/external_data_job.py:0: error: Unused "type: ignore" comment [unused-ignore]
posthog/temporal/data_imports/external_data_job.py:0: error: Argument "team_id" has incompatible type "int"; expected "str" [arg-type]
-posthog/temporal/data_imports/external_data_job.py:0: error: Argument 2 to "DataImportPipeline" has incompatible type "DltSource"; expected "DltResource" [arg-type]
posthog/hogql/transforms/lazy_tables.py:0: error: Incompatible default for argument "context" (default has type "None", argument has type "HogQLContext") [assignment]
posthog/hogql/transforms/lazy_tables.py:0: note: PEP 484 prohibits implicit Optional. Accordingly, mypy has changed its default to no_implicit_optional=True
posthog/hogql/transforms/lazy_tables.py:0: note: Use https://github.com/hauntsaninja/no_implicit_optional to automatically upgrade your codebase
@@ -339,20 +348,12 @@ posthog/hogql_queries/sessions_timeline_query_runner.py:0: error: Statement is u
posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr]
posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_histogram_bin_count" [union-attr]
posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown" [union-attr]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Argument 1 to "parse_expr" has incompatible type "str | float | list[str | float] | Any | None"; expected "str" [arg-type]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown" [union-attr]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Argument 1 to "parse_expr" has incompatible type "str | float | list[str | float] | Any | None"; expected "str" [arg-type]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Statement is unreachable [unreachable]
posthog/hogql_queries/insights/trends/breakdown.py:0: error: Argument "exprs" to "Or" has incompatible type "list[CompareOperation]"; expected "list[Expr]" [arg-type]
posthog/hogql_queries/insights/trends/breakdown.py:0: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance
posthog/hogql_queries/insights/trends/breakdown.py:0: note: Consider using "Sequence" instead, which is covariant
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Incompatible types in assignment (expression has type "float", variable has type "int") [assignment]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Incompatible types in assignment (expression has type "float", variable has type "int") [assignment]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment]
+posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr]
+posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown" [union-attr]
+posthog/hogql_queries/insights/trends/breakdown.py:0: error: Argument 1 to "parse_expr" has incompatible type "str | float | list[str | float] | Any | None"; expected "str" [arg-type]
posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr]
posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown" [union-attr]
posthog/hogql_queries/insights/trends/breakdown.py:0: error: Argument "breakdown_field" to "get_properties_chain" has incompatible type "str | float | list[str | float] | Any | None"; expected "str" [arg-type]
@@ -377,11 +378,11 @@ posthog/hogql_queries/insights/trends/trends_query_runner.py:0: error: Signature
posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: Superclass:
posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: def to_actors_query(self) -> SelectQuery | SelectUnionQuery
posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: Subclass:
-posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: def to_actors_query(self, time_frame: str | int | None, series_index: int, breakdown_value: str | int | None = ..., compare: Compare | None = ...) -> SelectQuery | SelectUnionQuery
+posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: def to_actors_query(self, time_frame: str | None, series_index: int, breakdown_value: str | int | None = ..., compare: Compare | None = ...) -> SelectQuery | SelectUnionQuery
posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: Superclass:
posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: def to_actors_query(self) -> SelectQuery | SelectUnionQuery
posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: Subclass:
-posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: def to_actors_query(self, time_frame: str | int | None, series_index: int, breakdown_value: str | int | None = ..., compare: Compare | None = ...) -> SelectQuery | SelectUnionQuery
+posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: def to_actors_query(self, time_frame: str | None, series_index: int, breakdown_value: str | int | None = ..., compare: Compare | None = ...) -> SelectQuery | SelectUnionQuery
posthog/hogql_queries/insights/trends/trends_query_runner.py:0: error: Statement is unreachable [unreachable]
posthog/hogql_queries/insights/trends/trends_query_runner.py:0: error: Argument 1 to "_event_property" of "TrendsQueryRunner" has incompatible type "str | float | list[str | float] | None"; expected "str" [arg-type]
posthog/hogql_queries/insights/retention_query_runner.py:0: error: Incompatible types in assignment (expression has type "Expr", variable has type "Call") [assignment]
@@ -542,7 +543,6 @@ posthog/hogql_queries/insights/trends/test/test_aggregation_operations.py:0: err
posthog/hogql_queries/insights/trends/test/test_aggregation_operations.py:0: error: Item "SelectQuery" of "SelectQuery | SelectUnionQuery | Field | Any | None" has no attribute "chain" [union-attr]
posthog/hogql_queries/insights/trends/test/test_aggregation_operations.py:0: error: Item "SelectUnionQuery" of "SelectQuery | SelectUnionQuery | Field | Any | None" has no attribute "chain" [union-attr]
posthog/hogql_queries/insights/trends/test/test_aggregation_operations.py:0: error: Item "None" of "SelectQuery | SelectUnionQuery | Field | Any | None" has no attribute "chain" [union-attr]
-posthog/hogql_queries/insights/test/test_paginators.py:0: error: Argument 2 to "execute_hogql_query" of "HogQLHasMorePaginator" has incompatible type "SelectQuery | SelectUnionQuery"; expected "SelectQuery" [arg-type]
posthog/hogql_queries/insights/test/test_paginators.py:0: error: Value of type "object" is not indexable [index]
posthog/hogql_queries/insights/test/test_paginators.py:0: error: Value of type "object" is not indexable [index]
posthog/hogql_queries/insights/test/test_paginators.py:0: error: Value of type "object" is not indexable [index]
diff --git a/package.json b/package.json
index 770be74997198..ca948c3607d16 100644
--- a/package.json
+++ b/package.json
@@ -140,11 +140,12 @@
"maplibre-gl": "^3.5.1",
"md5": "^2.3.0",
"monaco-editor": "^0.39.0",
+ "natural-orderby": "^3.0.2",
"papaparse": "^5.4.1",
"pmtiles": "^2.11.0",
"postcss": "^8.4.31",
"postcss-preset-env": "^9.3.0",
- "posthog-js": "1.116.3",
+ "posthog-js": "1.116.6",
"posthog-js-lite": "2.5.0",
"prettier": "^2.8.8",
"prop-types": "^15.7.2",
diff --git a/plugin-server/functional_tests/api.ts b/plugin-server/functional_tests/api.ts
index abbd770d7bb77..c6ff46bf5bf6d 100644
--- a/plugin-server/functional_tests/api.ts
+++ b/plugin-server/functional_tests/api.ts
@@ -106,6 +106,7 @@ export const capture = async ({
})
),
key: teamId ? teamId.toString() : '',
+ waitForAck: true,
})
}
diff --git a/plugin-server/functional_tests/jobs-consumer.test.ts b/plugin-server/functional_tests/jobs-consumer.test.ts
index 30e2abd9af282..353bd3518397e 100644
--- a/plugin-server/functional_tests/jobs-consumer.test.ts
+++ b/plugin-server/functional_tests/jobs-consumer.test.ts
@@ -43,7 +43,7 @@ describe('dlq handling', () => {
test.concurrent(`handles empty messages`, async () => {
const key = uuidv4()
- await produce({ topic: 'jobs', message: null, key })
+ await produce({ topic: 'jobs', message: null, key, waitForAck: true })
await waitForExpect(() => {
const messages = dlq.filter((message) => message.key?.toString() === key)
@@ -54,7 +54,7 @@ describe('dlq handling', () => {
test.concurrent(`handles invalid JSON`, async () => {
const key = uuidv4()
- await produce({ topic: 'jobs', message: Buffer.from('invalid json'), key })
+ await produce({ topic: 'jobs', message: Buffer.from('invalid json'), key, waitForAck: true })
await waitForExpect(() => {
const messages = dlq.filter((message) => message.key?.toString() === key)
@@ -72,7 +72,7 @@ describe('dlq handling', () => {
labels: { topic: 'jobs', partition: '0', groupId: 'jobs-inserter' },
})
- await produce({ topic: 'jobs', message: Buffer.from(''), key: '' })
+ await produce({ topic: 'jobs', message: Buffer.from(''), key: '', waitForAck: true })
await waitForExpect(async () => {
const metricAfter = await getMetric({
diff --git a/plugin-server/functional_tests/kafka.ts b/plugin-server/functional_tests/kafka.ts
index c2ab7ac87a6ab..f431488b290ac 100644
--- a/plugin-server/functional_tests/kafka.ts
+++ b/plugin-server/functional_tests/kafka.ts
@@ -36,7 +36,17 @@ export async function createKafkaProducer() {
return producer
}
-export async function produce({ topic, message, key }: { topic: string; message: Buffer | null; key: string }) {
+export async function produce({
+ topic,
+ message,
+ key,
+ waitForAck,
+}: {
+ topic: string
+ message: Buffer | null
+ key: string
+ waitForAck: boolean
+}) {
producer = producer ?? (await createKafkaProducer())
- await defaultProduce({ producer, topic, value: message, key: Buffer.from(key) })
+ await defaultProduce({ producer, topic, value: message, key: Buffer.from(key), waitForAck })
}
diff --git a/plugin-server/functional_tests/scheduled-tasks-runner.test.ts b/plugin-server/functional_tests/scheduled-tasks-runner.test.ts
index 3e3345245a644..48764ae7f90a7 100644
--- a/plugin-server/functional_tests/scheduled-tasks-runner.test.ts
+++ b/plugin-server/functional_tests/scheduled-tasks-runner.test.ts
@@ -43,7 +43,7 @@ describe('dlq handling', () => {
test.concurrent(`handles empty messages`, async () => {
const key = uuidv4()
- await produce({ topic: 'scheduled_tasks', message: null, key })
+ await produce({ topic: 'scheduled_tasks', message: null, key, waitForAck: true })
await waitForExpect(() => {
const messages = dlq.filter((message) => message.key?.toString() === key)
@@ -54,7 +54,7 @@ describe('dlq handling', () => {
test.concurrent(`handles invalid JSON`, async () => {
const key = uuidv4()
- await produce({ topic: 'scheduled_tasks', message: Buffer.from('invalid json'), key })
+ await produce({ topic: 'scheduled_tasks', message: Buffer.from('invalid json'), key, waitForAck: true })
await waitForExpect(() => {
const messages = dlq.filter((message) => message.key?.toString() === key)
@@ -69,6 +69,7 @@ describe('dlq handling', () => {
topic: 'scheduled_tasks',
message: Buffer.from(JSON.stringify({ taskType: 'invalidTaskType', pluginConfigId: 1 })),
key,
+ waitForAck: true,
})
await waitForExpect(() => {
@@ -84,6 +85,7 @@ describe('dlq handling', () => {
topic: 'scheduled_tasks',
message: Buffer.from(JSON.stringify({ taskType: 'runEveryMinute', pluginConfigId: 'asdf' })),
key,
+ waitForAck: true,
})
await waitForExpect(() => {
@@ -104,7 +106,7 @@ describe('dlq handling', () => {
// NOTE: we don't actually care too much about the contents of the
// message, just that it triggeres the consumer to try to process it.
- await produce({ topic: 'scheduled_tasks', message: Buffer.from(''), key: '' })
+ await produce({ topic: 'scheduled_tasks', message: Buffer.from(''), key: '', waitForAck: true })
await waitForExpect(async () => {
const metricAfter = await getMetric({
diff --git a/plugin-server/functional_tests/session-recordings.test.ts b/plugin-server/functional_tests/session-recordings.test.ts
index 62075bc6bd10f..783fbdbeb43cd 100644
--- a/plugin-server/functional_tests/session-recordings.test.ts
+++ b/plugin-server/functional_tests/session-recordings.test.ts
@@ -173,7 +173,12 @@ test.skip('consumer updates timestamp exported to prometheus', async () => {
},
})
- await produce({ topic: KAFKA_SESSION_RECORDING_SNAPSHOT_ITEM_EVENTS, message: Buffer.from(''), key: '' })
+ await produce({
+ topic: KAFKA_SESSION_RECORDING_SNAPSHOT_ITEM_EVENTS,
+ message: Buffer.from(''),
+ key: '',
+ waitForAck: true,
+ })
await waitForExpect(async () => {
const metricAfter = await getMetric({
@@ -245,6 +250,7 @@ test.skip(`handles message with no token or with token and no associated team_id
topic: KAFKA_SESSION_RECORDING_SNAPSHOT_ITEM_EVENTS,
message: Buffer.from(JSON.stringify({ uuid: noTokenUuid, data: JSON.stringify({}) })),
key: noTokenKey,
+ waitForAck: true,
})
await produce({
topic: KAFKA_SESSION_RECORDING_SNAPSHOT_ITEM_EVENTS,
@@ -252,6 +258,7 @@ test.skip(`handles message with no token or with token and no associated team_id
JSON.stringify({ uuid: noAssociatedTeamUuid, token: 'no associated team', data: JSON.stringify({}) })
),
key: noAssociatedTeamKey,
+ waitForAck: true,
})
await capture(makeSessionMessage(teamId, 'should be ingested'))
diff --git a/plugin-server/src/config/config.ts b/plugin-server/src/config/config.ts
index 8e9b50afb9528..def72eea474bb 100644
--- a/plugin-server/src/config/config.ts
+++ b/plugin-server/src/config/config.ts
@@ -163,6 +163,9 @@ export function getDefaultConfig(): PluginsServerConfig {
SESSION_RECORDING_DEBUG_PARTITION: undefined,
SESSION_RECORDING_KAFKA_DEBUG: undefined,
SESSION_RECORDING_MAX_PARALLEL_FLUSHES: 10,
+ SESSION_RECORDING_OVERFLOW_ENABLED: false,
+ SESSION_RECORDING_OVERFLOW_BUCKET_REPLENISH_RATE: 5_000_000, // 5MB/second uncompressed, sustained
+ SESSION_RECORDING_OVERFLOW_BUCKET_CAPACITY: 200_000_000, // 200MB burst
}
}
diff --git a/plugin-server/src/kafka/producer.ts b/plugin-server/src/kafka/producer.ts
index 7029a26c79fbd..062785f902bc4 100644
--- a/plugin-server/src/kafka/producer.ts
+++ b/plugin-server/src/kafka/producer.ts
@@ -7,6 +7,7 @@ import {
NumberNullUndefined,
ProducerGlobalConfig,
} from 'node-rdkafka'
+import { Summary } from 'prom-client'
import { getSpan } from '../sentry'
import { status } from '../utils/status'
@@ -17,6 +18,13 @@ export type KafkaProducerConfig = {
KAFKA_PRODUCER_QUEUE_BUFFERING_MAX_MESSAGES: number
}
+export const ingestEventKafkaProduceLatency = new Summary({
+ name: 'ingest_event_kafka_produce_latency',
+ help: 'Wait time for individual Kafka produces',
+ labelNames: ['topic', 'waitForAck'],
+ percentiles: [0.5, 0.9, 0.95, 0.99],
+})
+
// Kafka production related functions using node-rdkafka.
export const createKafkaProducer = async (globalConfig: ProducerGlobalConfig, producerConfig: KafkaProducerConfig) => {
const producer = new RdKafkaProducer({
@@ -71,18 +79,22 @@ export const produce = async ({
value,
key,
headers = [],
- waitForAck = true,
+ waitForAck,
}: {
producer: RdKafkaProducer
topic: string
value: MessageValue
key: MessageKey
headers?: MessageHeader[]
- waitForAck?: boolean
+ waitForAck: boolean
}): Promise => {
status.debug('📤', 'Producing message', { topic: topic })
const produceSpan = getSpan()?.startChild({ op: 'kafka_produce' })
return await new Promise((resolve, reject) => {
+ const produceTimer = ingestEventKafkaProduceLatency
+ .labels({ topic, waitForAck: waitForAck.toString() })
+ .startTimer()
+
if (waitForAck) {
producer.produce(
topic,
@@ -100,6 +112,7 @@ export const produce = async ({
resolve(offset)
}
+ produceTimer()
produceSpan?.finish()
}
)
@@ -112,6 +125,7 @@ export const produce = async ({
produceSpan?.finish()
})
resolve(undefined)
+ produceTimer()
}
})
}
diff --git a/plugin-server/src/main/graphile-worker/schedule.ts b/plugin-server/src/main/graphile-worker/schedule.ts
index d50c672cea428..16435d02c0466 100644
--- a/plugin-server/src/main/graphile-worker/schedule.ts
+++ b/plugin-server/src/main/graphile-worker/schedule.ts
@@ -56,8 +56,11 @@ export async function runScheduledTasks(
for (const pluginConfigId of server.pluginSchedule?.[taskType] || []) {
status.info('⏲️', 'queueing_schedule_task', { taskType, pluginConfigId })
await server.kafkaProducer.queueMessage({
- topic: KAFKA_SCHEDULED_TASKS,
- messages: [{ key: pluginConfigId.toString(), value: JSON.stringify({ taskType, pluginConfigId }) }],
+ kafkaMessage: {
+ topic: KAFKA_SCHEDULED_TASKS,
+ messages: [{ key: pluginConfigId.toString(), value: JSON.stringify({ taskType, pluginConfigId }) }],
+ },
+ waitForAck: true,
})
graphileScheduledTaskCounter.labels({ status: 'queued', task: taskType }).inc()
}
diff --git a/plugin-server/src/main/ingestion-queues/batch-processing/each-batch-ingestion.ts b/plugin-server/src/main/ingestion-queues/batch-processing/each-batch-ingestion.ts
index 749e41c18c335..588c2c92beb86 100644
--- a/plugin-server/src/main/ingestion-queues/batch-processing/each-batch-ingestion.ts
+++ b/plugin-server/src/main/ingestion-queues/batch-processing/each-batch-ingestion.ts
@@ -15,6 +15,7 @@ import { eventDroppedCounter, latestOffsetTimestampGauge } from '../metrics'
import {
ingestEventBatchingBatchCountSummary,
ingestEventBatchingInputLengthSummary,
+ ingestEventEachBatchKafkaAckWait,
ingestionOverflowingMessagesTotal,
ingestionParallelism,
ingestionParallelismPotential,
@@ -41,7 +42,7 @@ type IngestionSplitBatch = {
type IngestResult = {
// Promises that the batch handler should await on before committing offsets,
// contains the Kafka producer ACKs, to avoid blocking after every message.
- promises?: Array>
+ ackPromises?: Array>
}
async function handleProcessingError(
@@ -166,7 +167,7 @@ export async function eachBatchParallelIngestion(
return await runner.runEventPipeline(pluginEvent)
})) as IngestResult
- result.promises?.forEach((promise) =>
+ result.ackPromises?.forEach((promise) =>
processingPromises.push(
promise.catch(async (error) => {
await handleProcessingError(error, message, pluginEvent, queue)
@@ -227,7 +228,9 @@ export async function eachBatchParallelIngestion(
// impact the success. Delaying ACKs allows the producer to write in big batches for
// better throughput and lower broker load.
const awaitSpan = transaction.startChild({ op: 'awaitACKs', data: { promiseCount: processingPromises.length } })
+ const kafkaAckWaitMetric = ingestEventEachBatchKafkaAckWait.startTimer()
await Promise.all(processingPromises)
+ kafkaAckWaitMetric()
awaitSpan.finish()
for (const message of messages) {
diff --git a/plugin-server/src/main/ingestion-queues/batch-processing/metrics.ts b/plugin-server/src/main/ingestion-queues/batch-processing/metrics.ts
index 42c1b06a27b5d..60563b6cabaaa 100644
--- a/plugin-server/src/main/ingestion-queues/batch-processing/metrics.ts
+++ b/plugin-server/src/main/ingestion-queues/batch-processing/metrics.ts
@@ -41,3 +41,9 @@ export const ingestEventBatchingBatchCountSummary = new Summary({
help: 'Number of batches of events',
percentiles: [0.5, 0.9, 0.95, 0.99],
})
+
+export const ingestEventEachBatchKafkaAckWait = new Summary({
+ name: 'ingest_event_each_batch_kafka_ack_wait',
+ help: 'Wait time for the batch of Kafka ACKs at the end of eachBatchParallelIngestion',
+ percentiles: [0.5, 0.9, 0.95, 0.99],
+})
diff --git a/plugin-server/src/main/ingestion-queues/jobs-consumer.ts b/plugin-server/src/main/ingestion-queues/jobs-consumer.ts
index 94549340da4fe..605a812068c51 100644
--- a/plugin-server/src/main/ingestion-queues/jobs-consumer.ts
+++ b/plugin-server/src/main/ingestion-queues/jobs-consumer.ts
@@ -54,8 +54,11 @@ export const startJobsConsumer = async ({
})
// TODO: handle resolving offsets asynchronously
await producer.queueMessage({
- topic: KAFKA_JOBS_DLQ,
- messages: [{ value: message.value, key: message.key }],
+ kafkaMessage: {
+ topic: KAFKA_JOBS_DLQ,
+ messages: [{ value: message.value, key: message.key }],
+ },
+ waitForAck: true,
})
resolveOffset(message.offset)
continue
@@ -71,8 +74,11 @@ export const startJobsConsumer = async ({
})
// TODO: handle resolving offsets asynchronously
await producer.queueMessage({
- topic: KAFKA_JOBS_DLQ,
- messages: [{ value: message.value, key: message.key }],
+ kafkaMessage: {
+ topic: KAFKA_JOBS_DLQ,
+ messages: [{ value: message.value, key: message.key }],
+ },
+ waitForAck: true,
})
resolveOffset(message.offset)
continue
diff --git a/plugin-server/src/main/ingestion-queues/scheduled-tasks-consumer.ts b/plugin-server/src/main/ingestion-queues/scheduled-tasks-consumer.ts
index 3de544ce2d0a4..83ea62fdfdd6f 100644
--- a/plugin-server/src/main/ingestion-queues/scheduled-tasks-consumer.ts
+++ b/plugin-server/src/main/ingestion-queues/scheduled-tasks-consumer.ts
@@ -163,8 +163,11 @@ const getTasksFromBatch = async (batch: Batch, producer: KafkaProducerWrapper) =
value: message.value,
})
await producer.queueMessage({
- topic: KAFKA_SCHEDULED_TASKS_DLQ,
- messages: [{ value: message.value, key: message.key }],
+ kafkaMessage: {
+ topic: KAFKA_SCHEDULED_TASKS_DLQ,
+ messages: [{ value: message.value, key: message.key }],
+ },
+ waitForAck: true,
})
continue
}
@@ -181,8 +184,11 @@ const getTasksFromBatch = async (batch: Batch, producer: KafkaProducerWrapper) =
error: error.stack ?? error,
})
await producer.queueMessage({
- topic: KAFKA_SCHEDULED_TASKS_DLQ,
- messages: [{ value: message.value, key: message.key }],
+ kafkaMessage: {
+ topic: KAFKA_SCHEDULED_TASKS_DLQ,
+ messages: [{ value: message.value, key: message.key }],
+ },
+ waitForAck: true,
})
continue
}
@@ -190,8 +196,11 @@ const getTasksFromBatch = async (batch: Batch, producer: KafkaProducerWrapper) =
if (!taskTypes.includes(task.taskType) || isNaN(task.pluginConfigId)) {
status.warn('⚠️', `Invalid schema for partition ${batch.partition} offset ${message.offset}.`, task)
await producer.queueMessage({
- topic: KAFKA_SCHEDULED_TASKS_DLQ,
- messages: [{ value: message.value, key: message.key }],
+ kafkaMessage: {
+ topic: KAFKA_SCHEDULED_TASKS_DLQ,
+ messages: [{ value: message.value, key: message.key }],
+ },
+ waitForAck: true,
})
continue
}
diff --git a/plugin-server/src/main/ingestion-queues/session-recording/services/console-logs-ingester.ts b/plugin-server/src/main/ingestion-queues/session-recording/services/console-logs-ingester.ts
index 5729da5cb373e..1c581451e44ec 100644
--- a/plugin-server/src/main/ingestion-queues/session-recording/services/console-logs-ingester.ts
+++ b/plugin-server/src/main/ingestion-queues/session-recording/services/console-logs-ingester.ts
@@ -163,6 +163,7 @@ export class ConsoleLogsIngester {
topic: KAFKA_LOG_ENTRIES,
value: Buffer.from(JSON.stringify(cle)),
key: event.session_id,
+ waitForAck: true,
})
)
} catch (error) {
diff --git a/plugin-server/src/main/ingestion-queues/session-recording/services/overflow-detection.ts b/plugin-server/src/main/ingestion-queues/session-recording/services/overflow-detection.ts
new file mode 100644
index 0000000000000..8b478b781bc95
--- /dev/null
+++ b/plugin-server/src/main/ingestion-queues/session-recording/services/overflow-detection.ts
@@ -0,0 +1,45 @@
+import LRUCache from 'lru-cache'
+import { Gauge } from 'prom-client'
+
+import { Limiter } from '../../../../utils/token-bucket'
+
+export enum OverflowState {
+ Okay,
+ Triggered, // Recently triggered the overflow detection
+ Cooldown, // Already triggered the overflow detection earlier than cooldownSeconds
+}
+
+export const overflowTriggeredGauge = new Gauge({
+ name: 'overflow_detection_triggered_total',
+ help: 'Number of entities that triggered overflow detection.',
+})
+
+/**
+ * OverflowDetection handles consumer-side detection of hot partitions by
+ * accounting for data volumes per entity (a session_id, a distinct_id...).
+ *
+ * The first time that the observed spike crosses the thresholds set via burstCapacity
+ * and replenishRate, observe returns Triggered. Subsequent calls will return Cooldown
+ * until cooldownSeconds is reached.
+ */
+export class OverflowDetection {
+ private limiter: Limiter
+ private triggered: LRUCache
+
+ constructor(burstCapacity: number, replenishRate: number, cooldownSeconds: number) {
+ this.limiter = new Limiter(burstCapacity, replenishRate)
+ this.triggered = new LRUCache({ max: 1_000_000, maxAge: cooldownSeconds * 1000 })
+ }
+
+ public observe(key: string, quantity: number, now?: number): OverflowState {
+ if (this.triggered.has(key)) {
+ return OverflowState.Cooldown
+ }
+ if (this.limiter.consume(key, quantity, now)) {
+ return OverflowState.Okay
+ }
+ this.triggered.set(key, true)
+ overflowTriggeredGauge.inc(1)
+ return OverflowState.Triggered
+ }
+}
diff --git a/plugin-server/src/main/ingestion-queues/session-recording/services/replay-events-ingester.ts b/plugin-server/src/main/ingestion-queues/session-recording/services/replay-events-ingester.ts
index 632f695a158f5..029f28f20bb9a 100644
--- a/plugin-server/src/main/ingestion-queues/session-recording/services/replay-events-ingester.ts
+++ b/plugin-server/src/main/ingestion-queues/session-recording/services/replay-events-ingester.ts
@@ -171,6 +171,7 @@ export class ReplayEventsIngester {
topic: KAFKA_CLICKHOUSE_SESSION_REPLAY_EVENTS,
value: Buffer.from(JSON.stringify(replayRecord)),
key: event.session_id,
+ waitForAck: true,
}),
]
} catch (error) {
diff --git a/plugin-server/src/main/ingestion-queues/session-recording/session-recordings-consumer.ts b/plugin-server/src/main/ingestion-queues/session-recording/session-recordings-consumer.ts
index 30aaab4a023d5..491044652d80f 100644
--- a/plugin-server/src/main/ingestion-queues/session-recording/session-recordings-consumer.ts
+++ b/plugin-server/src/main/ingestion-queues/session-recording/session-recordings-consumer.ts
@@ -20,6 +20,7 @@ import { addSentryBreadcrumbsEventListeners } from '../kafka-metrics'
import { eventDroppedCounter } from '../metrics'
import { ConsoleLogsIngester } from './services/console-logs-ingester'
import { OffsetHighWaterMarker } from './services/offset-high-water-marker'
+import { OverflowDetection } from './services/overflow-detection'
import { RealtimeManager } from './services/realtime-manager'
import { ReplayEventsIngester } from './services/replay-events-ingester'
import { BUCKETS_KB_WRITTEN, SessionManager } from './services/session-manager'
@@ -128,6 +129,7 @@ export class SessionRecordingIngester {
sessionHighWaterMarker: OffsetHighWaterMarker
persistentHighWaterMarker: OffsetHighWaterMarker
realtimeManager: RealtimeManager
+ overflowDetection?: OverflowDetection
replayEventsIngester?: ReplayEventsIngester
consoleLogsIngester?: ConsoleLogsIngester
batchConsumer?: BatchConsumer
@@ -160,6 +162,14 @@ export class SessionRecordingIngester {
this.realtimeManager = new RealtimeManager(this.redisPool, this.config)
+ if (globalServerConfig.SESSION_RECORDING_OVERFLOW_ENABLED) {
+ this.overflowDetection = new OverflowDetection(
+ globalServerConfig.SESSION_RECORDING_OVERFLOW_BUCKET_CAPACITY,
+ globalServerConfig.SESSION_RECORDING_OVERFLOW_BUCKET_REPLENISH_RATE,
+ 24 * 3600 // One day
+ )
+ }
+
// We create a hash of the cluster to use as a unique identifier for the high-water marks
// This enables us to swap clusters without having to worry about resetting the high-water marks
const kafkaClusterIdentifier = crypto.createHash('md5').update(this.config.KAFKA_HOSTS).digest('hex')
@@ -275,6 +285,9 @@ export class SessionRecordingIngester {
return
}
+ // TODO: update Redis if this triggers
+ this.overflowDetection?.observe(key, event.metadata.rawSize, event.metadata.timestamp)
+
if (!this.sessions[key]) {
const { partition, topic } = event.metadata
diff --git a/plugin-server/src/main/ingestion-queues/session-recording/types.ts b/plugin-server/src/main/ingestion-queues/session-recording/types.ts
index 254e3f0897ee7..d61dadda9279e 100644
--- a/plugin-server/src/main/ingestion-queues/session-recording/types.ts
+++ b/plugin-server/src/main/ingestion-queues/session-recording/types.ts
@@ -6,6 +6,7 @@ export type IncomingRecordingMessage = {
metadata: {
topic: string
partition: number
+ rawSize: number
lowOffset: number
highOffset: number
timestamp: number
diff --git a/plugin-server/src/main/ingestion-queues/session-recording/utils.ts b/plugin-server/src/main/ingestion-queues/session-recording/utils.ts
index 4b4345d43b48d..53ce953e5bd92 100644
--- a/plugin-server/src/main/ingestion-queues/session-recording/utils.ts
+++ b/plugin-server/src/main/ingestion-queues/session-recording/utils.ts
@@ -225,6 +225,7 @@ export const parseKafkaMessage = async (
metadata: {
partition: message.partition,
topic: message.topic,
+ rawSize: message.size,
lowOffset: message.offset,
highOffset: message.offset,
timestamp: message.timestamp,
@@ -267,6 +268,7 @@ export const reduceRecordingMessages = (messages: IncomingRecordingMessage[]): I
existingMessage.eventsByWindowId[windowId] = events
}
}
+ existingMessage.metadata.rawSize += clonedMessage.metadata.rawSize
// Update the events ranges
existingMessage.metadata.lowOffset = Math.min(
diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts
index b8eeb5b296a9e..114547cfe605f 100644
--- a/plugin-server/src/types.ts
+++ b/plugin-server/src/types.ts
@@ -230,6 +230,10 @@ export interface PluginsServerConfig {
// a single partition which will output many more log messages to the console
// useful when that partition is lagging unexpectedly
SESSION_RECORDING_DEBUG_PARTITION: string | undefined
+ // overflow detection, updating Redis for capture to move the traffic away
+ SESSION_RECORDING_OVERFLOW_ENABLED: boolean
+ SESSION_RECORDING_OVERFLOW_BUCKET_CAPACITY: number
+ SESSION_RECORDING_OVERFLOW_BUCKET_REPLENISH_RATE: number
// Dedicated infra values
SESSION_RECORDING_KAFKA_HOSTS: string | undefined
diff --git a/plugin-server/src/utils/db/db.ts b/plugin-server/src/utils/db/db.ts
index 2baa10671a91e..c7b6ce86a895a 100644
--- a/plugin-server/src/utils/db/db.ts
+++ b/plugin-server/src/utils/db/db.ts
@@ -707,7 +707,7 @@ export class DB {
})
}
- await this.kafkaProducer.queueMessages(kafkaMessages)
+ await this.kafkaProducer.queueMessages({ kafkaMessages, waitForAck: true })
return person
}
@@ -759,7 +759,7 @@ export class DB {
if (tx) {
kafkaMessages.push(message)
} else {
- await this.kafkaProducer.queueMessage(message)
+ await this.kafkaProducer.queueMessage({ kafkaMessage: message, waitForAck: true })
}
status.debug(
@@ -829,7 +829,7 @@ export class DB {
public async addDistinctId(person: Person, distinctId: string): Promise {
const kafkaMessages = await this.addDistinctIdPooled(person, distinctId)
if (kafkaMessages.length) {
- await this.kafkaProducer.queueMessages(kafkaMessages)
+ await this.kafkaProducer.queueMessages({ kafkaMessages, waitForAck: true })
}
}
@@ -1072,15 +1072,15 @@ export class DB {
pluginLogEntryCounter.labels({ plugin_id: String(pluginConfig.plugin_id), source }).inc()
try {
- await this.kafkaProducer.queueSingleJsonMessage(
- KAFKA_PLUGIN_LOG_ENTRIES,
- parsedEntry.id,
- parsedEntry,
+ await this.kafkaProducer.queueSingleJsonMessage({
+ topic: KAFKA_PLUGIN_LOG_ENTRIES,
+ key: parsedEntry.id,
+ object: parsedEntry,
// For logs, we relax our durability requirements a little and
// do not wait for acks that Kafka has persisted the message to
// disk.
- false
- )
+ waitForAck: false,
+ })
} catch (e) {
captureException(e, { tags: { team_id: entry.pluginConfig.team_id } })
console.error('Failed to produce message', e, parsedEntry)
@@ -1409,19 +1409,22 @@ export class DB {
version: number
): Promise {
await this.kafkaProducer.queueMessage({
- topic: KAFKA_GROUPS,
- messages: [
- {
- value: JSON.stringify({
- group_type_index: groupTypeIndex,
- group_key: groupKey,
- team_id: teamId,
- group_properties: JSON.stringify(properties),
- created_at: castTimestampOrNow(createdAt, TimestampFormat.ClickHouseSecondPrecision),
- version,
- }),
- },
- ],
+ kafkaMessage: {
+ topic: KAFKA_GROUPS,
+ messages: [
+ {
+ value: JSON.stringify({
+ group_type_index: groupTypeIndex,
+ group_key: groupKey,
+ team_id: teamId,
+ group_properties: JSON.stringify(properties),
+ created_at: castTimestampOrNow(createdAt, TimestampFormat.ClickHouseSecondPrecision),
+ version,
+ }),
+ },
+ ],
+ },
+ waitForAck: true,
})
}
diff --git a/plugin-server/src/utils/db/hub.ts b/plugin-server/src/utils/db/hub.ts
index 0a50533a1dbdb..098a44e7d4aa6 100644
--- a/plugin-server/src/utils/db/hub.ts
+++ b/plugin-server/src/utils/db/hub.ts
@@ -159,13 +159,16 @@ export async function createHub(
// chained, and if we do not manage to produce then the chain will be
// broken.
await kafkaProducer.queueMessage({
- topic: KAFKA_JOBS,
- messages: [
- {
- value: Buffer.from(JSON.stringify(job)),
- key: Buffer.from(job.pluginConfigTeam.toString()),
- },
- ],
+ kafkaMessage: {
+ topic: KAFKA_JOBS,
+ messages: [
+ {
+ value: Buffer.from(JSON.stringify(job)),
+ key: Buffer.from(job.pluginConfigTeam.toString()),
+ },
+ ],
+ },
+ waitForAck: true,
})
}
diff --git a/plugin-server/src/utils/db/kafka-producer-wrapper.ts b/plugin-server/src/utils/db/kafka-producer-wrapper.ts
index 8f7cef4c06b30..0ea1e01c5099f 100644
--- a/plugin-server/src/utils/db/kafka-producer-wrapper.ts
+++ b/plugin-server/src/utils/db/kafka-producer-wrapper.ts
@@ -35,7 +35,7 @@ export class KafkaProducerWrapper {
key: MessageKey
topic: string
headers?: MessageHeader[]
- waitForAck?: boolean
+ waitForAck: boolean
}): Promise {
try {
kafkaProducerMessagesQueuedCounter.labels({ topic_name: topic }).inc()
@@ -66,7 +66,7 @@ export class KafkaProducerWrapper {
}
}
- async queueMessage(kafkaMessage: ProducerRecord, waitForAck?: boolean) {
+ async queueMessage({ kafkaMessage, waitForAck }: { kafkaMessage: ProducerRecord; waitForAck: boolean }) {
return await Promise.all(
kafkaMessage.messages.map((message) =>
this.produce({
@@ -80,23 +80,34 @@ export class KafkaProducerWrapper {
)
}
- async queueMessages(kafkaMessages: ProducerRecord[], waitForAck?: boolean): Promise {
- await Promise.all(kafkaMessages.map((message) => this.queueMessage(message, waitForAck)))
+ async queueMessages({
+ kafkaMessages,
+ waitForAck,
+ }: {
+ kafkaMessages: ProducerRecord[]
+ waitForAck: boolean
+ }): Promise {
+ await Promise.all(kafkaMessages.map((kafkaMessage) => this.queueMessage({ kafkaMessage, waitForAck })))
}
- async queueSingleJsonMessage(
- topic: string,
- key: Message['key'],
- object: Record,
- waitForAck?: boolean
- ): Promise {
- await this.queueMessage(
- {
+ async queueSingleJsonMessage({
+ topic,
+ key,
+ object,
+ waitForAck,
+ }: {
+ topic: string
+ key: Message['key']
+ object: Record
+ waitForAck: boolean
+ }): Promise {
+ await this.queueMessage({
+ kafkaMessage: {
topic,
messages: [{ key, value: JSON.stringify(object) }],
},
- waitForAck
- )
+ waitForAck,
+ })
}
public async flush() {
diff --git a/plugin-server/src/worker/ingestion/app-metrics.ts b/plugin-server/src/worker/ingestion/app-metrics.ts
index d8f52a7401150..5e0a83c92ae31 100644
--- a/plugin-server/src/worker/ingestion/app-metrics.ts
+++ b/plugin-server/src/worker/ingestion/app-metrics.ts
@@ -183,8 +183,11 @@ export class AppMetrics {
}))
await this.kafkaProducer.queueMessage({
- topic: KAFKA_APP_METRICS,
- messages: kafkaMessages,
+ kafkaMessage: {
+ topic: KAFKA_APP_METRICS,
+ messages: kafkaMessages,
+ },
+ waitForAck: true,
})
status.debug('🚽', `Finished flushing app metrics, took ${Date.now() - startTime}ms`)
}
diff --git a/plugin-server/src/worker/ingestion/event-pipeline/runner.ts b/plugin-server/src/worker/ingestion/event-pipeline/runner.ts
index 7d71548381b16..6ae2248513073 100644
--- a/plugin-server/src/worker/ingestion/event-pipeline/runner.ts
+++ b/plugin-server/src/worker/ingestion/event-pipeline/runner.ts
@@ -25,7 +25,7 @@ import { processPersonsStep } from './processPersonsStep'
export type EventPipelineResult = {
// Promises that the batch handler should await on before committing offsets,
// contains the Kafka producer ACKs, to avoid blocking after every message.
- promises?: Array>
+ ackPromises?: Array>
// Only used in tests
// TODO: update to test for side-effects of running the pipeline rather than
// this return type.
@@ -78,14 +78,14 @@ export class EventPipelineRunner {
drop_cause: 'disallowed',
})
.inc()
- return this.registerLastStep('eventDisallowedStep', null, [event])
+ return this.registerLastStep('eventDisallowedStep', [event])
}
let result: EventPipelineResult
const eventWithTeam = await this.runStep(populateTeamDataStep, [this, event], event.team_id || -1)
if (eventWithTeam != null) {
result = await this.runEventPipelineSteps(eventWithTeam)
} else {
- result = this.registerLastStep('populateTeamDataStep', null, [event])
+ result = this.registerLastStep('populateTeamDataStep', [event])
}
eventProcessedAndIngestedCounter.inc()
return result
@@ -120,7 +120,7 @@ export class EventPipelineRunner {
const processedEvent = await this.runStep(pluginsProcessEventStep, [this, event], event.team_id)
if (processedEvent == null) {
- return this.registerLastStep('pluginsProcessEventStep', event.team_id, [event])
+ return this.registerLastStep('pluginsProcessEventStep', [event])
}
const [normalizedEvent, person] = await this.runStep(processPersonsStep, [this, processedEvent], event.team_id)
@@ -132,17 +132,12 @@ export class EventPipelineRunner {
event.team_id
)
- return this.registerLastStep('createEventStep', event.team_id, [rawClickhouseEvent, person], [eventAck])
+ return this.registerLastStep('createEventStep', [rawClickhouseEvent, person], [eventAck])
}
- registerLastStep(
- stepName: string,
- teamId: number | null,
- args: any[],
- promises?: Array>
- ): EventPipelineResult {
+ registerLastStep(stepName: string, args: any[], ackPromises?: Array>): EventPipelineResult {
pipelineLastStepCounter.labels(stepName).inc()
- return { promises: promises, lastStep: stepName, args }
+ return { ackPromises, lastStep: stepName, args }
}
protected runStep any>(
@@ -218,7 +213,7 @@ export class EventPipelineRunner {
teamId,
`plugin_server_ingest_event:${currentStepName}`
)
- await this.hub.db.kafkaProducer!.queueMessage(message)
+ await this.hub.db.kafkaProducer!.queueMessage({ kafkaMessage: message, waitForAck: true })
} catch (dlqError) {
status.info('🔔', `Errored trying to add event to dead letter queue. Error: ${dlqError}`)
Sentry.captureException(dlqError, {
diff --git a/plugin-server/src/worker/ingestion/person-state.ts b/plugin-server/src/worker/ingestion/person-state.ts
index b2356f3652662..525bbbf84c910 100644
--- a/plugin-server/src/worker/ingestion/person-state.ts
+++ b/plugin-server/src/worker/ingestion/person-state.ts
@@ -453,7 +453,7 @@ export class PersonState {
olderCreatedAt, // Keep the oldest created_at (i.e. the first time we've seen either person)
properties
)
- await this.db.kafkaProducer.queueMessages(kafkaMessages)
+ await this.db.kafkaProducer.queueMessages({ kafkaMessages, waitForAck: true })
return mergedPerson
}
@@ -767,7 +767,7 @@ export class DeferredPersonOverrideWorker {
// Postgres for some reason -- the same row state should be
// generated each call, and the receiving ReplacingMergeTree will
// ensure we keep only the latest version after all writes settle.)
- await this.kafkaProducer.queueMessages(messages, true)
+ await this.kafkaProducer.queueMessages({ kafkaMessages: messages, waitForAck: true })
return rows.length
}
diff --git a/plugin-server/src/worker/ingestion/utils.ts b/plugin-server/src/worker/ingestion/utils.ts
index c52ef4ebba78e..9488ee759581b 100644
--- a/plugin-server/src/worker/ingestion/utils.ts
+++ b/plugin-server/src/worker/ingestion/utils.ts
@@ -80,18 +80,21 @@ export async function captureIngestionWarning(
const limiter_key = `${teamId}:${type}:${debounce?.key || ''}`
if (!!debounce?.alwaysSend || IngestionWarningLimiter.consume(limiter_key, 1)) {
await kafkaProducer.queueMessage({
- topic: KAFKA_INGESTION_WARNINGS,
- messages: [
- {
- value: JSON.stringify({
- team_id: teamId,
- type: type,
- source: 'plugin-server',
- details: JSON.stringify(details),
- timestamp: castTimestampOrNow(null, TimestampFormat.ClickHouse),
- }),
- },
- ],
+ kafkaMessage: {
+ topic: KAFKA_INGESTION_WARNINGS,
+ messages: [
+ {
+ value: JSON.stringify({
+ team_id: teamId,
+ type: type,
+ source: 'plugin-server',
+ details: JSON.stringify(details),
+ timestamp: castTimestampOrNow(null, TimestampFormat.ClickHouse),
+ }),
+ },
+ ],
+ },
+ waitForAck: true,
})
} else {
return Promise.resolve()
diff --git a/plugin-server/src/worker/vm/extensions/posthog.ts b/plugin-server/src/worker/vm/extensions/posthog.ts
index c7a0a7124c50d..34e9cb2befd1c 100644
--- a/plugin-server/src/worker/vm/extensions/posthog.ts
+++ b/plugin-server/src/worker/vm/extensions/posthog.ts
@@ -29,22 +29,25 @@ async function queueEvent(hub: Hub, pluginConfig: PluginConfig, data: InternalDa
const partitionKey = partitionKeyHash.digest('hex')
await hub.kafkaProducer.queueMessage({
- topic: hub.KAFKA_CONSUMPTION_TOPIC!,
- messages: [
- {
- key: partitionKey,
- value: JSON.stringify({
- distinct_id: data.distinct_id,
- ip: '',
- site_url: '',
- data: JSON.stringify(data),
- team_id: pluginConfig.team_id,
- now: data.timestamp,
- sent_at: data.timestamp,
- uuid: data.uuid,
- } as RawEventMessage),
- },
- ],
+ kafkaMessage: {
+ topic: hub.KAFKA_CONSUMPTION_TOPIC!,
+ messages: [
+ {
+ key: partitionKey,
+ value: JSON.stringify({
+ distinct_id: data.distinct_id,
+ ip: '',
+ site_url: '',
+ data: JSON.stringify(data),
+ team_id: pluginConfig.team_id,
+ now: data.timestamp,
+ sent_at: data.timestamp,
+ uuid: data.uuid,
+ } as RawEventMessage),
+ },
+ ],
+ },
+ waitForAck: true,
})
}
diff --git a/plugin-server/tests/main/db.test.ts b/plugin-server/tests/main/db.test.ts
index 2adc7567c8a5d..14448f196f9be 100644
--- a/plugin-server/tests/main/db.test.ts
+++ b/plugin-server/tests/main/db.test.ts
@@ -367,9 +367,10 @@ describe('DB', () => {
expect(updatedPerson.properties).toEqual({ c: 'aaa' })
// verify correct Kafka message was sent
- expect(db.kafkaProducer!.queueMessage).toHaveBeenLastCalledWith(
- generateKafkaPersonUpdateMessage(updatedPerson)
- )
+ expect(db.kafkaProducer!.queueMessage).toHaveBeenLastCalledWith({
+ kafkaMessage: generateKafkaPersonUpdateMessage(updatedPerson),
+ waitForAck: true,
+ })
})
})
@@ -416,7 +417,7 @@ describe('DB', () => {
await delayUntilEventIngested(fetchPersonsRows, 2)
const kafkaMessages = await db.deletePerson(person)
- await db.kafkaProducer.queueMessages(kafkaMessages)
+ await db.kafkaProducer.queueMessages({ kafkaMessages, waitForAck: true })
await db.kafkaProducer.flush()
const persons = await delayUntilEventIngested(fetchPersonsRows, 3)
diff --git a/plugin-server/tests/main/ingestion-queues/analytics-events-ingestion-overflow-consumer.test.ts b/plugin-server/tests/main/ingestion-queues/analytics-events-ingestion-overflow-consumer.test.ts
index 851bb23e2ac14..774475a5b34aa 100644
--- a/plugin-server/tests/main/ingestion-queues/analytics-events-ingestion-overflow-consumer.test.ts
+++ b/plugin-server/tests/main/ingestion-queues/analytics-events-ingestion-overflow-consumer.test.ts
@@ -97,20 +97,23 @@ describe('eachBatchParallelIngestion with overflow consume', () => {
expect(queue.pluginsServer.teamManager.getTeamForEvent).toHaveBeenCalledTimes(1)
expect(consume).toHaveBeenCalledWith('1:ingestion_capacity_overflow:id', 1)
expect(mockQueueMessage).toHaveBeenCalledWith({
- topic: 'clickhouse_ingestion_warnings_test',
- messages: [
- {
- value: JSON.stringify({
- team_id: 1,
- type: 'ingestion_capacity_overflow',
- source: 'plugin-server',
- details: JSON.stringify({
- overflowDistinctId: 'id',
+ kafkaMessage: {
+ topic: 'clickhouse_ingestion_warnings_test',
+ messages: [
+ {
+ value: JSON.stringify({
+ team_id: 1,
+ type: 'ingestion_capacity_overflow',
+ source: 'plugin-server',
+ details: JSON.stringify({
+ overflowDistinctId: 'id',
+ }),
+ timestamp: castTimestampOrNow(null, TimestampFormat.ClickHouse),
}),
- timestamp: castTimestampOrNow(null, TimestampFormat.ClickHouse),
- }),
- },
- ],
+ },
+ ],
+ },
+ waitForAck: true,
})
// Event is processed
diff --git a/plugin-server/tests/main/ingestion-queues/each-batch.test.ts b/plugin-server/tests/main/ingestion-queues/each-batch.test.ts
index 667c278d243f1..b0e61e62fdd66 100644
--- a/plugin-server/tests/main/ingestion-queues/each-batch.test.ts
+++ b/plugin-server/tests/main/ingestion-queues/each-batch.test.ts
@@ -408,7 +408,7 @@ describe('eachBatchX', () => {
const batch = createBatch(captureEndpointEvent)
runEventPipeline.mockImplementationOnce(() =>
Promise.resolve({
- promises: [Promise.resolve(), Promise.reject('deferred nopes out')],
+ ackPromises: [Promise.resolve(), Promise.reject('deferred nopes out')],
})
)
const tokenBlockList = buildStringMatcher('another_token,more_token', false)
diff --git a/plugin-server/tests/main/ingestion-queues/session-recording/__snapshots__/utils.test.ts.snap b/plugin-server/tests/main/ingestion-queues/session-recording/__snapshots__/utils.test.ts.snap
index 9962eb544bc6d..87ca515b22bd6 100644
--- a/plugin-server/tests/main/ingestion-queues/session-recording/__snapshots__/utils.test.ts.snap
+++ b/plugin-server/tests/main/ingestion-queues/session-recording/__snapshots__/utils.test.ts.snap
@@ -33,6 +33,7 @@ Array [
"highOffset": 3,
"lowOffset": 1,
"partition": 1,
+ "rawSize": 12,
"timestamp": 1,
"topic": "the_topic",
},
@@ -59,6 +60,7 @@ Array [
"highOffset": 4,
"lowOffset": 4,
"partition": 1,
+ "rawSize": 30,
"timestamp": 4,
"topic": "the_topic",
},
@@ -85,6 +87,7 @@ Array [
"highOffset": 5,
"lowOffset": 5,
"partition": 1,
+ "rawSize": 31,
"timestamp": 5,
"topic": "the_topic",
},
@@ -130,6 +133,7 @@ Object {
"highOffset": 1,
"lowOffset": 1,
"partition": 1,
+ "rawSize": 42,
"timestamp": 1,
"topic": "the_topic",
},
diff --git a/plugin-server/tests/main/ingestion-queues/session-recording/services/console-log-ingester.test.ts b/plugin-server/tests/main/ingestion-queues/session-recording/services/console-log-ingester.test.ts
index 42dfb9e55b5c1..6698b40a8ca6a 100644
--- a/plugin-server/tests/main/ingestion-queues/session-recording/services/console-log-ingester.test.ts
+++ b/plugin-server/tests/main/ingestion-queues/session-recording/services/console-log-ingester.test.ts
@@ -80,6 +80,7 @@ describe('console log ingester', () => {
timestamp: '1970-01-01 00:00:00.000',
})
),
+ waitForAck: true,
},
],
])
@@ -124,6 +125,7 @@ describe('console log ingester', () => {
timestamp: '1970-01-01 00:00:00.000',
})
),
+ waitForAck: true,
},
],
[
@@ -142,6 +144,7 @@ describe('console log ingester', () => {
timestamp: '1970-01-01 00:00:00.000',
})
),
+ waitForAck: true,
},
],
])
@@ -181,6 +184,7 @@ describe('console log ingester', () => {
timestamp: '1970-01-01 00:00:00.000',
})
),
+ waitForAck: true,
},
],
])
diff --git a/plugin-server/tests/main/ingestion-queues/session-recording/utils.test.ts b/plugin-server/tests/main/ingestion-queues/session-recording/utils.test.ts
index b8e6dc59284e7..c5a3851486d93 100644
--- a/plugin-server/tests/main/ingestion-queues/session-recording/utils.test.ts
+++ b/plugin-server/tests/main/ingestion-queues/session-recording/utils.test.ts
@@ -57,7 +57,7 @@ describe('session-recording utils', () => {
})
),
timestamp: 1,
- size: 1,
+ size: 42,
topic: 'the_topic',
offset: 1,
partition: 1,
@@ -257,7 +257,7 @@ describe('session-recording utils', () => {
distinct_id: '1',
eventsRange: { start: 1, end: 1 },
eventsByWindowId: { window_1: [{ timestamp: 1, type: 1, data: {} }] },
- metadata: { lowOffset: 1, highOffset: 1, partition: 1, timestamp: 1, topic: 'the_topic' },
+ metadata: { lowOffset: 1, highOffset: 1, partition: 1, timestamp: 1, topic: 'the_topic', rawSize: 5 },
session_id: '1',
team_id: 1,
snapshot_source: null,
@@ -266,7 +266,7 @@ describe('session-recording utils', () => {
distinct_id: '1',
eventsRange: { start: 2, end: 2 },
eventsByWindowId: { window_1: [{ timestamp: 2, type: 2, data: {} }] },
- metadata: { lowOffset: 2, highOffset: 2, partition: 1, timestamp: 2, topic: 'the_topic' },
+ metadata: { lowOffset: 2, highOffset: 2, partition: 1, timestamp: 2, topic: 'the_topic', rawSize: 4 },
session_id: '1',
team_id: 1,
snapshot_source: null,
@@ -276,7 +276,7 @@ describe('session-recording utils', () => {
distinct_id: '1',
eventsRange: { start: 3, end: 3 },
eventsByWindowId: { window_2: [{ timestamp: 3, type: 3, data: {} }] },
- metadata: { lowOffset: 3, highOffset: 3, partition: 1, timestamp: 3, topic: 'the_topic' },
+ metadata: { lowOffset: 3, highOffset: 3, partition: 1, timestamp: 3, topic: 'the_topic', rawSize: 3 },
session_id: '1',
team_id: 1,
snapshot_source: null,
@@ -286,7 +286,7 @@ describe('session-recording utils', () => {
distinct_id: '1',
eventsRange: { start: 4, end: 4 },
eventsByWindowId: { window_1: [{ timestamp: 4, type: 4, data: {} }] },
- metadata: { lowOffset: 4, highOffset: 4, partition: 1, timestamp: 4, topic: 'the_topic' },
+ metadata: { lowOffset: 4, highOffset: 4, partition: 1, timestamp: 4, topic: 'the_topic', rawSize: 30 },
session_id: '1',
team_id: 2,
snapshot_source: null,
@@ -296,7 +296,7 @@ describe('session-recording utils', () => {
distinct_id: '1',
eventsRange: { start: 5, end: 5 },
eventsByWindowId: { window_1: [{ timestamp: 5, type: 5, data: {} }] },
- metadata: { lowOffset: 5, highOffset: 5, partition: 1, timestamp: 5, topic: 'the_topic' },
+ metadata: { lowOffset: 5, highOffset: 5, partition: 1, timestamp: 5, topic: 'the_topic', rawSize: 31 },
session_id: '2',
team_id: 1,
snapshot_source: null,
diff --git a/plugin-server/tests/main/jobs/schedule.test.ts b/plugin-server/tests/main/jobs/schedule.test.ts
index 150d171f97d3b..b6b52c892f23d 100644
--- a/plugin-server/tests/main/jobs/schedule.test.ts
+++ b/plugin-server/tests/main/jobs/schedule.test.ts
@@ -37,120 +37,147 @@ describe('Graphile Worker schedule', () => {
} as any)
expect(mockHubWithPluginSchedule.kafkaProducer.queueMessage).toHaveBeenNthCalledWith(1, {
- topic: KAFKA_SCHEDULED_TASKS,
- messages: [
- {
- key: '1',
- value: JSON.stringify({
- taskType: 'runEveryMinute',
- pluginConfigId: 1,
- }),
- },
- ],
+ kafkaMessage: {
+ topic: KAFKA_SCHEDULED_TASKS,
+ messages: [
+ {
+ key: '1',
+ value: JSON.stringify({
+ taskType: 'runEveryMinute',
+ pluginConfigId: 1,
+ }),
+ },
+ ],
+ },
+ waitForAck: true,
})
expect(mockHubWithPluginSchedule.kafkaProducer.queueMessage).toHaveBeenNthCalledWith(2, {
- topic: KAFKA_SCHEDULED_TASKS,
- messages: [
- {
- key: '2',
- value: JSON.stringify({
- taskType: 'runEveryMinute',
- pluginConfigId: 2,
- }),
- },
- ],
+ kafkaMessage: {
+ topic: KAFKA_SCHEDULED_TASKS,
+ messages: [
+ {
+ key: '2',
+ value: JSON.stringify({
+ taskType: 'runEveryMinute',
+ pluginConfigId: 2,
+ }),
+ },
+ ],
+ },
+ waitForAck: true,
})
expect(mockHubWithPluginSchedule.kafkaProducer.queueMessage).toHaveBeenNthCalledWith(3, {
- topic: KAFKA_SCHEDULED_TASKS,
- messages: [
- {
- key: '3',
- value: JSON.stringify({
- taskType: 'runEveryMinute',
- pluginConfigId: 3,
- }),
- },
- ],
+ kafkaMessage: {
+ topic: KAFKA_SCHEDULED_TASKS,
+ messages: [
+ {
+ key: '3',
+ value: JSON.stringify({
+ taskType: 'runEveryMinute',
+ pluginConfigId: 3,
+ }),
+ },
+ ],
+ },
+ waitForAck: true,
})
await runScheduledTasks(mockHubWithPluginSchedule, mockPiscina as any, 'runEveryHour', {
job: { run_at: new Date() },
} as any)
expect(mockHubWithPluginSchedule.kafkaProducer.queueMessage).toHaveBeenNthCalledWith(4, {
- topic: KAFKA_SCHEDULED_TASKS,
- messages: [
- {
- key: '4',
- value: JSON.stringify({
- taskType: 'runEveryHour',
- pluginConfigId: 4,
- }),
- },
- ],
+ kafkaMessage: {
+ topic: KAFKA_SCHEDULED_TASKS,
+ messages: [
+ {
+ key: '4',
+ value: JSON.stringify({
+ taskType: 'runEveryHour',
+ pluginConfigId: 4,
+ }),
+ },
+ ],
+ },
+ waitForAck: true,
})
expect(mockHubWithPluginSchedule.kafkaProducer.queueMessage).toHaveBeenNthCalledWith(5, {
- topic: KAFKA_SCHEDULED_TASKS,
- messages: [
- {
- key: '5',
- value: JSON.stringify({
- taskType: 'runEveryHour',
- pluginConfigId: 5,
- }),
- },
- ],
+ kafkaMessage: {
+ topic: KAFKA_SCHEDULED_TASKS,
+ messages: [
+ {
+ key: '5',
+ value: JSON.stringify({
+ taskType: 'runEveryHour',
+ pluginConfigId: 5,
+ }),
+ },
+ ],
+ },
+ waitForAck: true,
})
expect(mockHubWithPluginSchedule.kafkaProducer.queueMessage).toHaveBeenNthCalledWith(6, {
- topic: KAFKA_SCHEDULED_TASKS,
- messages: [
- {
- key: '6',
- value: JSON.stringify({
- taskType: 'runEveryHour',
- pluginConfigId: 6,
- }),
- },
- ],
+ kafkaMessage: {
+ topic: KAFKA_SCHEDULED_TASKS,
+ messages: [
+ {
+ key: '6',
+ value: JSON.stringify({
+ taskType: 'runEveryHour',
+ pluginConfigId: 6,
+ }),
+ },
+ ],
+ },
+ waitForAck: true,
})
await runScheduledTasks(mockHubWithPluginSchedule, mockPiscina as any, 'runEveryDay', {
job: { run_at: new Date() },
} as any)
expect(mockHubWithPluginSchedule.kafkaProducer.queueMessage).toHaveBeenNthCalledWith(7, {
- topic: KAFKA_SCHEDULED_TASKS,
- messages: [
- {
- key: '7',
- value: JSON.stringify({
- taskType: 'runEveryDay',
- pluginConfigId: 7,
- }),
- },
- ],
+ kafkaMessage: {
+ topic: KAFKA_SCHEDULED_TASKS,
+ messages: [
+ {
+ key: '7',
+ value: JSON.stringify({
+ taskType: 'runEveryDay',
+ pluginConfigId: 7,
+ }),
+ },
+ ],
+ },
+ waitForAck: true,
})
expect(mockHubWithPluginSchedule.kafkaProducer.queueMessage).toHaveBeenNthCalledWith(8, {
- topic: KAFKA_SCHEDULED_TASKS,
- messages: [
- {
- key: '8',
- value: JSON.stringify({
- taskType: 'runEveryDay',
- pluginConfigId: 8,
- }),
- },
- ],
+ kafkaMessage: {
+ topic: KAFKA_SCHEDULED_TASKS,
+ messages: [
+ {
+ key: '8',
+ value: JSON.stringify({
+ taskType: 'runEveryDay',
+ pluginConfigId: 8,
+ }),
+ },
+ ],
+ },
+ waitForAck: true,
})
expect(mockHubWithPluginSchedule.kafkaProducer.queueMessage).toHaveBeenNthCalledWith(9, {
- topic: KAFKA_SCHEDULED_TASKS,
- messages: [
- {
- key: '9',
- value: JSON.stringify({
- taskType: 'runEveryDay',
- pluginConfigId: 9,
- }),
- },
- ],
+ kafkaMessage: {
+ topic: KAFKA_SCHEDULED_TASKS,
+ messages: [
+ {
+ key: '9',
+ value: JSON.stringify({
+ taskType: 'runEveryDay',
+ pluginConfigId: 9,
+ }),
+ },
+ ],
+ },
+ waitForAck: true,
})
})
})
diff --git a/plugin-server/tests/worker/console.test.ts b/plugin-server/tests/worker/console.test.ts
index 18c3480989700..4535f10f6e327 100644
--- a/plugin-server/tests/worker/console.test.ts
+++ b/plugin-server/tests/worker/console.test.ts
@@ -42,10 +42,10 @@ describe('console extension', () => {
await (console[typeMethod](...args) as unknown as Promise)
expect(queueSingleJsonMessageSpy).toHaveBeenCalledTimes(1)
- expect(queueSingleJsonMessageSpy).toHaveBeenCalledWith(
- KAFKA_PLUGIN_LOG_ENTRIES,
- expect.any(String),
- {
+ expect(queueSingleJsonMessageSpy).toHaveBeenCalledWith({
+ topic: KAFKA_PLUGIN_LOG_ENTRIES,
+ key: expect.any(String),
+ object: {
source: PluginLogEntrySource.Console,
type,
id: expect.any(String),
@@ -56,8 +56,8 @@ describe('console extension', () => {
message: expectedFinalMessage,
instance_id: hub.instanceId.toString(),
},
- false
- )
+ waitForAck: false,
+ })
})
})
})
diff --git a/plugin-server/tests/worker/ingestion/__snapshots__/app-metrics.test.ts.snap b/plugin-server/tests/worker/ingestion/__snapshots__/app-metrics.test.ts.snap
index 10cf219b7e43b..1894a82b49dbd 100644
--- a/plugin-server/tests/worker/ingestion/__snapshots__/app-metrics.test.ts.snap
+++ b/plugin-server/tests/worker/ingestion/__snapshots__/app-metrics.test.ts.snap
@@ -4,12 +4,15 @@ exports[`AppMetrics() flush() flushes queued messages 1`] = `
Array [
Array [
Object {
- "messages": Array [
- Object {
- "value": "{\\"timestamp\\":\\"1970-01-01 00:16:40.000\\",\\"team_id\\":2,\\"plugin_config_id\\":2,\\"job_id\\":\\"000-000\\",\\"category\\":\\"processEvent\\",\\"successes\\":1,\\"successes_on_retry\\":0,\\"failures\\":0}",
- },
- ],
- "topic": "clickhouse_app_metrics_test",
+ "kafkaMessage": Object {
+ "messages": Array [
+ Object {
+ "value": "{\\"timestamp\\":\\"1970-01-01 00:16:40.000\\",\\"team_id\\":2,\\"plugin_config_id\\":2,\\"job_id\\":\\"000-000\\",\\"category\\":\\"processEvent\\",\\"successes\\":1,\\"successes_on_retry\\":0,\\"failures\\":0}",
+ },
+ ],
+ "topic": "clickhouse_app_metrics_test",
+ },
+ "waitForAck": true,
},
],
]
diff --git a/plugin-server/tests/worker/ingestion/event-pipeline/runner.test.ts b/plugin-server/tests/worker/ingestion/event-pipeline/runner.test.ts
index 4bfc79f5e2379..364483f7c09a6 100644
--- a/plugin-server/tests/worker/ingestion/event-pipeline/runner.test.ts
+++ b/plugin-server/tests/worker/ingestion/event-pipeline/runner.test.ts
@@ -219,7 +219,9 @@ describe('EventPipelineRunner', () => {
await runner.runEventPipeline(pipelineEvent)
expect(hub.db.kafkaProducer.queueMessage).toHaveBeenCalledTimes(1)
- expect(JSON.parse(hub.db.kafkaProducer.queueMessage.mock.calls[0][0].messages[0].value)).toMatchObject({
+ expect(
+ JSON.parse(hub.db.kafkaProducer.queueMessage.mock.calls[0][0].kafkaMessage.messages[0].value)
+ ).toMatchObject({
team_id: 2,
distinct_id: 'my_id',
error: 'Event ingestion failed. Error: testError',
diff --git a/plugin-server/tests/worker/ingestion/postgres-parity.test.ts b/plugin-server/tests/worker/ingestion/postgres-parity.test.ts
index 5cdf1246c53f5..5c764e5809b40 100644
--- a/plugin-server/tests/worker/ingestion/postgres-parity.test.ts
+++ b/plugin-server/tests/worker/ingestion/postgres-parity.test.ts
@@ -339,7 +339,7 @@ describe('postgres parity', () => {
// move distinct ids from person to to anotherPerson
const kafkaMessages = await hub.db.moveDistinctIds(person, anotherPerson)
- await hub.db!.kafkaProducer!.queueMessages(kafkaMessages)
+ await hub.db!.kafkaProducer!.queueMessages({ kafkaMessages, waitForAck: true })
await delayUntilEventIngested(() => hub.db.fetchDistinctIdValues(anotherPerson, Database.ClickHouse), 2)
// it got added
@@ -395,7 +395,7 @@ describe('postgres parity', () => {
// delete person
await hub.db.postgres.transaction(PostgresUse.COMMON_WRITE, '', async (client) => {
const deletePersonMessage = await hub.db.deletePerson(person, client)
- await hub.db!.kafkaProducer!.queueMessage(deletePersonMessage[0])
+ await hub.db!.kafkaProducer!.queueMessage({ kafkaMessage: deletePersonMessage[0], waitForAck: true })
})
await delayUntilEventIngested(async () =>
diff --git a/plugin-server/tests/worker/vm.test.ts b/plugin-server/tests/worker/vm.test.ts
index 7e3769de61328..5f1f727d4dbeb 100644
--- a/plugin-server/tests/worker/vm.test.ts
+++ b/plugin-server/tests/worker/vm.test.ts
@@ -689,10 +689,10 @@ describe('vm tests', () => {
await vm.methods.processEvent!(event)
expect(queueSingleJsonMessageSpy).toHaveBeenCalledTimes(1)
- expect(queueSingleJsonMessageSpy).toHaveBeenCalledWith(
- KAFKA_PLUGIN_LOG_ENTRIES,
- expect.any(String),
- {
+ expect(queueSingleJsonMessageSpy).toHaveBeenCalledWith({
+ topic: KAFKA_PLUGIN_LOG_ENTRIES,
+ key: expect.any(String),
+ object: {
id: expect.any(String),
instance_id: hub.instanceId.toString(),
message: 'logged event',
@@ -703,8 +703,8 @@ describe('vm tests', () => {
timestamp: expect.any(String),
type: PluginLogEntryType.Log,
},
- false
- )
+ waitForAck: false,
+ })
})
test('fetch', async () => {
@@ -969,8 +969,8 @@ describe('vm tests', () => {
expect(response).toBe('haha')
expect(queueMessageSpy).toHaveBeenCalledTimes(1)
- expect(queueMessageSpy.mock.calls[0][0].topic).toEqual(KAFKA_EVENTS_PLUGIN_INGESTION)
- const parsedMessage = JSON.parse(queueMessageSpy.mock.calls[0][0].messages[0].value!.toString())
+ expect(queueMessageSpy.mock.calls[0][0].kafkaMessage.topic).toEqual(KAFKA_EVENTS_PLUGIN_INGESTION)
+ const parsedMessage = JSON.parse(queueMessageSpy.mock.calls[0][0].kafkaMessage.messages[0].value!.toString())
expect(JSON.parse(parsedMessage.data)).toMatchObject({
distinct_id: 'plugin-id-60',
event: 'my-new-event',
@@ -998,8 +998,8 @@ describe('vm tests', () => {
expect(response).toBe('haha')
expect(queueMessageSpy).toHaveBeenCalledTimes(1)
- expect(queueMessageSpy.mock.calls[0][0].topic).toEqual(KAFKA_EVENTS_PLUGIN_INGESTION)
- const parsedMessage = JSON.parse(queueMessageSpy.mock.calls[0][0].messages[0].value!.toString())
+ expect(queueMessageSpy.mock.calls[0][0].kafkaMessage.topic).toEqual(KAFKA_EVENTS_PLUGIN_INGESTION)
+ const parsedMessage = JSON.parse(queueMessageSpy.mock.calls[0][0].kafkaMessage.messages[0].value!.toString())
expect(JSON.parse(parsedMessage.data)).toMatchObject({
timestamp: '2020-02-23T02:15:00Z', // taken out of the properties
distinct_id: 'plugin-id-60',
@@ -1025,8 +1025,8 @@ describe('vm tests', () => {
expect(response).toBe('haha')
expect(response).toBe('haha')
expect(queueMessageSpy).toHaveBeenCalledTimes(1)
- expect(queueMessageSpy.mock.calls[0][0].topic).toEqual(KAFKA_EVENTS_PLUGIN_INGESTION)
- const parsedMessage = JSON.parse(queueMessageSpy.mock.calls[0][0].messages[0].value!.toString())
+ expect(queueMessageSpy.mock.calls[0][0].kafkaMessage.topic).toEqual(KAFKA_EVENTS_PLUGIN_INGESTION)
+ const parsedMessage = JSON.parse(queueMessageSpy.mock.calls[0][0].kafkaMessage.messages[0].value!.toString())
expect(JSON.parse(parsedMessage.data)).toMatchObject({
distinct_id: 'custom id',
event: 'my-new-event',
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 5f157ec8b039e..73481ed11db17 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -238,6 +238,9 @@ dependencies:
monaco-editor:
specifier: ^0.39.0
version: 0.39.0
+ natural-orderby:
+ specifier: ^3.0.2
+ version: 3.0.2
papaparse:
specifier: ^5.4.1
version: 5.4.1
@@ -251,8 +254,8 @@ dependencies:
specifier: ^9.3.0
version: 9.3.0(postcss@8.4.31)
posthog-js:
- specifier: 1.116.3
- version: 1.116.3
+ specifier: 1.116.6
+ version: 1.116.6
posthog-js-lite:
specifier: 2.5.0
version: 2.5.0
@@ -6793,7 +6796,7 @@ packages:
'@storybook/csf': 0.1.3
'@storybook/global': 5.0.0
'@storybook/types': 7.6.17
- '@types/qs': 6.9.13
+ '@types/qs': 6.9.14
dequal: 2.0.3
lodash: 4.17.21
memoizerific: 1.11.3
@@ -8197,8 +8200,8 @@ packages:
resolution: {integrity: sha512-bZcOkJ6uWrL0Qb2NAWKa7TBU+mJHPzhx9jjLL1KHF+XpzEcR7EXHvjbHlGtR/IsP1vyPrehuS6XqkmaePy//mg==}
dev: false
- /@types/qs@6.9.13:
- resolution: {integrity: sha512-iLR+1vTTJ3p0QaOUq6ACbY1mzKTODFDT/XedZI8BksOotFmL4ForwDfRQ/DZeuTHR7/2i4lI1D203gdfxuqTlA==}
+ /@types/qs@6.9.14:
+ resolution: {integrity: sha512-5khscbd3SwWMhFqylJBLQ0zIu7c1K6Vz0uBIt915BI3zV0q1nfjRQD3RqSBcPaO6PHEF4ov/t9y89fSiyThlPA==}
dev: true
/@types/query-selector-shadow-dom@1.0.0:
@@ -13630,7 +13633,7 @@ packages:
hogan.js: 3.0.2
htm: 3.1.1
instantsearch-ui-components: 0.3.0
- preact: 10.20.0
+ preact: 10.20.1
qs: 6.9.7
search-insights: 2.13.0
dev: false
@@ -15917,6 +15920,11 @@ packages:
resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==}
dev: true
+ /natural-orderby@3.0.2:
+ resolution: {integrity: sha512-x7ZdOwBxZCEm9MM7+eQCjkrNLrW3rkBKNHVr78zbtqnMGVNlnDi6C/eUEYgxHNrcbu0ymvjzcwIL/6H1iHri9g==}
+ engines: {node: '>=18'}
+ dev: false
+
/needle@3.3.1:
resolution: {integrity: sha512-6k0YULvhpw+RoLNiQCRKOl09Rv1dPLr8hHnVjHqdolKwDrdNyk+Hmrthi4lIGPPz3r39dLx0hsF5s40sZ3Us4Q==}
engines: {node: '>= 4.4.x'}
@@ -17446,19 +17454,19 @@ packages:
resolution: {integrity: sha512-Urvlp0Vu9h3td0BVFWt0QXFJDoOZcaAD83XM9d91NKMKTVPZtfU0ysoxstIf5mw/ce9ZfuMgpWPaagrZI4rmSg==}
dev: false
- /posthog-js@1.116.3:
- resolution: {integrity: sha512-KakGsQ8rS/K/U5Q/tiBrRrFRCgGrR0oI9VSYw9hwNCY00EClwAU3EuykUuQTFdQ1EuYMrZDIMWDD4NW6zgf7wQ==}
+ /posthog-js@1.116.6:
+ resolution: {integrity: sha512-rvt8HxzJD4c2B/xsUa4jle8ApdqljeBI2Qqjp4XJMohQf18DXRyM6b96H5/UMs8jxYuZG14Er0h/kEIWeU6Fmw==}
dependencies:
fflate: 0.4.8
- preact: 10.20.0
+ preact: 10.20.1
dev: false
/potpack@2.0.0:
resolution: {integrity: sha512-Q+/tYsFU9r7xoOJ+y/ZTtdVQwTWfzjbiXBDMM/JKUux3+QPP02iUuIoeBQ+Ot6oEDlC+/PGjB/5A3K7KKb7hcw==}
dev: false
- /preact@10.20.0:
- resolution: {integrity: sha512-wU7iZw2BjsaKDal3pDRDy/HpPB6cuFOnVUCcw9aIPKG98+ZrXx3F+szkos8BVME5bquyKDKvRlOJFG8kMkcAbg==}
+ /preact@10.20.1:
+ resolution: {integrity: sha512-JIFjgFg9B2qnOoGiYMVBtrcFxHqn+dNXbq76bVmcaHYJFYR4lW67AOcXgAYQQTDYXDOg/kTZrKPNCdRgJ2UJmw==}
dev: false
/prelude-ls@1.2.1:
diff --git a/posthog/api/decide.py b/posthog/api/decide.py
index 93234aadd6a1d..3a6e08bc7a7a0 100644
--- a/posthog/api/decide.py
+++ b/posthog/api/decide.py
@@ -49,9 +49,11 @@ def on_permitted_recording_domain(team: Team, request: HttpRequest) -> bool:
) or hostname_in_allowed_url_list(team.recording_domains, referer)
# TODO this is a short term fix for beta testers
# TODO we will match on the app identifier in the origin instead and allow users to auth those
- is_authorized_android_client: bool = user_agent is not None and "posthog-android" in user_agent
+ is_authorized_mobile_client: bool = user_agent is not None and any(
+ keyword in user_agent for keyword in ["posthog-android", "posthog-ios"]
+ )
- return is_authorized_web_client or is_authorized_android_client
+ return is_authorized_web_client or is_authorized_mobile_client
def hostname_in_allowed_url_list(allowed_url_list: Optional[List[str]], hostname: Optional[str]) -> bool:
diff --git a/posthog/api/query.py b/posthog/api/query.py
index d8f45531253a0..e30853655c749 100644
--- a/posthog/api/query.py
+++ b/posthog/api/query.py
@@ -67,9 +67,10 @@ def create(self, request, *args, **kwargs) -> Response:
if data.async_:
query_status = enqueue_process_query_task(
team_id=self.team.pk,
+ user_id=self.request.user.pk,
query_json=request.data["query"],
query_id=client_query_id,
- refresh_requested=data.refresh,
+ refresh_requested=data.refresh or False,
)
return Response(query_status.model_dump(), status=status.HTTP_202_ACCEPTED)
diff --git a/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr b/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr
index 642602f396f8d..9ae54e6e582eb 100644
--- a/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr
+++ b/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr
@@ -2762,6 +2762,24 @@
5 /* ... */) /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/
'''
# ---
+# name: TestDashboard.test_listing_dashboards_is_not_nplus1.57
+ '''
+ SELECT "posthog_sharingconfiguration"."id",
+ "posthog_sharingconfiguration"."team_id",
+ "posthog_sharingconfiguration"."dashboard_id",
+ "posthog_sharingconfiguration"."insight_id",
+ "posthog_sharingconfiguration"."recording_id",
+ "posthog_sharingconfiguration"."created_at",
+ "posthog_sharingconfiguration"."enabled",
+ "posthog_sharingconfiguration"."access_token"
+ FROM "posthog_sharingconfiguration"
+ WHERE "posthog_sharingconfiguration"."dashboard_id" IN (1,
+ 2,
+ 3,
+ 4,
+ 5 /* ... */) /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/
+ '''
+# ---
# name: TestDashboard.test_listing_dashboards_is_not_nplus1.6
'''
SELECT "posthog_team"."id",
@@ -11959,6 +11977,24 @@
5 /* ... */) /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/
'''
# ---
+# name: TestDashboard.test_retrieve_dashboard_list.33
+ '''
+ SELECT "posthog_sharingconfiguration"."id",
+ "posthog_sharingconfiguration"."team_id",
+ "posthog_sharingconfiguration"."dashboard_id",
+ "posthog_sharingconfiguration"."insight_id",
+ "posthog_sharingconfiguration"."recording_id",
+ "posthog_sharingconfiguration"."created_at",
+ "posthog_sharingconfiguration"."enabled",
+ "posthog_sharingconfiguration"."access_token"
+ FROM "posthog_sharingconfiguration"
+ WHERE "posthog_sharingconfiguration"."dashboard_id" IN (1,
+ 2,
+ 3,
+ 4,
+ 5 /* ... */) /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/
+ '''
+# ---
# name: TestDashboard.test_retrieve_dashboard_list.4
'''
SELECT "posthog_dashboardtile"."id"
diff --git a/posthog/api/test/test_decide.py b/posthog/api/test/test_decide.py
index e56d763ed869f..7cb58bbce2324 100644
--- a/posthog/api/test/test_decide.py
+++ b/posthog/api/test/test_decide.py
@@ -482,6 +482,20 @@ def test_user_session_recording_allowed_for_android(self, *args) -> None:
"networkPayloadCapture": None,
}
+ def test_user_session_recording_allowed_for_ios(self, *args) -> None:
+ self._update_team({"session_recording_opt_in": True, "recording_domains": ["https://my-website.io"]})
+
+ response = self._post_decide(origin="any.site.com", user_agent="posthog-ios/3.1.0").json()
+ assert response["sessionRecording"] == {
+ "endpoint": "/s/",
+ "recorderVersion": "v2",
+ "consoleLogRecordingEnabled": False,
+ "sampleRate": None,
+ "linkedFlag": None,
+ "minimumDurationMilliseconds": None,
+ "networkPayloadCapture": None,
+ }
+
def test_user_session_recording_allowed_when_permitted_domains_are_not_http_based(self, *args):
self._update_team(
{
diff --git a/posthog/batch_exports/service.py b/posthog/batch_exports/service.py
index c26be9a77ed1a..d51dfdb2fbc3c 100644
--- a/posthog/batch_exports/service.py
+++ b/posthog/batch_exports/service.py
@@ -90,6 +90,7 @@ class S3BatchExportInputs:
kms_key_id: str | None = None
batch_export_schema: BatchExportSchema | None = None
endpoint_url: str | None = None
+ file_format: str = "JSONLines"
@dataclass
@@ -439,8 +440,11 @@ def create_batch_export_run(
return run
-def update_batch_export_run_status(
- run_id: UUID, status: str, latest_error: str | None, records_completed: int = 0
+def update_batch_export_run(
+ run_id: UUID,
+ status: str,
+ latest_error: str | None,
+ records_completed: int = 0,
) -> BatchExportRun:
"""Update the status of an BatchExportRun with given id.
@@ -448,7 +452,14 @@ def update_batch_export_run_status(
id: The id of the BatchExportRun to update.
"""
model = BatchExportRun.objects.filter(id=run_id)
- updated = model.update(status=status, latest_error=latest_error, records_completed=records_completed)
+ update_at = dt.datetime.now()
+
+ updated = model.update(
+ status=status,
+ latest_error=latest_error,
+ records_completed=records_completed,
+ last_updated_at=update_at,
+ )
if not updated:
raise ValueError(f"BatchExportRun with id {run_id} not found.")
diff --git a/posthog/clickhouse/client/execute_async.py b/posthog/clickhouse/client/execute_async.py
index 06f7fc639f824..4671b0060299b 100644
--- a/posthog/clickhouse/client/execute_async.py
+++ b/posthog/clickhouse/client/execute_async.py
@@ -1,5 +1,6 @@
import datetime
import json
+from typing import Optional
import uuid
import structlog
@@ -69,11 +70,12 @@ def delete_query_status(self):
def execute_process_query(
- team_id,
- query_id,
- query_json,
- limit_context,
- refresh_requested,
+ team_id: int,
+ user_id: int,
+ query_id: str,
+ query_json: dict,
+ limit_context: Optional[LimitContext],
+ refresh_requested: bool,
):
manager = QueryStatusManager(query_id, team_id)
@@ -91,7 +93,7 @@ def execute_process_query(
QUERY_WAIT_TIME.observe(wait_duration)
try:
- tag_queries(client_query_id=query_id, team_id=team_id)
+ tag_queries(client_query_id=query_id, team_id=team_id, user_id=user_id)
results = process_query(
team=team, query_json=query_json, limit_context=limit_context, refresh_requested=refresh_requested
)
@@ -113,12 +115,13 @@ def execute_process_query(
def enqueue_process_query_task(
- team_id,
- query_json,
- query_id=None,
- refresh_requested=False,
- bypass_celery=False,
- force=False,
+ team_id: int,
+ user_id: int,
+ query_json: dict,
+ query_id: Optional[str] = None,
+ refresh_requested: bool = False,
+ force: bool = False,
+ _test_only_bypass_celery: bool = False,
) -> QueryStatus:
if not query_id:
query_id = uuid.uuid4().hex
@@ -136,14 +139,23 @@ def enqueue_process_query_task(
query_status = QueryStatus(id=query_id, team_id=team_id, start_time=datetime.datetime.now(datetime.timezone.utc))
manager.store_query_status(query_status)
- if bypass_celery:
- # Call directly ( for testing )
+ if _test_only_bypass_celery:
process_query_task(
- team_id, query_id, query_json, limit_context=LimitContext.QUERY_ASYNC, refresh_requested=refresh_requested
+ team_id,
+ user_id,
+ query_id,
+ query_json,
+ limit_context=LimitContext.QUERY_ASYNC,
+ refresh_requested=refresh_requested,
)
else:
task = process_query_task.delay(
- team_id, query_id, query_json, limit_context=LimitContext.QUERY_ASYNC, refresh_requested=refresh_requested
+ team_id,
+ user_id,
+ query_id,
+ query_json,
+ limit_context=LimitContext.QUERY_ASYNC,
+ refresh_requested=refresh_requested,
)
query_status.task_id = task.id
manager.store_query_status(query_status)
diff --git a/posthog/clickhouse/client/test/test_execute_async.py b/posthog/clickhouse/client/test/test_execute_async.py
index 0d7a7281e6a4b..085e7708b9232 100644
--- a/posthog/clickhouse/client/test/test_execute_async.py
+++ b/posthog/clickhouse/client/test/test_execute_async.py
@@ -24,6 +24,7 @@ def setUp(self):
self.organization = Organization.objects.create(name="test")
self.team = Team.objects.create(organization=self.organization)
self.team_id = self.team.pk
+ self.user_id = 1337
self.query_id = "test_query_id"
self.query_json = {}
self.limit_context = None
@@ -41,7 +42,9 @@ def test_execute_process_query(self, mock_process_query, mock_redis_client):
mock_process_query.return_value = [float("inf"), float("-inf"), float("nan"), 1.0, "👍"]
- execute_process_query(self.team_id, self.query_id, self.query_json, self.limit_context, self.refresh_requested)
+ execute_process_query(
+ self.team_id, self.user_id, self.query_id, self.query_json, self.limit_context, self.refresh_requested
+ )
mock_redis_client.assert_called_once()
mock_process_query.assert_called_once()
@@ -55,15 +58,16 @@ def test_execute_process_query(self, mock_process_query, mock_redis_client):
class ClickhouseClientTestCase(TestCase, ClickhouseTestMixin):
def setUp(self):
- self.organization = Organization.objects.create(name="test")
- self.team = Team.objects.create(organization=self.organization)
- self.team_id = self.team.pk
+ self.organization: Organization = Organization.objects.create(name="test")
+ self.team: Team = Team.objects.create(organization=self.organization)
+ self.team_id: int = self.team.pk
+ self.user_id: int = 2137
@snapshot_clickhouse_queries
def test_async_query_client(self):
query = build_query("SELECT 1+1")
team_id = self.team_id
- query_id = client.enqueue_process_query_task(team_id, query, bypass_celery=True).id
+ query_id = client.enqueue_process_query_task(team_id, self.user_id, query, _test_only_bypass_celery=True).id
result = client.get_query_status(team_id, query_id)
self.assertFalse(result.error, result.error_message)
self.assertTrue(result.complete)
@@ -74,11 +78,13 @@ def test_async_query_client_errors(self):
self.assertRaises(
HogQLException,
client.enqueue_process_query_task,
- **{"team_id": (self.team_id), "query_json": query, "bypass_celery": True},
+ **{"team_id": self.team_id, "user_id": self.user_id, "query_json": query, "_test_only_bypass_celery": True},
)
query_id = uuid.uuid4().hex
try:
- client.enqueue_process_query_task(self.team_id, query, query_id=query_id, bypass_celery=True)
+ client.enqueue_process_query_task(
+ self.team_id, self.user_id, query, query_id=query_id, _test_only_bypass_celery=True
+ )
except Exception:
pass
@@ -89,7 +95,7 @@ def test_async_query_client_errors(self):
def test_async_query_client_uuid(self):
query = build_query("SELECT toUUID('00000000-0000-0000-0000-000000000000')")
team_id = self.team_id
- query_id = client.enqueue_process_query_task(team_id, query, bypass_celery=True).id
+ query_id = client.enqueue_process_query_task(team_id, self.user_id, query, _test_only_bypass_celery=True).id
result = client.get_query_status(team_id, query_id)
self.assertFalse(result.error, result.error_message)
self.assertTrue(result.complete)
@@ -99,7 +105,7 @@ def test_async_query_client_does_not_leak(self):
query = build_query("SELECT 1+1")
team_id = self.team_id
wrong_team = 5
- query_id = client.enqueue_process_query_task(team_id, query, bypass_celery=True).id
+ query_id = client.enqueue_process_query_task(team_id, self.user_id, query, _test_only_bypass_celery=True).id
try:
client.get_query_status(wrong_team, query_id)
@@ -111,13 +117,19 @@ def test_async_query_client_is_lazy(self, execute_sync_mock):
query = build_query("SELECT 4 + 4")
query_id = uuid.uuid4().hex
team_id = self.team_id
- client.enqueue_process_query_task(team_id, query, query_id=query_id, bypass_celery=True)
+ client.enqueue_process_query_task(
+ team_id, self.user_id, query, query_id=query_id, _test_only_bypass_celery=True
+ )
# Try the same query again
- client.enqueue_process_query_task(team_id, query, query_id=query_id, bypass_celery=True)
+ client.enqueue_process_query_task(
+ team_id, self.user_id, query, query_id=query_id, _test_only_bypass_celery=True
+ )
# Try the same query again (for good measure!)
- client.enqueue_process_query_task(team_id, query, query_id=query_id, bypass_celery=True)
+ client.enqueue_process_query_task(
+ team_id, self.user_id, query, query_id=query_id, _test_only_bypass_celery=True
+ )
# Assert that we only called clickhouse once
execute_sync_mock.assert_called_once()
@@ -127,13 +139,19 @@ def test_async_query_client_is_lazy_but_not_too_lazy(self, execute_sync_mock):
query = build_query("SELECT 8 + 8")
query_id = uuid.uuid4().hex
team_id = self.team_id
- client.enqueue_process_query_task(team_id, query, query_id=query_id, bypass_celery=True)
+ client.enqueue_process_query_task(
+ team_id, self.user_id, query, query_id=query_id, _test_only_bypass_celery=True
+ )
# Try the same query again, but with force
- client.enqueue_process_query_task(team_id, query, query_id=query_id, bypass_celery=True, force=True)
+ client.enqueue_process_query_task(
+ team_id, self.user_id, query, query_id=query_id, _test_only_bypass_celery=True, force=True
+ )
# Try the same query again (for good measure!)
- client.enqueue_process_query_task(team_id, query, query_id=query_id, bypass_celery=True)
+ client.enqueue_process_query_task(
+ team_id, self.user_id, query, query_id=query_id, _test_only_bypass_celery=True
+ )
# Assert that we called clickhouse twice
self.assertEqual(execute_sync_mock.call_count, 2)
@@ -145,13 +163,19 @@ def test_async_query_client_manual_query_uuid(self, execute_sync_mock):
query = build_query("SELECT 8 + 8")
team_id = self.team_id
query_id = "I'm so unique"
- client.enqueue_process_query_task(team_id, query, query_id=query_id, bypass_celery=True)
+ client.enqueue_process_query_task(
+ team_id, self.user_id, query, query_id=query_id, _test_only_bypass_celery=True
+ )
# Try the same query again, but with force
- client.enqueue_process_query_task(team_id, query, query_id=query_id, bypass_celery=True, force=True)
+ client.enqueue_process_query_task(
+ team_id, self.user_id, query, query_id=query_id, _test_only_bypass_celery=True, force=True
+ )
# Try the same query again (for good measure!)
- client.enqueue_process_query_task(team_id, query, query_id=query_id, bypass_celery=True)
+ client.enqueue_process_query_task(
+ team_id, self.user_id, query, query_id=query_id, _test_only_bypass_celery=True
+ )
# Assert that we called clickhouse twice
self.assertEqual(execute_sync_mock.call_count, 2)
@@ -186,4 +210,4 @@ def test_client_strips_comments_from_request(self):
# Make sure it still includes the "annotation" comment that includes
# request routing information for debugging purposes
- self.assertIn("/* request:1 */", first_query)
+ self.assertIn(f"/* user_id:{self.user_id} request:1 */", first_query)
diff --git a/posthog/errors.py b/posthog/errors.py
index 39f07be762d00..a6e3536042a7f 100644
--- a/posthog/errors.py
+++ b/posthog/errors.py
@@ -1,6 +1,6 @@
from dataclasses import dataclass
import re
-from typing import Dict
+from typing import Dict, Optional
from clickhouse_driver.errors import ServerException
@@ -8,9 +8,10 @@
class InternalCHQueryError(ServerException):
- code_name: str
+ code_name: Optional[str]
+ """Can be null if re-raised from a thread (see `failhard_threadhook_context`)."""
- def __init__(self, message, *, code=None, nested=None, code_name):
+ def __init__(self, message, *, code=None, nested=None, code_name=None):
self.code_name = code_name
super().__init__(message, code, nested)
@@ -151,7 +152,7 @@ def look_up_error_code_meta(error: ServerException) -> ErrorCodeMeta:
60: ErrorCodeMeta("UNKNOWN_TABLE"),
61: ErrorCodeMeta("ONLY_FILTER_COLUMN_IN_BLOCK"),
62: ErrorCodeMeta("SYNTAX_ERROR"),
- 63: ErrorCodeMeta("UNKNOWN_AGGREGATE_FUNCTION"),
+ 63: ErrorCodeMeta("UNKNOWN_AGGREGATE_FUNCTION", user_safe=True),
64: ErrorCodeMeta("CANNOT_READ_AGGREGATE_FUNCTION_FROM_TEXT"),
65: ErrorCodeMeta("CANNOT_WRITE_AGGREGATE_FUNCTION_AS_TEXT"),
66: ErrorCodeMeta("NOT_A_COLUMN"),
diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py
index a459514f2524f..806226b8f1b9e 100644
--- a/posthog/hogql/ast.py
+++ b/posthog/hogql/ast.py
@@ -46,8 +46,17 @@ def resolve_constant_type(self, context: HogQLContext):
def resolve_database_field(self, context: HogQLContext):
if isinstance(self.type, FieldType):
return self.type.resolve_database_field(context)
+ if isinstance(self.type, PropertyType):
+ return self.type.field_type.resolve_database_field(context)
raise NotImplementedException("FieldAliasType.resolve_database_field not implemented")
+ def resolve_table_type(self, context: HogQLContext):
+ if isinstance(self.type, FieldType):
+ return self.type.table_type
+ if isinstance(self.type, PropertyType):
+ return self.type.field_type.table_type
+ raise NotImplementedException("FieldAliasType.resolve_table_type not implemented")
+
@dataclass(kw_only=True)
class BaseTableType(Type):
@@ -339,6 +348,9 @@ def get_child(self, name: str | int, context: HogQLContext) -> Type:
f'Can not access property "{name}" on field "{self.name}" of type: {type(database_field).__name__}'
)
+ def resolve_table_type(self, context: HogQLContext):
+ return self.table_type
+
@dataclass(kw_only=True)
class PropertyType(Type):
diff --git a/posthog/hogql/base.py b/posthog/hogql/base.py
index fbdafffb2d08c..e8a74025b78be 100644
--- a/posthog/hogql/base.py
+++ b/posthog/hogql/base.py
@@ -32,7 +32,7 @@ def accept(self, visitor):
return visit(self)
if hasattr(visitor, "visit_unknown"):
return visitor.visit_unknown(self)
- raise NotImplementedException(f"Visitor has no method {method_name}")
+ raise NotImplementedException(f"{visitor.__class__.__name__} has no method {method_name}")
@dataclass(kw_only=True)
diff --git a/posthog/hogql/database/__init__.py b/posthog/hogql/database/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/posthog/hogql/database/database.py b/posthog/hogql/database/database.py
index 6909211070e59..afeac3c26a143 100644
--- a/posthog/hogql/database/database.py
+++ b/posthog/hogql/database/database.py
@@ -31,6 +31,11 @@
from posthog.hogql.database.schema.events import EventsTable
from posthog.hogql.database.schema.groups import GroupsTable, RawGroupsTable
from posthog.hogql.database.schema.numbers import NumbersTable
+from posthog.hogql.database.schema.person_distinct_id_overrides import (
+ PersonDistinctIdOverridesTable,
+ RawPersonDistinctIdOverridesTable,
+ join_with_person_distinct_id_overrides_table,
+)
from posthog.hogql.database.schema.person_distinct_ids import (
PersonDistinctIdsTable,
RawPersonDistinctIdsTable,
@@ -53,7 +58,6 @@
from posthog.models.team.team import WeekStartDay
from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode
-
if TYPE_CHECKING:
from posthog.models import Team
@@ -66,6 +70,7 @@ class Database(BaseModel):
groups: GroupsTable = GroupsTable()
persons: PersonsTable = PersonsTable()
person_distinct_ids: PersonDistinctIdsTable = PersonDistinctIdsTable()
+ person_distinct_id_overrides: PersonDistinctIdOverridesTable = PersonDistinctIdOverridesTable()
person_overrides: PersonOverridesTable = PersonOverridesTable()
session_replay_events: SessionReplayEventsTable = SessionReplayEventsTable()
@@ -81,6 +86,7 @@ class Database(BaseModel):
raw_persons: RawPersonsTable = RawPersonsTable()
raw_groups: RawGroupsTable = RawGroupsTable()
raw_cohort_people: RawCohortPeople = RawCohortPeople()
+ raw_person_distinct_id_overrides: RawPersonDistinctIdOverridesTable = RawPersonDistinctIdOverridesTable()
raw_person_overrides: RawPersonOverridesTable = RawPersonOverridesTable()
raw_sessions: RawSessionsTable = RawSessionsTable()
@@ -186,6 +192,24 @@ def create_hogql_database(
database.events.fields["poe"].fields["id"] = database.events.fields["person_id"]
database.events.fields["person"] = FieldTraverser(chain=["poe"])
+ elif modifiers.personsOnEventsMode == PersonsOnEventsMode.v3_enabled:
+ database.events.fields["event_person_id"] = StringDatabaseField(name="person_id")
+ database.events.fields["override"] = LazyJoin(
+ from_field=["distinct_id"], # ???
+ join_table=PersonDistinctIdOverridesTable(),
+ join_function=join_with_person_distinct_id_overrides_table,
+ )
+ database.events.fields["person_id"] = ExpressionField(
+ name="person_id",
+ expr=parse_expr(
+ # NOTE: assumes `join_use_nulls = 0` (the default), as ``override.distinct_id`` is not Nullable
+ "if(not(empty(override.distinct_id)), override.person_id, event_person_id)",
+ start=None,
+ ),
+ )
+ database.events.fields["poe"].fields["id"] = database.events.fields["person_id"]
+ database.events.fields["person"] = FieldTraverser(chain=["poe"])
+
database.persons.fields["$virt_initial_referring_domain_type"] = create_initial_domain_type(
"$virt_initial_referring_domain_type"
)
@@ -209,10 +233,22 @@ def create_hogql_database(
)
if "timestamp" not in tables[warehouse_modifier.table_name].fields.keys():
- tables[warehouse_modifier.table_name].fields["timestamp"] = ExpressionField(
- name="timestamp",
- expr=ast.Call(name="toDateTime", args=[ast.Field(chain=[warehouse_modifier.timestamp_field])]),
- )
+ table_model = DataWarehouseTable.objects.filter(
+ team_id=team.pk, name=warehouse_modifier.table_name
+ ).latest("created_at")
+ timestamp_field_type = table_model.get_clickhouse_column_type(warehouse_modifier.timestamp_field)
+
+ # If field type is none or datetime, we can use the field directly
+ if timestamp_field_type is None or timestamp_field_type.startswith("DateTime"):
+ tables[warehouse_modifier.table_name].fields["timestamp"] = ExpressionField(
+ name="timestamp",
+ expr=ast.Field(chain=[warehouse_modifier.timestamp_field]),
+ )
+ else:
+ tables[warehouse_modifier.table_name].fields["timestamp"] = ExpressionField(
+ name="timestamp",
+ expr=ast.Call(name="toDateTime", args=[ast.Field(chain=[warehouse_modifier.timestamp_field])]),
+ )
# TODO: Need to decide how the distinct_id and person_id fields are going to be handled
if "distinct_id" not in tables[warehouse_modifier.table_name].fields.keys():
diff --git a/posthog/hogql/database/models.py b/posthog/hogql/database/models.py
index 95a00595c6472..d2da7868a7f9c 100644
--- a/posthog/hogql/database/models.py
+++ b/posthog/hogql/database/models.py
@@ -3,7 +3,6 @@
from posthog.hogql.base import Expr
from posthog.hogql.errors import HogQLException, NotImplementedException
-from posthog.schema import HogQLQueryModifiers
if TYPE_CHECKING:
from posthog.hogql.context import HogQLContext
@@ -126,12 +125,14 @@ def resolve_table(self, context: "HogQLContext") -> Table:
class LazyTable(Table):
"""
- A table that is replaced with a subquery returned from `lazy_select(requested_fields: Dict[name, chain], modifiers: HogQLQueryModifiers)`
+ A table that is replaced with a subquery returned from `lazy_select(requested_fields: Dict[name, chain], modifiers: HogQLQueryModifiers, node: SelectQuery)`
"""
model_config = ConfigDict(extra="forbid")
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers) -> Any:
+ def lazy_select(
+ self, requested_fields: Dict[str, List[str | int]], context: "HogQLContext", node: "SelectQuery"
+ ) -> Any:
raise NotImplementedException("LazyTable.lazy_select not overridden")
diff --git a/posthog/hogql/database/schema/__init__.py b/posthog/hogql/database/schema/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/posthog/hogql/database/schema/channel_type.py b/posthog/hogql/database/schema/channel_type.py
index 5dee575fc59a3..4954cc5be2b29 100644
--- a/posthog/hogql/database/schema/channel_type.py
+++ b/posthog/hogql/database/schema/channel_type.py
@@ -62,6 +62,9 @@ def create_channel_type_expr(
gclid: ast.Expr,
gad_source: ast.Expr,
) -> ast.Expr:
+ def wrap_with_null_if_empty(expr: ast.Expr) -> ast.Expr:
+ return ast.Call(name="nullIf", args=[expr, ast.Constant(value="")])
+
return parse_expr(
"""
multiIf(
@@ -95,8 +98,8 @@ def create_channel_type_expr(
(
{referring_domain} = '$direct'
- AND ({medium} IS NULL OR {medium} = '')
- AND ({source} IS NULL OR {source} IN ('', '(direct)', 'direct'))
+ AND ({medium} IS NULL)
+ AND ({source} IS NULL OR {source} IN ('(direct)', 'direct'))
),
'Direct',
@@ -122,11 +125,11 @@ def create_channel_type_expr(
)""",
start=None,
placeholders={
- "campaign": campaign,
- "medium": medium,
- "source": source,
+ "campaign": wrap_with_null_if_empty(campaign),
+ "medium": wrap_with_null_if_empty(medium),
+ "source": wrap_with_null_if_empty(source),
"referring_domain": referring_domain,
- "gclid": gclid,
- "gad_source": gad_source,
+ "gclid": wrap_with_null_if_empty(gclid),
+ "gad_source": wrap_with_null_if_empty(gad_source),
},
)
diff --git a/posthog/hogql/database/schema/cohort_people.py b/posthog/hogql/database/schema/cohort_people.py
index 72080419b7355..11723f0194619 100644
--- a/posthog/hogql/database/schema/cohort_people.py
+++ b/posthog/hogql/database/schema/cohort_people.py
@@ -9,7 +9,6 @@
FieldOrTable,
)
from posthog.hogql.database.schema.persons import join_with_persons_table
-from posthog.schema import HogQLQueryModifiers
COHORT_PEOPLE_FIELDS = {
"person_id": StringDatabaseField(name="person_id"),
@@ -67,7 +66,7 @@ def to_printed_hogql(self):
class CohortPeople(LazyTable):
fields: Dict[str, FieldOrTable] = COHORT_PEOPLE_FIELDS
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers):
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node):
return select_from_cohort_people_table(requested_fields)
def to_printed_clickhouse(self, context):
diff --git a/posthog/hogql/database/schema/groups.py b/posthog/hogql/database/schema/groups.py
index bb237d68e8070..3b9de7f08befc 100644
--- a/posthog/hogql/database/schema/groups.py
+++ b/posthog/hogql/database/schema/groups.py
@@ -13,7 +13,6 @@
FieldOrTable,
)
from posthog.hogql.errors import HogQLException
-from posthog.schema import HogQLQueryModifiers
GROUPS_TABLE_FIELDS = {
"index": IntegerDatabaseField(name="group_type_index"),
@@ -83,7 +82,7 @@ def to_printed_hogql(self):
class GroupsTable(LazyTable):
fields: Dict[str, FieldOrTable] = GROUPS_TABLE_FIELDS
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers):
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node):
return select_from_groups_table(requested_fields)
def to_printed_clickhouse(self, context):
diff --git a/posthog/hogql/database/schema/log_entries.py b/posthog/hogql/database/schema/log_entries.py
index c14e90e26da50..9f5dc816ac4b0 100644
--- a/posthog/hogql/database/schema/log_entries.py
+++ b/posthog/hogql/database/schema/log_entries.py
@@ -9,7 +9,6 @@
LazyTable,
FieldOrTable,
)
-from posthog.schema import HogQLQueryModifiers
LOG_ENTRIES_FIELDS: Dict[str, FieldOrTable] = {
"team_id": IntegerDatabaseField(name="team_id"),
@@ -35,7 +34,7 @@ def to_printed_hogql(self):
class ReplayConsoleLogsLogEntriesTable(LazyTable):
fields: Dict[str, FieldOrTable] = LOG_ENTRIES_FIELDS
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers):
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node):
fields: List[ast.Expr] = [ast.Field(chain=["log_entries"] + chain) for name, chain in requested_fields.items()]
return ast.SelectQuery(
@@ -58,7 +57,7 @@ def to_printed_hogql(self):
class BatchExportLogEntriesTable(LazyTable):
fields: Dict[str, FieldOrTable] = LOG_ENTRIES_FIELDS
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers):
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node):
fields: List[ast.Expr] = [ast.Field(chain=["log_entries"] + chain) for name, chain in requested_fields.items()]
return ast.SelectQuery(
diff --git a/posthog/hogql/database/schema/person_distinct_id_overrides.py b/posthog/hogql/database/schema/person_distinct_id_overrides.py
new file mode 100644
index 0000000000000..34df59655c24d
--- /dev/null
+++ b/posthog/hogql/database/schema/person_distinct_id_overrides.py
@@ -0,0 +1,91 @@
+from typing import Dict, List
+from posthog.hogql.ast import SelectQuery
+from posthog.hogql.context import HogQLContext
+
+from posthog.hogql.database.argmax import argmax_select
+from posthog.hogql.database.models import (
+ Table,
+ IntegerDatabaseField,
+ StringDatabaseField,
+ BooleanDatabaseField,
+ LazyJoin,
+ LazyTable,
+ FieldOrTable,
+)
+from posthog.hogql.database.schema.persons import join_with_persons_table
+from posthog.hogql.errors import HogQLException
+
+PERSON_DISTINCT_ID_OVERRIDES_FIELDS = {
+ "team_id": IntegerDatabaseField(name="team_id"),
+ "distinct_id": StringDatabaseField(name="distinct_id"),
+ "person_id": StringDatabaseField(name="person_id"),
+ "person": LazyJoin(
+ from_field=["person_id"],
+ join_table="persons",
+ join_function=join_with_persons_table,
+ ),
+}
+
+
+def select_from_person_distinct_id_overrides_table(requested_fields: Dict[str, List[str | int]]):
+ # Always include "person_id", as it's the key we use to make further joins, and it'd be great if it's available
+ if "person_id" not in requested_fields:
+ requested_fields = {**requested_fields, "person_id": ["person_id"]}
+ return argmax_select(
+ table_name="raw_person_distinct_id_overrides",
+ select_fields=requested_fields,
+ group_fields=["distinct_id"],
+ argmax_field="version",
+ deleted_field="is_deleted",
+ )
+
+
+def join_with_person_distinct_id_overrides_table(
+ from_table: str,
+ to_table: str,
+ requested_fields: Dict[str, List[str]],
+ context: HogQLContext,
+ node: SelectQuery,
+):
+ from posthog.hogql import ast
+
+ if not requested_fields:
+ raise HogQLException("No fields requested from person_distinct_id_overrides")
+ join_expr = ast.JoinExpr(table=select_from_person_distinct_id_overrides_table(requested_fields))
+ join_expr.join_type = "LEFT OUTER JOIN"
+ join_expr.alias = to_table
+ join_expr.constraint = ast.JoinConstraint(
+ expr=ast.CompareOperation(
+ op=ast.CompareOperationOp.Eq,
+ left=ast.Field(chain=[from_table, "distinct_id"]),
+ right=ast.Field(chain=[to_table, "distinct_id"]),
+ )
+ )
+ return join_expr
+
+
+class RawPersonDistinctIdOverridesTable(Table):
+ fields: Dict[str, FieldOrTable] = {
+ **PERSON_DISTINCT_ID_OVERRIDES_FIELDS,
+ "is_deleted": BooleanDatabaseField(name="is_deleted"),
+ "version": IntegerDatabaseField(name="version"),
+ }
+
+ def to_printed_clickhouse(self, context):
+ return "person_distinct_id_overrides"
+
+ def to_printed_hogql(self):
+ return "raw_person_distinct_id_overrides"
+
+
+class PersonDistinctIdOverridesTable(LazyTable):
+ fields: Dict[str, FieldOrTable] = PERSON_DISTINCT_ID_OVERRIDES_FIELDS
+
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context: HogQLContext, node: SelectQuery):
+ return select_from_person_distinct_id_overrides_table(requested_fields)
+
+ def to_printed_clickhouse(self, context):
+ return "person_distinct_id_overrides"
+
+ def to_printed_hogql(self):
+ return "person_distinct_id_overrides"
diff --git a/posthog/hogql/database/schema/person_distinct_ids.py b/posthog/hogql/database/schema/person_distinct_ids.py
index 02144b35fc3d8..3304eccda862e 100644
--- a/posthog/hogql/database/schema/person_distinct_ids.py
+++ b/posthog/hogql/database/schema/person_distinct_ids.py
@@ -14,7 +14,6 @@
)
from posthog.hogql.database.schema.persons import join_with_persons_table
from posthog.hogql.errors import HogQLException
-from posthog.schema import HogQLQueryModifiers
PERSON_DISTINCT_IDS_FIELDS = {
"team_id": IntegerDatabaseField(name="team_id"),
@@ -82,7 +81,7 @@ def to_printed_hogql(self):
class PersonDistinctIdsTable(LazyTable):
fields: Dict[str, FieldOrTable] = PERSON_DISTINCT_IDS_FIELDS
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers):
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node):
return select_from_person_distinct_ids_table(requested_fields)
def to_printed_clickhouse(self, context):
diff --git a/posthog/hogql/database/schema/persons.py b/posthog/hogql/database/schema/persons.py
index a248da56b7307..c7abdd89e14c6 100644
--- a/posthog/hogql/database/schema/persons.py
+++ b/posthog/hogql/database/schema/persons.py
@@ -123,8 +123,8 @@ def to_printed_hogql(self):
class PersonsTable(LazyTable):
fields: Dict[str, FieldOrTable] = PERSONS_FIELDS
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers):
- return select_from_persons_table(requested_fields, modifiers)
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node):
+ return select_from_persons_table(requested_fields, context.modifiers)
def to_printed_clickhouse(self, context):
return "person"
diff --git a/posthog/hogql/database/schema/persons_pdi.py b/posthog/hogql/database/schema/persons_pdi.py
index 9f476f407b4d2..195643b90c08c 100644
--- a/posthog/hogql/database/schema/persons_pdi.py
+++ b/posthog/hogql/database/schema/persons_pdi.py
@@ -10,7 +10,6 @@
FieldOrTable,
)
from posthog.hogql.errors import HogQLException
-from posthog.schema import HogQLQueryModifiers
# :NOTE: We already have person_distinct_ids.py, which most tables link to. This persons_pdi.py is a hack to
@@ -63,7 +62,7 @@ class PersonsPDITable(LazyTable):
"person_id": StringDatabaseField(name="person_id"),
}
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers):
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node):
return persons_pdi_select(requested_fields)
def to_printed_clickhouse(self, context):
diff --git a/posthog/hogql/database/schema/session_replay_events.py b/posthog/hogql/database/schema/session_replay_events.py
index c9d564c7d4588..baaecef89e049 100644
--- a/posthog/hogql/database/schema/session_replay_events.py
+++ b/posthog/hogql/database/schema/session_replay_events.py
@@ -15,7 +15,6 @@
PersonDistinctIdsTable,
join_with_person_distinct_ids_table,
)
-from posthog.schema import HogQLQueryModifiers
RAW_ONLY_FIELDS = ["min_first_timestamp", "max_last_timestamp"]
@@ -115,7 +114,7 @@ class SessionReplayEventsTable(LazyTable):
"first_url": StringDatabaseField(name="first_url"),
}
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers):
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node):
return select_from_session_replay_events_table(requested_fields)
def to_printed_clickhouse(self, context):
diff --git a/posthog/hogql/database/schema/sessions.py b/posthog/hogql/database/schema/sessions.py
index 2a4865798eeb8..770daceaa23c5 100644
--- a/posthog/hogql/database/schema/sessions.py
+++ b/posthog/hogql/database/schema/sessions.py
@@ -1,5 +1,7 @@
from typing import Dict, List, cast
+from posthog.hogql import ast
+from posthog.hogql.context import HogQLContext
from posthog.hogql.database.models import (
StringDatabaseField,
DateTimeDatabaseField,
@@ -11,7 +13,7 @@
LazyTable,
)
from posthog.hogql.database.schema.channel_type import create_channel_type_expr
-from posthog.schema import HogQLQueryModifiers
+from posthog.hogql.database.schema.util.session_where_clause_extractor import SessionMinTimestampWhereClauseExtractor
SESSIONS_COMMON_FIELDS: Dict[str, FieldOrTable] = {
@@ -62,7 +64,9 @@ def avoid_asterisk_fields(self) -> List[str]:
]
-def select_from_sessions_table(requested_fields: Dict[str, List[str | int]]):
+def select_from_sessions_table(
+ requested_fields: Dict[str, List[str | int]], node: ast.SelectQuery, context: HogQLContext
+):
from posthog.hogql import ast
table_name = "raw_sessions"
@@ -134,10 +138,13 @@ def select_from_sessions_table(requested_fields: Dict[str, List[str | int]]):
)
group_by_fields.append(ast.Field(chain=cast(list[str | int], [table_name]) + chain))
+ where = SessionMinTimestampWhereClauseExtractor(context).get_inner_where(node)
+
return ast.SelectQuery(
select=select_fields,
select_from=ast.JoinExpr(table=ast.Field(chain=[table_name])),
group_by=group_by_fields,
+ where=where,
)
@@ -148,8 +155,8 @@ class SessionsTable(LazyTable):
"channel_type": StringDatabaseField(name="channel_type"),
}
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers):
- return select_from_sessions_table(requested_fields)
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node: ast.SelectQuery):
+ return select_from_sessions_table(requested_fields, node, context)
def to_printed_clickhouse(self, context):
return "sessions"
diff --git a/posthog/hogql/database/schema/test/test_channel_type.py b/posthog/hogql/database/schema/test/test_channel_type.py
index 89e026ff3aed0..10cd4ea4ae009 100644
--- a/posthog/hogql/database/schema/test/test_channel_type.py
+++ b/posthog/hogql/database/schema/test/test_channel_type.py
@@ -106,6 +106,21 @@ def test_direct(self):
),
)
+ def test_direct_empty_string(self):
+ self.assertEqual(
+ "Direct",
+ self._get_initial_channel_type(
+ {
+ "$initial_referring_domain": "$direct",
+ "$initial_utm_source": "",
+ "$initial_utm_medium": "",
+ "$initial_utm_campaign": "",
+ "$initial_gclid": "",
+ "$initial_gad_source": "",
+ }
+ ),
+ )
+
def test_cross_network(self):
self.assertEqual(
"Cross Network",
diff --git a/posthog/hogql/database/schema/util/__init__.py b/posthog/hogql/database/schema/util/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/posthog/hogql/database/schema/util/session_where_clause_extractor.py b/posthog/hogql/database/schema/util/session_where_clause_extractor.py
new file mode 100644
index 0000000000000..83933bdde8b85
--- /dev/null
+++ b/posthog/hogql/database/schema/util/session_where_clause_extractor.py
@@ -0,0 +1,398 @@
+from dataclasses import dataclass
+from typing import Optional
+
+from posthog.hogql import ast
+from posthog.hogql.ast import CompareOperationOp, ArithmeticOperationOp
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.models import DatabaseField
+
+from posthog.hogql.visitor import clone_expr, CloningVisitor, Visitor
+
+SESSION_BUFFER_DAYS = 3
+
+
+@dataclass
+class SessionMinTimestampWhereClauseExtractor(CloningVisitor):
+ """This class extracts the Where clause from the lazy sessions table, to the clickhouse sessions table.
+
+ The sessions table in Clickhouse is an AggregatingMergeTree, and will have one row per session per day. This means that
+ when we want to query sessions, we need to pre-group these rows, so that we only have one row per session.
+
+ We hide this detail using a lazy table, but to make querying the underlying Clickhouse table faster, we can inline the
+ min_timestamp where conditions from the select on the outer lazy table to the select on the inner real table.
+
+ This class is called on the select query of the lazy table, and will return the where clause that should be applied to
+ the inner table.
+
+ As a query can be unreasonably complex, we only handle simple cases, but this class is designed to fail-safe. If it
+ can't reason about a particular expression, it will just return a constant True, i.e. fetch more rows than necessary.
+
+ This means that we can incrementally add support for more complex queries, without breaking existing queries, by
+ handling more cases.
+
+ Some examples of failing-safe:
+
+ `SELECT * FROM sessions where min_timestamp > '2022-01-01' AND f(session_id)`
+ only the` min_timestamp > '2022-01-01'` part is relevant, so we can ignore the `f(session_id)` part, and it is safe
+ to replace it with a constant True, which collapses the AND to just the `min_timestamp > '2022-01-01'` part.
+
+ `SELECT * FROM sessions where min_timestamp > '2022-01-01' OR f(session_id)`
+ only the` min_timestamp > '2022-01-01'` part is relevant, and turning the `f(session_id)` part into a constant True
+ would collapse the OR to True. In this case we return None as no pre-filtering is possible.
+
+ All min_timestamp comparisons are given a buffer of SESSION_BUFFER_DAYS each side, to ensure that we collect all the
+ relevant rows for each session.
+ """
+
+ context: HogQLContext
+ clear_types: bool = False
+ clear_locations: bool = False
+
+ def get_inner_where(self, parsed_query: ast.SelectQuery) -> Optional[ast.Expr]:
+ if not parsed_query.where:
+ return None
+
+ # visit the where clause
+ where = self.visit(parsed_query.where)
+
+ if isinstance(where, ast.Constant):
+ return None
+
+ return clone_expr(where, clear_types=True, clear_locations=True)
+
+ def visit_compare_operation(self, node: ast.CompareOperation) -> ast.Expr:
+ is_left_constant = is_time_or_interval_constant(node.left)
+ is_right_constant = is_time_or_interval_constant(node.right)
+ is_left_timestamp_field = is_simple_timestamp_field_expression(node.left, self.context)
+ is_right_timestamp_field = is_simple_timestamp_field_expression(node.right, self.context)
+
+ if is_left_constant and is_right_constant:
+ # just ignore this comparison
+ return ast.Constant(value=True)
+
+ # handle the left side being a min_timestamp expression and the right being constant
+ if is_left_timestamp_field and is_right_constant:
+ if node.op == CompareOperationOp.Eq:
+ return ast.And(
+ exprs=[
+ ast.CompareOperation(
+ op=ast.CompareOperationOp.LtEq,
+ left=ast.ArithmeticOperation(
+ op=ast.ArithmeticOperationOp.Sub,
+ left=rewrite_timestamp_field(node.left, self.context),
+ right=ast.Call(name="toIntervalDay", args=[ast.Constant(value=SESSION_BUFFER_DAYS)]),
+ ),
+ right=node.right,
+ ),
+ ast.CompareOperation(
+ op=ast.CompareOperationOp.GtEq,
+ left=ast.ArithmeticOperation(
+ op=ast.ArithmeticOperationOp.Add,
+ left=rewrite_timestamp_field(node.left, self.context),
+ right=ast.Call(name="toIntervalDay", args=[ast.Constant(value=SESSION_BUFFER_DAYS)]),
+ ),
+ right=node.right,
+ ),
+ ]
+ )
+ elif node.op == CompareOperationOp.Gt or node.op == CompareOperationOp.GtEq:
+ return ast.CompareOperation(
+ op=ast.CompareOperationOp.GtEq,
+ left=ast.ArithmeticOperation(
+ op=ast.ArithmeticOperationOp.Add,
+ left=rewrite_timestamp_field(node.left, self.context),
+ right=ast.Call(name="toIntervalDay", args=[ast.Constant(value=SESSION_BUFFER_DAYS)]),
+ ),
+ right=node.right,
+ )
+ elif node.op == CompareOperationOp.Lt or node.op == CompareOperationOp.LtEq:
+ return ast.CompareOperation(
+ op=ast.CompareOperationOp.LtEq,
+ left=ast.ArithmeticOperation(
+ op=ast.ArithmeticOperationOp.Sub,
+ left=rewrite_timestamp_field(node.left, self.context),
+ right=ast.Call(name="toIntervalDay", args=[ast.Constant(value=SESSION_BUFFER_DAYS)]),
+ ),
+ right=node.right,
+ )
+ elif is_right_timestamp_field and is_left_constant:
+ # let's not duplicate the logic above, instead just flip and it and recurse
+ if node.op in [
+ CompareOperationOp.Eq,
+ CompareOperationOp.Lt,
+ CompareOperationOp.LtEq,
+ CompareOperationOp.Gt,
+ CompareOperationOp.GtEq,
+ ]:
+ return self.visit(
+ ast.CompareOperation(
+ op=CompareOperationOp.Eq
+ if node.op == CompareOperationOp.Eq
+ else CompareOperationOp.Lt
+ if node.op == CompareOperationOp.Gt
+ else CompareOperationOp.LtEq
+ if node.op == CompareOperationOp.GtEq
+ else CompareOperationOp.Gt
+ if node.op == CompareOperationOp.Lt
+ else CompareOperationOp.GtEq,
+ left=node.right,
+ right=node.left,
+ )
+ )
+
+ return ast.Constant(value=True)
+
+ def visit_arithmetic_operation(self, node: ast.ArithmeticOperation) -> ast.Expr:
+ # don't even try to handle complex logic
+ return ast.Constant(value=True)
+
+ def visit_not(self, node: ast.Not) -> ast.Expr:
+ return ast.Constant(value=True)
+
+ def visit_call(self, node: ast.Call) -> ast.Expr:
+ if node.name == "and":
+ return self.visit_and(ast.And(exprs=node.args))
+ elif node.name == "or":
+ return self.visit_or(ast.Or(exprs=node.args))
+ return ast.Constant(value=True)
+
+ def visit_field(self, node: ast.Field) -> ast.Expr:
+ return ast.Constant(value=True)
+
+ def visit_constant(self, node: ast.Constant) -> ast.Expr:
+ return ast.Constant(value=True)
+
+ def visit_placeholder(self, node: ast.Placeholder) -> ast.Expr:
+ raise Exception() # this should never happen, as placeholders should be resolved before this runs
+
+ def visit_and(self, node: ast.And) -> ast.Expr:
+ exprs = [self.visit(expr) for expr in node.exprs]
+
+ flattened = []
+ for expr in exprs:
+ if isinstance(expr, ast.And):
+ flattened.extend(expr.exprs)
+ else:
+ flattened.append(expr)
+
+ if any(isinstance(expr, ast.Constant) and expr.value is False for expr in flattened):
+ return ast.Constant(value=False)
+
+ filtered = [expr for expr in flattened if not isinstance(expr, ast.Constant) or expr.value is not True]
+ if len(filtered) == 0:
+ return ast.Constant(value=True)
+ elif len(filtered) == 1:
+ return filtered[0]
+ else:
+ return ast.And(exprs=filtered)
+
+ def visit_or(self, node: ast.Or) -> ast.Expr:
+ exprs = [self.visit(expr) for expr in node.exprs]
+
+ flattened = []
+ for expr in exprs:
+ if isinstance(expr, ast.Or):
+ flattened.extend(expr.exprs)
+ else:
+ flattened.append(expr)
+
+ if any(isinstance(expr, ast.Constant) and expr.value is True for expr in flattened):
+ return ast.Constant(value=True)
+
+ filtered = [expr for expr in flattened if not isinstance(expr, ast.Constant) or expr.value is not False]
+ if len(filtered) == 0:
+ return ast.Constant(value=False)
+ elif len(filtered) == 1:
+ return filtered[0]
+ else:
+ return ast.Or(exprs=filtered)
+
+ def visit_alias(self, node: ast.Alias) -> ast.Expr:
+ return self.visit(node.expr)
+
+
+def is_time_or_interval_constant(expr: ast.Expr) -> bool:
+ return IsTimeOrIntervalConstantVisitor().visit(expr)
+
+
+class IsTimeOrIntervalConstantVisitor(Visitor[bool]):
+ def visit_constant(self, node: ast.Constant) -> bool:
+ return True
+
+ def visit_compare_operation(self, node: ast.CompareOperation) -> bool:
+ return self.visit(node.left) and self.visit(node.right)
+
+ def visit_arithmetic_operation(self, node: ast.ArithmeticOperation) -> bool:
+ return self.visit(node.left) and self.visit(node.right)
+
+ def visit_call(self, node: ast.Call) -> bool:
+ # some functions just return a constant
+ if node.name in ["today", "now"]:
+ return True
+ # some functions return a constant if the first argument is a constant
+ if node.name in [
+ "parseDateTime64BestEffortOrNull",
+ "toDateTime",
+ "toTimeZone",
+ "assumeNotNull",
+ "toIntervalYear",
+ "toIntervalMonth",
+ "toIntervalWeek",
+ "toIntervalDay",
+ "toIntervalHour",
+ "toIntervalMinute",
+ "toIntervalSecond",
+ "toStartOfDay",
+ "toStartOfWeek",
+ "toStartOfMonth",
+ "toStartOfQuarter",
+ "toStartOfYear",
+ ]:
+ return self.visit(node.args[0])
+
+ if node.name in ["minus", "add"]:
+ return all(self.visit(arg) for arg in node.args)
+
+ # otherwise we don't know, so return False
+ return False
+
+ def visit_field(self, node: ast.Field) -> bool:
+ return False
+
+ def visit_and(self, node: ast.And) -> bool:
+ return False
+
+ def visit_or(self, node: ast.Or) -> bool:
+ return False
+
+ def visit_not(self, node: ast.Not) -> bool:
+ return False
+
+ def visit_placeholder(self, node: ast.Placeholder) -> bool:
+ raise Exception()
+
+ def visit_alias(self, node: ast.Alias) -> bool:
+ return self.visit(node.expr)
+
+
+def is_simple_timestamp_field_expression(expr: ast.Expr, context: HogQLContext) -> bool:
+ return IsSimpleTimestampFieldExpressionVisitor(context).visit(expr)
+
+
+@dataclass
+class IsSimpleTimestampFieldExpressionVisitor(Visitor[bool]):
+ context: HogQLContext
+
+ def visit_constant(self, node: ast.Constant) -> bool:
+ return False
+
+ def visit_field(self, node: ast.Field) -> bool:
+ if node.type and isinstance(node.type, ast.FieldType):
+ resolved_field = node.type.resolve_database_field(self.context)
+ if resolved_field and isinstance(resolved_field, DatabaseField) and resolved_field:
+ return resolved_field.name in ["min_timestamp", "timestamp"]
+ # no type information, so just use the name of the field
+ return node.chain[-1] in ["min_timestamp", "timestamp"]
+
+ def visit_arithmetic_operation(self, node: ast.ArithmeticOperation) -> bool:
+ # only allow the min_timestamp field to be used on one side of the arithmetic operation
+ return (
+ self.visit(node.left)
+ and is_time_or_interval_constant(node.right)
+ or (self.visit(node.right) and is_time_or_interval_constant(node.left))
+ )
+
+ def visit_call(self, node: ast.Call) -> bool:
+ # some functions count as a timestamp field expression if their first argument is
+ if node.name in [
+ "parseDateTime64BestEffortOrNull",
+ "toDateTime",
+ "toTimeZone",
+ "assumeNotNull",
+ "toStartOfDay",
+ "toStartOfWeek",
+ "toStartOfMonth",
+ "toStartOfQuarter",
+ "toStartOfYear",
+ ]:
+ return self.visit(node.args[0])
+
+ if node.name in ["minus", "add"]:
+ return self.visit_arithmetic_operation(
+ ast.ArithmeticOperation(
+ op=ArithmeticOperationOp.Sub if node.name == "minus" else ArithmeticOperationOp.Add,
+ left=node.args[0],
+ right=node.args[1],
+ )
+ )
+
+ # otherwise we don't know, so return False
+ return False
+
+ def visit_compare_operation(self, node: ast.CompareOperation) -> bool:
+ return False
+
+ def visit_and(self, node: ast.And) -> bool:
+ return False
+
+ def visit_or(self, node: ast.Or) -> bool:
+ return False
+
+ def visit_not(self, node: ast.Not) -> bool:
+ return False
+
+ def visit_placeholder(self, node: ast.Placeholder) -> bool:
+ raise Exception()
+
+ def visit_alias(self, node: ast.Alias) -> bool:
+ from posthog.hogql.database.schema.events import EventsTable
+ from posthog.hogql.database.schema.sessions import SessionsTable
+
+ if node.type and isinstance(node.type, ast.FieldAliasType):
+ resolved_field = node.type.resolve_database_field(self.context)
+ table_type = node.type.resolve_table_type(self.context)
+ if not table_type:
+ return False
+ return (
+ isinstance(table_type, ast.TableType)
+ and isinstance(table_type.table, EventsTable)
+ and resolved_field.name == "timestamp"
+ ) or (
+ isinstance(table_type, ast.LazyTableType)
+ and isinstance(table_type.table, SessionsTable)
+ and resolved_field.name == "min_timestamp"
+ )
+
+ return self.visit(node.expr)
+
+
+def rewrite_timestamp_field(expr: ast.Expr, context: HogQLContext) -> ast.Expr:
+ return RewriteTimestampFieldVisitor(context).visit(expr)
+
+
+class RewriteTimestampFieldVisitor(CloningVisitor):
+ context: HogQLContext
+
+ def __init__(self, context: HogQLContext, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.context = context
+
+ def visit_field(self, node: ast.Field) -> ast.Field:
+ from posthog.hogql.database.schema.events import EventsTable
+ from posthog.hogql.database.schema.sessions import SessionsTable
+
+ if node.type and isinstance(node.type, ast.FieldType):
+ resolved_field = node.type.resolve_database_field(self.context)
+ table = node.type.resolve_table_type(self.context).table
+ if resolved_field and isinstance(resolved_field, DatabaseField):
+ if (isinstance(table, EventsTable) and resolved_field.name == "timestamp") or (
+ isinstance(table, SessionsTable) and resolved_field.name == "min_timestamp"
+ ):
+ return ast.Field(chain=["raw_sessions", "min_timestamp"])
+ # no type information, so just use the name of the field
+ if node.chain[-1] in ["min_timestamp", "timestamp"]:
+ return ast.Field(chain=["raw_sessions", "min_timestamp"])
+ return node
+
+ def visit_alias(self, node: ast.Alias) -> ast.Expr:
+ return self.visit(node.expr)
diff --git a/posthog/hogql/database/schema/util/test/test_session_where_clause_extractor.py b/posthog/hogql/database/schema/util/test/test_session_where_clause_extractor.py
new file mode 100644
index 0000000000000..bc5324e739ad9
--- /dev/null
+++ b/posthog/hogql/database/schema/util/test/test_session_where_clause_extractor.py
@@ -0,0 +1,284 @@
+from typing import Union, Optional, Dict
+
+from posthog.hogql import ast
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.schema.util.session_where_clause_extractor import SessionMinTimestampWhereClauseExtractor
+from posthog.hogql.modifiers import create_default_modifiers_for_team
+from posthog.hogql.parser import parse_select, parse_expr
+from posthog.hogql.printer import prepare_ast_for_printing, print_prepared_ast
+from posthog.hogql.visitor import clone_expr
+from posthog.test.base import ClickhouseTestMixin, APIBaseTest
+
+
+def f(s: Union[str, ast.Expr, None], placeholders: Optional[dict[str, ast.Expr]] = None) -> Union[ast.Expr, None]:
+ if s is None:
+ return None
+ if isinstance(s, str):
+ expr = parse_expr(s, placeholders=placeholders)
+ else:
+ expr = s
+ return clone_expr(expr, clear_types=True, clear_locations=True)
+
+
+def parse(
+ s: str,
+ placeholders: Optional[Dict[str, ast.Expr]] = None,
+) -> ast.SelectQuery:
+ parsed = parse_select(s, placeholders=placeholders)
+ assert isinstance(parsed, ast.SelectQuery)
+ return parsed
+
+
+class TestSessionTimestampInliner(ClickhouseTestMixin, APIBaseTest):
+ @property
+ def inliner(self):
+ team = self.team
+ modifiers = create_default_modifiers_for_team(team)
+ context = HogQLContext(
+ team_id=team.pk,
+ team=team,
+ enable_select_queries=True,
+ modifiers=modifiers,
+ )
+ return SessionMinTimestampWhereClauseExtractor(context)
+
+ def test_handles_select_with_no_where_claus(self):
+ inner_where = self.inliner.get_inner_where(parse("SELECT * FROM sessions"))
+ assert inner_where is None
+
+ def test_handles_select_with_eq(self):
+ actual = f(self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE min_timestamp = '2021-01-01'")))
+ expected = f(
+ "((raw_sessions.min_timestamp - toIntervalDay(3)) <= '2021-01-01') AND ((raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-01')"
+ )
+ assert expected == actual
+
+ def test_handles_select_with_eq_flipped(self):
+ actual = f(self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE '2021-01-01' = min_timestamp")))
+ expected = f(
+ "((raw_sessions.min_timestamp - toIntervalDay(3)) <= '2021-01-01') AND ((raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-01')"
+ )
+ assert expected == actual
+
+ def test_handles_select_with_simple_gt(self):
+ actual = f(self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE min_timestamp > '2021-01-01'")))
+ expected = f("((raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-01')")
+ assert expected == actual
+
+ def test_handles_select_with_simple_gte(self):
+ actual = f(self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE min_timestamp >= '2021-01-01'")))
+ expected = f("((raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-01')")
+ assert expected == actual
+
+ def test_handles_select_with_simple_lt(self):
+ actual = f(self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE min_timestamp < '2021-01-01'")))
+ expected = f("((raw_sessions.min_timestamp - toIntervalDay(3)) <= '2021-01-01')")
+ assert expected == actual
+
+ def test_handles_select_with_simple_lte(self):
+ actual = f(self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE min_timestamp <= '2021-01-01'")))
+ expected = f("((raw_sessions.min_timestamp - toIntervalDay(3)) <= '2021-01-01')")
+ assert expected == actual
+
+ def test_select_with_placeholder(self):
+ actual = f(
+ self.inliner.get_inner_where(
+ parse(
+ "SELECT * FROM sessions WHERE min_timestamp > {timestamp}",
+ placeholders={"timestamp": ast.Constant(value="2021-01-01")},
+ )
+ )
+ )
+ expected = f("((raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-01')")
+ assert expected == actual
+
+ def test_unrelated_equals(self):
+ actual = self.inliner.get_inner_where(
+ parse("SELECT * FROM sessions WHERE initial_utm_campaign = initial_utm_source")
+ )
+ assert actual is None
+
+ def test_timestamp_and(self):
+ actual = f(
+ self.inliner.get_inner_where(
+ parse("SELECT * FROM sessions WHERE and(min_timestamp >= '2021-01-01', min_timestamp <= '2021-01-03')")
+ )
+ )
+ expected = f(
+ "((raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-01') AND ((raw_sessions.min_timestamp - toIntervalDay(3)) <= '2021-01-03')"
+ )
+ assert expected == actual
+
+ def test_timestamp_or(self):
+ actual = f(
+ self.inliner.get_inner_where(
+ parse("SELECT * FROM sessions WHERE and(min_timestamp <= '2021-01-01', min_timestamp >= '2021-01-03')")
+ )
+ )
+ expected = f(
+ "((raw_sessions.min_timestamp - toIntervalDay(3)) <= '2021-01-01') AND ((raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-03')"
+ )
+ assert expected == actual
+
+ def test_unrelated_function(self):
+ actual = f(self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE like('a', 'b')")))
+ assert actual is None
+
+ def test_timestamp_unrelated_function(self):
+ actual = f(
+ self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE like(toString(min_timestamp), 'b')"))
+ )
+ assert actual is None
+
+ def test_timestamp_unrelated_function_timestamp(self):
+ actual = f(
+ self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE like(toString(min_timestamp), 'b')"))
+ )
+ assert actual is None
+
+ def test_ambiguous_or(self):
+ actual = f(
+ self.inliner.get_inner_where(
+ parse(
+ "SELECT * FROM sessions WHERE or(min_timestamp > '2021-01-03', like(toString(min_timestamp), 'b'))"
+ )
+ )
+ )
+ assert actual is None
+
+ def test_ambiguous_and(self):
+ actual = f(
+ self.inliner.get_inner_where(
+ parse(
+ "SELECT * FROM sessions WHERE and(min_timestamp > '2021-01-03', like(toString(min_timestamp), 'b'))"
+ )
+ )
+ )
+ assert actual == f("(raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-03'")
+
+ def test_join(self):
+ actual = f(
+ self.inliner.get_inner_where(
+ parse(
+ "SELECT * FROM events JOIN sessions ON events.session_id = raw_sessions.session_id WHERE min_timestamp > '2021-01-03'"
+ )
+ )
+ )
+ expected = f("((raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-03')")
+ assert expected == actual
+
+ def test_join_using_events_timestamp_filter(self):
+ actual = f(
+ self.inliner.get_inner_where(
+ parse(
+ "SELECT * FROM events JOIN sessions ON events.session_id = raw_sessions.session_id WHERE timestamp > '2021-01-03'"
+ )
+ )
+ )
+ expected = f("((raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-03')")
+ assert expected == actual
+
+ def test_minus(self):
+ actual = f(self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE min_timestamp >= today() - 2")))
+ expected = f("((raw_sessions.min_timestamp + toIntervalDay(3)) >= (today() - 2))")
+ assert expected == actual
+
+ def test_minus_function(self):
+ actual = f(
+ self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE min_timestamp >= minus(today() , 2)"))
+ )
+ expected = f("((raw_sessions.min_timestamp + toIntervalDay(3)) >= minus(today(), 2))")
+ assert expected == actual
+
+ def test_real_example(self):
+ actual = f(
+ self.inliner.get_inner_where(
+ parse(
+ "SELECT * FROM events JOIN sessions ON events.session_id = raw_sessions.session_id WHERE event = '$pageview' AND toTimeZone(timestamp, 'US/Pacific') >= toDateTime('2024-03-12 00:00:00', 'US/Pacific') AND toTimeZone(timestamp, 'US/Pacific') <= toDateTime('2024-03-19 23:59:59', 'US/Pacific')"
+ )
+ )
+ )
+ expected = f(
+ "(toTimeZone(raw_sessions.min_timestamp, 'US/Pacific') + toIntervalDay(3)) >= toDateTime('2024-03-12 00:00:00', 'US/Pacific') AND (toTimeZone(raw_sessions.min_timestamp, 'US/Pacific') - toIntervalDay(3)) <= toDateTime('2024-03-19 23:59:59', 'US/Pacific') "
+ )
+ assert expected == actual
+
+ def test_collapse_and(self):
+ actual = f(
+ self.inliner.get_inner_where(
+ parse(
+ "SELECT * FROM sesions WHERE event = '$pageview' AND (TRUE AND (TRUE AND TRUE AND (timestamp >= '2024-03-12' AND TRUE)))"
+ )
+ )
+ )
+ expected = f("(raw_sessions.min_timestamp + toIntervalDay(3)) >= '2024-03-12'")
+ assert expected == actual
+
+
+class TestSessionsQueriesHogQLToClickhouse(ClickhouseTestMixin, APIBaseTest):
+ def print_query(self, query: str) -> str:
+ team = self.team
+ modifiers = create_default_modifiers_for_team(team)
+ context = HogQLContext(
+ team_id=team.pk,
+ team=team,
+ enable_select_queries=True,
+ modifiers=modifiers,
+ )
+ prepared_ast = prepare_ast_for_printing(node=parse(query), context=context, dialect="clickhouse")
+ pretty = print_prepared_ast(prepared_ast, context=context, dialect="clickhouse", pretty=True)
+ return pretty
+
+ def test_select_with_timestamp(self):
+ actual = self.print_query("SELECT session_id FROM sessions WHERE min_timestamp > '2021-01-01'")
+ expected = f"""SELECT
+ sessions.session_id AS session_id
+FROM
+ (SELECT
+ sessions.session_id AS session_id,
+ min(sessions.min_timestamp) AS min_timestamp
+ FROM
+ sessions
+ WHERE
+ and(equals(sessions.team_id, {self.team.id}), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, %(hogql_val_0)s), toIntervalDay(3)), %(hogql_val_1)s), 0))
+ GROUP BY
+ sessions.session_id,
+ sessions.session_id) AS sessions
+WHERE
+ ifNull(greater(toTimeZone(sessions.min_timestamp, %(hogql_val_2)s), %(hogql_val_3)s), 0)
+LIMIT 10000"""
+ assert expected == actual
+
+ def test_join_with_events(self):
+ actual = self.print_query(
+ """
+SELECT
+ sessions.session_id,
+ uniq(uuid)
+FROM events
+JOIN sessions
+ON events.$session_id = sessions.session_id
+WHERE events.timestamp > '2021-01-01'
+GROUP BY sessions.session_id
+"""
+ )
+ expected = f"""SELECT
+ sessions.session_id AS session_id,
+ uniq(events.uuid)
+FROM
+ events
+ JOIN (SELECT
+ sessions.session_id AS session_id
+ FROM
+ sessions
+ WHERE
+ and(equals(sessions.team_id, {self.team.id}), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, %(hogql_val_0)s), toIntervalDay(3)), %(hogql_val_1)s), 0))
+ GROUP BY
+ sessions.session_id,
+ sessions.session_id) AS sessions ON equals(events.`$session_id`, sessions.session_id)
+WHERE
+ and(equals(events.team_id, {self.team.id}), greater(toTimeZone(events.timestamp, %(hogql_val_2)s), %(hogql_val_3)s))
+GROUP BY
+ sessions.session_id
+LIMIT 10000"""
+ assert expected == actual
diff --git a/posthog/hogql/database/test/__snapshots__/test_database.ambr b/posthog/hogql/database/test/__snapshots__/test_database.ambr
index db4dfc8f6df9f..63c2d16ce87aa 100644
--- a/posthog/hogql/database/test/__snapshots__/test_database.ambr
+++ b/posthog/hogql/database/test/__snapshots__/test_database.ambr
@@ -304,6 +304,31 @@
]
}
],
+ "person_distinct_id_overrides": [
+ {
+ "key": "distinct_id",
+ "type": "string"
+ },
+ {
+ "key": "person_id",
+ "type": "string"
+ },
+ {
+ "key": "person",
+ "type": "lazy_table",
+ "table": "persons",
+ "fields": [
+ "id",
+ "created_at",
+ "team_id",
+ "properties",
+ "is_identified",
+ "pdi",
+ "$virt_initial_referring_domain_type",
+ "$virt_initial_channel_type"
+ ]
+ }
+ ],
"person_overrides": [
{
"key": "old_person_id",
@@ -790,6 +815,39 @@
"type": "integer"
}
],
+ "raw_person_distinct_id_overrides": [
+ {
+ "key": "distinct_id",
+ "type": "string"
+ },
+ {
+ "key": "person_id",
+ "type": "string"
+ },
+ {
+ "key": "person",
+ "type": "lazy_table",
+ "table": "persons",
+ "fields": [
+ "id",
+ "created_at",
+ "team_id",
+ "properties",
+ "is_identified",
+ "pdi",
+ "$virt_initial_referring_domain_type",
+ "$virt_initial_channel_type"
+ ]
+ },
+ {
+ "key": "is_deleted",
+ "type": "boolean"
+ },
+ {
+ "key": "version",
+ "type": "integer"
+ }
+ ],
"raw_person_overrides": [
{
"key": "old_person_id",
@@ -1155,6 +1213,31 @@
]
}
],
+ "person_distinct_id_overrides": [
+ {
+ "key": "distinct_id",
+ "type": "string"
+ },
+ {
+ "key": "person_id",
+ "type": "string"
+ },
+ {
+ "key": "person",
+ "type": "lazy_table",
+ "table": "persons",
+ "fields": [
+ "id",
+ "created_at",
+ "team_id",
+ "properties",
+ "is_identified",
+ "pdi",
+ "$virt_initial_referring_domain_type",
+ "$virt_initial_channel_type"
+ ]
+ }
+ ],
"person_overrides": [
{
"key": "old_person_id",
@@ -1641,6 +1724,39 @@
"type": "integer"
}
],
+ "raw_person_distinct_id_overrides": [
+ {
+ "key": "distinct_id",
+ "type": "string"
+ },
+ {
+ "key": "person_id",
+ "type": "string"
+ },
+ {
+ "key": "person",
+ "type": "lazy_table",
+ "table": "persons",
+ "fields": [
+ "id",
+ "created_at",
+ "team_id",
+ "properties",
+ "is_identified",
+ "pdi",
+ "$virt_initial_referring_domain_type",
+ "$virt_initial_channel_type"
+ ]
+ },
+ {
+ "key": "is_deleted",
+ "type": "boolean"
+ },
+ {
+ "key": "version",
+ "type": "integer"
+ }
+ ],
"raw_person_overrides": [
{
"key": "old_person_id",
diff --git a/posthog/hogql/query.py b/posthog/hogql/query.py
index e7bc3f7984205..f47c14c5cef86 100644
--- a/posthog/hogql/query.py
+++ b/posthog/hogql/query.py
@@ -28,6 +28,7 @@
def execute_hogql_query(
query: Union[str, ast.SelectQuery, ast.SelectUnionQuery],
team: Team,
+ *,
query_type: str = "hogql_query",
filters: Optional[HogQLFilters] = None,
placeholders: Optional[Dict[str, ast.Expr]] = None,
@@ -147,6 +148,7 @@ def execute_hogql_query(
has_joins="JOIN" in clickhouse_sql,
has_json_operations="JSONExtract" in clickhouse_sql or "JSONHas" in clickhouse_sql,
timings=timings_dict,
+ modifiers={k: v for k, v in modifiers.model_dump().items() if v is not None} if modifiers else {},
)
error = None
diff --git a/posthog/hogql/test/test_bytecode.py b/posthog/hogql/test/test_bytecode.py
index cf0b8113b574d..f7d810700e74a 100644
--- a/posthog/hogql/test/test_bytecode.py
+++ b/posthog/hogql/test/test_bytecode.py
@@ -130,7 +130,7 @@ def test_bytecode_create(self):
def test_bytecode_create_error(self):
with self.assertRaises(NotImplementedException) as e:
to_bytecode("(select 1)")
- self.assertEqual(str(e.exception), "Visitor has no method visit_select_query")
+ self.assertEqual(str(e.exception), "BytecodeBuilder has no method visit_select_query")
with self.assertRaises(NotImplementedException) as e:
to_bytecode("1 in cohort 2")
diff --git a/posthog/hogql/test/test_modifiers.py b/posthog/hogql/test/test_modifiers.py
index eba1f5195ab3d..b2b0ef1e40630 100644
--- a/posthog/hogql/test/test_modifiers.py
+++ b/posthog/hogql/test/test_modifiers.py
@@ -74,6 +74,13 @@ def test_modifiers_persons_on_events_mode_mapping(self):
"events.person_properties AS properties",
"toTimeZone(events.person_created_at, %(hogql_val_1)s) AS created_at",
),
+ (
+ PersonsOnEventsMode.v3_enabled,
+ "events.event AS event",
+ "if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS id",
+ "events.person_properties AS properties",
+ "toTimeZone(events.person_created_at, %(hogql_val_0)s) AS created_at",
+ ),
]
for mode, *expected in test_cases:
diff --git a/posthog/hogql/test/test_visitor.py b/posthog/hogql/test/test_visitor.py
index 8aa6689328fbf..a01193f788d5f 100644
--- a/posthog/hogql/test/test_visitor.py
+++ b/posthog/hogql/test/test_visitor.py
@@ -125,7 +125,7 @@ def visit_arithmetic_operation(self, node: ast.ArithmeticOperation):
with self.assertRaises(HogQLException) as e:
UnknownNotDefinedVisitor().visit(parse_expr("1 + 3 / 'asd2'"))
- self.assertEqual(str(e.exception), "Visitor has no method visit_constant")
+ self.assertEqual(str(e.exception), "UnknownNotDefinedVisitor has no method visit_constant")
def test_hogql_exception_start_end(self):
class EternalVisitor(TraversingVisitor):
diff --git a/posthog/hogql/transforms/lazy_tables.py b/posthog/hogql/transforms/lazy_tables.py
index bdbb322d54397..df8ce6962259c 100644
--- a/posthog/hogql/transforms/lazy_tables.py
+++ b/posthog/hogql/transforms/lazy_tables.py
@@ -309,7 +309,7 @@ def create_override(table_name: str, field_chain: List[str | int]) -> None:
# For all the collected tables, create the subqueries, and add them to the table.
for table_name, table_to_add in tables_to_add.items():
- subquery = table_to_add.lazy_table.lazy_select(table_to_add.fields_accessed, self.context.modifiers)
+ subquery = table_to_add.lazy_table.lazy_select(table_to_add.fields_accessed, self.context, node=node)
subquery = cast(ast.SelectQuery, clone_expr(subquery, clear_locations=True))
subquery = cast(ast.SelectQuery, resolve_types(subquery, self.context, self.dialect, [node.type]))
old_table_type = select_type.tables[table_name]
diff --git a/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr b/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr
index 9ff7f8ee0ab49..e0f5ea847110d 100644
--- a/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr
+++ b/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr
@@ -31,7 +31,7 @@
FROM events LEFT JOIN (
SELECT person_static_cohort.person_id AS cohort_person_id, 1 AS matched, person_static_cohort.cohort_id AS cohort_id
FROM person_static_cohort
- WHERE and(equals(person_static_cohort.team_id, 420), in(person_static_cohort.cohort_id, [12]))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id)
+ WHERE and(equals(person_static_cohort.team_id, 420), in(person_static_cohort.cohort_id, [11]))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id)
WHERE and(equals(events.team_id, 420), 1, ifNull(equals(__in_cohort.matched, 1), 0))
LIMIT 100
SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1
@@ -42,7 +42,7 @@
FROM events LEFT JOIN (
SELECT person_id AS cohort_person_id, 1 AS matched, cohort_id
FROM static_cohort_people
- WHERE in(cohort_id, [12])) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id)
+ WHERE in(cohort_id, [11])) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id)
WHERE and(1, equals(__in_cohort.matched, 1))
LIMIT 100
'''
@@ -55,7 +55,7 @@
FROM events LEFT JOIN (
SELECT person_static_cohort.person_id AS cohort_person_id, 1 AS matched, person_static_cohort.cohort_id AS cohort_id
FROM person_static_cohort
- WHERE and(equals(person_static_cohort.team_id, 420), in(person_static_cohort.cohort_id, [13]))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id)
+ WHERE and(equals(person_static_cohort.team_id, 420), in(person_static_cohort.cohort_id, [12]))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id)
WHERE and(equals(events.team_id, 420), 1, ifNull(equals(__in_cohort.matched, 1), 0))
LIMIT 100
SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1
@@ -66,7 +66,7 @@
FROM events LEFT JOIN (
SELECT person_id AS cohort_person_id, 1 AS matched, cohort_id
FROM static_cohort_people
- WHERE in(cohort_id, [13])) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id)
+ WHERE in(cohort_id, [12])) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id)
WHERE and(1, equals(__in_cohort.matched, 1))
LIMIT 100
'''
diff --git a/posthog/hogql/visitor.py b/posthog/hogql/visitor.py
index c11856169297f..2bf968abf2ab0 100644
--- a/posthog/hogql/visitor.py
+++ b/posthog/hogql/visitor.py
@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Optional, TypeVar, Generic, Any
from posthog.hogql import ast
from posthog.hogql.base import AST, Expr
@@ -14,8 +14,11 @@ def clear_locations(expr: Expr) -> Expr:
return CloningVisitor(clear_locations=True).visit(expr)
-class Visitor(object):
- def visit(self, node: AST):
+T = TypeVar("T")
+
+
+class Visitor(Generic[T]):
+ def visit(self, node: AST) -> T:
if node is None:
return node
@@ -28,7 +31,7 @@ def visit(self, node: AST):
raise e
-class TraversingVisitor(Visitor):
+class TraversingVisitor(Visitor[None]):
"""Visitor that traverses the AST tree without returning anything"""
def visit_expr(self, node: Expr):
@@ -258,7 +261,7 @@ def visit_hogqlx_attribute(self, node: ast.HogQLXAttribute):
self.visit(node.value)
-class CloningVisitor(Visitor):
+class CloningVisitor(Visitor[Any]):
"""Visitor that traverses and clones the AST tree. Clears types."""
def __init__(
diff --git a/posthog/hogql_queries/insights/funnels/base.py b/posthog/hogql_queries/insights/funnels/base.py
index ef8782fade54a..4e97d79b94534 100644
--- a/posthog/hogql_queries/insights/funnels/base.py
+++ b/posthog/hogql_queries/insights/funnels/base.py
@@ -284,6 +284,7 @@ def _get_breakdown_expr(self) -> ast.Expr:
properties_column = f"group_{breakdownFilter.breakdown_group_type_index}.properties"
return get_breakdown_expr(breakdown, properties_column)
elif breakdownType == "hogql":
+ assert isinstance(breakdown, list)
return ast.Alias(
alias="value",
expr=ast.Array(exprs=[parse_expr(str(value)) for value in breakdown]),
@@ -530,6 +531,7 @@ def _add_breakdown_attribution_subquery(self, inner_query: ast.SelectQuery) -> a
# so just select that. Except for the empty case, where we select the default.
if self._query_has_array_breakdown():
+ assert isinstance(breakdown, list)
default_breakdown_value = f"""[{','.join(["''" for _ in range(len(breakdown or []))])}]"""
# default is [''] when dealing with a single breakdown array, otherwise ['', '', ...., '']
breakdown_selector = parse_expr(
@@ -613,7 +615,7 @@ def _build_step_query(
event_expr = ast.Constant(value=True)
else:
# event
- event_expr = parse_expr(f"event = '{entity.event}'")
+ event_expr = parse_expr("event = {event}", {"event": ast.Constant(value=entity.event)})
if entity.properties is not None and entity.properties != []:
# add property filters
@@ -657,11 +659,15 @@ def _get_funnel_person_step_condition(self) -> ast.Expr:
raise ValueError("Missing both funnelStep and funnelCustomSteps")
if funnelStepBreakdown is not None:
- breakdown_prop_value = funnelStepBreakdown
- if isinstance(breakdown_prop_value, int) and breakdownType != "cohort":
- breakdown_prop_value = str(breakdown_prop_value)
+ if isinstance(funnelStepBreakdown, int) and breakdownType != "cohort":
+ funnelStepBreakdown = str(funnelStepBreakdown)
- conditions.append(parse_expr(f"arrayFlatten(array(prop)) = arrayFlatten(array({breakdown_prop_value}))"))
+ conditions.append(
+ parse_expr(
+ "arrayFlatten(array(prop)) = arrayFlatten(array({funnelStepBreakdown}))",
+ {"funnelStepBreakdown": ast.Constant(value=funnelStepBreakdown)},
+ )
+ )
return ast.And(exprs=conditions)
@@ -898,7 +904,12 @@ def _get_breakdown_prop_expr(self, group_remaining=False) -> List[ast.Expr]:
BreakdownType.group,
]:
breakdown_values = self._get_breakdown_conditions()
- return [parse_expr(f"if(has({breakdown_values}, prop), prop, {other_aggregation}) as prop")]
+ return [
+ parse_expr(
+ f"if(has({{breakdown_values}}, prop), prop, {other_aggregation}) as prop",
+ {"breakdown_values": ast.Constant(value=breakdown_values)},
+ )
+ ]
else:
# Cohorts don't have "Other" aggregation
return [ast.Field(chain=["prop"])]
diff --git a/posthog/hogql_queries/insights/funnels/funnel_correlation_query_runner.py b/posthog/hogql_queries/insights/funnels/funnel_correlation_query_runner.py
index 2fef78a372324..bcb362ff3d4f9 100644
--- a/posthog/hogql_queries/insights/funnels/funnel_correlation_query_runner.py
+++ b/posthog/hogql_queries/insights/funnels/funnel_correlation_query_runner.py
@@ -236,6 +236,7 @@ def _calculate(self) -> tuple[List[EventOddsRatio], bool, str, HogQLQueryRespons
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
assert response.results
diff --git a/posthog/hogql_queries/insights/funnels/funnel_query_context.py b/posthog/hogql_queries/insights/funnels/funnel_query_context.py
index 66a0d28ad3d7f..3b777e3ff8026 100644
--- a/posthog/hogql_queries/insights/funnels/funnel_query_context.py
+++ b/posthog/hogql_queries/insights/funnels/funnel_query_context.py
@@ -25,7 +25,7 @@ class FunnelQueryContext(QueryContext):
interval: IntervalType
- breakdown: List[Union[str, int]] | None
+ breakdown: List[Union[str, int]] | str | int | None
breakdownType: BreakdownType
breakdownAttributionType: BreakdownAttributionType
diff --git a/posthog/hogql_queries/insights/funnels/funnel_trends.py b/posthog/hogql_queries/insights/funnels/funnel_trends.py
index 5c370512a20e8..9d486f1b06196 100644
--- a/posthog/hogql_queries/insights/funnels/funnel_trends.py
+++ b/posthog/hogql_queries/insights/funnels/funnel_trends.py
@@ -203,7 +203,16 @@ def get_query(self) -> ast.SelectQuery:
[
ast.Alias(
alias="breakdown_value",
- expr=ast.Array(exprs=[parse_expr(str(value)) for value in self.breakdown_values]),
+ expr=ast.Array(
+ exprs=[
+ (
+ ast.Array(exprs=[ast.Constant(value=sub_value) for sub_value in value])
+ if isinstance(value, list)
+ else ast.Constant(value=value)
+ )
+ for value in self.breakdown_values
+ ]
+ ),
hidden=False,
)
]
diff --git a/posthog/hogql_queries/insights/funnels/funnels_query_runner.py b/posthog/hogql_queries/insights/funnels/funnels_query_runner.py
index 38e04603f2725..b1ca8dfd6dd54 100644
--- a/posthog/hogql_queries/insights/funnels/funnels_query_runner.py
+++ b/posthog/hogql_queries/insights/funnels/funnels_query_runner.py
@@ -92,6 +92,7 @@ def calculate(self):
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
results = self.funnel_class._format_results(response.results)
diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel.py b/posthog/hogql_queries/insights/funnels/test/test_funnel.py
index 98f4d060fb905..89382bebfb994 100644
--- a/posthog/hogql_queries/insights/funnels/test/test_funnel.py
+++ b/posthog/hogql_queries/insights/funnels/test/test_funnel.py
@@ -18,7 +18,14 @@
from posthog.models.group_type_mapping import GroupTypeMapping
from posthog.models.property_definition import PropertyDefinition
from posthog.queries.funnels import ClickhouseFunnelActors
-from posthog.schema import ActorsQuery, EventsNode, FunnelsActorsQuery, FunnelsQuery
+from posthog.schema import (
+ ActorsQuery,
+ BreakdownFilter,
+ DateRange,
+ EventsNode,
+ FunnelsActorsQuery,
+ FunnelsQuery,
+)
from posthog.test.base import (
APIBaseTest,
BaseTest,
@@ -3576,6 +3583,72 @@ def test_funnel_window_ignores_dst_transition(self):
self.assertEqual(results[1]["average_conversion_time"], 1_207_020)
self.assertEqual(results[1]["median_conversion_time"], 1_207_020)
+ def test_parses_breakdowns_correctly(self):
+ _create_person(
+ distinct_ids=[f"user_1"],
+ team=self.team,
+ )
+
+ events_by_person = {
+ "user_1": [
+ {
+ "event": "$pageview",
+ "timestamp": datetime(2024, 3, 22, 13, 46),
+ "properties": {"utm_medium": "test''123"},
+ },
+ {
+ "event": "$pageview",
+ "timestamp": datetime(2024, 3, 22, 13, 47),
+ "properties": {"utm_medium": "test''123"},
+ },
+ ],
+ }
+ journeys_for(events_by_person, self.team)
+
+ query = FunnelsQuery(
+ series=[EventsNode(event="$pageview"), EventsNode(event="$pageview")],
+ dateRange=DateRange(
+ date_from="2024-03-22",
+ date_to="2024-03-22",
+ ),
+ breakdownFilter=BreakdownFilter(breakdown="utm_medium"),
+ )
+ results = FunnelsQueryRunner(query=query, team=self.team).calculate().results
+
+ self.assertEqual(results[0][1]["breakdown_value"], ["test'123"])
+ self.assertEqual(results[0][1]["count"], 1)
+
+ def test_funnel_parses_event_names_correctly(self):
+ _create_person(
+ distinct_ids=[f"user_1"],
+ team=self.team,
+ )
+
+ events_by_person = {
+ "user_1": [
+ {
+ "event": "test''1",
+ "timestamp": datetime(2024, 3, 22, 13, 46),
+ },
+ {
+ "event": "test''2",
+ "timestamp": datetime(2024, 3, 22, 13, 47),
+ },
+ ],
+ }
+ journeys_for(events_by_person, self.team)
+
+ query = FunnelsQuery(
+ series=[EventsNode(event="test'1"), EventsNode()],
+ dateRange=DateRange(
+ date_from="2024-03-22",
+ date_to="2024-03-22",
+ ),
+ )
+ results = FunnelsQueryRunner(query=query, team=self.team).calculate().results
+
+ self.assertEqual(results[0]["count"], 1)
+
return TestGetFunnel
diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_persons.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_persons.py
index 4c342d2f2926c..dec7bdd933b3e 100644
--- a/posthog/hogql_queries/insights/funnels/test/test_funnel_persons.py
+++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_persons.py
@@ -626,3 +626,45 @@ def test_funnel_person_recordings(self):
}
],
)
+
+ def test_parses_step_breakdown_correctly(self):
+ person1 = _create_person(
+ distinct_ids=["person1"],
+ team_id=self.team.pk,
+ properties={"$country": "PL"},
+ )
+ journeys_for(
+ {
+ "person1": [
+ {
+ "event": "sign up",
+ "timestamp": datetime(2020, 1, 1, 12),
+ "properties": {"$browser": "test''123"},
+ },
+ {
+ "event": "play movie",
+ "timestamp": datetime(2020, 1, 1, 13),
+ "properties": {"$browser": "test''123"},
+ },
+ ],
+ },
+ self.team,
+ create_people=False,
+ )
+
+ filters = {
+ "insight": INSIGHT_FUNNELS,
+ "date_from": "2020-01-01",
+ "date_to": "2020-01-08",
+ "interval": "day",
+ "funnel_window_days": 7,
+ "events": [
+ {"id": "sign up", "order": 0},
+ {"id": "play movie", "order": 1},
+ ],
+ "breakdown_type": "event",
+ "breakdown": "$browser",
+ }
+
+ results = get_actors(filters, self.team, funnelStep=1, funnelStepBreakdown=["test'123"])
+ self.assertCountEqual([results[0][0]], [person1.uuid])
diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py
index 6ca333b036f14..f9c7b107074de 100644
--- a/posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py
+++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py
@@ -1387,3 +1387,43 @@ def test_trend_for_hour_based_conversion_window(self):
results = FunnelsQueryRunner(query=query, team=self.team, just_summarize=True).calculate().results
conversion_rates = [row["conversion_rate"] for row in results]
self.assertEqual(conversion_rates, [50.0, 0.0, 0.0, 0.0, 0.0, 0.0])
+
+ def test_parses_breakdown_correctly(self):
+ journeys_for(
+ {
+ "user_one": [
+ {
+ "event": "step one",
+ "timestamp": datetime(2021, 5, 1),
+ "properties": {"$browser": "test''123"},
+ },
+ {
+ "event": "step two",
+ "timestamp": datetime(2021, 5, 3),
+ "properties": {"$browser": "test''123"},
+ },
+ ],
+ },
+ self.team,
+ )
+
+ filters = {
+ "insight": INSIGHT_FUNNELS,
+ "funnel_viz_type": "trends",
+ "display": TRENDS_LINEAR,
+ "interval": "day",
+ "date_from": "2021-05-01 00:00:00",
+ "date_to": "2021-05-13 23:59:59",
+ "funnel_window_days": 7,
+ "events": [
+ {"id": "step one", "order": 0},
+ {"id": "step two", "order": 1},
+ ],
+ "breakdown_type": "event",
+ "breakdown": "$browser",
+ }
+
+ query = cast(FunnelsQuery, filter_to_query(filters))
+ results = FunnelsQueryRunner(query=query, team=self.team).calculate().results
+
+ self.assertEqual(len(results), 1)
diff --git a/posthog/hogql_queries/insights/funnels/utils.py b/posthog/hogql_queries/insights/funnels/utils.py
index 47c1487e5fbcc..cdccce0251a33 100644
--- a/posthog/hogql_queries/insights/funnels/utils.py
+++ b/posthog/hogql_queries/insights/funnels/utils.py
@@ -61,23 +61,26 @@ def funnel_window_interval_unit_to_sql(
def get_breakdown_expr(
- breakdown: List[str | int] | None, properties_column: str, normalize_url: bool | None = False
+ breakdowns: List[str | int] | str | int, properties_column: str, normalize_url: bool | None = False
) -> ast.Expr:
- if isinstance(breakdown, str) or isinstance(breakdown, int) or breakdown is None:
- return parse_expr(f"ifNull({properties_column}.\"{breakdown}\", '')")
+ if isinstance(breakdowns, str) or isinstance(breakdowns, int) or breakdowns is None:
+ return ast.Call(
+ name="ifNull", args=[ast.Field(chain=[*properties_column.split("."), breakdowns]), ast.Constant(value="")]
+ )
else:
exprs = []
- for b in breakdown:
- expr = parse_expr(normalize_url_breakdown(f"ifNull({properties_column}.\"{b}\", '')", normalize_url))
+ for breakdown in breakdowns:
+ expr: ast.Expr = ast.Call(
+ name="ifNull",
+ args=[ast.Field(chain=[*properties_column.split("."), breakdown]), ast.Constant(value="")],
+ )
+ if normalize_url:
+ regex = "[\\\\/?#]*$"
+ expr = parse_expr(
+ f"if( empty( replaceRegexpOne({{breakdown_value}}, '{regex}', '') ), '/', replaceRegexpOne({{breakdown_value}}, '{regex}', ''))",
+ {"breakdown_value": expr},
+ )
exprs.append(expr)
expression = ast.Array(exprs=exprs)
return expression
-
-
-def normalize_url_breakdown(breakdown_value, breakdown_normalize_url: bool | None):
- if breakdown_normalize_url:
- regex = "[\\\\/?#]*$"
- return f"if( empty( replaceRegexpOne({breakdown_value}, '{regex}', '') ), '/', replaceRegexpOne({breakdown_value}, '{regex}', ''))"
-
- return breakdown_value
diff --git a/posthog/hogql_queries/insights/insight_actors_query_runner.py b/posthog/hogql_queries/insights/insight_actors_query_runner.py
index 782dd5b054a0e..d58f36cb6f7ee 100644
--- a/posthog/hogql_queries/insights/insight_actors_query_runner.py
+++ b/posthog/hogql_queries/insights/insight_actors_query_runner.py
@@ -1,5 +1,5 @@
from datetime import timedelta
-from typing import cast
+from typing import cast, Optional
from posthog.hogql import ast
from posthog.hogql.query import execute_hogql_query
@@ -37,7 +37,7 @@ def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery:
trends_runner = cast(TrendsQueryRunner, self.source_runner)
query = cast(InsightActorsQuery, self.query)
return trends_runner.to_actors_query(
- time_frame=query.day,
+ time_frame=cast(Optional[str], query.day), # Other runner accept day as int, but not this one
series_index=query.series or 0,
breakdown_value=query.breakdown,
compare=query.compare,
@@ -102,6 +102,7 @@ def calculate(self) -> HogQLQueryResponse:
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
def _is_stale(self, cached_result_package):
diff --git a/posthog/hogql_queries/insights/lifecycle_query_runner.py b/posthog/hogql_queries/insights/lifecycle_query_runner.py
index 24bbe36f1c6bf..ea883eec542bc 100644
--- a/posthog/hogql_queries/insights/lifecycle_query_runner.py
+++ b/posthog/hogql_queries/insights/lifecycle_query_runner.py
@@ -126,7 +126,7 @@ def to_actors_query(
def to_actors_query_options(self) -> InsightActorsQueryOptionsResponse:
return InsightActorsQueryOptionsResponse(
- day=[{"label": day, "value": day} for day in self.query_date_range.all_values()],
+ day=[{"label": format_label_date(value), "value": value} for value in self.query_date_range.all_values()],
status=[
{
"label": "Dormant",
@@ -157,6 +157,7 @@ def calculate(self) -> LifecycleQueryResponse:
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
# TODO: can we move the data conversion part into the query as well? It would make it easier to swap
diff --git a/posthog/hogql_queries/insights/paginators.py b/posthog/hogql_queries/insights/paginators.py
index 6dbdb1543b929..0dfda79ced617 100644
--- a/posthog/hogql_queries/insights/paginators.py
+++ b/posthog/hogql_queries/insights/paginators.py
@@ -54,8 +54,9 @@ def trim_results(self) -> list[Any]:
def execute_hogql_query(
self,
- query_type: str,
query: ast.SelectQuery,
+ *,
+ query_type: str,
**kwargs,
) -> HogQLQueryResponse:
self.response = cast(
diff --git a/posthog/hogql_queries/insights/paths_query_runner.py b/posthog/hogql_queries/insights/paths_query_runner.py
index c10a5a2320207..c454feb8e56ac 100644
--- a/posthog/hogql_queries/insights/paths_query_runner.py
+++ b/posthog/hogql_queries/insights/paths_query_runner.py
@@ -725,6 +725,7 @@ def calculate(self) -> PathsQueryResponse:
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
response.results = self.validate_results(response.results)
diff --git a/posthog/hogql_queries/insights/retention_query_runner.py b/posthog/hogql_queries/insights/retention_query_runner.py
index 221cb976757d2..3ac2c5b4b5462 100644
--- a/posthog/hogql_queries/insights/retention_query_runner.py
+++ b/posthog/hogql_queries/insights/retention_query_runner.py
@@ -313,6 +313,7 @@ def calculate(self) -> RetentionQueryResponse:
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
result_dict = {
diff --git a/posthog/hogql_queries/insights/stickiness_query_runner.py b/posthog/hogql_queries/insights/stickiness_query_runner.py
index d0b4b65c67f9b..184e3c0af02df 100644
--- a/posthog/hogql_queries/insights/stickiness_query_runner.py
+++ b/posthog/hogql_queries/insights/stickiness_query_runner.py
@@ -212,6 +212,7 @@ def calculate(self):
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
if response.timings is not None:
diff --git a/posthog/hogql_queries/insights/test/test_paginators.py b/posthog/hogql_queries/insights/test/test_paginators.py
index ac83efb45b353..6698115c46535 100644
--- a/posthog/hogql_queries/insights/test/test_paginators.py
+++ b/posthog/hogql_queries/insights/test/test_paginators.py
@@ -1,3 +1,5 @@
+from typing import cast
+from posthog.hogql.ast import SelectQuery
from posthog.hogql.constants import (
LimitContext,
get_default_limit_for_context,
@@ -136,8 +138,8 @@ def test_response_params_consistency(self):
"""Test consistency of response_params method."""
paginator = HogQLHasMorePaginator(limit=5, offset=10)
paginator.response = paginator.execute_hogql_query(
- "test_query",
- parse_select("SELECT * FROM persons"),
+ cast(SelectQuery, parse_select("SELECT * FROM persons")),
+ query_type="test_query",
team=self.team,
)
params = paginator.response_params()
diff --git a/posthog/hogql_queries/insights/test/test_retention_query_runner.py b/posthog/hogql_queries/insights/test/test_retention_query_runner.py
index 30edb32102f76..04c108dd779f1 100644
--- a/posthog/hogql_queries/insights/test/test_retention_query_runner.py
+++ b/posthog/hogql_queries/insights/test/test_retention_query_runner.py
@@ -1,3 +1,5 @@
+from typing import Optional
+from unittest.mock import MagicMock, patch
import uuid
from datetime import datetime
@@ -6,11 +8,14 @@
from django.test import override_settings
from rest_framework import status
+from posthog.clickhouse.client.execute import sync_execute
from posthog.constants import (
RETENTION_FIRST_TIME,
TREND_FILTER_TYPE_ACTIONS,
TREND_FILTER_TYPE_EVENTS,
)
+from posthog.hogql.constants import LimitContext
+from posthog.hogql.query import INCREASED_MAX_EXECUTION_TIME
from posthog.hogql_queries.insights.retention_query_runner import RetentionQueryRunner
from posthog.hogql_queries.actors_query_runner import ActorsQueryRunner
from posthog.models import Action, ActionStep
@@ -1685,10 +1690,10 @@ def test_day_interval_sampled(self):
class TestClickhouseRetentionGroupAggregation(ClickhouseTestMixin, APIBaseTest):
- def run_query(self, query):
+ def run_query(self, query, *, limit_context: Optional[LimitContext] = None):
if not query.get("retentionFilter"):
query["retentionFilter"] = {}
- runner = RetentionQueryRunner(team=self.team, query=query)
+ runner = RetentionQueryRunner(team=self.team, query=query, limit_context=limit_context)
return runner.calculate().model_dump()["results"]
def run_actors_query(self, interval, query, select=None, actor="person"):
@@ -1920,3 +1925,10 @@ def test_groups_aggregating_person_on_events(self):
[1],
],
)
+
+ @patch("posthog.hogql.query.sync_execute", wraps=sync_execute)
+ def test_limit_is_context_aware(self, mock_sync_execute: MagicMock):
+ self.run_query(query={}, limit_context=LimitContext.QUERY_ASYNC)
+
+ mock_sync_execute.assert_called_once()
+ self.assertIn(f" max_execution_time={INCREASED_MAX_EXECUTION_TIME},", mock_sync_execute.call_args[0][0])
diff --git a/posthog/hogql_queries/insights/test/test_stickiness_query_runner.py b/posthog/hogql_queries/insights/test/test_stickiness_query_runner.py
index 3de1fb6ce865e..6e25827e6ecba 100644
--- a/posthog/hogql_queries/insights/test/test_stickiness_query_runner.py
+++ b/posthog/hogql_queries/insights/test/test_stickiness_query_runner.py
@@ -1,8 +1,12 @@
from dataclasses import dataclass
from typing import Dict, List, Optional, Union
+from unittest.mock import MagicMock, patch
from django.test import override_settings
from freezegun import freeze_time
+from posthog.clickhouse.client.execute import sync_execute
+from posthog.hogql.constants import LimitContext
+from posthog.hogql.query import INCREASED_MAX_EXECUTION_TIME
from posthog.hogql_queries.insights.stickiness_query_runner import StickinessQueryRunner
from posthog.models.action.action import Action
from posthog.models.action_step import ActionStep
@@ -197,6 +201,7 @@ def _run_query(
properties: Optional[StickinessProperties] = None,
filters: Optional[StickinessFilter] = None,
filter_test_accounts: Optional[bool] = False,
+ limit_context: Optional[LimitContext] = None,
):
query_series: List[EventsNode | ActionsNode] = [EventsNode(event="$pageview")] if series is None else series
query_date_from = date_from or self.default_date_from
@@ -211,7 +216,7 @@ def _run_query(
stickinessFilter=filters,
filterTestAccounts=filter_test_accounts,
)
- return StickinessQueryRunner(team=self.team, query=query).calculate()
+ return StickinessQueryRunner(team=self.team, query=query, limit_context=limit_context).calculate()
def test_stickiness_runs(self):
self._create_test_events()
@@ -580,3 +585,10 @@ def test_hogql_aggregations(self):
1,
0,
]
+
+ @patch("posthog.hogql.query.sync_execute", wraps=sync_execute)
+ def test_limit_is_context_aware(self, mock_sync_execute: MagicMock):
+ self._run_query(limit_context=LimitContext.QUERY_ASYNC)
+
+ mock_sync_execute.assert_called_once()
+ self.assertIn(f" max_execution_time={INCREASED_MAX_EXECUTION_TIME},", mock_sync_execute.call_args[0][0])
diff --git a/posthog/hogql_queries/insights/trends/breakdown.py b/posthog/hogql_queries/insights/trends/breakdown.py
index 45a3a8421e8d8..bde2cd807b6a7 100644
--- a/posthog/hogql_queries/insights/trends/breakdown.py
+++ b/posthog/hogql_queries/insights/trends/breakdown.py
@@ -3,9 +3,7 @@
from posthog.hogql.parser import parse_expr
from posthog.hogql.timings import HogQLTimings
from posthog.hogql_queries.insights.trends.breakdown_values import (
- BREAKDOWN_NULL_NUMERIC_LABEL,
BREAKDOWN_NULL_STRING_LABEL,
- BREAKDOWN_OTHER_NUMERIC_LABEL,
BREAKDOWN_OTHER_STRING_LABEL,
BreakdownValues,
)
@@ -19,6 +17,10 @@
from posthog.schema import ActionsNode, EventsNode, DataWarehouseNode, HogQLQueryModifiers, InCohortVia, TrendsQuery
+def hogql_to_string(expr: ast.Expr) -> ast.Call:
+ return ast.Call(name="toString", args=[expr])
+
+
class Breakdown:
query: TrendsQuery
team: Team
@@ -27,7 +29,7 @@ class Breakdown:
timings: HogQLTimings
modifiers: HogQLQueryModifiers
events_filter: ast.Expr
- breakdown_values_override: Optional[List[str | int | float]]
+ breakdown_values_override: Optional[List[str]]
def __init__(
self,
@@ -38,7 +40,7 @@ def __init__(
timings: HogQLTimings,
modifiers: HogQLQueryModifiers,
events_filter: ast.Expr,
- breakdown_values_override: Optional[List[str | int | float]] = None,
+ breakdown_values_override: Optional[List[str]] = None,
):
self.team = team
self.query = query
@@ -70,19 +72,15 @@ def placeholders(self) -> Dict[str, ast.Expr]:
return {"cross_join_breakdown_values": ast.Alias(alias="breakdown_value", expr=values)}
- def column_expr(self) -> ast.Expr:
+ def column_expr(self) -> ast.Alias:
if self.is_histogram_breakdown:
return ast.Alias(alias="breakdown_value", expr=self._get_breakdown_histogram_multi_if())
- elif self.query.breakdownFilter.breakdown_type == "hogql":
- return ast.Alias(
- alias="breakdown_value",
- expr=parse_expr(self.query.breakdownFilter.breakdown),
- )
- elif self.query.breakdownFilter.breakdown_type == "cohort":
+
+ if self.query.breakdownFilter.breakdown_type == "cohort":
if self.modifiers.inCohortVia == InCohortVia.leftjoin_conjoined:
return ast.Alias(
alias="breakdown_value",
- expr=ast.Field(chain=["__in_cohort", "cohort_id"]),
+ expr=hogql_to_string(ast.Field(chain=["__in_cohort", "cohort_id"])),
)
cohort_breakdown = (
@@ -90,19 +88,9 @@ def column_expr(self) -> ast.Expr:
)
return ast.Alias(
alias="breakdown_value",
- expr=ast.Constant(value=cohort_breakdown),
- )
-
- if self.query.breakdownFilter.breakdown_type == "hogql":
- return ast.Alias(
- alias="breakdown_value",
- expr=parse_expr(self.query.breakdownFilter.breakdown),
+ expr=hogql_to_string(ast.Constant(value=cohort_breakdown)),
)
- # If there's no breakdown values
- if len(self._breakdown_values) == 1 and self._breakdown_values[0] is None:
- return ast.Alias(alias="breakdown_value", expr=ast.Field(chain=self._properties_chain))
-
return ast.Alias(alias="breakdown_value", expr=self._get_breakdown_transform_func)
def events_where_filter(self) -> ast.Expr | None:
@@ -148,15 +136,14 @@ def events_where_filter(self) -> ast.Expr | None:
else:
left = ast.Field(chain=self._properties_chain)
+ if not self.is_histogram_breakdown:
+ left = hogql_to_string(left)
+
compare_ops = []
for _value in self._breakdown_values:
- value: Optional[str | int | float] = _value
+ value: Optional[str] = str(_value) # non-cohorts are always strings
# If the value is one of the "other" values, then use the `transform()` func
- if (
- value == BREAKDOWN_OTHER_STRING_LABEL
- or value == BREAKDOWN_OTHER_NUMERIC_LABEL
- or value == float(BREAKDOWN_OTHER_NUMERIC_LABEL)
- ):
+ if value == BREAKDOWN_OTHER_STRING_LABEL:
transform_func = self._get_breakdown_transform_func
compare_ops.append(
ast.CompareOperation(
@@ -164,11 +151,7 @@ def events_where_filter(self) -> ast.Expr | None:
)
)
else:
- if (
- value == BREAKDOWN_NULL_STRING_LABEL
- or value == BREAKDOWN_NULL_NUMERIC_LABEL
- or value == float(BREAKDOWN_NULL_NUMERIC_LABEL)
- ):
+ if value == BREAKDOWN_NULL_STRING_LABEL:
value = None
compare_ops.append(
@@ -184,30 +167,25 @@ def events_where_filter(self) -> ast.Expr | None:
@cached_property
def _get_breakdown_transform_func(self) -> ast.Call:
- values = self._breakdown_values
- all_values_are_ints_or_none = all(isinstance(value, int) or value is None for value in values)
- all_values_are_floats_or_none = all(isinstance(value, float) or value is None for value in values)
-
- if all_values_are_ints_or_none:
- breakdown_other_value = BREAKDOWN_OTHER_NUMERIC_LABEL
- breakdown_null_value = BREAKDOWN_NULL_NUMERIC_LABEL
- elif all_values_are_floats_or_none:
- breakdown_other_value = float(BREAKDOWN_OTHER_NUMERIC_LABEL)
- breakdown_null_value = float(BREAKDOWN_NULL_NUMERIC_LABEL)
- else:
- breakdown_other_value = BREAKDOWN_OTHER_STRING_LABEL
- breakdown_null_value = BREAKDOWN_NULL_STRING_LABEL
+ if self.query.breakdownFilter.breakdown_type == "hogql":
+ return self._get_breakdown_values_transform(parse_expr(self.query.breakdownFilter.breakdown))
+ return self._get_breakdown_values_transform(ast.Field(chain=self._properties_chain))
+ def _get_breakdown_values_transform(self, node: ast.Expr) -> ast.Call:
+ breakdown_values = self._breakdown_values_ast
return ast.Call(
name="transform",
args=[
ast.Call(
name="ifNull",
- args=[ast.Field(chain=self._properties_chain), ast.Constant(value=breakdown_null_value)],
+ args=[
+ hogql_to_string(node),
+ ast.Constant(value=BREAKDOWN_NULL_STRING_LABEL),
+ ],
),
- self._breakdown_values_ast,
- self._breakdown_values_ast,
- ast.Constant(value=breakdown_other_value),
+ breakdown_values,
+ breakdown_values,
+ ast.Constant(value=BREAKDOWN_OTHER_STRING_LABEL),
],
)
@@ -220,15 +198,21 @@ def _breakdown_buckets_ast(self) -> ast.Array:
return ast.Array(exprs=list(map(lambda v: ast.Constant(value=v), values)))
- @cached_property
+ @property
def _breakdown_values_ast(self) -> ast.Array:
- return ast.Array(exprs=[ast.Constant(value=v) for v in self._breakdown_values])
+ exprs: list[ast.Expr] = []
+ for value in self._breakdown_values:
+ if isinstance(value, str):
+ exprs.append(ast.Constant(value=value))
+ else:
+ exprs.append(hogql_to_string(ast.Constant(value=value)))
+ return ast.Array(exprs=exprs)
@cached_property
- def _all_breakdown_values(self) -> List[str | int | float | None]:
+ def _all_breakdown_values(self) -> List[str | int | None]:
# Used in the actors query
if self.breakdown_values_override is not None:
- return cast(List[str | int | float | None], self.breakdown_values_override)
+ return cast(List[str | int | None], self.breakdown_values_override)
if self.query.breakdownFilter is None:
return []
@@ -243,25 +227,12 @@ def _all_breakdown_values(self) -> List[str | int | float | None]:
query_date_range=self.query_date_range,
modifiers=self.modifiers,
)
- return cast(List[str | int | float | None], breakdown.get_breakdown_values())
+ return cast(List[str | int | None], breakdown.get_breakdown_values())
@cached_property
- def _breakdown_values(self) -> List[str | int | float]:
- values = self._all_breakdown_values
- if len(values) == 0 or all(value is None for value in values):
- return []
-
- if None in values:
- all_values_are_ints_or_none = all(isinstance(value, int) or value is None for value in values)
- all_values_are_floats_or_none = all(isinstance(value, float) or value is None for value in values)
-
- if all_values_are_ints_or_none:
- values = [v if v is not None else BREAKDOWN_NULL_NUMERIC_LABEL for v in values]
- elif all_values_are_floats_or_none:
- values = [v if v is not None else float(BREAKDOWN_NULL_NUMERIC_LABEL) for v in values]
- else:
- values = [v if v is not None else BREAKDOWN_NULL_STRING_LABEL for v in values]
- return cast(List[str | int | float], values)
+ def _breakdown_values(self) -> List[str | int]:
+ values = [BREAKDOWN_NULL_STRING_LABEL if v is None else v for v in self._all_breakdown_values]
+ return cast(List[str | int], values)
@cached_property
def has_breakdown_values(self) -> bool:
diff --git a/posthog/hogql_queries/insights/trends/breakdown_values.py b/posthog/hogql_queries/insights/trends/breakdown_values.py
index 7b1522d5f25c5..d9ab11891f210 100644
--- a/posthog/hogql_queries/insights/trends/breakdown_values.py
+++ b/posthog/hogql_queries/insights/trends/breakdown_values.py
@@ -97,6 +97,9 @@ def get_breakdown_values(self) -> List[str | int]:
),
)
+ if not self.histogram_bin_count:
+ select_field.expr = ast.Call(name="toString", args=[select_field.expr])
+
if self.chart_display_type == ChartDisplayType.WorldMap:
breakdown_limit = BREAKDOWN_VALUES_LIMIT_FOR_COUNTRIES
else:
@@ -211,23 +214,9 @@ def get_breakdown_values(self) -> List[str | int]:
# Add "other" value if "other" is not hidden and we're not bucketing numeric values
if self.hide_other_aggregation is not True and self.histogram_bin_count is None:
- all_values_are_ints_or_none = all(isinstance(value, int) or value is None for value in values)
- all_values_are_floats_or_none = all(isinstance(value, float) or value is None for value in values)
- all_values_are_string_or_none = all(isinstance(value, str) or value is None for value in values)
-
- if all_values_are_string_or_none:
- values = [BREAKDOWN_NULL_STRING_LABEL if value in (None, "") else value for value in values]
- if needs_other:
- values.insert(0, BREAKDOWN_OTHER_STRING_LABEL)
- elif all_values_are_ints_or_none or all_values_are_floats_or_none:
- if all_values_are_ints_or_none:
- values = [BREAKDOWN_NULL_NUMERIC_LABEL if value is None else value for value in values]
- if needs_other:
- values.insert(0, BREAKDOWN_OTHER_NUMERIC_LABEL)
- else:
- values = [float(BREAKDOWN_NULL_NUMERIC_LABEL) if value is None else value for value in values]
- if needs_other:
- values.insert(0, float(BREAKDOWN_OTHER_NUMERIC_LABEL))
+ values = [BREAKDOWN_NULL_STRING_LABEL if value in (None, "") else value for value in values]
+ if needs_other:
+ values = [BREAKDOWN_OTHER_STRING_LABEL] + values
if len(values) == 0:
values.insert(0, None)
diff --git a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr
index f6eb3748afb2b..e2ec22fb9fb1c 100644
--- a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr
+++ b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr
@@ -187,7 +187,7 @@
# ---
# name: TestTrends.test_breakdown_by_group_props_person_on_events
'''
- SELECT e__group_0.properties___industry AS value,
+ SELECT toString(e__group_0.properties___industry) AS value,
count(e.uuid) AS count
FROM events AS e
LEFT JOIN
@@ -210,7 +210,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -231,7 +231,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.uuid) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(e__group_0.properties___industry, '$$_posthog_breakdown_null_$$'), ['finance', 'technology'], ['finance', 'technology'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(e__group_0.properties___industry), '$$_posthog_breakdown_null_$$'), ['finance', 'technology'], ['finance', 'technology'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
LEFT JOIN
(SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), groups._timestamp) AS properties___industry,
@@ -241,7 +241,7 @@
WHERE and(equals(groups.team_id, 2), ifNull(equals(index, 0), 0))
GROUP BY groups.group_type_index,
groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(e__group_0.properties___industry, 'finance'), 0), ifNull(equals(e__group_0.properties___industry, 'technology'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(toString(e__group_0.properties___industry), 'finance'), 0), ifNull(equals(toString(e__group_0.properties___industry), 'technology'), 0)))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -287,7 +287,7 @@
# ---
# name: TestTrends.test_breakdown_by_group_props_with_person_filter_person_on_events
'''
- SELECT e__group_0.properties___industry AS value,
+ SELECT toString(e__group_0.properties___industry) AS value,
count(e.uuid) AS count
FROM events AS e
LEFT JOIN
@@ -310,7 +310,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -331,7 +331,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.uuid) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(e__group_0.properties___industry, '$$_posthog_breakdown_null_$$'), ['finance'], ['finance'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(e__group_0.properties___industry), '$$_posthog_breakdown_null_$$'), ['finance'], ['finance'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
LEFT JOIN
(SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), groups._timestamp) AS properties___industry,
@@ -341,7 +341,7 @@
WHERE and(equals(groups.team_id, 2), ifNull(equals(index, 0), 0))
GROUP BY groups.group_type_index,
groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, 'key'), ''), 'null'), '^"|"$', ''), 'value'), 0), ifNull(equals(e__group_0.properties___industry, 'finance'), 0))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, 'key'), ''), 'null'), '^"|"$', ''), 'value'), 0), ifNull(equals(toString(e__group_0.properties___industry), 'finance'), 0))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -356,7 +356,7 @@
# ---
# name: TestTrends.test_breakdown_filtering_with_properties_in_new_format
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Firefox'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Windows'), 0)), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0)))
@@ -371,7 +371,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -392,9 +392,9 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.uuid) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['second url'], ['second url'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['second url'], ['second url'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Firefox'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Windows'), 0)), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'second url'), 0))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Firefox'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Windows'), 0)), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '')), 'second url'), 0))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -409,7 +409,7 @@
# ---
# name: TestTrends.test_breakdown_filtering_with_properties_in_new_format.2
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Firefox'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Windows'), 0)), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0)))
@@ -423,24 +423,38 @@
# name: TestTrends.test_breakdown_filtering_with_properties_in_new_format.3
'''
SELECT groupArray(day_start) AS date,
- groupArray(count) AS total
+ groupArray(count) AS total,
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
- day_start AS day_start
+ day_start AS day_start,
+ breakdown_value AS breakdown_value
FROM
(SELECT 0 AS total,
- minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start
- FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), 0)) AS numbers
- UNION ALL SELECT 0 AS total,
- toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC'))) AS day_start
+ ticks.day_start AS day_start,
+ sec.breakdown_value AS breakdown_value
+ FROM
+ (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start
+ FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), 0)) AS numbers
+ UNION ALL SELECT toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC'))) AS day_start) AS ticks
+ CROSS JOIN
+ (SELECT breakdown_value
+ FROM
+ (SELECT ['$$_posthog_breakdown_null_$$'] AS breakdown_value) ARRAY
+ JOIN breakdown_value AS breakdown_value) AS sec
+ ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.uuid) AS total,
- toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start
+ toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$'], ['$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Firefox'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Windows'), 0)), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0))
- GROUP BY day_start)
- GROUP BY day_start
- ORDER BY day_start ASC)
- ORDER BY sum(count) DESC
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Firefox'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Windows'), 0)), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''))))
+ GROUP BY day_start,
+ breakdown_value)
+ GROUP BY day_start,
+ breakdown_value
+ ORDER BY day_start ASC, breakdown_value ASC)
+ GROUP BY breakdown_value
+ ORDER BY sum(count) DESC, breakdown_value ASC
LIMIT 10000 SETTINGS readonly=2,
max_execution_time=60,
allow_experimental_object_type=1
@@ -448,7 +462,7 @@
# ---
# name: TestTrends.test_breakdown_weekly_active_users_aggregated
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')) AS value,
count(DISTINCT e__pdi.person_id) AS count
FROM events AS e
INNER JOIN
@@ -480,7 +494,7 @@
CROSS JOIN
(SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp,
e__pdi.person_id AS actor_id,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['val', 'bor'], ['val', 'bor'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['val', 'bor'], ['val', 'bor'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -489,7 +503,7 @@
WHERE equals(person_distinct_id2.team_id, 2)
GROUP BY person_distinct_id2.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
- WHERE and(equals(e.team_id, 2), and(equals(e.event, '$pageview'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'bor'), 0))), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 00:00:00', 6, 'UTC')), toIntervalDay(7))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC'))), 0))
+ WHERE and(equals(e.team_id, 2), and(equals(e.event, '$pageview'), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), 'val'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), 'bor'), 0))), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 00:00:00', 6, 'UTC')), toIntervalDay(7))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC'))), 0))
GROUP BY timestamp, actor_id,
breakdown_value) AS e
WHERE and(ifNull(lessOrEquals(e.timestamp, plus(d.timestamp, toIntervalDay(1))), 0), ifNull(greater(e.timestamp, minus(d.timestamp, toIntervalDay(6))), 0))
@@ -506,7 +520,7 @@
# ---
# name: TestTrends.test_breakdown_weekly_active_users_aggregated_materialized
'''
- SELECT nullIf(nullIf(e.mat_key, ''), 'null') AS value,
+ SELECT toString(nullIf(nullIf(e.mat_key, ''), 'null')) AS value,
count(DISTINCT e__pdi.person_id) AS count
FROM events AS e
INNER JOIN
@@ -538,7 +552,7 @@
CROSS JOIN
(SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp,
e__pdi.person_id AS actor_id,
- transform(ifNull(nullIf(nullIf(e.mat_key, ''), 'null'), '$$_posthog_breakdown_null_$$'), ['val', 'bor'], ['val', 'bor'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(nullIf(nullIf(e.mat_key, ''), 'null')), '$$_posthog_breakdown_null_$$'), ['val', 'bor'], ['val', 'bor'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -547,7 +561,7 @@
WHERE equals(person_distinct_id2.team_id, 2)
GROUP BY person_distinct_id2.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
- WHERE and(equals(e.team_id, 2), and(equals(e.event, '$pageview'), or(ifNull(equals(nullIf(nullIf(e.mat_key, ''), 'null'), 'val'), 0), ifNull(equals(nullIf(nullIf(e.mat_key, ''), 'null'), 'bor'), 0))), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 00:00:00', 6, 'UTC')), toIntervalDay(7))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC'))), 0))
+ WHERE and(equals(e.team_id, 2), and(equals(e.event, '$pageview'), or(ifNull(equals(toString(nullIf(nullIf(e.mat_key, ''), 'null')), 'val'), 0), ifNull(equals(toString(nullIf(nullIf(e.mat_key, ''), 'null')), 'bor'), 0))), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 00:00:00', 6, 'UTC')), toIntervalDay(7))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC'))), 0))
GROUP BY timestamp, actor_id,
breakdown_value) AS e
WHERE and(ifNull(lessOrEquals(e.timestamp, plus(d.timestamp, toIntervalDay(1))), 0), ifNull(greater(e.timestamp, minus(d.timestamp, toIntervalDay(6))), 0))
@@ -584,7 +598,7 @@
# ---
# name: TestTrends.test_breakdown_weekly_active_users_daily_based_on_action.2
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')) AS value,
count(DISTINCT e__pdi.person_id) AS count
FROM events AS e
INNER JOIN
@@ -622,7 +636,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -654,7 +668,7 @@
CROSS JOIN
(SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp,
e__pdi.person_id AS actor_id,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['val'], ['val'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['val'], ['val'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -679,7 +693,7 @@
FROM cohortpeople
WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 2))
GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version
- HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0))), 0))), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0)), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), toIntervalDay(7))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), 0))
+ HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0))), 0))), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), 'val'), 0)), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), toIntervalDay(7))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), 0))
GROUP BY timestamp, actor_id,
breakdown_value) AS e
WHERE and(ifNull(lessOrEquals(e.timestamp, plus(d.timestamp, toIntervalDay(1))), 0), ifNull(greater(e.timestamp, minus(d.timestamp, toIntervalDay(6))), 0))
@@ -699,7 +713,7 @@
# ---
# name: TestTrends.test_breakdown_with_filter_groups_person_on_events
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
LEFT JOIN
@@ -722,7 +736,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -743,7 +757,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.uuid) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['uh', 'oh'], ['uh', 'oh'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['uh', 'oh'], ['uh', 'oh'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
LEFT JOIN
(SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), groups._timestamp) AS properties___industry,
@@ -753,7 +767,7 @@
WHERE and(equals(groups.team_id, 2), ifNull(equals(index, 0), 0))
GROUP BY groups.group_type_index,
groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(e__group_0.properties___industry, 'finance'), 0), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'uh'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'oh'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(e__group_0.properties___industry, 'finance'), 0), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), 'uh'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), 'oh'), 0)))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -782,7 +796,7 @@
# ---
# name: TestTrends.test_breakdown_with_filter_groups_person_on_events_v2.1
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
LEFT JOIN
@@ -805,7 +819,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -826,7 +840,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT ifNull(nullIf(e__override.override_person_id, '00000000-0000-0000-0000-000000000000'), e.person_id)) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['uh', 'oh'], ['uh', 'oh'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['uh', 'oh'], ['uh', 'oh'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
LEFT OUTER JOIN
(SELECT argMax(person_overrides.override_person_id, person_overrides.version) AS override_person_id,
@@ -842,7 +856,7 @@
WHERE and(equals(groups.team_id, 2), ifNull(equals(index, 0), 0))
GROUP BY groups.group_type_index,
groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(e__group_0.properties___industry, 'finance'), 0), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'uh'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'oh'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(e__group_0.properties___industry, 'finance'), 0), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), 'uh'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), 'oh'), 0)))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -857,7 +871,7 @@
# ---
# name: TestTrends.test_dau_with_breakdown_filtering_with_sampling
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')))
@@ -872,7 +886,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -893,7 +907,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT e__pdi.person_id) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['other_value', '$$_posthog_breakdown_null_$$', 'value'], ['other_value', '$$_posthog_breakdown_null_$$', 'value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['other_value', '$$_posthog_breakdown_null_$$', 'value'], ['other_value', '$$_posthog_breakdown_null_$$', 'value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1.0
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -902,7 +916,7 @@
WHERE equals(person_distinct_id2.team_id, 2)
GROUP BY person_distinct_id2.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'other_value'), 0), isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'other_value'), 0), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''))), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value'), 0)))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -917,7 +931,7 @@
# ---
# name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.2
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')))
@@ -932,7 +946,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -953,7 +967,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT e__pdi.person_id) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['other_value', '$$_posthog_breakdown_null_$$', 'value'], ['other_value', '$$_posthog_breakdown_null_$$', 'value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['other_value', '$$_posthog_breakdown_null_$$', 'value'], ['other_value', '$$_posthog_breakdown_null_$$', 'value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1.0
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -962,7 +976,7 @@
WHERE equals(person_distinct_id2.team_id, 2)
GROUP BY person_distinct_id2.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'other_value'), 0), isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'other_value'), 0), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''))), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value'), 0)))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -1242,7 +1256,7 @@
# ---
# name: TestTrends.test_mau_with_breakdown_filtering_and_prop_filter
'''
- SELECT e__pdi__person.`properties___$some_prop` AS value,
+ SELECT toString(e__pdi__person.`properties___$some_prop`) AS value,
count(DISTINCT e__pdi.person_id) AS count
FROM events AS e
INNER JOIN
@@ -1276,7 +1290,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -1308,7 +1322,7 @@
CROSS JOIN
(SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp,
e__pdi.person_id AS actor_id,
- transform(ifNull(e__pdi__person.`properties___$some_prop`, '$$_posthog_breakdown_null_$$'), ['some_val2', 'some_val'], ['some_val2', 'some_val'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(e__pdi__person.`properties___$some_prop`), '$$_posthog_breakdown_null_$$'), ['some_val2', 'some_val'], ['some_val2', 'some_val'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -1329,7 +1343,7 @@
WHERE equals(person.team_id, 2)
GROUP BY person.id
HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id)
- WHERE and(equals(e.team_id, 2), and(equals(e.event, 'sign up'), ifNull(equals(e__pdi__person.properties___filter_prop, 'filter_val'), 0), or(ifNull(equals(e__pdi__person.`properties___$some_prop`, 'some_val2'), 0), ifNull(equals(e__pdi__person.`properties___$some_prop`, 'some_val'), 0))), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), toIntervalDay(30))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0))
+ WHERE and(equals(e.team_id, 2), and(equals(e.event, 'sign up'), ifNull(equals(e__pdi__person.properties___filter_prop, 'filter_val'), 0), or(ifNull(equals(toString(e__pdi__person.`properties___$some_prop`), 'some_val2'), 0), ifNull(equals(toString(e__pdi__person.`properties___$some_prop`), 'some_val'), 0))), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), toIntervalDay(30))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0))
GROUP BY timestamp, actor_id,
breakdown_value) AS e
WHERE and(ifNull(lessOrEquals(e.timestamp, plus(d.timestamp, toIntervalDay(1))), 0), ifNull(greater(e.timestamp, minus(d.timestamp, toIntervalDay(29))), 0))
@@ -1349,7 +1363,7 @@
# ---
# name: TestTrends.test_mau_with_breakdown_filtering_and_prop_filter_poe_v2
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$some_prop'), ''), 'null'), '^"|"$', '')) AS value,
count(DISTINCT ifNull(nullIf(e__override.override_person_id, '00000000-0000-0000-0000-000000000000'), e.person_id)) AS count
FROM events AS e
LEFT OUTER JOIN
@@ -1370,7 +1384,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -1402,7 +1416,7 @@
CROSS JOIN
(SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp,
ifNull(nullIf(e__override.override_person_id, '00000000-0000-0000-0000-000000000000'), e.person_id) AS actor_id,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$some_prop'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['some_val2', 'some_val'], ['some_val2', 'some_val'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$some_prop'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['some_val2', 'some_val'], ['some_val2', 'some_val'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
LEFT OUTER JOIN
(SELECT argMax(person_overrides.override_person_id, person_overrides.version) AS override_person_id,
@@ -1410,7 +1424,7 @@
FROM person_overrides
WHERE equals(person_overrides.team_id, 2)
GROUP BY person_overrides.old_person_id) AS e__override ON equals(e.person_id, e__override.old_person_id)
- WHERE and(equals(e.team_id, 2), and(equals(e.event, 'sign up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, 'filter_prop'), ''), 'null'), '^"|"$', ''), 'filter_val'), 0), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$some_prop'), ''), 'null'), '^"|"$', ''), 'some_val2'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$some_prop'), ''), 'null'), '^"|"$', ''), 'some_val'), 0))), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), toIntervalDay(30))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0))
+ WHERE and(equals(e.team_id, 2), and(equals(e.event, 'sign up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, 'filter_prop'), ''), 'null'), '^"|"$', ''), 'filter_val'), 0), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$some_prop'), ''), 'null'), '^"|"$', '')), 'some_val2'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$some_prop'), ''), 'null'), '^"|"$', '')), 'some_val'), 0))), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), toIntervalDay(30))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0))
GROUP BY timestamp, actor_id,
breakdown_value) AS e
WHERE and(ifNull(lessOrEquals(e.timestamp, plus(d.timestamp, toIntervalDay(1))), 0), ifNull(greater(e.timestamp, minus(d.timestamp, toIntervalDay(29))), 0))
@@ -1476,7 +1490,7 @@
# ---
# name: TestTrends.test_person_filtering_in_cohort_in_action.2
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
INNER JOIN
@@ -1503,7 +1517,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -1524,7 +1538,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.uuid) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -1538,7 +1552,7 @@
FROM cohortpeople
WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 2))
GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version
- HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0))), 0)), or(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'other_value'), 0)))
+ HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0))), 0)), or(isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''))), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'other_value'), 0)))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -1573,7 +1587,7 @@
# ---
# name: TestTrends.test_person_filtering_in_cohort_in_action_poe_v2.2
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
LEFT OUTER JOIN
@@ -1599,7 +1613,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -1620,7 +1634,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.uuid) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
LEFT OUTER JOIN
(SELECT argMax(person_overrides.override_person_id, person_overrides.version) AS override_person_id,
@@ -1633,7 +1647,7 @@
FROM cohortpeople
WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 2))
GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version
- HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0))), 0)), or(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'other_value'), 0)))
+ HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0))), 0)), or(isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''))), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'other_value'), 0)))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -2217,7 +2231,7 @@
# ---
# name: TestTrends.test_timezones_daily.4
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')))
@@ -2232,7 +2246,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -2253,7 +2267,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT e__pdi.person_id) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['Mac'], ['Mac'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['Mac'], ['Mac'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -2262,7 +2276,7 @@
WHERE equals(person_distinct_id2.team_id, 2)
GROUP BY person_distinct_id2.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')), 'Mac'), 0))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -2408,7 +2422,7 @@
# ---
# name: TestTrends.test_timezones_daily_minus_utc.4
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'America/Phoenix')))), lessOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'America/Phoenix')))), lessOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix'))), equals(e.event, 'sign up')))
@@ -2423,7 +2437,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -2444,7 +2458,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT e__pdi.person_id) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'America/Phoenix')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['Mac'], ['Mac'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['Mac'], ['Mac'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -2453,7 +2467,7 @@
WHERE equals(person_distinct_id2.team_id, 2)
GROUP BY person_distinct_id2.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'America/Phoenix')))), lessOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix'))), equals(e.event, 'sign up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'America/Phoenix')))), lessOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix'))), equals(e.event, 'sign up'), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')), 'Mac'), 0))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -2599,7 +2613,7 @@
# ---
# name: TestTrends.test_timezones_daily_plus_utc.4
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'Asia/Tokyo')))), lessOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'Asia/Tokyo')))), lessOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo'))), equals(e.event, 'sign up')))
@@ -2614,7 +2628,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -2635,7 +2649,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT e__pdi.person_id) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'Asia/Tokyo')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['Mac'], ['Mac'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['Mac'], ['Mac'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -2644,7 +2658,7 @@
WHERE equals(person_distinct_id2.team_id, 2)
GROUP BY person_distinct_id2.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'Asia/Tokyo')))), lessOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo'))), equals(e.event, 'sign up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'Asia/Tokyo')))), lessOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo'))), equals(e.event, 'sign up'), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')), 'Mac'), 0))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -2992,7 +3006,7 @@
# ---
# name: TestTrends.test_trend_breakdown_user_props_with_filter_with_partial_property_pushdowns
'''
- SELECT e__pdi__person.properties___email AS value,
+ SELECT toString(e__pdi__person.properties___email) AS value,
count(e.uuid) AS count
FROM events AS e
INNER JOIN
@@ -3027,7 +3041,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -3048,7 +3062,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.uuid) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(e__pdi__person.properties___email, '$$_posthog_breakdown_null_$$'), ['test2@posthog.com', 'test@gmail.com', 'test5@posthog.com', 'test4@posthog.com', 'test3@posthog.com'], ['test2@posthog.com', 'test@gmail.com', 'test5@posthog.com', 'test4@posthog.com', 'test3@posthog.com'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(e__pdi__person.properties___email), '$$_posthog_breakdown_null_$$'), ['test2@posthog.com', 'test@gmail.com', 'test5@posthog.com', 'test4@posthog.com', 'test3@posthog.com'], ['test2@posthog.com', 'test@gmail.com', 'test5@posthog.com', 'test4@posthog.com', 'test3@posthog.com'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id,
@@ -3070,7 +3084,7 @@
WHERE equals(person.team_id, 2)
GROUP BY person.id
HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(or(ifNull(notILike(e__pdi__person.properties___email, '%@posthog.com%'), 1), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0)), or(ifNull(equals(e__pdi__person.`properties___$os`, 'android'), 0), ifNull(equals(e__pdi__person.`properties___$browser`, 'safari'), 0))), or(ifNull(equals(e__pdi__person.properties___email, 'test2@posthog.com'), 0), ifNull(equals(e__pdi__person.properties___email, 'test@gmail.com'), 0), ifNull(equals(e__pdi__person.properties___email, 'test5@posthog.com'), 0), ifNull(equals(e__pdi__person.properties___email, 'test4@posthog.com'), 0), ifNull(equals(e__pdi__person.properties___email, 'test3@posthog.com'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(or(ifNull(notILike(e__pdi__person.properties___email, '%@posthog.com%'), 1), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0)), or(ifNull(equals(e__pdi__person.`properties___$os`, 'android'), 0), ifNull(equals(e__pdi__person.`properties___$browser`, 'safari'), 0))), or(ifNull(equals(toString(e__pdi__person.properties___email), 'test2@posthog.com'), 0), ifNull(equals(toString(e__pdi__person.properties___email), 'test@gmail.com'), 0), ifNull(equals(toString(e__pdi__person.properties___email), 'test5@posthog.com'), 0), ifNull(equals(toString(e__pdi__person.properties___email), 'test4@posthog.com'), 0), ifNull(equals(toString(e__pdi__person.properties___email), 'test3@posthog.com'), 0)))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -3085,7 +3099,7 @@
# ---
# name: TestTrends.test_trend_breakdown_user_props_with_filter_with_partial_property_pushdowns.2
'''
- SELECT e__pdi__person.properties___email AS value,
+ SELECT toString(e__pdi__person.properties___email) AS value,
count(e.uuid) AS count
FROM events AS e
INNER JOIN
@@ -3120,7 +3134,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -3141,7 +3155,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.uuid) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(e__pdi__person.properties___email, '$$_posthog_breakdown_null_$$'), ['test2@posthog.com'], ['test2@posthog.com'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(e__pdi__person.properties___email), '$$_posthog_breakdown_null_$$'), ['test2@posthog.com'], ['test2@posthog.com'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id,
@@ -3163,7 +3177,7 @@
WHERE equals(person.team_id, 2)
GROUP BY person.id
HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(ifNull(equals(e__pdi__person.`properties___$os`, 'android'), 0), ifNull(equals(e__pdi__person.`properties___$browser`, 'chrome'), 0)), and(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0), ifNull(ilike(e__pdi__person.properties___email, '%@posthog.com%'), 0)), ifNull(equals(e__pdi__person.properties___email, 'test2@posthog.com'), 0))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(ifNull(equals(e__pdi__person.`properties___$os`, 'android'), 0), ifNull(equals(e__pdi__person.`properties___$browser`, 'chrome'), 0)), and(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0), ifNull(ilike(e__pdi__person.properties___email, '%@posthog.com%'), 0)), ifNull(equals(toString(e__pdi__person.properties___email), 'test2@posthog.com'), 0))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -3248,7 +3262,7 @@
# ---
# name: TestTrends.test_trends_aggregate_by_distinct_id.2
'''
- SELECT e__pdi__person.`properties___$some_prop` AS value,
+ SELECT toString(e__pdi__person.`properties___$some_prop`) AS value,
count(e.uuid) AS count
FROM events AS e
INNER JOIN
@@ -3281,7 +3295,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -3302,7 +3316,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT e.distinct_id) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(e__pdi__person.`properties___$some_prop`, '$$_posthog_breakdown_null_$$'), ['some_val', '$$_posthog_breakdown_null_$$'], ['some_val', '$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(e__pdi__person.`properties___$some_prop`), '$$_posthog_breakdown_null_$$'), ['some_val', '$$_posthog_breakdown_null_$$'], ['some_val', '$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id,
@@ -3322,7 +3336,7 @@
WHERE equals(person.team_id, 2)
GROUP BY person.id
HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(e__pdi__person.`properties___$some_prop`, 'some_val'), 0), isNull(e__pdi__person.`properties___$some_prop`)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(toString(e__pdi__person.`properties___$some_prop`), 'some_val'), 0), isNull(toString(e__pdi__person.`properties___$some_prop`))))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -3415,7 +3429,7 @@
# ---
# name: TestTrends.test_trends_aggregate_by_distinct_id.6
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_prop'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')))
@@ -3430,7 +3444,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -3451,9 +3465,9 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT e.distinct_id) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_prop'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$'], ['$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_prop'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$'], ['$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_prop'), ''), 'null'), '^"|"$', '')))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_prop'), ''), 'null'), '^"|"$', ''))))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -3520,7 +3534,7 @@
# ---
# name: TestTrends.test_trends_breakdown_cumulative
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')))
@@ -3535,7 +3549,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT day_start AS day_start,
sum(count) OVER (PARTITION BY breakdown_value
@@ -3561,7 +3575,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT e__pdi.person_id) AS total,
min(toStartOfDay(toTimeZone(e.timestamp, 'UTC'))) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -3570,7 +3584,7 @@
WHERE equals(person_distinct_id2.team_id, 2)
GROUP BY person_distinct_id2.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'other_value'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''))), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'other_value'), 0)))
GROUP BY e__pdi.person_id,
breakdown_value)
GROUP BY day_start,
@@ -3585,7 +3599,7 @@
# ---
# name: TestTrends.test_trends_breakdown_cumulative_poe_v2
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')))
@@ -3600,7 +3614,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT day_start AS day_start,
sum(count) OVER (PARTITION BY breakdown_value
@@ -3626,7 +3640,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT ifNull(nullIf(e__override.override_person_id, '00000000-0000-0000-0000-000000000000'), e.person_id)) AS total,
min(toStartOfDay(toTimeZone(e.timestamp, 'UTC'))) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
LEFT OUTER JOIN
(SELECT argMax(person_overrides.override_person_id, person_overrides.version) AS override_person_id,
@@ -3634,7 +3648,7 @@
FROM person_overrides
WHERE equals(person_overrides.team_id, 2)
GROUP BY person_overrides.old_person_id) AS e__override ON equals(e.person_id, e__override.old_person_id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'other_value'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''))), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'other_value'), 0)))
GROUP BY ifNull(nullIf(e__override.override_person_id, '00000000-0000-0000-0000-000000000000'), e.person_id),
breakdown_value)
GROUP BY day_start,
@@ -3649,7 +3663,7 @@
# ---
# name: TestTrends.test_trends_breakdown_with_session_property_single_aggregate_math_and_breakdown
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
max(e__session.duration) AS count
FROM events AS e
INNER JOIN
@@ -3672,7 +3686,7 @@
breakdown_value AS breakdown_value
FROM
(SELECT any(e__session.duration) AS session_duration,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['value2', 'value1', '$$_posthog_breakdown_null_$$'], ['value2', 'value1', '$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['value2', 'value1', '$$_posthog_breakdown_null_$$'], ['value2', 'value1', '$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT events.`$session_id` AS id,
@@ -3680,7 +3694,7 @@
FROM events
WHERE and(equals(events.team_id, 2), greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), ifNull(notEquals(id, ''), 1))
GROUP BY id) AS e__session ON equals(e.`$session_id`, e__session.id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value2'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value1'), 0), isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''))))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value2'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value1'), 0), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')))))
GROUP BY e__session.id,
breakdown_value)
GROUP BY breakdown_value
@@ -3691,7 +3705,7 @@
# ---
# name: TestTrends.test_trends_breakdown_with_session_property_single_aggregate_math_and_breakdown.2
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
max(e__session.duration) AS count
FROM events AS e
INNER JOIN
@@ -3714,7 +3728,7 @@
breakdown_value AS breakdown_value
FROM
(SELECT any(e__session.duration) AS session_duration,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['value2', 'value1', '$$_posthog_breakdown_null_$$'], ['value2', 'value1', '$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['value2', 'value1', '$$_posthog_breakdown_null_$$'], ['value2', 'value1', '$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT events.`$session_id` AS id,
@@ -3722,7 +3736,7 @@
FROM events
WHERE and(equals(events.team_id, 2), greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), ifNull(notEquals(id, ''), 1))
GROUP BY id) AS e__session ON equals(e.`$session_id`, e__session.id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value2'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value1'), 0), isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''))))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value2'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value1'), 0), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')))))
GROUP BY e__session.id,
breakdown_value)
GROUP BY breakdown_value
@@ -3854,7 +3868,7 @@
# ---
# name: TestTrends.test_trends_count_per_user_average_aggregated_with_event_property_breakdown_with_sampling
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC')))), equals(e.event, 'viewed video'))
@@ -3874,7 +3888,7 @@
breakdown_value AS breakdown_value
FROM
(SELECT count(e.uuid) AS total,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['red', 'blue', '$$_posthog_breakdown_null_$$'], ['red', 'blue', '$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['red', 'blue', '$$_posthog_breakdown_null_$$'], ['red', 'blue', '$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1.0
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -3883,7 +3897,7 @@
WHERE equals(person_distinct_id2.team_id, 2)
GROUP BY person_distinct_id2.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
- WHERE and(equals(e.team_id, 2), and(equals(e.event, 'viewed video'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', ''), 'red'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', ''), 'blue'), 0), isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', '')))), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), toIntervalDay(0))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC'))))
+ WHERE and(equals(e.team_id, 2), and(equals(e.event, 'viewed video'), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', '')), 'red'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', '')), 'blue'), 0), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', ''))))), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), toIntervalDay(0))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC'))))
GROUP BY e__pdi.person_id,
breakdown_value)
GROUP BY breakdown_value)
@@ -4098,7 +4112,7 @@
# ---
# name: TestTrends.test_trends_person_breakdown_with_session_property_single_aggregate_math_and_breakdown
'''
- SELECT e__pdi__person.`properties___$some_prop` AS value,
+ SELECT toString(e__pdi__person.`properties___$some_prop`) AS value,
max(e__session.duration) AS count
FROM events AS e
INNER JOIN
@@ -4139,7 +4153,7 @@
breakdown_value AS breakdown_value
FROM
(SELECT any(e__session.duration) AS session_duration,
- transform(ifNull(e__pdi__person.`properties___$some_prop`, '$$_posthog_breakdown_null_$$'), ['some_val', 'another_val'], ['some_val', 'another_val'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(e__pdi__person.`properties___$some_prop`), '$$_posthog_breakdown_null_$$'), ['some_val', 'another_val'], ['some_val', 'another_val'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT events.`$session_id` AS id,
@@ -4165,7 +4179,7 @@
WHERE equals(person.team_id, 2)
GROUP BY person.id
HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(e__pdi__person.`properties___$some_prop`, 'some_val'), 0), ifNull(equals(e__pdi__person.`properties___$some_prop`, 'another_val'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(toString(e__pdi__person.`properties___$some_prop`), 'some_val'), 0), ifNull(equals(toString(e__pdi__person.`properties___$some_prop`), 'another_val'), 0)))
GROUP BY e__session.id,
breakdown_value)
GROUP BY breakdown_value
@@ -4316,7 +4330,7 @@
# ---
# name: TestTrends.test_trends_with_session_property_total_volume_math_with_breakdowns
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
max(e__session.duration) AS count
FROM events AS e
INNER JOIN
@@ -4337,7 +4351,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -4361,7 +4375,7 @@
breakdown_value AS breakdown_value
FROM
(SELECT any(e__session.duration) AS session_duration,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['value2', 'value1'], ['value2', 'value1'], '$$_posthog_breakdown_other_$$') AS breakdown_value,
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['value2', 'value1'], ['value2', 'value1'], '$$_posthog_breakdown_other_$$') AS breakdown_value,
toStartOfWeek(toTimeZone(e.timestamp, 'UTC'), 0) AS day_start
FROM events AS e SAMPLE 1
INNER JOIN
@@ -4370,7 +4384,7 @@
FROM events
WHERE and(equals(events.team_id, 2), greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), ifNull(notEquals(id, ''), 1))
GROUP BY id) AS e__session ON equals(e.`$session_id`, e__session.id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value2'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value1'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value2'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value1'), 0)))
GROUP BY day_start,
e__session.id,
breakdown_value,
@@ -4389,7 +4403,7 @@
# ---
# name: TestTrends.test_trends_with_session_property_total_volume_math_with_breakdowns.2
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
max(e__session.duration) AS count
FROM events AS e
INNER JOIN
@@ -4410,7 +4424,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -4434,7 +4448,7 @@
breakdown_value AS breakdown_value
FROM
(SELECT any(e__session.duration) AS session_duration,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['value2', 'value1'], ['value2', 'value1'], '$$_posthog_breakdown_other_$$') AS breakdown_value,
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['value2', 'value1'], ['value2', 'value1'], '$$_posthog_breakdown_other_$$') AS breakdown_value,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start
FROM events AS e SAMPLE 1
INNER JOIN
@@ -4443,7 +4457,7 @@
FROM events
WHERE and(equals(events.team_id, 2), greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), ifNull(notEquals(id, ''), 1))
GROUP BY id) AS e__session ON equals(e.`$session_id`, e__session.id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value2'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value1'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value2'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value1'), 0)))
GROUP BY day_start,
e__session.id,
breakdown_value,
diff --git a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends_data_warehouse_query.ambr b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends_data_warehouse_query.ambr
index db9e8e1d45000..bd7142030fe3a 100644
--- a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends_data_warehouse_query.ambr
+++ b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends_data_warehouse_query.ambr
@@ -1,10 +1,10 @@
# serializer version: 1
# name: TestTrendsDataWarehouseQuery.test_trends_breakdown
'''
- SELECT e.prop_1 AS value,
+ SELECT toString(e.prop_1) AS value,
count(e.id) AS count
FROM s3('http://host.docker.internal:19000/posthog/test_storage_bucket-posthog.hogql.datawarehouse.trendquery/*.parquet', 'object_storage_root_user', 'object_storage_root_password', 'Parquet', 'id String, prop_1 String, prop_2 String, created DateTime64(3, \'UTC\')') AS e
- WHERE and(and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0)), and(ifNull(greaterOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0)))
+ WHERE and(and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0)), and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0)))
GROUP BY value
ORDER BY count DESC, value DESC
LIMIT 26 SETTINGS readonly=2,
@@ -16,7 +16,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -36,10 +36,10 @@
JOIN breakdown_value AS breakdown_value) AS sec
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.id) AS total,
- toStartOfDay(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC')) AS day_start,
- transform(ifNull(e.prop_1, '$$_posthog_breakdown_null_$$'), ['d', 'c', 'b', 'a'], ['d', 'c', 'b', 'a'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ toStartOfDay(toTimeZone(e.created, 'UTC')) AS day_start,
+ transform(ifNull(toString(e.prop_1), '$$_posthog_breakdown_null_$$'), ['d', 'c', 'b', 'a'], ['d', 'c', 'b', 'a'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM s3('http://host.docker.internal:19000/posthog/test_storage_bucket-posthog.hogql.datawarehouse.trendquery/*.parquet', 'object_storage_root_user', 'object_storage_root_password', 'Parquet', 'id String, prop_1 String, prop_2 String, created DateTime64(3, \'UTC\')') AS e
- WHERE and(ifNull(greaterOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), or(equals(e.prop_1, 'd'), equals(e.prop_1, 'c'), equals(e.prop_1, 'b'), equals(e.prop_1, 'a')))
+ WHERE and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), or(ifNull(equals(toString(e.prop_1), 'd'), 0), ifNull(equals(toString(e.prop_1), 'c'), 0), ifNull(equals(toString(e.prop_1), 'b'), 0), ifNull(equals(toString(e.prop_1), 'a'), 0)))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -54,10 +54,10 @@
# ---
# name: TestTrendsDataWarehouseQuery.test_trends_breakdown_with_property
'''
- SELECT e.prop_1 AS value,
+ SELECT toString(e.prop_1) AS value,
count(e.id) AS count
FROM s3('http://host.docker.internal:19000/posthog/test_storage_bucket-posthog.hogql.datawarehouse.trendquery/*.parquet', 'object_storage_root_user', 'object_storage_root_password', 'Parquet', 'id String, prop_1 String, prop_2 String, created DateTime64(3, \'UTC\')') AS e
- WHERE and(and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0)), and(ifNull(greaterOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), equals(e.prop_1, 'a')))
+ WHERE and(and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0)), and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), equals(e.prop_1, 'a')))
GROUP BY value
ORDER BY count DESC, value DESC
LIMIT 26 SETTINGS readonly=2,
@@ -69,7 +69,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -89,10 +89,10 @@
JOIN breakdown_value AS breakdown_value) AS sec
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.id) AS total,
- toStartOfDay(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC')) AS day_start,
- transform(ifNull(e.prop_1, '$$_posthog_breakdown_null_$$'), ['a'], ['a'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ toStartOfDay(toTimeZone(e.created, 'UTC')) AS day_start,
+ transform(ifNull(toString(e.prop_1), '$$_posthog_breakdown_null_$$'), ['a'], ['a'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM s3('http://host.docker.internal:19000/posthog/test_storage_bucket-posthog.hogql.datawarehouse.trendquery/*.parquet', 'object_storage_root_user', 'object_storage_root_password', 'Parquet', 'id String, prop_1 String, prop_2 String, created DateTime64(3, \'UTC\')') AS e
- WHERE and(ifNull(greaterOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), equals(e.prop_1, 'a'), equals(e.prop_1, 'a'))
+ WHERE and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), equals(e.prop_1, 'a'), ifNull(equals(toString(e.prop_1), 'a'), 0))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -119,9 +119,9 @@
UNION ALL SELECT 0 AS total,
toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC'))) AS day_start
UNION ALL SELECT count(e.id) AS total,
- toStartOfDay(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC')) AS day_start
+ toStartOfDay(toTimeZone(e.created, 'UTC')) AS day_start
FROM s3('http://host.docker.internal:19000/posthog/test_storage_bucket-posthog.hogql.datawarehouse.trendquery/*.parquet', 'object_storage_root_user', 'object_storage_root_password', 'Parquet', 'id String, prop_1 String, prop_2 String, created DateTime64(3, \'UTC\')') AS e
- WHERE and(ifNull(greaterOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0))
+ WHERE and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0))
GROUP BY day_start)
GROUP BY day_start
ORDER BY day_start ASC)
@@ -145,9 +145,9 @@
UNION ALL SELECT 0 AS total,
toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC'))) AS day_start
UNION ALL SELECT count(e.id) AS total,
- toStartOfDay(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC')) AS day_start
+ toStartOfDay(toTimeZone(e.created, 'UTC')) AS day_start
FROM s3('http://host.docker.internal:19000/posthog/test_storage_bucket-posthog.hogql.datawarehouse.trendquery/*.parquet', 'object_storage_root_user', 'object_storage_root_password', 'Parquet', 'id String, prop_1 String, prop_2 String, created DateTime64(3, \'UTC\')') AS e
- WHERE and(ifNull(greaterOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), equals(e.prop_1, 'a'))
+ WHERE and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), equals(e.prop_1, 'a'))
GROUP BY day_start)
GROUP BY day_start
ORDER BY day_start ASC)
@@ -171,9 +171,9 @@
UNION ALL SELECT 0 AS total,
toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC'))) AS day_start
UNION ALL SELECT count(e.id) AS total,
- toStartOfDay(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC')) AS day_start
+ toStartOfDay(toTimeZone(e.created, 'UTC')) AS day_start
FROM s3('http://host.docker.internal:19000/posthog/test_storage_bucket-posthog.hogql.datawarehouse.trendquery/*.parquet', 'object_storage_root_user', 'object_storage_root_password', 'Parquet', 'id String, prop_1 String, prop_2 String, created DateTime64(3, \'UTC\')') AS e
- WHERE and(ifNull(greaterOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), equals(e.prop_1, 'a'))
+ WHERE and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), equals(e.prop_1, 'a'))
GROUP BY day_start)
GROUP BY day_start
ORDER BY day_start ASC)
diff --git a/posthog/hogql_queries/insights/trends/test/test_trends.py b/posthog/hogql_queries/insights/trends/test/test_trends.py
index 1ac54e16de629..9e885fbadcc1d 100644
--- a/posthog/hogql_queries/insights/trends/test/test_trends.py
+++ b/posthog/hogql_queries/insights/trends/test/test_trends.py
@@ -5180,7 +5180,9 @@ def test_breakdown_filtering_with_properties_in_new_format(self):
)
response = sorted(response, key=lambda x: x["label"])
- self.assertEqual(len(response), 0)
+ self.assertEqual(len(response), 1)
+ self.assertEqual(response[0]["label"], "$$_posthog_breakdown_null_$$")
+ self.assertEqual(response[0]["count"], 0)
@also_test_with_person_on_events_v2
@snapshot_clickhouse_queries
diff --git a/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py b/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py
index 104e232a01406..6bb41b19c79cf 100644
--- a/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py
+++ b/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py
@@ -1,11 +1,15 @@
+import zoneinfo
from dataclasses import dataclass
+from datetime import datetime
from typing import Dict, List, Optional
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
from django.test import override_settings
from freezegun import freeze_time
+from posthog.clickhouse.client.execute import sync_execute
from posthog.hogql import ast
-from posthog.hogql.constants import MAX_SELECT_RETURNED_ROWS
+from posthog.hogql.constants import MAX_SELECT_RETURNED_ROWS, LimitContext
from posthog.hogql.modifiers import create_default_modifiers_for_team
+from posthog.hogql.query import INCREASED_MAX_EXECUTION_TIME
from posthog.hogql_queries.insights.trends.trends_query_runner import TrendsQueryRunner
from posthog.models.cohort.cohort import Cohort
from posthog.models.property_definition import PropertyDefinition
@@ -175,6 +179,7 @@ def _create_query_runner(
breakdown: Optional[BreakdownFilter] = None,
filter_test_accounts: Optional[bool] = None,
hogql_modifiers: Optional[HogQLQueryModifiers] = None,
+ limit_context: Optional[LimitContext] = None,
) -> TrendsQueryRunner:
query_series: List[EventsNode | ActionsNode] = [EventsNode(event="$pageview")] if series is None else series
query = TrendsQuery(
@@ -185,7 +190,7 @@ def _create_query_runner(
breakdownFilter=breakdown,
filterTestAccounts=filter_test_accounts,
)
- return TrendsQueryRunner(team=self.team, query=query, modifiers=hogql_modifiers)
+ return TrendsQueryRunner(team=self.team, query=query, modifiers=hogql_modifiers, limit_context=limit_context)
def _run_trends_query(
self,
@@ -195,8 +200,10 @@ def _run_trends_query(
series: Optional[List[EventsNode | ActionsNode]],
trends_filters: Optional[TrendsFilter] = None,
breakdown: Optional[BreakdownFilter] = None,
+ *,
filter_test_accounts: Optional[bool] = None,
hogql_modifiers: Optional[HogQLQueryModifiers] = None,
+ limit_context: Optional[LimitContext] = None,
):
return self._create_query_runner(
date_from=date_from,
@@ -207,6 +214,7 @@ def _run_trends_query(
breakdown=breakdown,
filter_test_accounts=filter_test_accounts,
hogql_modifiers=hogql_modifiers,
+ limit_context=limit_context,
).calculate()
def test_trends_query_label(self):
@@ -364,6 +372,19 @@ def test_trends_query_formula(self):
self.assertEqual("Formula (A+B)", response.results[0]["label"])
self.assertEqual([1, 0, 2, 4, 4, 0, 2, 1, 1, 0, 1], response.results[0]["data"])
+ def test_trends_query_formula_breakdown_no_data(self):
+ self._create_test_events()
+
+ response = self._run_trends_query(
+ self.default_date_from,
+ self.default_date_to,
+ IntervalType.day,
+ [EventsNode(event="$pageviewxxx"), EventsNode(event="$pageleavexxx")],
+ TrendsFilter(formula="A+B"),
+ BreakdownFilter(breakdown_type=BreakdownType.person, breakdown="$browser"),
+ )
+ self.assertEqual([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], response.results[0]["data"])
+
def test_trends_query_formula_aggregate(self):
self._create_test_events()
@@ -695,16 +716,7 @@ def test_trends_breakdowns_multiple_hogql(self):
breakdown_labels = [result["breakdown_value"] for result in response.results]
assert len(response.results) == 8
- assert breakdown_labels == [
- "Chrome",
- "Firefox",
- "Edge",
- "Safari",
- "Chrome",
- "Edge",
- "Firefox",
- "Safari",
- ]
+ assert breakdown_labels == ["Chrome", "Firefox", "Edge", "Safari", "Chrome", "Edge", "Firefox", "Safari"]
assert response.results[0]["label"] == f"$pageview - Chrome"
assert response.results[1]["label"] == f"$pageview - Firefox"
assert response.results[2]["label"] == f"$pageview - Edge"
@@ -804,6 +816,7 @@ def test_trends_breakdown_and_aggregation_query_orchestration(self):
10,
0,
]
+
assert response.results[1]["data"] == [
20,
0,
@@ -1114,6 +1127,38 @@ def test_breakdown_values_limit(self):
)
self.assertEqual(len(response.results), 11)
+ def test_breakdown_values_unknown_property(self):
+ # same as above test, just without creating the property definition
+ for value in list(range(30)):
+ _create_event(
+ team=self.team,
+ event="$pageview",
+ distinct_id=f"person_{value}",
+ timestamp="2020-01-11T12:00:00Z",
+ properties={"breakdown_value": f"{value}"},
+ )
+
+ response = self._run_trends_query(
+ "2020-01-09",
+ "2020-01-20",
+ IntervalType.day,
+ [EventsNode(event="$pageview")],
+ TrendsFilter(display=ChartDisplayType.ActionsLineGraph),
+ BreakdownFilter(breakdown="breakdown_value", breakdown_type=BreakdownType.event),
+ )
+
+ self.assertEqual(len(response.results), 26)
+
+ response = self._run_trends_query(
+ "2020-01-09",
+ "2020-01-20",
+ IntervalType.day,
+ [EventsNode(event="$pageview")],
+ TrendsFilter(display=ChartDisplayType.ActionsLineGraph),
+ BreakdownFilter(breakdown="breakdown_value", breakdown_type=BreakdownType.event, breakdown_limit=10),
+ )
+ self.assertEqual(len(response.results), 11)
+
def test_breakdown_values_world_map_limit(self):
PropertyDefinition.objects.create(team=self.team, name="breakdown_value", property_type="String")
@@ -1435,18 +1480,18 @@ def test_to_actors_query_options(self):
response = runner.to_actors_query_options()
assert response.day == [
- DayItem(label="2020-01-09", value="2020-01-09"),
- DayItem(label="2020-01-10", value="2020-01-10"),
- DayItem(label="2020-01-11", value="2020-01-11"),
- DayItem(label="2020-01-12", value="2020-01-12"),
- DayItem(label="2020-01-13", value="2020-01-13"),
- DayItem(label="2020-01-14", value="2020-01-14"),
- DayItem(label="2020-01-15", value="2020-01-15"),
- DayItem(label="2020-01-16", value="2020-01-16"),
- DayItem(label="2020-01-17", value="2020-01-17"),
- DayItem(label="2020-01-18", value="2020-01-18"),
- DayItem(label="2020-01-19", value="2020-01-19"),
- DayItem(label="2020-01-20", value="2020-01-20"),
+ DayItem(label="9-Jan-2020", value=datetime(2020, 1, 9, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="10-Jan-2020", value=datetime(2020, 1, 10, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="11-Jan-2020", value=datetime(2020, 1, 11, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="12-Jan-2020", value=datetime(2020, 1, 12, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="13-Jan-2020", value=datetime(2020, 1, 13, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="14-Jan-2020", value=datetime(2020, 1, 14, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="15-Jan-2020", value=datetime(2020, 1, 15, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="16-Jan-2020", value=datetime(2020, 1, 16, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="17-Jan-2020", value=datetime(2020, 1, 17, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="18-Jan-2020", value=datetime(2020, 1, 18, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="19-Jan-2020", value=datetime(2020, 1, 19, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="20-Jan-2020", value=datetime(2020, 1, 20, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
]
assert response.breakdown is None
@@ -1470,18 +1515,18 @@ def test_to_actors_query_options_compare(self):
response = runner.to_actors_query_options()
assert response.day == [
- DayItem(label="2020-01-09", value="2020-01-09"),
- DayItem(label="2020-01-10", value="2020-01-10"),
- DayItem(label="2020-01-11", value="2020-01-11"),
- DayItem(label="2020-01-12", value="2020-01-12"),
- DayItem(label="2020-01-13", value="2020-01-13"),
- DayItem(label="2020-01-14", value="2020-01-14"),
- DayItem(label="2020-01-15", value="2020-01-15"),
- DayItem(label="2020-01-16", value="2020-01-16"),
- DayItem(label="2020-01-17", value="2020-01-17"),
- DayItem(label="2020-01-18", value="2020-01-18"),
- DayItem(label="2020-01-19", value="2020-01-19"),
- DayItem(label="2020-01-20", value="2020-01-20"),
+ DayItem(label="9-Jan-2020", value=datetime(2020, 1, 9, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="10-Jan-2020", value=datetime(2020, 1, 10, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="11-Jan-2020", value=datetime(2020, 1, 11, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="12-Jan-2020", value=datetime(2020, 1, 12, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="13-Jan-2020", value=datetime(2020, 1, 13, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="14-Jan-2020", value=datetime(2020, 1, 14, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="15-Jan-2020", value=datetime(2020, 1, 15, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="16-Jan-2020", value=datetime(2020, 1, 16, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="17-Jan-2020", value=datetime(2020, 1, 17, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="18-Jan-2020", value=datetime(2020, 1, 18, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="19-Jan-2020", value=datetime(2020, 1, 19, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="20-Jan-2020", value=datetime(2020, 1, 20, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
]
assert response.breakdown is None
@@ -1555,9 +1600,8 @@ def test_to_actors_query_options_breakdowns_boolean(self):
assert response.series == [InsightActorsQuerySeries(label="$pageview", value=0)]
assert response.breakdown == [
- # BreakdownItem(label="Other", value="$$_posthog_breakdown_other_$$"), # TODO: Add when "Other" works
- BreakdownItem(label="true", value=1),
- BreakdownItem(label="false", value=0),
+ BreakdownItem(label="true", value="true"),
+ BreakdownItem(label="false", value="false"),
]
def test_to_actors_query_options_breakdowns_histogram(self):
@@ -1649,3 +1693,16 @@ def test_to_actors_query_options_breakdowns_hogql(self):
BreakdownItem(label="Safari", value="Safari"),
BreakdownItem(label="Edge", value="Edge"),
]
+
+ @patch("posthog.hogql.query.sync_execute", wraps=sync_execute)
+ def test_limit_is_context_aware(self, mock_sync_execute: MagicMock):
+ self._run_trends_query(
+ "2020-01-09",
+ "2020-01-20",
+ IntervalType.day,
+ [EventsNode(event="$pageview")],
+ limit_context=LimitContext.QUERY_ASYNC,
+ )
+
+ mock_sync_execute.assert_called_once()
+ self.assertIn(f" max_execution_time={INCREASED_MAX_EXECUTION_TIME},", mock_sync_execute.call_args[0][0])
diff --git a/posthog/hogql_queries/insights/trends/trends_query_builder.py b/posthog/hogql_queries/insights/trends/trends_query_builder.py
index ed5d867b48b75..a911e4bf8302a 100644
--- a/posthog/hogql_queries/insights/trends/trends_query_builder.py
+++ b/posthog/hogql_queries/insights/trends/trends_query_builder.py
@@ -14,6 +14,7 @@
from posthog.models.action.action import Action
from posthog.models.filters.mixins.utils import cached_property
from posthog.models.team.team import Team
+from posthog.queries.trends.breakdown import BREAKDOWN_NULL_STRING_LABEL
from posthog.schema import (
ActionsNode,
DataWarehouseNode,
@@ -68,14 +69,18 @@ def build_query(self) -> ast.SelectQuery | ast.SelectUnionQuery:
return full_query
def build_actors_query(
- self, time_frame: Optional[str | int] = None, breakdown_filter: Optional[str | int] = None
+ self, time_frame: Optional[str] = None, breakdown_filter: Optional[str] = None
) -> ast.SelectQuery | ast.SelectUnionQuery:
breakdown = self._breakdown(is_actors_query=True, breakdown_values_override=breakdown_filter)
return parse_select(
"""
- SELECT DISTINCT actor_id
+ SELECT
+ actor_id,
+ count() as event_count,
+ groupUniqArray(100)((timestamp, uuid, $session_id, $window_id)) as matching_events
FROM {subquery}
+ GROUP BY actor_id
""",
placeholders={
"subquery": self._get_events_subquery(
@@ -165,7 +170,7 @@ def _get_events_subquery(
is_actors_query: bool,
breakdown: Breakdown,
breakdown_values_override: Optional[str | int] = None,
- actors_query_time_frame: Optional[str | int] = None,
+ actors_query_time_frame: Optional[str] = None,
) -> ast.SelectQuery:
day_start = ast.Alias(
alias="day_start",
@@ -182,31 +187,16 @@ def _get_events_subquery(
actors_query_time_frame=actors_query_time_frame,
)
- default_query = cast(
- ast.SelectQuery,
- parse_select(
- """
- SELECT
- {aggregation_operation} AS total
- FROM {table} AS e
- WHERE {events_filter}
- """
- if isinstance(self.series, DataWarehouseNode)
- else """
- SELECT
- {aggregation_operation} AS total
- FROM {table} AS e
- SAMPLE {sample}
- WHERE {events_filter}
- """,
- placeholders={
- "table": self._table_expr,
- "events_filter": events_filter,
- "aggregation_operation": self._aggregation_operation.select_aggregation(),
- "sample": self._sample_value(),
- },
- ),
+ default_query = ast.SelectQuery(
+ select=[ast.Alias(alias="total", expr=self._aggregation_operation.select_aggregation())],
+ select_from=ast.JoinExpr(table=self._table_expr, alias="e"),
+ where=events_filter,
)
+ if not isinstance(self.series, DataWarehouseNode):
+ assert default_query.select_from is not None
+ default_query.select_from.sample = ast.SampleExpr(
+ sample_value=self._sample_value(),
+ )
default_query.group_by = []
@@ -225,8 +215,13 @@ def _get_events_subquery(
# TODO: Move this logic into the below branches when working on adding breakdown support for the person modal
if is_actors_query:
- default_query.select = [ast.Alias(alias="actor_id", expr=self._aggregation_operation.actor_id())]
- default_query.distinct = True
+ default_query.select = [
+ ast.Alias(alias="actor_id", expr=self._aggregation_operation.actor_id()),
+ ast.Field(chain=["e", "timestamp"]),
+ ast.Field(chain=["e", "uuid"]),
+ ast.Field(chain=["e", "$session_id"]),
+ ast.Field(chain=["e", "$window_id"]),
+ ]
default_query.group_by = []
# No breakdowns and no complex series aggregation
@@ -298,7 +293,8 @@ def _get_events_subquery(
# Just breakdowns
elif breakdown.enabled:
if not is_actors_query:
- default_query.select.append(breakdown.column_expr())
+ breakdown_expr = breakdown.column_expr()
+ default_query.select.append(breakdown_expr)
default_query.group_by.append(ast.Field(chain=["breakdown_value"]))
# Just session duration math property
elif self._aggregation_operation.aggregating_on_session_duration():
@@ -375,7 +371,7 @@ def _outer_select_query(self, breakdown: Breakdown, inner_query: ast.SelectQuery
name="ifNull",
args=[
ast.Call(name="toString", args=[ast.Field(chain=["breakdown_value"])]),
- ast.Constant(value=""),
+ ast.Constant(value=BREAKDOWN_NULL_STRING_LABEL),
],
),
)
@@ -454,20 +450,27 @@ def _events_filter(
breakdown: Breakdown | None,
ignore_breakdowns: bool = False,
breakdown_values_override: Optional[str | int] = None,
- actors_query_time_frame: Optional[str | int] = None,
+ actors_query_time_frame: Optional[str] = None,
) -> ast.Expr:
series = self.series
filters: List[ast.Expr] = []
# Dates
if is_actors_query and actors_query_time_frame is not None:
- to_start_of_time_frame = f"toStartOf{self.query_date_range.interval_name.capitalize()}"
- filters.append(
- ast.CompareOperation(
- left=ast.Call(name=to_start_of_time_frame, args=[ast.Field(chain=["timestamp"])]),
- op=ast.CompareOperationOp.Eq,
- right=ast.Call(name="toDateTime", args=[ast.Constant(value=actors_query_time_frame)]),
- )
+ actors_from, actors_to = self.query_date_range.interval_bounds_from_str(actors_query_time_frame)
+ filters.extend(
+ [
+ ast.CompareOperation(
+ left=ast.Field(chain=["timestamp"]),
+ op=ast.CompareOperationOp.GtEq,
+ right=ast.Constant(value=actors_from),
+ ),
+ ast.CompareOperation(
+ left=ast.Field(chain=["timestamp"]),
+ op=ast.CompareOperationOp.Lt,
+ right=ast.Constant(value=actors_to),
+ ),
+ ]
)
elif not self._aggregation_operation.requires_query_orchestration():
filters.extend(
@@ -564,7 +567,7 @@ def session_duration_math_property_wrapper(self, default_query: ast.SelectQuery)
query.group_by = []
return query
- def _breakdown(self, is_actors_query: bool, breakdown_values_override: Optional[str | int] = None):
+ def _breakdown(self, is_actors_query: bool, breakdown_values_override: Optional[str] = None):
return Breakdown(
team=self.team,
query=self.query,
diff --git a/posthog/hogql_queries/insights/trends/trends_query_runner.py b/posthog/hogql_queries/insights/trends/trends_query_runner.py
index 29d29b55e8b0f..67e160084e68e 100644
--- a/posthog/hogql_queries/insights/trends/trends_query_runner.py
+++ b/posthog/hogql_queries/insights/trends/trends_query_runner.py
@@ -23,9 +23,7 @@
from posthog.hogql.query import execute_hogql_query
from posthog.hogql.timings import HogQLTimings
from posthog.hogql_queries.insights.trends.breakdown_values import (
- BREAKDOWN_NULL_NUMERIC_LABEL,
BREAKDOWN_NULL_STRING_LABEL,
- BREAKDOWN_OTHER_NUMERIC_LABEL,
BREAKDOWN_OTHER_STRING_LABEL,
)
from posthog.hogql_queries.insights.trends.display import TrendsDisplay
@@ -142,7 +140,7 @@ def to_queries(self) -> List[ast.SelectQuery | ast.SelectUnionQuery]:
def to_actors_query(
self,
- time_frame: Optional[str | int],
+ time_frame: Optional[str],
series_index: int,
breakdown_value: Optional[str | int] = None,
compare: Optional[Compare] = None,
@@ -175,7 +173,7 @@ def to_actors_query(
modifiers=self.modifiers,
)
- query = query_builder.build_actors_query(time_frame=time_frame, breakdown_filter=breakdown_value)
+ query = query_builder.build_actors_query(time_frame=time_frame, breakdown_filter=str(breakdown_value))
return query
@@ -185,7 +183,13 @@ def to_actors_query_options(self) -> InsightActorsQueryOptionsResponse:
res_compare: List[CompareItem] | None = None
# Days
- res_days: List[DayItem] = [DayItem(label=day, value=day) for day in self.query_date_range.all_values()]
+ res_days: list[DayItem] = [
+ DayItem(
+ label=format_label_date(value, self.query_date_range.interval_name),
+ value=value,
+ )
+ for value in self.query_date_range.all_values()
+ ]
# Series
for index, series in enumerate(self.query.series):
@@ -240,14 +244,10 @@ def to_actors_query_options(self) -> InsightActorsQueryOptionsResponse:
cohort_name = "all users" if str(value) == "0" else Cohort.objects.get(pk=value).name
label = cohort_name
value = value
- elif value == BREAKDOWN_OTHER_STRING_LABEL or value == BREAKDOWN_OTHER_NUMERIC_LABEL:
- # label = "Other"
- # value = BREAKDOWN_OTHER_STRING_LABEL
- continue # TODO: Add support for "other" breakdowns
- elif value == BREAKDOWN_NULL_STRING_LABEL or value == BREAKDOWN_NULL_NUMERIC_LABEL:
- # label = "Null"
- # value = BREAKDOWN_NULL_STRING_LABEL
- continue # TODO: Add support for "null" breakdowns
+ elif value == BREAKDOWN_OTHER_STRING_LABEL:
+ label = "Other (Groups all remaining values)"
+ elif value == BREAKDOWN_NULL_STRING_LABEL:
+ label = "None (No value)"
elif is_boolean_breakdown:
label = self._convert_boolean(value)
else:
@@ -292,6 +292,7 @@ def run(index: int, query: ast.SelectQuery | ast.SelectUnionQuery, is_parallel:
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
timings_matrix[index] = response.timings
@@ -500,18 +501,6 @@ def get_value(name: str, val: Any):
series_object["breakdown_value"] = remapped_label
- # If the breakdown value is the numeric "other", then set it to the string version
- if (
- remapped_label == BREAKDOWN_OTHER_NUMERIC_LABEL
- or remapped_label == str(BREAKDOWN_OTHER_NUMERIC_LABEL)
- or remapped_label == float(BREAKDOWN_OTHER_NUMERIC_LABEL)
- ):
- series_object["breakdown_value"] = BREAKDOWN_OTHER_STRING_LABEL
- if real_series_count > 1 or self._is_breakdown_field_boolean():
- series_object["label"] = "{} - {}".format(series_label or "All events", "Other")
- else:
- series_object["label"] = "Other"
-
res.append(series_object)
return res
@@ -666,25 +655,27 @@ def apply_formula(self, formula: str, results: List[Dict[str, Any]]) -> List[Dic
res.append(new_result)
return res
- if self._trends_display.should_aggregate_values():
- series_data = list(map(lambda s: [s["aggregated_value"]], results))
- new_series_data = FormulaAST(series_data).call(formula)
+ if len(results) > 0:
+ if self._trends_display.should_aggregate_values():
+ series_data = list(map(lambda s: [s["aggregated_value"]], results))
+ new_series_data = FormulaAST(series_data).call(formula)
- new_result = results[0]
- new_result["aggregated_value"] = float(sum(new_series_data))
- new_result["data"] = None
- new_result["count"] = 0
- new_result["label"] = f"Formula ({formula})"
- else:
- series_data = list(map(lambda s: s["data"], results))
- new_series_data = FormulaAST(series_data).call(formula)
+ new_result = results[0]
+ new_result["aggregated_value"] = float(sum(new_series_data))
+ new_result["data"] = None
+ new_result["count"] = 0
+ new_result["label"] = f"Formula ({formula})"
+ else:
+ series_data = list(map(lambda s: s["data"], results))
+ new_series_data = FormulaAST(series_data).call(formula)
- new_result = results[0]
- new_result["data"] = new_series_data
- new_result["count"] = float(sum(new_series_data))
- new_result["label"] = f"Formula ({formula})"
+ new_result = results[0]
+ new_result["data"] = new_series_data
+ new_result["count"] = float(sum(new_series_data))
+ new_result["label"] = f"Formula ({formula})"
- return [new_result]
+ return [new_result]
+ return []
def _is_breakdown_field_boolean(self):
if not self.query.breakdownFilter or not self.query.breakdownFilter.breakdown_type:
@@ -741,13 +732,19 @@ def _event_property(
field: str,
field_type: PropertyDefinition.Type,
group_type_index: Optional[int],
- ):
- return PropertyDefinition.objects.get(
- name=field,
- team=self.team,
- type=field_type,
- group_type_index=group_type_index if field_type == PropertyDefinition.Type.GROUP else None,
- ).property_type
+ ) -> str:
+ try:
+ return (
+ PropertyDefinition.objects.get(
+ name=field,
+ team=self.team,
+ type=field_type,
+ group_type_index=group_type_index if field_type == PropertyDefinition.Type.GROUP else None,
+ ).property_type
+ or "String"
+ )
+ except PropertyDefinition.DoesNotExist:
+ return "String"
# TODO: Move this to posthog/hogql_queries/legacy_compatibility/query_to_filter.py
def _query_to_filter(self) -> Dict[str, Any]:
diff --git a/posthog/hogql_queries/sessions_timeline_query_runner.py b/posthog/hogql_queries/sessions_timeline_query_runner.py
index d920ec7cf94fd..cda9433d63efa 100644
--- a/posthog/hogql_queries/sessions_timeline_query_runner.py
+++ b/posthog/hogql_queries/sessions_timeline_query_runner.py
@@ -135,6 +135,7 @@ def calculate(self) -> SessionsTimelineQueryResponse:
query_type="SessionsTimelineQuery",
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
assert query_result.results is not None
timeline_entries_map: Dict[str, TimelineEntry] = {}
diff --git a/posthog/hogql_queries/utils/query_date_range.py b/posthog/hogql_queries/utils/query_date_range.py
index f2e5cef3d82a3..b6386ac85f4ed 100644
--- a/posthog/hogql_queries/utils/query_date_range.py
+++ b/posthog/hogql_queries/utils/query_date_range.py
@@ -1,9 +1,10 @@
import re
from datetime import datetime, timedelta
from functools import cached_property
-from typing import Literal, Optional, Dict, List
+from typing import Literal, Optional, Dict
from zoneinfo import ZoneInfo
+from dateutil.parser import parse
from dateutil.relativedelta import relativedelta
from posthog.hogql.errors import HogQLException
@@ -116,36 +117,38 @@ def interval_type(self) -> IntervalType:
def interval_name(self) -> Literal["hour", "day", "week", "month"]:
return self.interval_type.name
- def all_values(self) -> List[str]:
- start: datetime = self.date_from()
- end: datetime = self.date_to()
- interval = self.interval_name
-
- if interval == "hour":
- start = start.replace(minute=0, second=0, microsecond=0)
- elif interval == "day":
- start = start.replace(hour=0, minute=0, second=0, microsecond=0)
- elif interval == "week":
+ def align_with_interval(self, start: datetime) -> datetime:
+ if self.interval_name == "hour":
+ return start.replace(minute=0, second=0, microsecond=0)
+ elif self.interval_name == "day":
+ return start.replace(hour=0, minute=0, second=0, microsecond=0)
+ elif self.interval_name == "week":
start = start.replace(hour=0, minute=0, second=0, microsecond=0)
week_start_alignment_days = start.isoweekday() % 7
if self._team.week_start_day == WeekStartDay.MONDAY:
week_start_alignment_days = start.weekday()
start -= timedelta(days=week_start_alignment_days)
- elif interval == "month":
- start = start.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
+ return start
+ elif self.interval_name == "month":
+ return start.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
+
+ def interval_relativedelta(self) -> relativedelta:
+ return relativedelta(
+ days=1 if self.interval_name == "day" else 0,
+ weeks=1 if self.interval_name == "week" else 0,
+ months=1 if self.interval_name == "month" else 0,
+ hours=1 if self.interval_name == "hour" else 0,
+ )
- values: List[str] = []
+ def all_values(self) -> list[datetime]:
+ start = self.align_with_interval(self.date_from())
+ end: datetime = self.date_to()
+ delta = self.interval_relativedelta()
+
+ values: list[datetime] = []
while start <= end:
- if interval == "hour":
- values.append(start.strftime("%Y-%m-%d %H:%M:%S"))
- else:
- values.append(start.strftime("%Y-%m-%d"))
- start += relativedelta(
- days=1 if interval == "day" else 0,
- weeks=1 if interval == "week" else 0,
- months=1 if interval == "month" else 0,
- hours=1 if interval == "hour" else 0,
- )
+ values.append(start)
+ start += delta
return values
def date_to_as_hogql(self) -> ast.Expr:
@@ -257,6 +260,11 @@ def to_placeholders(self) -> Dict[str, ast.Expr]:
else self.date_from_as_hogql(),
}
+ def interval_bounds_from_str(self, time_frame: str) -> tuple[datetime, datetime]:
+ date_from = parse(time_frame, tzinfos={None: self._team.timezone_info})
+ date_to = date_from + self.interval_relativedelta()
+ return date_from, date_to
+
class QueryDateRangeWithIntervals(QueryDateRange):
def __init__(
diff --git a/posthog/hogql_queries/utils/test/test_query_date_range.py b/posthog/hogql_queries/utils/test/test_query_date_range.py
index fd38ef700e137..f377e06880bbe 100644
--- a/posthog/hogql_queries/utils/test/test_query_date_range.py
+++ b/posthog/hogql_queries/utils/test/test_query_date_range.py
@@ -61,32 +61,47 @@ def test_all_values(self):
QueryDateRange(
team=self.team, date_range=DateRange(date_from="-20h"), interval=IntervalType.day, now=now
).all_values(),
- ["2021-08-24", "2021-08-25"],
+ [parser.isoparse("2021-08-24T00:00:00Z"), parser.isoparse("2021-08-25T00:00:00Z")],
)
self.assertEqual(
QueryDateRange(
team=self.team, date_range=DateRange(date_from="-20d"), interval=IntervalType.week, now=now
).all_values(),
- ["2021-08-01", "2021-08-08", "2021-08-15", "2021-08-22"],
+ [
+ parser.isoparse("2021-08-01T00:00:00Z"),
+ parser.isoparse("2021-08-08T00:00:00Z"),
+ parser.isoparse("2021-08-15T00:00:00Z"),
+ parser.isoparse("2021-08-22T00:00:00Z"),
+ ],
)
self.team.week_start_day = WeekStartDay.MONDAY
self.assertEqual(
QueryDateRange(
team=self.team, date_range=DateRange(date_from="-20d"), interval=IntervalType.week, now=now
).all_values(),
- ["2021-08-02", "2021-08-09", "2021-08-16", "2021-08-23"],
+ [
+ parser.isoparse("2021-08-02T00:00:00Z"),
+ parser.isoparse("2021-08-09T00:00:00Z"),
+ parser.isoparse("2021-08-16T00:00:00Z"),
+ parser.isoparse("2021-08-23T00:00:00Z"),
+ ],
)
self.assertEqual(
QueryDateRange(
team=self.team, date_range=DateRange(date_from="-50d"), interval=IntervalType.month, now=now
).all_values(),
- ["2021-07-01", "2021-08-01"],
+ [parser.isoparse("2021-07-01T00:00:00Z"), parser.isoparse("2021-08-01T00:00:00Z")],
)
self.assertEqual(
QueryDateRange(
team=self.team, date_range=DateRange(date_from="-3h"), interval=IntervalType.hour, now=now
).all_values(),
- ["2021-08-24 21:00:00", "2021-08-24 22:00:00", "2021-08-24 23:00:00", "2021-08-25 00:00:00"],
+ [
+ parser.isoparse("2021-08-24T21:00:00Z"),
+ parser.isoparse("2021-08-24T22:00:00Z"),
+ parser.isoparse("2021-08-24T23:00:00Z"),
+ parser.isoparse("2021-08-25T00:00:00Z"),
+ ],
)
diff --git a/posthog/hogql_queries/web_analytics/test/test_web_overview.py b/posthog/hogql_queries/web_analytics/test/test_web_overview.py
index 63a26ffea9233..dcafe660fc72d 100644
--- a/posthog/hogql_queries/web_analytics/test/test_web_overview.py
+++ b/posthog/hogql_queries/web_analytics/test/test_web_overview.py
@@ -1,6 +1,11 @@
+from typing import Optional
+from unittest.mock import MagicMock, patch
from freezegun import freeze_time
from parameterized import parameterized
+from posthog.clickhouse.client.execute import sync_execute
+from posthog.hogql.constants import LimitContext
+from posthog.hogql.query import INCREASED_MAX_EXECUTION_TIME
from posthog.hogql_queries.web_analytics.web_overview import WebOverviewQueryRunner
from posthog.schema import WebOverviewQuery, DateRange
from posthog.test.base import (
@@ -36,14 +41,21 @@ def _create_events(self, data, event="$pageview"):
)
return person_result
- def _run_web_overview_query(self, date_from, date_to, use_sessions_table=False, compare=True):
+ def _run_web_overview_query(
+ self,
+ date_from: str,
+ date_to: str,
+ use_sessions_table: bool = False,
+ compare: bool = True,
+ limit_context: Optional[LimitContext] = None,
+ ):
query = WebOverviewQuery(
dateRange=DateRange(date_from=date_from, date_to=date_to),
properties=[],
compare=compare,
useSessionsTable=use_sessions_table,
)
- runner = WebOverviewQueryRunner(team=self.team, query=query)
+ runner = WebOverviewQueryRunner(team=self.team, query=query, limit_context=limit_context)
return runner.calculate()
@parameterized.expand([(True,), (False,)])
@@ -185,3 +197,10 @@ def test_correctly_counts_pageviews_in_long_running_session(self, use_sessions_t
sessions = results[2]
self.assertEqual(1, sessions.value)
+
+ @patch("posthog.hogql.query.sync_execute", wraps=sync_execute)
+ def test_limit_is_context_aware(self, mock_sync_execute: MagicMock):
+ self._run_web_overview_query("2023-12-01", "2023-12-03", limit_context=LimitContext.QUERY_ASYNC)
+
+ mock_sync_execute.assert_called_once()
+ self.assertIn(f" max_execution_time={INCREASED_MAX_EXECUTION_TIME},", mock_sync_execute.call_args[0][0])
diff --git a/posthog/hogql_queries/web_analytics/top_clicks.py b/posthog/hogql_queries/web_analytics/top_clicks.py
index 3218e68975f7a..192d7b279b704 100644
--- a/posthog/hogql_queries/web_analytics/top_clicks.py
+++ b/posthog/hogql_queries/web_analytics/top_clicks.py
@@ -51,6 +51,7 @@ def calculate(self):
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
return WebTopClicksQueryResponse(
diff --git a/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py b/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py
index da4f98edcbf32..12ef703271c51 100644
--- a/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py
+++ b/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py
@@ -211,6 +211,7 @@ def _get_or_calculate_sample_ratio(self) -> SamplingRate:
query=event_count,
team=self.team,
timings=self.timings,
+ limit_context=self.limit_context,
)
if not response.results or not response.results[0] or not response.results[0][0]:
diff --git a/posthog/hogql_queries/web_analytics/web_overview.py b/posthog/hogql_queries/web_analytics/web_overview.py
index 38388315c8f0b..2da015a60ac4e 100644
--- a/posthog/hogql_queries/web_analytics/web_overview.py
+++ b/posthog/hogql_queries/web_analytics/web_overview.py
@@ -285,6 +285,7 @@ def calculate(self):
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
assert response.results
diff --git a/posthog/management/commands/backfill_sessions_table.py b/posthog/management/commands/backfill_sessions_table.py
index 798a501eb5b60..c01f4b6159749 100644
--- a/posthog/management/commands/backfill_sessions_table.py
+++ b/posthog/management/commands/backfill_sessions_table.py
@@ -16,6 +16,10 @@
TARGET_TABLE = "sessions"
+SETTINGS = {
+ "max_execution_time": 3600 # 1 hour
+}
+
@dataclass
class BackfillQuery:
@@ -26,6 +30,7 @@ class BackfillQuery:
def execute(
self,
dry_run: bool = True,
+ print_counts: bool = True,
) -> None:
def source_column(column_name: str) -> str:
return get_property_string_expr(
@@ -108,13 +113,14 @@ def select_query(select_date: Optional[datetime] = None) -> str:
"""
# print the count of entries in the main sessions table
- count_query = f"SELECT count(), uniq(session_id) FROM {TARGET_TABLE}"
- [(sessions_row_count, sessions_event_count)] = sync_execute(count_query)
- logger.info(f"{sessions_row_count} rows and {sessions_event_count} unique session_ids in sessions table")
+ if print_counts:
+ count_query = f"SELECT count(), uniq(session_id) FROM {TARGET_TABLE}"
+ [(sessions_row_count, sessions_event_count)] = sync_execute(count_query, settings=SETTINGS)
+ logger.info(f"{sessions_row_count} rows and {sessions_event_count} unique session_ids in sessions table")
if dry_run:
count_query = f"SELECT count(), uniq(session_id) FROM ({select_query()})"
- [(events_count, sessions_count)] = sync_execute(count_query)
+ [(events_count, sessions_count)] = sync_execute(count_query, settings=SETTINGS)
logger.info(f"{events_count} events and {sessions_count} sessions to backfill for")
logger.info(f"The first select query would be:\n{select_query(self.start_date)}")
return
@@ -125,12 +131,14 @@ def select_query(select_date: Optional[datetime] = None) -> str:
sync_execute(
query=f"""INSERT INTO writable_sessions {select_query(select_date=date)} SETTINGS max_execution_time=3600""",
workload=Workload.OFFLINE if self.use_offline_workload else Workload.DEFAULT,
+ settings=SETTINGS,
)
# print the count of entries in the main sessions table
- count_query = f"SELECT count(), uniq(session_id) FROM {TARGET_TABLE}"
- [(sessions_row_count, sessions_event_count)] = sync_execute(count_query)
- logger.info(f"{sessions_row_count} rows and {sessions_event_count} unique session_ids in sessions table")
+ if print_counts:
+ count_query = f"SELECT count(), uniq(session_id) FROM {TARGET_TABLE}"
+ [(sessions_row_count, sessions_event_count)] = sync_execute(count_query, settings=SETTINGS)
+ logger.info(f"{sessions_row_count} rows and {sessions_event_count} unique session_ids in sessions table")
class Command(BaseCommand):
@@ -149,11 +157,25 @@ def add_arguments(self, parser):
parser.add_argument(
"--use-offline-workload", action="store_true", help="actually execute INSERT queries (default is dry-run)"
)
+ parser.add_argument(
+ "--print-counts", action="store_true", help="print events and session count beforehand and afterwards"
+ )
- def handle(self, *, live_run: bool, start_date: str, end_date: str, use_offline_workload: bool, **options):
+ def handle(
+ self,
+ *,
+ live_run: bool,
+ start_date: str,
+ end_date: str,
+ use_offline_workload: bool,
+ print_counts: bool,
+ **options,
+ ):
logger.setLevel(logging.INFO)
start_datetime = datetime.strptime(start_date, "%Y-%m-%d")
end_datetime = datetime.strptime(end_date, "%Y-%m-%d")
- BackfillQuery(start_datetime, end_datetime, use_offline_workload).execute(dry_run=not live_run)
+ BackfillQuery(start_datetime, end_datetime, use_offline_workload).execute(
+ dry_run=not live_run, print_counts=print_counts
+ )
diff --git a/posthog/migrations/0398_alter_externaldatasource_source_type.py b/posthog/migrations/0398_alter_externaldatasource_source_type.py
new file mode 100644
index 0000000000000..af95cd44eef98
--- /dev/null
+++ b/posthog/migrations/0398_alter_externaldatasource_source_type.py
@@ -0,0 +1,25 @@
+# Generated by Django 4.1.13 on 2024-03-21 13:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("posthog", "0397_projects_backfill"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="externaldatasource",
+ name="source_type",
+ field=models.CharField(
+ choices=[
+ ("Stripe", "Stripe"),
+ ("Hubspot", "Hubspot"),
+ ("Postgres", "Postgres"),
+ ("Zendesk", "Zendesk"),
+ ],
+ max_length=128,
+ ),
+ ),
+ ]
diff --git a/posthog/schema.py b/posthog/schema.py
index dc77da163db17..9d83587351683 100644
--- a/posthog/schema.py
+++ b/posthog/schema.py
@@ -180,6 +180,10 @@ class DateRange(BaseModel):
date_to: Optional[str] = None
+class DatetimeDay(RootModel[AwareDatetime]):
+ root: AwareDatetime
+
+
class Day(RootModel[int]):
root: int
@@ -418,6 +422,7 @@ class PersonsOnEventsMode(str, Enum):
v1_enabled = "v1_enabled"
v1_mixed = "v1_mixed"
v2_enabled = "v2_enabled"
+ v3_enabled = "v3_enabled"
class HogQLQueryModifiers(BaseModel):
@@ -457,7 +462,7 @@ class DayItem(BaseModel):
extra="forbid",
)
label: str
- value: Union[str, int]
+ value: Union[str, AwareDatetime, int]
class IntervalItem(BaseModel):
diff --git a/posthog/settings/feature_flags.py b/posthog/settings/feature_flags.py
index 5e1ad234e6de4..371f497376663 100644
--- a/posthog/settings/feature_flags.py
+++ b/posthog/settings/feature_flags.py
@@ -8,5 +8,4 @@
"simplify-actions",
"historical-exports-v2",
"ingestion-warnings-enabled",
- "role-based-access",
]
diff --git a/posthog/tasks/tasks.py b/posthog/tasks/tasks.py
index 5eff6afd33fe2..bead27cbd1eec 100644
--- a/posthog/tasks/tasks.py
+++ b/posthog/tasks/tasks.py
@@ -1,5 +1,5 @@
import time
-from typing import Any, Optional
+from typing import Optional
from uuid import UUID
from celery import shared_task
@@ -9,6 +9,7 @@
from prometheus_client import Gauge
from posthog.cloud_utils import is_cloud
+from posthog.hogql.constants import LimitContext
from posthog.metrics import pushed_metrics_registry
from posthog.ph_client import get_ph_client
from posthog.redis import get_client
@@ -33,7 +34,12 @@ def redis_heartbeat() -> None:
@shared_task(ignore_result=True, queue=CeleryQueue.ANALYTICS_QUERIES.value)
def process_query_task(
- team_id: str, query_id: str, query_json: Any, limit_context: Any = None, refresh_requested: bool = False
+ team_id: int,
+ user_id: int,
+ query_id: str,
+ query_json: dict,
+ limit_context: Optional[LimitContext] = None,
+ refresh_requested: bool = False,
) -> None:
"""
Kick off query
@@ -43,6 +49,7 @@ def process_query_task(
execute_process_query(
team_id=team_id,
+ user_id=user_id,
query_id=query_id,
query_json=query_json,
limit_context=limit_context,
diff --git a/posthog/temporal/batch_exports/batch_exports.py b/posthog/temporal/batch_exports/batch_exports.py
index c776e1f245ef3..88cf9e32f274f 100644
--- a/posthog/temporal/batch_exports/batch_exports.py
+++ b/posthog/temporal/batch_exports/batch_exports.py
@@ -1,15 +1,10 @@
import collections.abc
-import csv
import dataclasses
import datetime as dt
-import gzip
-import tempfile
import typing
import uuid
from string import Template
-import brotli
-import orjson
import pyarrow as pa
from asgiref.sync import sync_to_async
from django.conf import settings
@@ -22,7 +17,7 @@
create_batch_export_backfill,
create_batch_export_run,
update_batch_export_backfill_status,
- update_batch_export_run_status,
+ update_batch_export_run,
)
from posthog.temporal.batch_exports.metrics import (
get_export_finished_metric,
@@ -286,202 +281,6 @@ def get_data_interval(interval: str, data_interval_end: str | None) -> tuple[dt.
return (data_interval_start_dt, data_interval_end_dt)
-def json_dumps_bytes(d) -> bytes:
- return orjson.dumps(d, default=str)
-
-
-class BatchExportTemporaryFile:
- """A TemporaryFile used to as an intermediate step while exporting data.
-
- This class does not implement the file-like interface but rather passes any calls
- to the underlying tempfile.NamedTemporaryFile. We do override 'write' methods
- to allow tracking bytes and records.
- """
-
- def __init__(
- self,
- mode: str = "w+b",
- buffering=-1,
- compression: str | None = None,
- encoding: str | None = None,
- newline: str | None = None,
- suffix: str | None = None,
- prefix: str | None = None,
- dir: str | None = None,
- *,
- errors: str | None = None,
- ):
- self._file = tempfile.NamedTemporaryFile(
- mode=mode,
- encoding=encoding,
- newline=newline,
- buffering=buffering,
- suffix=suffix,
- prefix=prefix,
- dir=dir,
- errors=errors,
- )
- self.compression = compression
- self.bytes_total = 0
- self.records_total = 0
- self.bytes_since_last_reset = 0
- self.records_since_last_reset = 0
- self._brotli_compressor = None
-
- def __getattr__(self, name):
- """Pass get attr to underlying tempfile.NamedTemporaryFile."""
- return self._file.__getattr__(name)
-
- def __enter__(self):
- """Context-manager protocol enter method."""
- self._file.__enter__()
- return self
-
- def __exit__(self, exc, value, tb):
- """Context-manager protocol exit method."""
- return self._file.__exit__(exc, value, tb)
-
- def __iter__(self):
- yield from self._file
-
- @property
- def brotli_compressor(self):
- if self._brotli_compressor is None:
- self._brotli_compressor = brotli.Compressor()
- return self._brotli_compressor
-
- def compress(self, content: bytes | str) -> bytes:
- if isinstance(content, str):
- encoded = content.encode("utf-8")
- else:
- encoded = content
-
- match self.compression:
- case "gzip":
- return gzip.compress(encoded)
- case "brotli":
- self.brotli_compressor.process(encoded)
- return self.brotli_compressor.flush()
- case None:
- return encoded
- case _:
- raise ValueError(f"Unsupported compression: '{self.compression}'")
-
- def write(self, content: bytes | str):
- """Write bytes to underlying file keeping track of how many bytes were written."""
- compressed_content = self.compress(content)
-
- if "b" in self.mode:
- result = self._file.write(compressed_content)
- else:
- result = self._file.write(compressed_content.decode("utf-8"))
-
- self.bytes_total += result
- self.bytes_since_last_reset += result
-
- return result
-
- def write_record_as_bytes(self, record: bytes):
- result = self.write(record)
-
- self.records_total += 1
- self.records_since_last_reset += 1
-
- return result
-
- def write_records_to_jsonl(self, records):
- """Write records to a temporary file as JSONL."""
- if len(records) == 1:
- jsonl_dump = orjson.dumps(records[0], option=orjson.OPT_APPEND_NEWLINE, default=str)
- else:
- jsonl_dump = b"\n".join(map(json_dumps_bytes, records))
-
- result = self.write(jsonl_dump)
-
- self.records_total += len(records)
- self.records_since_last_reset += len(records)
-
- return result
-
- def write_records_to_csv(
- self,
- records,
- fieldnames: None | collections.abc.Sequence[str] = None,
- extrasaction: typing.Literal["raise", "ignore"] = "ignore",
- delimiter: str = ",",
- quotechar: str = '"',
- escapechar: str | None = "\\",
- lineterminator: str = "\n",
- quoting=csv.QUOTE_NONE,
- ):
- """Write records to a temporary file as CSV."""
- if len(records) == 0:
- return
-
- if fieldnames is None:
- fieldnames = list(records[0].keys())
-
- writer = csv.DictWriter(
- self,
- fieldnames=fieldnames,
- extrasaction=extrasaction,
- delimiter=delimiter,
- quotechar=quotechar,
- escapechar=escapechar,
- quoting=quoting,
- lineterminator=lineterminator,
- )
- writer.writerows(records)
-
- self.records_total += len(records)
- self.records_since_last_reset += len(records)
-
- def write_records_to_tsv(
- self,
- records,
- fieldnames: None | list[str] = None,
- extrasaction: typing.Literal["raise", "ignore"] = "ignore",
- quotechar: str = '"',
- escapechar: str | None = "\\",
- lineterminator: str = "\n",
- quoting=csv.QUOTE_NONE,
- ):
- """Write records to a temporary file as TSV."""
- return self.write_records_to_csv(
- records,
- fieldnames=fieldnames,
- extrasaction=extrasaction,
- delimiter="\t",
- quotechar=quotechar,
- escapechar=escapechar,
- quoting=quoting,
- lineterminator=lineterminator,
- )
-
- def rewind(self):
- """Rewind the file before reading it."""
- if self.compression == "brotli":
- result = self._file.write(self.brotli_compressor.finish())
-
- self.bytes_total += result
- self.bytes_since_last_reset += result
-
- self._brotli_compressor = None
-
- self._file.seek(0)
-
- def reset(self):
- """Reset underlying file by truncating it.
-
- Also resets the tracker attributes for bytes and records since last reset.
- """
- self._file.seek(0)
- self._file.truncate()
-
- self.bytes_since_last_reset = 0
- self.records_since_last_reset = 0
-
-
@dataclasses.dataclass
class CreateBatchExportRunInputs:
"""Inputs to the create_export_run activity.
@@ -542,7 +341,7 @@ async def update_export_run_status(inputs: UpdateBatchExportRunStatusInputs) ->
"""Activity that updates the status of an BatchExportRun."""
logger = await bind_temporal_worker_logger(team_id=inputs.team_id)
- batch_export_run = await sync_to_async(update_batch_export_run_status)(
+ batch_export_run = await sync_to_async(update_batch_export_run)(
run_id=uuid.UUID(inputs.id),
status=inputs.status,
latest_error=inputs.latest_error,
diff --git a/posthog/temporal/batch_exports/bigquery_batch_export.py b/posthog/temporal/batch_exports/bigquery_batch_export.py
index a0469de79bb9e..b754a7add16b4 100644
--- a/posthog/temporal/batch_exports/bigquery_batch_export.py
+++ b/posthog/temporal/batch_exports/bigquery_batch_export.py
@@ -15,7 +15,6 @@
from posthog.batch_exports.service import BatchExportField, BatchExportSchema, BigQueryBatchExportInputs
from posthog.temporal.batch_exports.base import PostHogWorkflow
from posthog.temporal.batch_exports.batch_exports import (
- BatchExportTemporaryFile,
CreateBatchExportRunInputs,
UpdateBatchExportRunStatusInputs,
create_export_run,
@@ -29,6 +28,9 @@
get_bytes_exported_metric,
get_rows_exported_metric,
)
+from posthog.temporal.batch_exports.temporary_file import (
+ BatchExportTemporaryFile,
+)
from posthog.temporal.batch_exports.utils import peek_first_and_rewind
from posthog.temporal.common.clickhouse import get_client
from posthog.temporal.common.logger import bind_temporal_worker_logger
diff --git a/posthog/temporal/batch_exports/http_batch_export.py b/posthog/temporal/batch_exports/http_batch_export.py
index 8aca65c80ff38..2866d50c99876 100644
--- a/posthog/temporal/batch_exports/http_batch_export.py
+++ b/posthog/temporal/batch_exports/http_batch_export.py
@@ -13,7 +13,6 @@
from posthog.models import BatchExportRun
from posthog.temporal.batch_exports.base import PostHogWorkflow
from posthog.temporal.batch_exports.batch_exports import (
- BatchExportTemporaryFile,
CreateBatchExportRunInputs,
UpdateBatchExportRunStatusInputs,
create_export_run,
@@ -21,12 +20,15 @@
get_data_interval,
get_rows_count,
iter_records,
- json_dumps_bytes,
)
from posthog.temporal.batch_exports.metrics import (
get_bytes_exported_metric,
get_rows_exported_metric,
)
+from posthog.temporal.batch_exports.temporary_file import (
+ BatchExportTemporaryFile,
+ json_dumps_bytes,
+)
from posthog.temporal.common.clickhouse import get_client
from posthog.temporal.common.logger import bind_temporal_worker_logger
diff --git a/posthog/temporal/batch_exports/postgres_batch_export.py b/posthog/temporal/batch_exports/postgres_batch_export.py
index 5dbfc6faa4acf..98969ee78de79 100644
--- a/posthog/temporal/batch_exports/postgres_batch_export.py
+++ b/posthog/temporal/batch_exports/postgres_batch_export.py
@@ -17,7 +17,6 @@
from posthog.batch_exports.service import BatchExportField, BatchExportSchema, PostgresBatchExportInputs
from posthog.temporal.batch_exports.base import PostHogWorkflow
from posthog.temporal.batch_exports.batch_exports import (
- BatchExportTemporaryFile,
CreateBatchExportRunInputs,
UpdateBatchExportRunStatusInputs,
create_export_run,
@@ -31,6 +30,9 @@
get_bytes_exported_metric,
get_rows_exported_metric,
)
+from posthog.temporal.batch_exports.temporary_file import (
+ BatchExportTemporaryFile,
+)
from posthog.temporal.batch_exports.utils import peek_first_and_rewind
from posthog.temporal.common.clickhouse import get_client
from posthog.temporal.common.logger import bind_temporal_worker_logger
diff --git a/posthog/temporal/batch_exports/s3_batch_export.py b/posthog/temporal/batch_exports/s3_batch_export.py
index 4d99cbeffd7c3..e83fe3f12915d 100644
--- a/posthog/temporal/batch_exports/s3_batch_export.py
+++ b/posthog/temporal/batch_exports/s3_batch_export.py
@@ -1,4 +1,5 @@
import asyncio
+import collections.abc
import contextlib
import datetime as dt
import io
@@ -8,6 +9,8 @@
from dataclasses import dataclass
import aioboto3
+import orjson
+import pyarrow as pa
from django.conf import settings
from temporalio import activity, workflow
from temporalio.common import RetryPolicy
@@ -16,7 +19,6 @@
from posthog.batch_exports.service import BatchExportField, BatchExportSchema, S3BatchExportInputs
from posthog.temporal.batch_exports.base import PostHogWorkflow
from posthog.temporal.batch_exports.batch_exports import (
- BatchExportTemporaryFile,
CreateBatchExportRunInputs,
UpdateBatchExportRunStatusInputs,
create_export_run,
@@ -30,6 +32,15 @@
get_bytes_exported_metric,
get_rows_exported_metric,
)
+from posthog.temporal.batch_exports.temporary_file import (
+ BatchExportTemporaryFile,
+ BatchExportWriter,
+ FlushCallable,
+ JSONLBatchExportWriter,
+ ParquetBatchExportWriter,
+ UnsupportedFileFormatError,
+)
+from posthog.temporal.batch_exports.utils import peek_first_and_rewind
from posthog.temporal.common.clickhouse import get_client
from posthog.temporal.common.logger import bind_temporal_worker_logger
@@ -50,19 +61,31 @@ def get_allowed_template_variables(inputs) -> dict[str, str]:
}
+FILE_FORMAT_EXTENSIONS = {
+ "Parquet": "parquet",
+ "JSONLines": "jsonl",
+}
+
+COMPRESSION_EXTENSIONS = {
+ "gzip": "gz",
+ "snappy": "sz",
+ "brotli": "br",
+ "ztsd": "zst",
+ "lz4": "lz4",
+}
+
+
def get_s3_key(inputs) -> str:
"""Return an S3 key given S3InsertInputs."""
template_variables = get_allowed_template_variables(inputs)
key_prefix = inputs.prefix.format(**template_variables)
+ file_extension = FILE_FORMAT_EXTENSIONS[inputs.file_format]
base_file_name = f"{inputs.data_interval_start}-{inputs.data_interval_end}"
- match inputs.compression:
- case "gzip":
- file_name = base_file_name + ".jsonl.gz"
- case "brotli":
- file_name = base_file_name + ".jsonl.br"
- case _:
- file_name = base_file_name + ".jsonl"
+ if inputs.compression is not None:
+ file_name = base_file_name + f".{file_extension}.{COMPRESSION_EXTENSIONS[inputs.compression]}"
+ else:
+ file_name = base_file_name + f".{file_extension}"
key = posixpath.join(key_prefix, file_name)
@@ -311,6 +334,8 @@ class S3InsertInputs:
kms_key_id: str | None = None
batch_export_schema: BatchExportSchema | None = None
endpoint_url: str | None = None
+ # TODO: In Python 3.11, this could be a enum.StrEnum.
+ file_format: str = "JSONLines"
async def initialize_and_resume_multipart_upload(inputs: S3InsertInputs) -> tuple[S3MultiPartUpload, str]:
@@ -451,7 +476,7 @@ async def insert_into_s3_activity(inputs: S3InsertInputs) -> int:
last_uploaded_part_timestamp: str | None = None
- async def worker_shutdown_handler():
+ async def worker_shutdown_handler() -> None:
"""Handle the Worker shutting down by heart-beating our latest status."""
await activity.wait_for_worker_shutdown()
logger.warn(
@@ -466,50 +491,147 @@ async def worker_shutdown_handler():
asyncio.create_task(worker_shutdown_handler())
- record = None
-
async with s3_upload as s3_upload:
- with BatchExportTemporaryFile(compression=inputs.compression) as local_results_file:
+
+ async def flush_to_s3(
+ local_results_file,
+ records_since_last_flush: int,
+ bytes_since_last_flush: int,
+ last_inserted_at: dt.datetime,
+ last: bool,
+ ):
+ nonlocal last_uploaded_part_timestamp
+
+ logger.debug(
+ "Uploading %s part %s containing %s records with size %s bytes",
+ "last " if last else "",
+ s3_upload.part_number + 1,
+ records_since_last_flush,
+ bytes_since_last_flush,
+ )
+
+ await s3_upload.upload_part(local_results_file)
+ rows_exported.add(records_since_last_flush)
+ bytes_exported.add(bytes_since_last_flush)
+
+ last_uploaded_part_timestamp = str(last_inserted_at)
+ activity.heartbeat(last_uploaded_part_timestamp, s3_upload.to_state())
+
+ first_record_batch, record_iterator = peek_first_and_rewind(record_iterator)
+ first_record_batch = cast_record_batch_json_columns(first_record_batch)
+ column_names = first_record_batch.column_names
+ column_names.pop(column_names.index("_inserted_at"))
+
+ schema = pa.schema(
+ # NOTE: For some reason, some batches set non-nullable fields as non-nullable, whereas other
+ # record batches have them as nullable.
+ # Until we figure it out, we set all fields to nullable. There are some fields we know
+ # are not nullable, but I'm opting for the more flexible option until we out why schemas differ
+ # between batches.
+ [field.with_nullable(True) for field in first_record_batch.select(column_names).schema]
+ )
+
+ writer = get_batch_export_writer(
+ inputs,
+ flush_callable=flush_to_s3,
+ max_bytes=settings.BATCH_EXPORT_S3_UPLOAD_CHUNK_SIZE_BYTES,
+ schema=schema,
+ )
+
+ async with writer.open_temporary_file():
rows_exported = get_rows_exported_metric()
bytes_exported = get_bytes_exported_metric()
- async def flush_to_s3(last_uploaded_part_timestamp: str, last=False):
- logger.debug(
- "Uploading %s part %s containing %s records with size %s bytes",
- "last " if last else "",
- s3_upload.part_number + 1,
- local_results_file.records_since_last_reset,
- local_results_file.bytes_since_last_reset,
- )
+ for record_batch in record_iterator:
+ record_batch = cast_record_batch_json_columns(record_batch)
- await s3_upload.upload_part(local_results_file)
- rows_exported.add(local_results_file.records_since_last_reset)
- bytes_exported.add(local_results_file.bytes_since_last_reset)
+ await writer.write_record_batch(record_batch)
- activity.heartbeat(last_uploaded_part_timestamp, s3_upload.to_state())
+ await s3_upload.complete()
- for record_batch in record_iterator:
- for record in record_batch.to_pylist():
- for json_column in ("properties", "person_properties", "set", "set_once"):
- if (json_str := record.get(json_column, None)) is not None:
- record[json_column] = json.loads(json_str)
+ return writer.records_total
- inserted_at = record.pop("_inserted_at")
- local_results_file.write_records_to_jsonl([record])
+def get_batch_export_writer(
+ inputs: S3InsertInputs, flush_callable: FlushCallable, max_bytes: int, schema: pa.Schema | None = None
+) -> BatchExportWriter:
+ """Return the `BatchExportWriter` corresponding to configured `file_format`.
- if local_results_file.tell() > settings.BATCH_EXPORT_S3_UPLOAD_CHUNK_SIZE_BYTES:
- last_uploaded_part_timestamp = str(inserted_at)
- await flush_to_s3(last_uploaded_part_timestamp)
- local_results_file.reset()
+ Raises:
+ UnsupportedFileFormatError: If no writer exists for given `file_format`.
+ """
+ writer: BatchExportWriter
- if local_results_file.tell() > 0 and record is not None:
- last_uploaded_part_timestamp = str(inserted_at)
- await flush_to_s3(last_uploaded_part_timestamp, last=True)
+ if inputs.file_format == "Parquet":
+ writer = ParquetBatchExportWriter(
+ max_bytes=max_bytes,
+ flush_callable=flush_callable,
+ compression=inputs.compression,
+ schema=schema,
+ )
+ elif inputs.file_format == "JSONLines":
+ writer = JSONLBatchExportWriter(
+ max_bytes=settings.BATCH_EXPORT_S3_UPLOAD_CHUNK_SIZE_BYTES,
+ flush_callable=flush_callable,
+ compression=inputs.compression,
+ )
+ else:
+ raise UnsupportedFileFormatError(inputs.file_format, "S3")
- await s3_upload.complete()
+ return writer
+
+
+def cast_record_batch_json_columns(
+ record_batch: pa.RecordBatch,
+ json_columns: collections.abc.Sequence = ("properties", "person_properties", "set", "set_once"),
+) -> pa.RecordBatch:
+ """Cast json_columns in record_batch to JsonType.
+
+ We return a new RecordBatch with any json_columns replaced by fields casted to JsonType.
+ Casting is not copying the underlying array buffers, so memory usage does not increase when creating
+ the new array or the new record batch.
+ """
+ column_names = set(record_batch.column_names)
+ intersection = column_names & set(json_columns)
+
+ casted_arrays = []
+ for array in record_batch.select(intersection):
+ if pa.types.is_string(array.type):
+ casted_array = array.cast(JsonType())
+ casted_arrays.append(casted_array)
+
+ remaining_column_names = list(column_names - intersection)
+ return pa.RecordBatch.from_arrays(
+ record_batch.select(remaining_column_names).columns + casted_arrays,
+ names=remaining_column_names + list(intersection),
+ )
+
+
+class JsonScalar(pa.ExtensionScalar):
+ """Represents a JSON binary string."""
+
+ def as_py(self) -> dict | None:
+ if self.value:
+ return orjson.loads(self.value.as_py().encode("utf-8"))
+ else:
+ return None
+
+
+class JsonType(pa.ExtensionType):
+ """Type for JSON binary strings."""
+
+ def __init__(self):
+ super().__init__(pa.string(), "json")
+
+ def __arrow_ext_serialize__(self):
+ return b""
+
+ @classmethod
+ def __arrow_ext_deserialize__(self, storage_type, serialized):
+ return JsonType()
- return local_results_file.records_total
+ def __arrow_ext_scalar_class__(self):
+ return JsonScalar
@workflow.defn(name="s3-export")
@@ -572,6 +694,7 @@ async def run(self, inputs: S3BatchExportInputs):
encryption=inputs.encryption,
kms_key_id=inputs.kms_key_id,
batch_export_schema=inputs.batch_export_schema,
+ file_format=inputs.file_format,
)
await execute_batch_export_insert_activity(
diff --git a/posthog/temporal/batch_exports/snowflake_batch_export.py b/posthog/temporal/batch_exports/snowflake_batch_export.py
index be94eca89a799..9053f3e1006ad 100644
--- a/posthog/temporal/batch_exports/snowflake_batch_export.py
+++ b/posthog/temporal/batch_exports/snowflake_batch_export.py
@@ -18,7 +18,6 @@
from posthog.batch_exports.service import BatchExportField, BatchExportSchema, SnowflakeBatchExportInputs
from posthog.temporal.batch_exports.base import PostHogWorkflow
from posthog.temporal.batch_exports.batch_exports import (
- BatchExportTemporaryFile,
CreateBatchExportRunInputs,
UpdateBatchExportRunStatusInputs,
create_export_run,
@@ -32,6 +31,9 @@
get_bytes_exported_metric,
get_rows_exported_metric,
)
+from posthog.temporal.batch_exports.temporary_file import (
+ BatchExportTemporaryFile,
+)
from posthog.temporal.batch_exports.utils import peek_first_and_rewind
from posthog.temporal.common.clickhouse import get_client
from posthog.temporal.common.logger import bind_temporal_worker_logger
diff --git a/posthog/temporal/batch_exports/temporary_file.py b/posthog/temporal/batch_exports/temporary_file.py
new file mode 100644
index 0000000000000..f955f45553727
--- /dev/null
+++ b/posthog/temporal/batch_exports/temporary_file.py
@@ -0,0 +1,528 @@
+"""This module contains a temporary file to stage data in batch exports."""
+import abc
+import collections.abc
+import contextlib
+import csv
+import datetime as dt
+import gzip
+import tempfile
+import typing
+
+import brotli
+import orjson
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+
+def json_dumps_bytes(d) -> bytes:
+ return orjson.dumps(d, default=str)
+
+
+class BatchExportTemporaryFile:
+ """A TemporaryFile used to as an intermediate step while exporting data.
+
+ This class does not implement the file-like interface but rather passes any calls
+ to the underlying tempfile.NamedTemporaryFile. We do override 'write' methods
+ to allow tracking bytes and records.
+ """
+
+ def __init__(
+ self,
+ mode: str = "w+b",
+ buffering=-1,
+ compression: str | None = None,
+ encoding: str | None = None,
+ newline: str | None = None,
+ suffix: str | None = None,
+ prefix: str | None = None,
+ dir: str | None = None,
+ *,
+ errors: str | None = None,
+ ):
+ self._file = tempfile.NamedTemporaryFile(
+ mode=mode,
+ encoding=encoding,
+ newline=newline,
+ buffering=buffering,
+ suffix=suffix,
+ prefix=prefix,
+ dir=dir,
+ errors=errors,
+ )
+ self.compression = compression
+ self.bytes_total = 0
+ self.records_total = 0
+ self.bytes_since_last_reset = 0
+ self.records_since_last_reset = 0
+ self._brotli_compressor = None
+
+ def __getattr__(self, name):
+ """Pass get attr to underlying tempfile.NamedTemporaryFile."""
+ return self._file.__getattr__(name)
+
+ def __enter__(self):
+ """Context-manager protocol enter method."""
+ self._file.__enter__()
+ return self
+
+ def __exit__(self, exc, value, tb):
+ """Context-manager protocol exit method."""
+ return self._file.__exit__(exc, value, tb)
+
+ def __iter__(self):
+ yield from self._file
+
+ @property
+ def brotli_compressor(self):
+ if self._brotli_compressor is None:
+ self._brotli_compressor = brotli.Compressor()
+ return self._brotli_compressor
+
+ def finish_brotli_compressor(self):
+ """Flush remaining brotli bytes."""
+ # TODO: Move compression out of `BatchExportTemporaryFile` to a standard class for all writers.
+ if self.compression != "brotli":
+ raise ValueError(f"Compression is '{self.compression}', not 'brotli'")
+
+ result = self._file.write(self.brotli_compressor.finish())
+ self.bytes_total += result
+ self.bytes_since_last_reset += result
+ self._brotli_compressor = None
+
+ def compress(self, content: bytes | str) -> bytes:
+ if isinstance(content, str):
+ encoded = content.encode("utf-8")
+ else:
+ encoded = content
+
+ match self.compression:
+ case "gzip":
+ return gzip.compress(encoded)
+ case "brotli":
+ self.brotli_compressor.process(encoded)
+ return self.brotli_compressor.flush()
+ case None:
+ return encoded
+ case _:
+ raise ValueError(f"Unsupported compression: '{self.compression}'")
+
+ def write(self, content: bytes | str):
+ """Write bytes to underlying file keeping track of how many bytes were written."""
+ compressed_content = self.compress(content)
+
+ if "b" in self.mode:
+ result = self._file.write(compressed_content)
+ else:
+ result = self._file.write(compressed_content.decode("utf-8"))
+
+ self.bytes_total += result
+ self.bytes_since_last_reset += result
+
+ return result
+
+ def write_record_as_bytes(self, record: bytes):
+ result = self.write(record)
+
+ self.records_total += 1
+ self.records_since_last_reset += 1
+
+ return result
+
+ def write_records_to_jsonl(self, records):
+ """Write records to a temporary file as JSONL."""
+ if len(records) == 1:
+ jsonl_dump = orjson.dumps(records[0], option=orjson.OPT_APPEND_NEWLINE, default=str)
+ else:
+ jsonl_dump = b"\n".join(map(json_dumps_bytes, records))
+
+ result = self.write(jsonl_dump)
+
+ self.records_total += len(records)
+ self.records_since_last_reset += len(records)
+
+ return result
+
+ def write_records_to_csv(
+ self,
+ records,
+ fieldnames: None | collections.abc.Sequence[str] = None,
+ extrasaction: typing.Literal["raise", "ignore"] = "ignore",
+ delimiter: str = ",",
+ quotechar: str = '"',
+ escapechar: str | None = "\\",
+ lineterminator: str = "\n",
+ quoting=csv.QUOTE_NONE,
+ ):
+ """Write records to a temporary file as CSV."""
+ if len(records) == 0:
+ return
+
+ if fieldnames is None:
+ fieldnames = list(records[0].keys())
+
+ writer = csv.DictWriter(
+ self,
+ fieldnames=fieldnames,
+ extrasaction=extrasaction,
+ delimiter=delimiter,
+ quotechar=quotechar,
+ escapechar=escapechar,
+ quoting=quoting,
+ lineterminator=lineterminator,
+ )
+ writer.writerows(records)
+
+ self.records_total += len(records)
+ self.records_since_last_reset += len(records)
+
+ def write_records_to_tsv(
+ self,
+ records,
+ fieldnames: None | list[str] = None,
+ extrasaction: typing.Literal["raise", "ignore"] = "ignore",
+ quotechar: str = '"',
+ escapechar: str | None = "\\",
+ lineterminator: str = "\n",
+ quoting=csv.QUOTE_NONE,
+ ):
+ """Write records to a temporary file as TSV."""
+ return self.write_records_to_csv(
+ records,
+ fieldnames=fieldnames,
+ extrasaction=extrasaction,
+ delimiter="\t",
+ quotechar=quotechar,
+ escapechar=escapechar,
+ quoting=quoting,
+ lineterminator=lineterminator,
+ )
+
+ def rewind(self):
+ """Rewind the file before reading it."""
+ self._file.seek(0)
+
+ def reset(self):
+ """Reset underlying file by truncating it.
+
+ Also resets the tracker attributes for bytes and records since last reset.
+ """
+ self._file.seek(0)
+ self._file.truncate()
+
+ self.bytes_since_last_reset = 0
+ self.records_since_last_reset = 0
+
+
+LastInsertedAt = dt.datetime
+IsLast = bool
+RecordsSinceLastFlush = int
+BytesSinceLastFlush = int
+FlushCallable = collections.abc.Callable[
+ [BatchExportTemporaryFile, RecordsSinceLastFlush, BytesSinceLastFlush, LastInsertedAt, IsLast],
+ collections.abc.Awaitable[None],
+]
+
+
+class UnsupportedFileFormatError(Exception):
+ """Raised when a writer for an unsupported file format is requested."""
+
+ def __init__(self, file_format: str, destination: str):
+ super().__init__(f"{file_format} is not a supported format for {destination} batch exports.")
+
+
+class BatchExportWriter(abc.ABC):
+ """A temporary file writer to be used by batch export workflows.
+
+ Subclasses should define `_write_record_batch` with the particular intricacies
+ of the format they are writing as.
+
+ Actual writing calls are passed to the underlying `batch_export_file`.
+
+ Attributes:
+ _batch_export_file: The temporary file we are writing to.
+ max_bytes: Flush the temporary file with the provided `flush_callable`
+ upon reaching or surpassing this threshold. Keep in mind we write on a RecordBatch
+ per RecordBatch basis, which means the threshold will be surpassed by at most the
+ size of a RecordBatch before a flush occurs.
+ flush_callable: A callback to flush the temporary file when `max_bytes` is reached.
+ The temporary file will be reset after calling `flush_callable`. When calling
+ `flush_callable` the following positional arguments will be passed: The temporary file
+ that must be flushed, the number of records since the last flush, the number of bytes
+ since the last flush, the latest recorded `_inserted_at`, and a `bool` indicating if
+ this is the last flush (when exiting the context manager).
+ file_kwargs: Optional keyword arguments passed when initializing `_batch_export_file`.
+ last_inserted_at: Latest `_inserted_at` written. This attribute leaks some implementation
+ details, as we are assuming assume `_inserted_at` is present, as it's added to all
+ batch export queries.
+ records_total: The total number of records (not RecordBatches!) written.
+ records_since_last_flush: The number of records written since last flush.
+ bytes_total: The total number of bytes written.
+ bytes_since_last_flush: The number of bytes written since last flush.
+ """
+
+ def __init__(
+ self,
+ flush_callable: FlushCallable,
+ max_bytes: int,
+ file_kwargs: collections.abc.Mapping[str, typing.Any] | None = None,
+ ):
+ self.flush_callable = flush_callable
+ self.max_bytes = max_bytes
+ self.file_kwargs: collections.abc.Mapping[str, typing.Any] = file_kwargs or {}
+
+ self._batch_export_file: BatchExportTemporaryFile | None = None
+ self.reset_writer_tracking()
+
+ def reset_writer_tracking(self):
+ """Reset this writer's tracking state."""
+ self.last_inserted_at: dt.datetime | None = None
+ self.records_total = 0
+ self.records_since_last_flush = 0
+ self.bytes_total = 0
+ self.bytes_since_last_flush = 0
+
+ @contextlib.asynccontextmanager
+ async def open_temporary_file(self):
+ """Explicitly open the temporary file this writer is writing to.
+
+ The underlying `BatchExportTemporaryFile` is only accessible within this context manager. This helps
+ us separate the lifetime of the underlying temporary file from the writer: The writer may still be
+ accessed even after the temporary file is closed, while on the other hand we ensure the file and all
+ its data is flushed and not leaked outside the context. Any relevant tracking information is copied
+ to the writer.
+ """
+ self.reset_writer_tracking()
+
+ with BatchExportTemporaryFile(**self.file_kwargs) as temp_file:
+ self._batch_export_file = temp_file
+
+ try:
+ yield
+ finally:
+ self.track_bytes_written(temp_file)
+
+ if self.last_inserted_at is not None and self.bytes_since_last_flush > 0:
+ # `bytes_since_last_flush` should be 0 unless:
+ # 1. The last batch wasn't flushed as it didn't reach `max_bytes`.
+ # 2. The last batch was flushed but there was another write after the last call to
+ # `write_record_batch`. For example, footer bytes.
+ await self.flush(self.last_inserted_at, is_last=True)
+
+ self._batch_export_file = None
+
+ @property
+ def batch_export_file(self):
+ """Property for underlying temporary file.
+
+ Raises:
+ ValueError: if attempting to access the temporary file before it has been opened.
+ """
+ if self._batch_export_file is None:
+ raise ValueError("Batch export file is closed. Did you forget to call 'open_temporary_file'?")
+ return self._batch_export_file
+
+ @abc.abstractmethod
+ def _write_record_batch(self, record_batch: pa.RecordBatch) -> None:
+ """Write a record batch to the underlying `BatchExportTemporaryFile`.
+
+ Subclasses must override this to provide the actual implementation according to the supported
+ file format.
+ """
+ pass
+
+ def track_records_written(self, record_batch: pa.RecordBatch) -> None:
+ """Update this writer's state with the number of records in `record_batch`."""
+ self.records_total += record_batch.num_rows
+ self.records_since_last_flush += record_batch.num_rows
+
+ def track_bytes_written(self, batch_export_file: BatchExportTemporaryFile) -> None:
+ """Update this writer's state with the bytes in `batch_export_file`."""
+ self.bytes_total = batch_export_file.bytes_total
+ self.bytes_since_last_flush = batch_export_file.bytes_since_last_reset
+
+ async def write_record_batch(self, record_batch: pa.RecordBatch) -> None:
+ """Issue a record batch write tracking progress and flushing if required."""
+ record_batch = record_batch.sort_by("_inserted_at")
+ last_inserted_at = record_batch.column("_inserted_at")[-1].as_py()
+
+ column_names = record_batch.column_names
+ column_names.pop(column_names.index("_inserted_at"))
+
+ self._write_record_batch(record_batch.select(column_names))
+
+ self.last_inserted_at = last_inserted_at
+ self.track_records_written(record_batch)
+ self.track_bytes_written(self.batch_export_file)
+
+ if self.bytes_since_last_flush >= self.max_bytes:
+ await self.flush(last_inserted_at)
+
+ async def flush(self, last_inserted_at: dt.datetime, is_last: bool = False) -> None:
+ """Call the provided `flush_callable` and reset underlying file.
+
+ The underlying batch export temporary file will be reset after calling `flush_callable`.
+ """
+ if is_last is True and self.batch_export_file.compression == "brotli":
+ self.batch_export_file.finish_brotli_compressor()
+
+ self.batch_export_file.seek(0)
+
+ await self.flush_callable(
+ self.batch_export_file,
+ self.records_since_last_flush,
+ self.bytes_since_last_flush,
+ last_inserted_at,
+ is_last,
+ )
+ self.batch_export_file.reset()
+
+ self.records_since_last_flush = 0
+ self.bytes_since_last_flush = 0
+
+
+class JSONLBatchExportWriter(BatchExportWriter):
+ """A `BatchExportWriter` for JSONLines format.
+
+ Attributes:
+ default: The default function to use to cast non-serializable Python objects to serializable objects.
+ By default, non-serializable objects will be cast to string via `str()`.
+ """
+
+ def __init__(
+ self,
+ max_bytes: int,
+ flush_callable: FlushCallable,
+ compression: None | str = None,
+ default: typing.Callable = str,
+ ):
+ super().__init__(
+ max_bytes=max_bytes,
+ flush_callable=flush_callable,
+ file_kwargs={"compression": compression},
+ )
+
+ self.default = default
+
+ def write(self, content: bytes) -> int:
+ """Write a single row of JSONL."""
+ n = self.batch_export_file.write(orjson.dumps(content, default=str) + b"\n")
+ return n
+
+ def _write_record_batch(self, record_batch: pa.RecordBatch) -> None:
+ """Write records to a temporary file as JSONL."""
+ for record in record_batch.to_pylist():
+ self.write(record)
+
+
+class CSVBatchExportWriter(BatchExportWriter):
+ """A `BatchExportWriter` for CSV format."""
+
+ def __init__(
+ self,
+ max_bytes: int,
+ flush_callable: FlushCallable,
+ field_names: collections.abc.Sequence[str],
+ extras_action: typing.Literal["raise", "ignore"] = "ignore",
+ delimiter: str = ",",
+ quote_char: str = '"',
+ escape_char: str | None = "\\",
+ line_terminator: str = "\n",
+ quoting=csv.QUOTE_NONE,
+ compression: str | None = None,
+ ):
+ super().__init__(
+ max_bytes=max_bytes,
+ flush_callable=flush_callable,
+ file_kwargs={"compression": compression},
+ )
+ self.field_names = field_names
+ self.extras_action: typing.Literal["raise", "ignore"] = extras_action
+ self.delimiter = delimiter
+ self.quote_char = quote_char
+ self.escape_char = escape_char
+ self.line_terminator = line_terminator
+ self.quoting = quoting
+
+ self._csv_writer: csv.DictWriter | None = None
+
+ @property
+ def csv_writer(self) -> csv.DictWriter:
+ if self._csv_writer is None:
+ self._csv_writer = csv.DictWriter(
+ self.batch_export_file,
+ fieldnames=self.field_names,
+ extrasaction=self.extras_action,
+ delimiter=self.delimiter,
+ quotechar=self.quote_char,
+ escapechar=self.escape_char,
+ quoting=self.quoting,
+ lineterminator=self.line_terminator,
+ )
+
+ return self._csv_writer
+
+ def _write_record_batch(self, record_batch: pa.RecordBatch) -> None:
+ """Write records to a temporary file as CSV."""
+ self.csv_writer.writerows(record_batch.to_pylist())
+
+
+class ParquetBatchExportWriter(BatchExportWriter):
+ """A `BatchExportWriter` for Apache Parquet format.
+
+ We utilize and wrap a `pyarrow.parquet.ParquetWriter` to do the actual writing. We default to their
+ defaults for most parameters; however this class could be extended with more attributes to pass along
+ to `pyarrow.parquet.ParquetWriter`.
+
+ See the pyarrow docs for more details on what parameters can the writer be configured with:
+ https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html
+
+ In contrast to other writers, instead of us handling compression we let `pyarrow.parquet.ParquetWriter`
+ handle it, so `BatchExportTemporaryFile` is always initialized with `compression=None`.
+
+ Attributes:
+ schema: The schema used by the Parquet file. Should match the schema of written RecordBatches.
+ compression: Compression codec passed to underlying `pyarrow.parquet.ParquetWriter`.
+ """
+
+ def __init__(
+ self,
+ max_bytes: int,
+ flush_callable: FlushCallable,
+ schema: pa.Schema,
+ compression: str | None = "snappy",
+ ):
+ super().__init__(
+ max_bytes=max_bytes,
+ flush_callable=flush_callable,
+ file_kwargs={"compression": None}, # ParquetWriter handles compression
+ )
+ self.schema = schema
+ self.compression = compression
+
+ self._parquet_writer: pq.ParquetWriter | None = None
+
+ @property
+ def parquet_writer(self) -> pq.ParquetWriter:
+ if self._parquet_writer is None:
+ self._parquet_writer = pq.ParquetWriter(
+ self.batch_export_file,
+ schema=self.schema,
+ compression="none" if self.compression is None else self.compression,
+ )
+ return self._parquet_writer
+
+ @contextlib.asynccontextmanager
+ async def open_temporary_file(self):
+ """Ensure underlying Parquet writer is closed before flushing and closing temporary file."""
+ async with super().open_temporary_file():
+ try:
+ yield
+ finally:
+ if self._parquet_writer is not None:
+ self._parquet_writer.writer.close()
+ self._parquet_writer = None
+
+ def _write_record_batch(self, record_batch: pa.RecordBatch) -> None:
+ """Write records to a temporary file as Parquet."""
+
+ self.parquet_writer.write_batch(record_batch.select(self.parquet_writer.schema.names))
diff --git a/posthog/temporal/common/utils.py b/posthog/temporal/common/utils.py
index 1b61a356dc898..022c8270d7748 100644
--- a/posthog/temporal/common/utils.py
+++ b/posthog/temporal/common/utils.py
@@ -119,10 +119,9 @@ async def should_resume_from_activity_heartbeat(
heartbeat_details = heartbeat_type.from_activity(activity)
except EmptyHeartbeatError:
- # We don't log this as a warning/error because it's the expected exception when heartbeat is empty.
+ # We don't log this as it's the expected exception when heartbeat is empty.
heartbeat_details = None
received = False
- logger.debug("Did not receive details from previous activity execution")
except NotEnoughHeartbeatValuesError:
heartbeat_details = None
diff --git a/posthog/temporal/data_imports/external_data_job.py b/posthog/temporal/data_imports/external_data_job.py
index db99eeb1de315..bf78c99e9d9e0 100644
--- a/posthog/temporal/data_imports/external_data_job.py
+++ b/posthog/temporal/data_imports/external_data_job.py
@@ -10,6 +10,7 @@
# TODO: remove dependency
from posthog.temporal.batch_exports.base import PostHogWorkflow
+from posthog.temporal.data_imports.pipelines.zendesk.credentials import ZendeskCredentialsToken
from posthog.warehouse.data_load.source_templates import create_warehouse_templates_for_source
from posthog.warehouse.data_load.validate_schema import validate_schema_and_update_table
@@ -220,7 +221,20 @@ async def run_external_data_job(inputs: ExternalDataJobInputs) -> TSchemaTables:
schema=schema,
table_names=endpoints,
)
+ elif model.pipeline.source_type == ExternalDataSource.Type.ZENDESK:
+ from posthog.temporal.data_imports.pipelines.zendesk.helpers import zendesk_support
+ credentials = ZendeskCredentialsToken()
+ credentials.token = model.pipeline.job_inputs.get("zendesk_api_key")
+ credentials.subdomain = model.pipeline.job_inputs.get("zendesk_subdomain")
+ credentials.email = model.pipeline.job_inputs.get("zendesk_email_address")
+
+ data_support = zendesk_support(credentials=credentials, endpoints=tuple(endpoints), team_id=inputs.team_id)
+ # Uncomment to support zendesk chat and talk
+ # data_chat = zendesk_chat()
+ # data_talk = zendesk_talk()
+
+ source = data_support
else:
raise ValueError(f"Source type {model.pipeline.source_type} not supported")
diff --git a/posthog/temporal/data_imports/pipelines/pipeline.py b/posthog/temporal/data_imports/pipelines/pipeline.py
index 5297f2e39ac29..6a922d2d96a67 100644
--- a/posthog/temporal/data_imports/pipelines/pipeline.py
+++ b/posthog/temporal/data_imports/pipelines/pipeline.py
@@ -1,4 +1,5 @@
from dataclasses import dataclass
+from typing import Literal
from uuid import UUID
import dlt
@@ -9,7 +10,7 @@
import os
from posthog.settings.base_variables import TEST
from structlog.typing import FilteringBoundLogger
-from dlt.sources import DltResource
+from dlt.sources import DltSource
@dataclass
@@ -23,9 +24,9 @@ class PipelineInputs:
class DataImportPipeline:
- loader_file_format = "parquet"
+ loader_file_format: Literal["parquet"] = "parquet"
- def __init__(self, inputs: PipelineInputs, source: DltResource, logger: FilteringBoundLogger):
+ def __init__(self, inputs: PipelineInputs, source: DltSource, logger: FilteringBoundLogger):
self.inputs = inputs
self.logger = logger
self.source = source
@@ -47,6 +48,7 @@ def _get_destination(self):
credentials = {
"aws_access_key_id": settings.AIRBYTE_BUCKET_KEY,
"aws_secret_access_key": settings.AIRBYTE_BUCKET_SECRET,
+ "region_name": settings.AIRBYTE_BUCKET_REGION,
}
return dlt.destinations.filesystem(
diff --git a/posthog/temporal/data_imports/pipelines/schemas.py b/posthog/temporal/data_imports/pipelines/schemas.py
index 371f8087b7966..1caea1364899a 100644
--- a/posthog/temporal/data_imports/pipelines/schemas.py
+++ b/posthog/temporal/data_imports/pipelines/schemas.py
@@ -1,3 +1,4 @@
+from posthog.temporal.data_imports.pipelines.zendesk.settings import BASE_ENDPOINTS, SUPPORT_ENDPOINTS
from posthog.warehouse.models import ExternalDataSource
from posthog.temporal.data_imports.pipelines.stripe.settings import ENDPOINTS as STRIPE_ENDPOINTS
from posthog.temporal.data_imports.pipelines.hubspot.settings import ENDPOINTS as HUBSPOT_ENDPOINTS
@@ -5,5 +6,8 @@
PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING = {
ExternalDataSource.Type.STRIPE: STRIPE_ENDPOINTS,
ExternalDataSource.Type.HUBSPOT: HUBSPOT_ENDPOINTS,
+ ExternalDataSource.Type.ZENDESK: tuple(
+ list(BASE_ENDPOINTS) + [resource for resource, endpoint_url, data_key, cursor_paginated in SUPPORT_ENDPOINTS]
+ ),
ExternalDataSource.Type.POSTGRES: (),
}
diff --git a/posthog/temporal/data_imports/pipelines/zendesk/api_helpers.py b/posthog/temporal/data_imports/pipelines/zendesk/api_helpers.py
new file mode 100644
index 0000000000000..c6e4eb4809ee7
--- /dev/null
+++ b/posthog/temporal/data_imports/pipelines/zendesk/api_helpers.py
@@ -0,0 +1,103 @@
+from typing import Optional, TypedDict, Dict
+
+from dlt.common import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import DictStrAny, DictStrStr, TDataItem
+
+
+class TCustomFieldInfo(TypedDict):
+ title: str
+ options: DictStrStr
+
+
+def _parse_date_or_none(value: Optional[str]) -> Optional[pendulum.DateTime]:
+ if not value:
+ return None
+ return ensure_pendulum_datetime(value)
+
+
+def process_ticket(
+ ticket: DictStrAny,
+ custom_fields: Dict[str, TCustomFieldInfo],
+ pivot_custom_fields: bool = True,
+) -> DictStrAny:
+ """
+ Helper function that processes a ticket object and returns a dictionary of ticket data.
+
+ Args:
+ ticket: The ticket dict object returned by a Zendesk API call.
+ custom_fields: A dictionary containing all the custom fields available for tickets.
+ pivot_custom_fields: A boolean indicating whether to pivot all custom fields or not.
+ Defaults to True.
+
+ Returns:
+ DictStrAny: A dictionary containing cleaned data about a ticket.
+ """
+ # Commented out due to how slow this processing code is, and how often it'd break the pipeline.
+ # to be revisited on whether we want/need this pre-processing and figure out the best way to do it.
+
+ # pivot custom field if indicated as such
+ # get custom fields
+ # pivoted_fields = set()
+ # for custom_field in ticket.get("custom_fields", []):
+ # if pivot_custom_fields:
+ # cus_field_id = str(custom_field["id"])
+ # field = custom_fields.get(cus_field_id, None)
+ # if field is None:
+ # logger.warning(
+ # "Custom field with ID %s does not exist in fields state. It may have been created after the pipeline run started.",
+ # cus_field_id,
+ # )
+ # custom_field["ticket_id"] = ticket["id"]
+ # continue
+
+ # pivoted_fields.add(cus_field_id)
+ # field_name = field["title"]
+ # current_value = custom_field["value"]
+ # options = field["options"]
+ # # Map dropdown values to labels
+ # if not current_value or not options:
+ # ticket[field_name] = current_value
+ # elif isinstance(current_value, list): # Multiple choice field has a list of values
+ # ticket[field_name] = [options.get(key, key) for key in current_value]
+ # else:
+ # ticket[field_name] = options.get(current_value)
+ # else:
+ # custom_field["ticket_id"] = ticket["id"]
+ # # delete fields that are not needed for pivoting
+ # if pivot_custom_fields:
+ # ticket["custom_fields"] = [f for f in ticket.get("custom_fields", []) if str(f["id"]) not in pivoted_fields]
+ # if not ticket.get("custom_fields"):
+ # del ticket["custom_fields"]
+ # del ticket["fields"]
+
+ # modify dates to return datetime objects instead
+ ticket["updated_at"] = _parse_date_or_none(ticket["updated_at"])
+ ticket["created_at"] = _parse_date_or_none(ticket["created_at"])
+ ticket["due_at"] = _parse_date_or_none(ticket["due_at"])
+ return ticket
+
+
+def process_ticket_field(field: DictStrAny, custom_fields_state: Dict[str, TCustomFieldInfo]) -> TDataItem:
+ """Update custom field mapping in dlt state for the given field."""
+ # grab id and update state dict
+ # if the id is new, add a new key to indicate that this is the initial value for title
+ # New dropdown options are added to existing field but existing options are not changed
+ return_dict = field.copy()
+ field_id = str(field["id"])
+
+ options = field.get("custom_field_options", [])
+ new_options = {o["value"]: o["name"] for o in options}
+ existing_field = custom_fields_state.get(field_id)
+ if existing_field:
+ existing_options = existing_field["options"]
+ if return_options := return_dict.get("custom_field_options"):
+ for item in return_options:
+ item["name"] = existing_options.get(item["value"], item["name"])
+ for key, value in new_options.items():
+ if key not in existing_options:
+ existing_options[key] = value
+ else:
+ custom_fields_state[field_id] = dict(title=field["title"], options=new_options)
+ return_dict["initial_title"] = field["title"]
+ return return_dict
diff --git a/posthog/temporal/data_imports/pipelines/zendesk/credentials.py b/posthog/temporal/data_imports/pipelines/zendesk/credentials.py
new file mode 100644
index 0000000000000..1f8110ae9b911
--- /dev/null
+++ b/posthog/temporal/data_imports/pipelines/zendesk/credentials.py
@@ -0,0 +1,49 @@
+"""
+This module handles how credentials are read in dlt sources
+"""
+from typing import ClassVar, List, Union
+from dlt.common.configuration import configspec
+from dlt.common.configuration.specs import CredentialsConfiguration
+from dlt.common.typing import TSecretValue
+
+
+@configspec
+class ZendeskCredentialsBase(CredentialsConfiguration):
+ """
+ The Base version of all the ZendeskCredential classes.
+ """
+
+ subdomain: str
+ __config_gen_annotations__: ClassVar[List[str]] = []
+
+
+@configspec
+class ZendeskCredentialsEmailPass(ZendeskCredentialsBase):
+ """
+ This class is used to store credentials for Email + Password Authentication
+ """
+
+ email: str
+ password: TSecretValue
+
+
+@configspec
+class ZendeskCredentialsOAuth(ZendeskCredentialsBase):
+ """
+ This class is used to store credentials for OAuth Token Authentication
+ """
+
+ oauth_token: TSecretValue
+
+
+@configspec
+class ZendeskCredentialsToken(ZendeskCredentialsBase):
+ """
+ This class is used to store credentials for Token Authentication
+ """
+
+ email: str
+ token: TSecretValue
+
+
+TZendeskCredentials = Union[ZendeskCredentialsEmailPass, ZendeskCredentialsToken, ZendeskCredentialsOAuth]
diff --git a/posthog/temporal/data_imports/pipelines/zendesk/helpers.py b/posthog/temporal/data_imports/pipelines/zendesk/helpers.py
new file mode 100644
index 0000000000000..a3e0328c8ab28
--- /dev/null
+++ b/posthog/temporal/data_imports/pipelines/zendesk/helpers.py
@@ -0,0 +1,444 @@
+from typing import Iterator, List, Optional, Iterable, Tuple
+from itertools import chain
+
+import dlt
+from dlt.common import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TDataItem, TAnyDateTime, TDataItems
+from dlt.sources import DltResource
+
+from .api_helpers import process_ticket, process_ticket_field
+from .talk_api import PaginationType, ZendeskAPIClient
+from .credentials import TZendeskCredentials, ZendeskCredentialsOAuth
+
+from .settings import (
+ DEFAULT_START_DATE,
+ CUSTOM_FIELDS_STATE_KEY,
+ SUPPORT_ENDPOINTS,
+ TALK_ENDPOINTS,
+ INCREMENTAL_TALK_ENDPOINTS,
+ SUPPORT_EXTRA_ENDPOINTS,
+)
+
+
+@dlt.source(max_table_nesting=0)
+def zendesk_talk(
+ credentials: TZendeskCredentials = dlt.secrets.value,
+ start_date: Optional[TAnyDateTime] = DEFAULT_START_DATE,
+ end_date: Optional[TAnyDateTime] = None,
+) -> Iterable[DltResource]:
+ """
+ Retrieves data from Zendesk Talk for phone calls and voicemails.
+
+ `start_date` argument can be used on its own or together with `end_date`. When both are provided
+ data is limited to items updated in that time range.
+ The range is "half-open", meaning elements equal and higher than `start_date` and elements lower than `end_date` are included.
+ All resources opt-in to use Airflow scheduler if run as Airflow task
+
+ Args:
+ credentials: The credentials for authentication. Defaults to the value in the `dlt.secrets` object.
+ start_date: The start time of the range for which to load. Defaults to January 1st 2000.
+ end_date: The end time of the range for which to load data.
+ If end time is not provided, the incremental loading will be enabled and after initial run, only new data will be retrieved
+ Yields:
+ DltResource: Data resources from Zendesk Talk.
+ """
+
+ # use the credentials to authenticate with the ZendeskClient
+ zendesk_client = ZendeskAPIClient(credentials)
+ start_date_obj = ensure_pendulum_datetime(start_date)
+ end_date_obj = ensure_pendulum_datetime(end_date) if end_date else None
+
+ # regular endpoints
+ for key, talk_endpoint, item_name, cursor_paginated in TALK_ENDPOINTS:
+ yield dlt.resource(
+ talk_resource(
+ zendesk_client,
+ key,
+ item_name or talk_endpoint,
+ PaginationType.CURSOR if cursor_paginated else PaginationType.OFFSET,
+ ),
+ name=key,
+ write_disposition="replace",
+ )
+
+ # adding incremental endpoints
+ for key, talk_incremental_endpoint in INCREMENTAL_TALK_ENDPOINTS.items():
+ yield dlt.resource(
+ talk_incremental_resource,
+ name=f"{key}_incremental",
+ primary_key="id",
+ write_disposition="merge",
+ )(
+ zendesk_client=zendesk_client,
+ talk_endpoint_name=key,
+ talk_endpoint=talk_incremental_endpoint,
+ updated_at=dlt.sources.incremental[str](
+ "updated_at",
+ initial_value=start_date_obj.isoformat(),
+ end_value=end_date_obj.isoformat() if end_date_obj else None,
+ allow_external_schedulers=True,
+ ),
+ )
+
+
+def talk_resource(
+ zendesk_client: ZendeskAPIClient,
+ talk_endpoint_name: str,
+ talk_endpoint: str,
+ pagination_type: PaginationType,
+) -> Iterator[TDataItem]:
+ """
+ Loads data from a Zendesk Talk endpoint.
+
+ Args:
+ zendesk_client: An instance of ZendeskAPIClient for making API calls to Zendesk Talk.
+ talk_endpoint_name: The name of the talk_endpoint.
+ talk_endpoint: The actual URL ending of the endpoint.
+ pagination: Type of pagination type used by endpoint
+
+ Yields:
+ TDataItem: Dictionary containing the data from the endpoint.
+ """
+ # send query and process it
+ yield from zendesk_client.get_pages(talk_endpoint, talk_endpoint_name, pagination_type)
+
+
+def talk_incremental_resource(
+ zendesk_client: ZendeskAPIClient,
+ talk_endpoint_name: str,
+ talk_endpoint: str,
+ updated_at: dlt.sources.incremental[str],
+) -> Iterator[TDataItem]:
+ """
+ Loads data from a Zendesk Talk endpoint with incremental loading.
+
+ Args:
+ zendesk_client: An instance of ZendeskAPIClient for making API calls to Zendesk Talk.
+ talk_endpoint_name: The name of the talk_endpoint.
+ talk_endpoint: The actual URL ending of the endpoint.
+ updated_at: Source for the last updated timestamp.
+
+ Yields:
+ TDataItem: Dictionary containing the data from the endpoint.
+ """
+ # send the request and process it
+ for page in zendesk_client.get_pages(
+ talk_endpoint,
+ talk_endpoint_name,
+ PaginationType.START_TIME,
+ params={"start_time": ensure_pendulum_datetime(updated_at.last_value).int_timestamp},
+ ):
+ yield page
+ if updated_at.end_out_of_range:
+ return
+
+
+@dlt.source(max_table_nesting=0)
+def zendesk_chat(
+ credentials: ZendeskCredentialsOAuth = dlt.secrets.value,
+ start_date: Optional[TAnyDateTime] = DEFAULT_START_DATE,
+ end_date: Optional[TAnyDateTime] = None,
+) -> Iterable[DltResource]:
+ """
+ Retrieves data from Zendesk Chat for chat interactions.
+
+ `start_date` argument can be used on its own or together with `end_date`. When both are provided
+ data is limited to items updated in that time range.
+ The range is "half-open", meaning elements equal and higher than `start_date` and elements lower than `end_date` are included.
+ All resources opt-in to use Airflow scheduler if run as Airflow task
+
+ Args:
+ credentials: The credentials for authentication. Defaults to the value in the `dlt.secrets` object.
+ start_date: The start time of the range for which to load. Defaults to January 1st 2000.
+ end_date: The end time of the range for which to load data.
+ If end time is not provided, the incremental loading will be enabled and after initial run, only new data will be retrieved
+
+ Yields:
+ DltResource: Data resources from Zendesk Chat.
+ """
+
+ # Authenticate
+ zendesk_client = ZendeskAPIClient(credentials, url_prefix="https://www.zopim.com")
+ start_date_obj = ensure_pendulum_datetime(start_date)
+ end_date_obj = ensure_pendulum_datetime(end_date) if end_date else None
+
+ yield dlt.resource(chats_table_resource, name="chats", write_disposition="merge")(
+ zendesk_client,
+ dlt.sources.incremental[str](
+ "update_timestamp|updated_timestamp",
+ initial_value=start_date_obj.isoformat(),
+ end_value=end_date_obj.isoformat() if end_date_obj else None,
+ allow_external_schedulers=True,
+ ),
+ )
+
+
+def chats_table_resource(
+ zendesk_client: ZendeskAPIClient,
+ update_timestamp: dlt.sources.incremental[str],
+) -> Iterator[TDataItems]:
+ """
+ Resource for Chats
+
+ Args:
+ zendesk_client: The Zendesk API client instance, used to make calls to Zendesk API.
+ update_timestamp: Incremental source specifying the timestamp for incremental loading.
+
+ Yields:
+ dict: A dictionary representing each row of data.
+ """
+ chat_pages = zendesk_client.get_pages(
+ "/api/v2/incremental/chats",
+ "chats",
+ PaginationType.START_TIME,
+ params={
+ "start_time": ensure_pendulum_datetime(update_timestamp.last_value).int_timestamp,
+ "fields": "chats(*)",
+ },
+ )
+ for page in chat_pages:
+ yield page
+
+ if update_timestamp.end_out_of_range:
+ return
+
+
+@dlt.source(max_table_nesting=0)
+def zendesk_support(
+ team_id: int,
+ credentials: TZendeskCredentials = dlt.secrets.value,
+ endpoints: Tuple[str, ...] = (),
+ pivot_ticket_fields: bool = True,
+ start_date: Optional[TAnyDateTime] = DEFAULT_START_DATE,
+ end_date: Optional[TAnyDateTime] = None,
+) -> Iterable[DltResource]:
+ """
+ Retrieves data from Zendesk Support for tickets, users, brands, organizations, and groups.
+
+ `start_date` argument can be used on its own or together with `end_date`. When both are provided
+ data is limited to items updated in that time range.
+ The range is "half-open", meaning elements equal and higher than `start_date` and elements lower than `end_date` are included.
+ All resources opt-in to use Airflow scheduler if run as Airflow task
+
+ Args:
+ credentials: The credentials for authentication. Defaults to the value in the `dlt.secrets` object.
+ load_all: Whether to load extra resources for the API. Defaults to True.
+ pivot_ticket_fields: Whether to pivot the custom fields in tickets. Defaults to True.
+ start_date: The start time of the range for which to load. Defaults to January 1st 2000.
+ end_date: The end time of the range for which to load data.
+ If end time is not provided, the incremental loading will be enabled and after initial run, only new data will be retrieved
+
+ Returns:
+ Sequence[DltResource]: Multiple dlt resources.
+ """
+
+ start_date_obj = ensure_pendulum_datetime(start_date)
+ end_date_obj = ensure_pendulum_datetime(end_date) if end_date else None
+
+ start_date_ts = start_date_obj.int_timestamp
+ start_date_iso_str = start_date_obj.isoformat()
+ end_date_ts: Optional[int] = None
+ end_date_iso_str: Optional[str] = None
+ if end_date_obj:
+ end_date_ts = end_date_obj.int_timestamp
+ end_date_iso_str = end_date_obj.isoformat()
+
+ @dlt.resource(name="ticket_events", primary_key="id", write_disposition="append")
+ def ticket_events(
+ zendesk_client: ZendeskAPIClient,
+ timestamp: dlt.sources.incremental[int] = dlt.sources.incremental( # noqa: B008
+ "timestamp",
+ initial_value=start_date_ts,
+ end_value=end_date_ts,
+ allow_external_schedulers=True,
+ ),
+ ) -> Iterator[TDataItem]:
+ # URL For ticket events
+ # 'https://d3v-dlthub.zendesk.com/api/v2/incremental/ticket_events.json?start_time=946684800'
+ event_pages = zendesk_client.get_pages(
+ "/api/v2/incremental/ticket_events.json",
+ "ticket_events",
+ PaginationType.STREAM,
+ params={"start_time": timestamp.last_value},
+ )
+ for page in event_pages:
+ yield page
+ if timestamp.end_out_of_range:
+ return
+
+ @dlt.resource(
+ name="tickets",
+ primary_key="id",
+ write_disposition="merge",
+ columns={
+ "tags": {"data_type": "complex"},
+ "custom_fields": {"data_type": "complex"},
+ },
+ )
+ def ticket_table(
+ zendesk_client: ZendeskAPIClient,
+ pivot_fields: bool = True,
+ updated_at: dlt.sources.incremental[pendulum.DateTime] = dlt.sources.incremental( # noqa: B008
+ "updated_at",
+ initial_value=start_date_obj,
+ end_value=end_date_obj,
+ allow_external_schedulers=True,
+ ),
+ ) -> Iterator[TDataItem]:
+ """
+ Resource for tickets table. Uses DLT state to handle column renaming of custom fields to prevent changing the names of said columns.
+ This resource uses pagination, loading and side loading to make API calls more efficient.
+
+ Args:
+ zendesk_client: The Zendesk API client instance, used to make calls to Zendesk API.
+ pivot_fields: Indicates whether to pivot the custom fields in tickets. Defaults to True.
+ per_page: The number of Ticket objects to load per page. Defaults to 1000.
+ updated_at: Incremental source for the 'updated_at' column.
+ Defaults to dlt.sources.incremental("updated_at", initial_value=start_date).
+
+ Yields:
+ TDataItem: Dictionary containing the ticket data.
+ """
+ # grab the custom fields from dlt state if any
+ if pivot_fields:
+ load_ticket_fields_state(zendesk_client)
+ fields_dict = dlt.current.source_state().setdefault(CUSTOM_FIELDS_STATE_KEY, {})
+ # include_objects = ["users", "groups", "organisation", "brands"]
+ ticket_pages = zendesk_client.get_pages(
+ "/api/v2/incremental/tickets",
+ "tickets",
+ PaginationType.STREAM,
+ params={"start_time": updated_at.last_value.int_timestamp},
+ )
+ for page in ticket_pages:
+ yield [process_ticket(ticket, fields_dict, pivot_custom_fields=pivot_fields) for ticket in page]
+
+ # stop loading when using end_value and end is reached
+ if updated_at.end_out_of_range:
+ return
+
+ @dlt.resource(name="ticket_metric_events", primary_key="id", write_disposition="append")
+ def ticket_metric_table(
+ zendesk_client: ZendeskAPIClient,
+ time: dlt.sources.incremental[str] = dlt.sources.incremental( # noqa: B008
+ "time",
+ initial_value=start_date_iso_str,
+ end_value=end_date_iso_str,
+ allow_external_schedulers=True,
+ ),
+ ) -> Iterator[TDataItem]:
+ """
+ Resource for ticket metric events table. Returns all the ticket metric events from the starting date,
+ with the default starting date being January 1st of the current year.
+
+ Args:
+ zendesk_client: The Zendesk API client instance, used to make calls to Zendesk API.
+ time: Incremental source for the 'time' column,
+ indicating the starting date for retrieving ticket metric events.
+ Defaults to dlt.sources.incremental("time", initial_value=start_date_iso_str).
+
+ Yields:
+ TDataItem: Dictionary containing the ticket metric event data.
+ """
+ # "https://example.zendesk.com/api/v2/incremental/ticket_metric_events?start_time=1332034771"
+ metric_event_pages = zendesk_client.get_pages(
+ "/api/v2/incremental/ticket_metric_events",
+ "ticket_metric_events",
+ PaginationType.CURSOR,
+ params={
+ "start_time": ensure_pendulum_datetime(time.last_value).int_timestamp,
+ },
+ )
+ for page in metric_event_pages:
+ yield page
+
+ if time.end_out_of_range:
+ return
+
+ def ticket_fields_table(zendesk_client: ZendeskAPIClient) -> Iterator[TDataItem]:
+ """
+ Loads ticket fields data from Zendesk API.
+
+ Args:
+ zendesk_client: The Zendesk API client instance, used to make calls to Zendesk API.
+
+ Yields:
+ TDataItem: Dictionary containing the ticket fields data.
+ """
+ # get dlt state
+ ticket_custom_fields = dlt.current.source_state().setdefault(CUSTOM_FIELDS_STATE_KEY, {})
+ # get all custom fields and update state if needed, otherwise just load dicts into tables
+ all_fields = list(
+ chain.from_iterable(
+ zendesk_client.get_pages("/api/v2/ticket_fields.json", "ticket_fields", PaginationType.OFFSET)
+ )
+ )
+ # all_fields = zendesk_client.ticket_fields()
+ for field in all_fields:
+ yield process_ticket_field(field, ticket_custom_fields)
+
+ def load_ticket_fields_state(
+ zendesk_client: ZendeskAPIClient,
+ ) -> None:
+ for _ in ticket_fields_table(zendesk_client):
+ pass
+
+ ticket_fields_resource = dlt.resource(name="ticket_fields", write_disposition="replace")(ticket_fields_table)
+
+ # Authenticate
+ zendesk_client = ZendeskAPIClient(credentials)
+
+ all_endpoints = SUPPORT_ENDPOINTS + SUPPORT_EXTRA_ENDPOINTS
+ resource_list: List[DltResource] = []
+
+ for endpoint in endpoints:
+ # loading base tables
+ if endpoint == "ticket_fields":
+ resource_list.append(ticket_fields_resource(zendesk_client=zendesk_client))
+ elif endpoint == "ticket_events":
+ resource_list.append(ticket_events(zendesk_client=zendesk_client))
+ elif endpoint == "tickets":
+ resource_list.append(ticket_table(zendesk_client=zendesk_client, pivot_fields=pivot_ticket_fields))
+ elif endpoint == "ticket_metric_events":
+ resource_list.append(ticket_metric_table(zendesk_client=zendesk_client))
+ else:
+ # other tables to be loaded
+ for resource, endpoint_url, data_key, cursor_paginated in all_endpoints:
+ if endpoint == resource:
+ resource_list.append(
+ dlt.resource(
+ basic_resource(zendesk_client, endpoint_url, data_key or resource, cursor_paginated),
+ name=resource,
+ write_disposition="replace",
+ )
+ )
+ break
+
+ return resource_list
+
+
+def basic_resource(
+ zendesk_client: ZendeskAPIClient,
+ endpoint_url: str,
+ data_key: str,
+ cursor_paginated: bool,
+) -> Iterator[TDataItem]:
+ """
+ Basic loader for most endpoints offered by Zenpy. Supports pagination. Expects to be called as a DLT Resource.
+
+ Args:
+ zendesk_client: The Zendesk API client instance, used to make calls to Zendesk API.
+ resource: The Zenpy endpoint to retrieve data from, usually directly linked to a Zendesk API endpoint.
+ cursor_paginated: Tells to use CURSOR pagination or OFFSET/no pagination
+
+ Yields:
+ TDataItem: Dictionary containing the resource data.
+ """
+
+ pages = zendesk_client.get_pages(
+ endpoint_url,
+ data_key,
+ PaginationType.CURSOR if cursor_paginated else PaginationType.OFFSET,
+ )
+ yield from pages
diff --git a/posthog/temporal/data_imports/pipelines/zendesk/settings.py b/posthog/temporal/data_imports/pipelines/zendesk/settings.py
new file mode 100644
index 0000000000000..aa44df7c20297
--- /dev/null
+++ b/posthog/temporal/data_imports/pipelines/zendesk/settings.py
@@ -0,0 +1,73 @@
+"""Zendesk source settings and constants"""
+
+from dlt.common import pendulum
+
+DEFAULT_START_DATE = pendulum.datetime(year=2000, month=1, day=1)
+PAGE_SIZE = 100
+INCREMENTAL_PAGE_SIZE = 1000
+
+
+CUSTOM_FIELDS_STATE_KEY = "ticket_custom_fields_v2"
+
+# Resources that will always get pulled
+BASE_ENDPOINTS = ["ticket_fields", "ticket_events", "tickets", "ticket_metric_events"]
+
+# Tuples of (Resource name, endpoint URL, data_key, supports pagination)
+# data_key is the key which data list is nested under in responses
+# if the data key is None it is assumed to be the same as the resource name
+# The last element of the tuple says if endpoint supports cursor pagination
+SUPPORT_ENDPOINTS = [
+ ("users", "/api/v2/users.json", "users", True),
+ ("sla_policies", "/api/v2/slas/policies.json", None, False),
+ ("groups", "/api/v2/groups.json", None, True),
+ ("organizations", "/api/v2/organizations.json", None, True),
+ ("brands", "/api/v2/brands.json", None, True),
+]
+
+SUPPORT_EXTRA_ENDPOINTS = [
+ ("activities", "/api/v2/activities.json", None, True),
+ ("automations", "/api/v2/automations.json", None, True),
+ ("custom_agent_roles", "/api/v2/custom_roles.json", "custom_roles", False),
+ ("dynamic_content", "/api/v2/dynamic_content/items.json", "items", True),
+ ("group_memberships", "/api/v2/group_memberships.json", None, True),
+ ("job_status", "/api/v2/job_statuses.json", "job_statuses", True),
+ ("macros", "/api/v2/macros.json", None, True),
+ ("organization_fields", "/api/v2/organization_fields.json", None, True),
+ ("organization_memberships", "/api/v2/organization_memberships.json", None, True),
+ ("recipient_addresses", "/api/v2/recipient_addresses.json", None, True),
+ ("requests", "/api/v2/requests.json", None, True),
+ ("satisfaction_ratings", "/api/v2/satisfaction_ratings.json", None, True),
+ ("sharing_agreements", "/api/v2/sharing_agreements.json", None, False),
+ ("skips", "/api/v2/skips.json", None, True),
+ ("suspended_tickets", "/api/v2/suspended_tickets.json", None, True),
+ ("targets", "/api/v2/targets.json", None, False),
+ ("ticket_forms", "/api/v2/ticket_forms.json", None, False),
+ ("ticket_metrics", "/api/v2/ticket_metrics.json", None, True),
+ ("triggers", "/api/v2/triggers.json", None, True),
+ ("user_fields", "/api/v2/user_fields.json", None, True),
+ ("views", "/api/v2/views.json", None, True),
+ ("tags", "/api/v2/tags.json", None, True),
+]
+
+TALK_ENDPOINTS = [
+ ("calls", "/api/v2/channels/voice/calls", None, False),
+ ("addresses", "/api/v2/channels/voice/addresses", None, False),
+ ("greeting_categories", "/api/v2/channels/voice/greeting_categories", None, False),
+ ("greetings", "/api/v2/channels/voice/greetings", None, False),
+ ("ivrs", "/api/v2/channels/voice/ivr", None, False),
+ ("phone_numbers", "/api/v2/channels/voice/phone_numbers", None, False),
+ ("settings", "/api/v2/channels/voice/settings", None, False),
+ ("lines", "/api/v2/channels/voice/lines", None, False),
+ ("agents_activity", "/api/v2/channels/voice/stats/agents_activity", None, False),
+ (
+ "current_queue_activity",
+ "/api/v2/channels/voice/stats/current_queue_activity",
+ None,
+ False,
+ ),
+]
+
+INCREMENTAL_TALK_ENDPOINTS = {
+ "calls": "/api/v2/channels/voice/stats/incremental/calls.json",
+ "legs": "/api/v2/channels/voice/stats/incremental/legs.json",
+}
diff --git a/posthog/temporal/data_imports/pipelines/zendesk/talk_api.py b/posthog/temporal/data_imports/pipelines/zendesk/talk_api.py
new file mode 100644
index 0000000000000..5db9a28eafc74
--- /dev/null
+++ b/posthog/temporal/data_imports/pipelines/zendesk/talk_api.py
@@ -0,0 +1,114 @@
+from enum import Enum
+from typing import Dict, Iterator, Optional, Tuple, Any
+from dlt.common.typing import DictStrStr, TDataItems, TSecretValue
+from dlt.sources.helpers.requests import client
+
+from . import settings
+from .credentials import (
+ ZendeskCredentialsEmailPass,
+ ZendeskCredentialsOAuth,
+ ZendeskCredentialsToken,
+ TZendeskCredentials,
+)
+
+
+class PaginationType(Enum):
+ OFFSET = 0
+ CURSOR = 1
+ STREAM = 2
+ START_TIME = 3
+
+
+class ZendeskAPIClient:
+ """
+ API client used to make requests to Zendesk talk, support and chat API
+ """
+
+ subdomain: str = ""
+ url: str = ""
+ headers: Optional[DictStrStr]
+ auth: Optional[Tuple[str, TSecretValue]]
+
+ def __init__(self, credentials: TZendeskCredentials, url_prefix: Optional[str] = None) -> None:
+ """
+ Initializer for the API client which is then used to make API calls to the ZendeskAPI
+
+ Args:
+ credentials: ZendeskCredentials object which contains the necessary credentials to authenticate to ZendeskAPI
+ """
+ # oauth token is the preferred way to authenticate, followed by api token and then email + password combo
+ # fill headers and auth for every possibility of credentials given, raise error if credentials are of incorrect type
+ if isinstance(credentials, ZendeskCredentialsOAuth):
+ self.headers = {"Authorization": f"Bearer {credentials.oauth_token}"}
+ self.auth = None
+ elif isinstance(credentials, ZendeskCredentialsToken):
+ self.headers = None
+ self.auth = (f"{credentials.email}/token", credentials.token)
+ elif isinstance(credentials, ZendeskCredentialsEmailPass):
+ self.auth = (credentials.email, credentials.password)
+ self.headers = None
+ else:
+ raise TypeError(
+ "Wrong credentials type provided to ZendeskAPIClient. The credentials need to be of type: ZendeskCredentialsOAuth, ZendeskCredentialsToken or ZendeskCredentialsEmailPass"
+ )
+
+ # If url_prefix is set it overrides the default API URL (e.g. chat api uses zopim.com domain)
+ if url_prefix:
+ self.url = url_prefix
+ else:
+ self.subdomain = credentials.subdomain
+ self.url = f"https://{self.subdomain}.zendesk.com"
+
+ def get_pages(
+ self,
+ endpoint: str,
+ data_point_name: str,
+ pagination: PaginationType,
+ params: Optional[Dict[str, Any]] = None,
+ ) -> Iterator[TDataItems]:
+ """
+ Makes a request to a paginated endpoint and returns a generator of data items per page.
+
+ Args:
+ endpoint: The url to the endpoint, e.g. /api/v2/calls
+ data_point_name: The key which data items are nested under in the response object (e.g. calls)
+ params: Optional dict of query params to include in the request
+ pagination: Type of pagination type used by endpoint
+
+ Returns:
+ Generator of pages, each page is a list of dict data items
+ """
+
+ # update the page size to enable cursor pagination
+ params = params or {}
+ if pagination == PaginationType.CURSOR:
+ params["page[size]"] = settings.PAGE_SIZE
+ elif pagination == PaginationType.STREAM:
+ params["per_page"] = settings.INCREMENTAL_PAGE_SIZE
+ elif pagination == PaginationType.START_TIME:
+ params["limit"] = settings.INCREMENTAL_PAGE_SIZE
+
+ # make request and keep looping until there is no next page
+ get_url = f"{self.url}{endpoint}"
+ while get_url:
+ response = client.get(get_url, headers=self.headers, auth=self.auth, params=params)
+ response.raise_for_status()
+ response_json = response.json()
+ result = response_json[data_point_name]
+ yield result
+
+ get_url = None
+ if pagination == PaginationType.CURSOR:
+ if response_json["meta"]["has_more"]:
+ get_url = response_json["links"]["next"]
+ elif pagination == PaginationType.OFFSET:
+ get_url = response_json.get("next_page", None)
+ elif pagination == PaginationType.STREAM:
+ # See https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/#json-format
+ if not response_json["end_of_stream"]:
+ get_url = response_json["next_page"]
+ elif pagination == PaginationType.START_TIME:
+ if response_json["count"] > 0:
+ get_url = response_json["next_page"]
+
+ params = {}
diff --git a/posthog/temporal/tests/batch_exports/test_batch_exports.py b/posthog/temporal/tests/batch_exports/test_batch_exports.py
index 0afbfcabb71cb..756c07e442e4f 100644
--- a/posthog/temporal/tests/batch_exports/test_batch_exports.py
+++ b/posthog/temporal/tests/batch_exports/test_batch_exports.py
@@ -1,6 +1,4 @@
-import csv
import datetime as dt
-import io
import json
import operator
from random import randint
@@ -9,11 +7,9 @@
from django.test import override_settings
from posthog.temporal.batch_exports.batch_exports import (
- BatchExportTemporaryFile,
get_data_interval,
get_rows_count,
iter_records,
- json_dumps_bytes,
)
from posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse
@@ -558,181 +554,3 @@ def test_get_data_interval(interval, data_interval_end, expected):
"""Test get_data_interval returns the expected data interval tuple."""
result = get_data_interval(interval, data_interval_end)
assert result == expected
-
-
-@pytest.mark.parametrize(
- "to_write",
- [
- (b"",),
- (b"", b""),
- (b"12345",),
- (b"12345", b"12345"),
- (b"abbcccddddeeeee",),
- (b"abbcccddddeeeee", b"abbcccddddeeeee"),
- ],
-)
-def test_batch_export_temporary_file_tracks_bytes(to_write):
- """Test the bytes written by BatchExportTemporaryFile match expected."""
- with BatchExportTemporaryFile() as be_file:
- for content in to_write:
- be_file.write(content)
-
- assert be_file.bytes_total == sum(len(content) for content in to_write)
- assert be_file.bytes_since_last_reset == sum(len(content) for content in to_write)
-
- be_file.reset()
-
- assert be_file.bytes_total == sum(len(content) for content in to_write)
- assert be_file.bytes_since_last_reset == 0
-
-
-TEST_RECORDS = [
- [],
- [
- {"id": "record-1", "property": "value", "property_int": 1},
- {"id": "record-2", "property": "another-value", "property_int": 2},
- {
- "id": "record-3",
- "property": {"id": "nested-record", "property": "nested-value"},
- "property_int": 3,
- },
- ],
-]
-
-
-@pytest.mark.parametrize(
- "records",
- TEST_RECORDS,
-)
-def test_batch_export_temporary_file_write_records_to_jsonl(records):
- """Test JSONL records written by BatchExportTemporaryFile match expected."""
- jsonl_dump = b"\n".join(map(json_dumps_bytes, records))
-
- with BatchExportTemporaryFile() as be_file:
- be_file.write_records_to_jsonl(records)
-
- assert be_file.bytes_total == len(jsonl_dump)
- assert be_file.bytes_since_last_reset == len(jsonl_dump)
- assert be_file.records_total == len(records)
- assert be_file.records_since_last_reset == len(records)
-
- be_file.seek(0)
- lines = be_file.readlines()
- assert len(lines) == len(records)
-
- for line_index, jsonl_record in enumerate(lines):
- json_loaded = json.loads(jsonl_record)
- assert json_loaded == records[line_index]
-
- be_file.reset()
-
- assert be_file.bytes_total == len(jsonl_dump)
- assert be_file.bytes_since_last_reset == 0
- assert be_file.records_total == len(records)
- assert be_file.records_since_last_reset == 0
-
-
-@pytest.mark.parametrize(
- "records",
- TEST_RECORDS,
-)
-def test_batch_export_temporary_file_write_records_to_csv(records):
- """Test CSV written by BatchExportTemporaryFile match expected."""
- in_memory_file_obj = io.StringIO()
- writer = csv.DictWriter(
- in_memory_file_obj,
- fieldnames=records[0].keys() if len(records) > 0 else [],
- delimiter=",",
- quotechar='"',
- escapechar="\\",
- lineterminator="\n",
- quoting=csv.QUOTE_NONE,
- )
- writer.writerows(records)
-
- with BatchExportTemporaryFile(mode="w+") as be_file:
- be_file.write_records_to_csv(records)
-
- assert be_file.bytes_total == in_memory_file_obj.tell()
- assert be_file.bytes_since_last_reset == in_memory_file_obj.tell()
- assert be_file.records_total == len(records)
- assert be_file.records_since_last_reset == len(records)
-
- be_file.seek(0)
- reader = csv.reader(
- be_file._file,
- delimiter=",",
- quotechar='"',
- escapechar="\\",
- quoting=csv.QUOTE_NONE,
- )
-
- rows = [row for row in reader]
- assert len(rows) == len(records)
-
- for row_index, csv_record in enumerate(rows):
- for value_index, value in enumerate(records[row_index].values()):
- # Everything returned by csv.reader is a str.
- # This means type information is lost when writing to CSV
- # but this just a limitation of the format.
- assert csv_record[value_index] == str(value)
-
- be_file.reset()
-
- assert be_file.bytes_total == in_memory_file_obj.tell()
- assert be_file.bytes_since_last_reset == 0
- assert be_file.records_total == len(records)
- assert be_file.records_since_last_reset == 0
-
-
-@pytest.mark.parametrize(
- "records",
- TEST_RECORDS,
-)
-def test_batch_export_temporary_file_write_records_to_tsv(records):
- """Test TSV written by BatchExportTemporaryFile match expected."""
- in_memory_file_obj = io.StringIO()
- writer = csv.DictWriter(
- in_memory_file_obj,
- fieldnames=records[0].keys() if len(records) > 0 else [],
- delimiter="\t",
- quotechar='"',
- escapechar="\\",
- lineterminator="\n",
- quoting=csv.QUOTE_NONE,
- )
- writer.writerows(records)
-
- with BatchExportTemporaryFile(mode="w+") as be_file:
- be_file.write_records_to_tsv(records)
-
- assert be_file.bytes_total == in_memory_file_obj.tell()
- assert be_file.bytes_since_last_reset == in_memory_file_obj.tell()
- assert be_file.records_total == len(records)
- assert be_file.records_since_last_reset == len(records)
-
- be_file.seek(0)
- reader = csv.reader(
- be_file._file,
- delimiter="\t",
- quotechar='"',
- escapechar="\\",
- quoting=csv.QUOTE_NONE,
- )
-
- rows = [row for row in reader]
- assert len(rows) == len(records)
-
- for row_index, csv_record in enumerate(rows):
- for value_index, value in enumerate(records[row_index].values()):
- # Everything returned by csv.reader is a str.
- # This means type information is lost when writing to CSV
- # but this just a limitation of the format.
- assert csv_record[value_index] == str(value)
-
- be_file.reset()
-
- assert be_file.bytes_total == in_memory_file_obj.tell()
- assert be_file.bytes_since_last_reset == 0
- assert be_file.records_total == len(records)
- assert be_file.records_since_last_reset == 0
diff --git a/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py b/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py
index e04e345d11245..e6583d049e2a8 100644
--- a/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py
+++ b/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py
@@ -10,10 +10,12 @@
import aioboto3
import botocore.exceptions
import brotli
+import pyarrow.parquet as pq
import pytest
import pytest_asyncio
from django.conf import settings
from django.test import override_settings
+from pyarrow import fs
from temporalio import activity
from temporalio.client import WorkflowFailureError
from temporalio.common import RetryPolicy
@@ -27,6 +29,7 @@
update_export_run_status,
)
from posthog.temporal.batch_exports.s3_batch_export import (
+ FILE_FORMAT_EXTENSIONS,
HeartbeatDetails,
S3BatchExportInputs,
S3BatchExportWorkflow,
@@ -107,6 +110,15 @@ def s3_key_prefix():
return f"posthog-events-{str(uuid4())}"
+@pytest.fixture
+def file_format(request) -> str:
+ """S3 file format."""
+ try:
+ return request.param
+ except AttributeError:
+ return f"JSONLines"
+
+
async def delete_all_from_s3(minio_client, bucket_name: str, key_prefix: str):
"""Delete all objects in bucket_name under key_prefix."""
response = await minio_client.list_objects_v2(Bucket=bucket_name, Prefix=key_prefix)
@@ -138,6 +150,61 @@ async def minio_client(bucket_name):
await minio_client.delete_bucket(Bucket=bucket_name)
+async def read_parquet_from_s3(bucket_name: str, key: str, json_columns) -> list:
+ async with aioboto3.Session().client("sts") as sts:
+ try:
+ await sts.get_caller_identity()
+ except botocore.exceptions.NoCredentialsError:
+ s3 = fs.S3FileSystem(
+ access_key="object_storage_root_user",
+ secret_key="object_storage_root_password",
+ endpoint_override=settings.OBJECT_STORAGE_ENDPOINT,
+ )
+
+ else:
+ if os.getenv("S3_TEST_BUCKET") is not None:
+ s3 = fs.S3FileSystem()
+ else:
+ s3 = fs.S3FileSystem(
+ access_key="object_storage_root_user",
+ secret_key="object_storage_root_password",
+ endpoint_override=settings.OBJECT_STORAGE_ENDPOINT,
+ )
+
+ table = pq.read_table(f"{bucket_name}/{key}", filesystem=s3)
+
+ parquet_data = []
+ for batch in table.to_batches():
+ for record in batch.to_pylist():
+ casted_record = {}
+ for k, v in record.items():
+ if isinstance(v, dt.datetime):
+ # We read data from clickhouse as string, but parquet already casts them as dates.
+ # To facilitate comparison, we isoformat the dates.
+ casted_record[k] = v.isoformat()
+ elif k in json_columns and v is not None:
+ # Parquet doesn't have a variable map type, so JSON fields are just strings.
+ casted_record[k] = json.loads(v)
+ else:
+ casted_record[k] = v
+ parquet_data.append(casted_record)
+
+ return parquet_data
+
+
+def read_s3_data_as_json(data: bytes, compression: str | None) -> list:
+ match compression:
+ case "gzip":
+ data = gzip.decompress(data)
+ case "brotli":
+ data = brotli.decompress(data)
+ case _:
+ pass
+
+ json_data = [json.loads(line) for line in data.decode("utf-8").split("\n") if line]
+ return json_data
+
+
async def assert_clickhouse_records_in_s3(
s3_compatible_client,
clickhouse_client: ClickHouseClient,
@@ -150,6 +217,7 @@ async def assert_clickhouse_records_in_s3(
include_events: list[str] | None = None,
batch_export_schema: BatchExportSchema | None = None,
compression: str | None = None,
+ file_format: str = "JSONLines",
):
"""Assert ClickHouse records are written to JSON in key_prefix in S3 bucket_name.
@@ -175,28 +243,24 @@ async def assert_clickhouse_records_in_s3(
# Get the object.
key = objects["Contents"][0].get("Key")
assert key
- s3_object = await s3_compatible_client.get_object(Bucket=bucket_name, Key=key)
- data = await s3_object["Body"].read()
- # Check that the data is correct.
- match compression:
- case "gzip":
- data = gzip.decompress(data)
- case "brotli":
- data = brotli.decompress(data)
- case _:
- pass
+ json_columns = ("properties", "person_properties", "set", "set_once")
- json_data = [json.loads(line) for line in data.decode("utf-8").split("\n") if line]
- # Pull out the fields we inserted only
+ if file_format == "Parquet":
+ s3_data = await read_parquet_from_s3(bucket_name, key, json_columns)
+
+ elif file_format == "JSONLines":
+ s3_object = await s3_compatible_client.get_object(Bucket=bucket_name, Key=key)
+ data = await s3_object["Body"].read()
+ s3_data = read_s3_data_as_json(data, compression)
+ else:
+ raise ValueError(f"Unsupported file format: {file_format}")
if batch_export_schema is not None:
schema_column_names = [field["alias"] for field in batch_export_schema["fields"]]
else:
schema_column_names = [field["alias"] for field in s3_default_fields()]
- json_columns = ("properties", "person_properties", "set", "set_once")
-
expected_records = []
for record_batch in iter_records(
client=clickhouse_client,
@@ -225,9 +289,9 @@ async def assert_clickhouse_records_in_s3(
expected_records.append(expected_record)
- assert len(json_data) == len(expected_records)
- assert json_data[0] == expected_records[0]
- assert json_data == expected_records
+ assert len(s3_data) == len(expected_records)
+ assert s3_data[0] == expected_records[0]
+ assert s3_data == expected_records
TEST_S3_SCHEMAS: list[BatchExportSchema | None] = [
@@ -255,6 +319,7 @@ async def assert_clickhouse_records_in_s3(
@pytest.mark.parametrize("compression", [None, "gzip", "brotli"], indirect=True)
@pytest.mark.parametrize("exclude_events", [None, ["test-exclude"]], indirect=True)
@pytest.mark.parametrize("batch_export_schema", TEST_S3_SCHEMAS)
+@pytest.mark.parametrize("file_format", FILE_FORMAT_EXTENSIONS.keys())
async def test_insert_into_s3_activity_puts_data_into_s3(
clickhouse_client,
bucket_name,
@@ -262,6 +327,7 @@ async def test_insert_into_s3_activity_puts_data_into_s3(
activity_environment,
compression,
exclude_events,
+ file_format,
batch_export_schema: BatchExportSchema | None,
):
"""Test that the insert_into_s3_activity function ends up with data into S3.
@@ -339,12 +405,15 @@ async def test_insert_into_s3_activity_puts_data_into_s3(
compression=compression,
exclude_events=exclude_events,
batch_export_schema=batch_export_schema,
+ file_format=file_format,
)
with override_settings(
BATCH_EXPORT_S3_UPLOAD_CHUNK_SIZE_BYTES=5 * 1024**2
): # 5MB, the minimum for Multipart uploads
- await activity_environment.run(insert_into_s3_activity, insert_inputs)
+ records_total = await activity_environment.run(insert_into_s3_activity, insert_inputs)
+
+ assert records_total == 10005
await assert_clickhouse_records_in_s3(
s3_compatible_client=minio_client,
@@ -358,6 +427,7 @@ async def test_insert_into_s3_activity_puts_data_into_s3(
exclude_events=exclude_events,
include_events=None,
compression=compression,
+ file_format=file_format,
)
@@ -371,6 +441,7 @@ async def s3_batch_export(
exclude_events,
temporal_client,
encryption,
+ file_format,
):
destination_data = {
"type": "S3",
@@ -385,6 +456,7 @@ async def s3_batch_export(
"exclude_events": exclude_events,
"encryption": encryption,
"kms_key_id": os.getenv("S3_TEST_KMS_KEY_ID") if encryption == "aws:kms" else None,
+ "file_format": file_format,
},
}
@@ -410,6 +482,7 @@ async def s3_batch_export(
@pytest.mark.parametrize("compression", [None, "gzip", "brotli"], indirect=True)
@pytest.mark.parametrize("exclude_events", [None, ["test-exclude"]], indirect=True)
@pytest.mark.parametrize("batch_export_schema", TEST_S3_SCHEMAS)
+@pytest.mark.parametrize("file_format", FILE_FORMAT_EXTENSIONS.keys(), indirect=True)
async def test_s3_export_workflow_with_minio_bucket(
clickhouse_client,
minio_client,
@@ -421,6 +494,7 @@ async def test_s3_export_workflow_with_minio_bucket(
exclude_events,
s3_key_prefix,
batch_export_schema,
+ file_format,
):
"""Test S3BatchExport Workflow end-to-end by using a local MinIO bucket instead of S3.
@@ -508,6 +582,7 @@ async def test_s3_export_workflow_with_minio_bucket(
batch_export_schema=batch_export_schema,
exclude_events=exclude_events,
compression=compression,
+ file_format=file_format,
)
@@ -537,6 +612,7 @@ async def s3_client(bucket_name, s3_key_prefix):
@pytest.mark.parametrize("encryption", [None, "AES256", "aws:kms"], indirect=True)
@pytest.mark.parametrize("bucket_name", [os.getenv("S3_TEST_BUCKET")], indirect=True)
@pytest.mark.parametrize("batch_export_schema", TEST_S3_SCHEMAS)
+@pytest.mark.parametrize("file_format", FILE_FORMAT_EXTENSIONS.keys(), indirect=True)
async def test_s3_export_workflow_with_s3_bucket(
s3_client,
clickhouse_client,
@@ -549,6 +625,7 @@ async def test_s3_export_workflow_with_s3_bucket(
exclude_events,
ateam,
batch_export_schema,
+ file_format,
):
"""Test S3 Export Workflow end-to-end by using an S3 bucket.
@@ -646,6 +723,7 @@ async def test_s3_export_workflow_with_s3_bucket(
exclude_events=exclude_events,
include_events=None,
compression=compression,
+ file_format=file_format,
)
@@ -1206,6 +1284,49 @@ async def never_finish_activity(_: S3InsertInputs) -> str:
),
"nested/prefix/2023-01-01 00:00:00-2023-01-01 01:00:00.jsonl.br",
),
+ (
+ S3InsertInputs(
+ prefix="/nested/prefix/",
+ data_interval_start="2023-01-01 00:00:00",
+ data_interval_end="2023-01-01 01:00:00",
+ file_format="Parquet",
+ compression="snappy",
+ **base_inputs, # type: ignore
+ ),
+ "nested/prefix/2023-01-01 00:00:00-2023-01-01 01:00:00.parquet.sz",
+ ),
+ (
+ S3InsertInputs(
+ prefix="/nested/prefix/",
+ data_interval_start="2023-01-01 00:00:00",
+ data_interval_end="2023-01-01 01:00:00",
+ file_format="Parquet",
+ **base_inputs, # type: ignore
+ ),
+ "nested/prefix/2023-01-01 00:00:00-2023-01-01 01:00:00.parquet",
+ ),
+ (
+ S3InsertInputs(
+ prefix="/nested/prefix/",
+ data_interval_start="2023-01-01 00:00:00",
+ data_interval_end="2023-01-01 01:00:00",
+ compression="gzip",
+ file_format="Parquet",
+ **base_inputs, # type: ignore
+ ),
+ "nested/prefix/2023-01-01 00:00:00-2023-01-01 01:00:00.parquet.gz",
+ ),
+ (
+ S3InsertInputs(
+ prefix="/nested/prefix/",
+ data_interval_start="2023-01-01 00:00:00",
+ data_interval_end="2023-01-01 01:00:00",
+ compression="brotli",
+ file_format="Parquet",
+ **base_inputs, # type: ignore
+ ),
+ "nested/prefix/2023-01-01 00:00:00-2023-01-01 01:00:00.parquet.br",
+ ),
],
)
def test_get_s3_key(inputs, expected):
@@ -1271,7 +1392,7 @@ def assert_heartbeat_details(*details):
endpoint_url=settings.OBJECT_STORAGE_ENDPOINT,
)
- with override_settings(BATCH_EXPORT_S3_UPLOAD_CHUNK_SIZE_BYTES=5 * 1024**2):
+ with override_settings(BATCH_EXPORT_S3_UPLOAD_CHUNK_SIZE_BYTES=1, CLICKHOUSE_MAX_BLOCK_SIZE_DEFAULT=1):
await activity_environment.run(insert_into_s3_activity, insert_inputs)
# This checks that the assert_heartbeat_details function was actually called.
diff --git a/posthog/temporal/tests/batch_exports/test_temporary_file.py b/posthog/temporal/tests/batch_exports/test_temporary_file.py
new file mode 100644
index 0000000000000..4fd7e69c0c12f
--- /dev/null
+++ b/posthog/temporal/tests/batch_exports/test_temporary_file.py
@@ -0,0 +1,389 @@
+import csv
+import datetime as dt
+import io
+import json
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pytest
+
+from posthog.temporal.batch_exports.temporary_file import (
+ BatchExportTemporaryFile,
+ CSVBatchExportWriter,
+ JSONLBatchExportWriter,
+ ParquetBatchExportWriter,
+ json_dumps_bytes,
+)
+
+
+@pytest.mark.parametrize(
+ "to_write",
+ [
+ (b"",),
+ (b"", b""),
+ (b"12345",),
+ (b"12345", b"12345"),
+ (b"abbcccddddeeeee",),
+ (b"abbcccddddeeeee", b"abbcccddddeeeee"),
+ ],
+)
+def test_batch_export_temporary_file_tracks_bytes(to_write):
+ """Test the bytes written by BatchExportTemporaryFile match expected."""
+ with BatchExportTemporaryFile() as be_file:
+ for content in to_write:
+ be_file.write(content)
+
+ assert be_file.bytes_total == sum(len(content) for content in to_write)
+ assert be_file.bytes_since_last_reset == sum(len(content) for content in to_write)
+
+ be_file.reset()
+
+ assert be_file.bytes_total == sum(len(content) for content in to_write)
+ assert be_file.bytes_since_last_reset == 0
+
+
+TEST_RECORDS = [
+ [],
+ [
+ {"id": "record-1", "property": "value", "property_int": 1},
+ {"id": "record-2", "property": "another-value", "property_int": 2},
+ {
+ "id": "record-3",
+ "property": {"id": "nested-record", "property": "nested-value"},
+ "property_int": 3,
+ },
+ ],
+]
+
+
+@pytest.mark.parametrize(
+ "records",
+ TEST_RECORDS,
+)
+def test_batch_export_temporary_file_write_records_to_jsonl(records):
+ """Test JSONL records written by BatchExportTemporaryFile match expected."""
+ jsonl_dump = b"\n".join(map(json_dumps_bytes, records))
+
+ with BatchExportTemporaryFile() as be_file:
+ be_file.write_records_to_jsonl(records)
+
+ assert be_file.bytes_total == len(jsonl_dump)
+ assert be_file.bytes_since_last_reset == len(jsonl_dump)
+ assert be_file.records_total == len(records)
+ assert be_file.records_since_last_reset == len(records)
+
+ be_file.seek(0)
+ lines = be_file.readlines()
+ assert len(lines) == len(records)
+
+ for line_index, jsonl_record in enumerate(lines):
+ json_loaded = json.loads(jsonl_record)
+ assert json_loaded == records[line_index]
+
+ be_file.reset()
+
+ assert be_file.bytes_total == len(jsonl_dump)
+ assert be_file.bytes_since_last_reset == 0
+ assert be_file.records_total == len(records)
+ assert be_file.records_since_last_reset == 0
+
+
+@pytest.mark.parametrize(
+ "records",
+ TEST_RECORDS,
+)
+def test_batch_export_temporary_file_write_records_to_csv(records):
+ """Test CSV written by BatchExportTemporaryFile match expected."""
+ in_memory_file_obj = io.StringIO()
+ writer = csv.DictWriter(
+ in_memory_file_obj,
+ fieldnames=records[0].keys() if len(records) > 0 else [],
+ delimiter=",",
+ quotechar='"',
+ escapechar="\\",
+ lineterminator="\n",
+ quoting=csv.QUOTE_NONE,
+ )
+ writer.writerows(records)
+
+ with BatchExportTemporaryFile(mode="w+") as be_file:
+ be_file.write_records_to_csv(records)
+
+ assert be_file.bytes_total == in_memory_file_obj.tell()
+ assert be_file.bytes_since_last_reset == in_memory_file_obj.tell()
+ assert be_file.records_total == len(records)
+ assert be_file.records_since_last_reset == len(records)
+
+ be_file.seek(0)
+ reader = csv.reader(
+ be_file._file,
+ delimiter=",",
+ quotechar='"',
+ escapechar="\\",
+ quoting=csv.QUOTE_NONE,
+ )
+
+ rows = [row for row in reader]
+ assert len(rows) == len(records)
+
+ for row_index, csv_record in enumerate(rows):
+ for value_index, value in enumerate(records[row_index].values()):
+ # Everything returned by csv.reader is a str.
+ # This means type information is lost when writing to CSV
+ # but this just a limitation of the format.
+ assert csv_record[value_index] == str(value)
+
+ be_file.reset()
+
+ assert be_file.bytes_total == in_memory_file_obj.tell()
+ assert be_file.bytes_since_last_reset == 0
+ assert be_file.records_total == len(records)
+ assert be_file.records_since_last_reset == 0
+
+
+@pytest.mark.parametrize(
+ "records",
+ TEST_RECORDS,
+)
+def test_batch_export_temporary_file_write_records_to_tsv(records):
+ """Test TSV written by BatchExportTemporaryFile match expected."""
+ in_memory_file_obj = io.StringIO()
+ writer = csv.DictWriter(
+ in_memory_file_obj,
+ fieldnames=records[0].keys() if len(records) > 0 else [],
+ delimiter="\t",
+ quotechar='"',
+ escapechar="\\",
+ lineterminator="\n",
+ quoting=csv.QUOTE_NONE,
+ )
+ writer.writerows(records)
+
+ with BatchExportTemporaryFile(mode="w+") as be_file:
+ be_file.write_records_to_tsv(records)
+
+ assert be_file.bytes_total == in_memory_file_obj.tell()
+ assert be_file.bytes_since_last_reset == in_memory_file_obj.tell()
+ assert be_file.records_total == len(records)
+ assert be_file.records_since_last_reset == len(records)
+
+ be_file.seek(0)
+ reader = csv.reader(
+ be_file._file,
+ delimiter="\t",
+ quotechar='"',
+ escapechar="\\",
+ quoting=csv.QUOTE_NONE,
+ )
+
+ rows = [row for row in reader]
+ assert len(rows) == len(records)
+
+ for row_index, csv_record in enumerate(rows):
+ for value_index, value in enumerate(records[row_index].values()):
+ # Everything returned by csv.reader is a str.
+ # This means type information is lost when writing to CSV
+ # but this just a limitation of the format.
+ assert csv_record[value_index] == str(value)
+
+ be_file.reset()
+
+ assert be_file.bytes_total == in_memory_file_obj.tell()
+ assert be_file.bytes_since_last_reset == 0
+ assert be_file.records_total == len(records)
+ assert be_file.records_since_last_reset == 0
+
+
+TEST_RECORD_BATCHES = [
+ pa.RecordBatch.from_pydict(
+ {
+ "event": pa.array(["test-event-0", "test-event-1", "test-event-2"]),
+ "properties": pa.array(['{"prop_0": 1, "prop_1": 2}', "{}", "null"]),
+ "_inserted_at": pa.array([0, 1, 2]),
+ }
+ )
+]
+
+
+@pytest.mark.parametrize(
+ "record_batch",
+ TEST_RECORD_BATCHES,
+)
+@pytest.mark.asyncio
+async def test_jsonl_writer_writes_record_batches(record_batch):
+ """Test record batches are written as valid JSONL."""
+ in_memory_file_obj = io.BytesIO()
+ inserted_ats_seen = []
+
+ async def store_in_memory_on_flush(
+ batch_export_file, records_since_last_flush, bytes_since_last_flush, last_inserted_at, is_last
+ ):
+ in_memory_file_obj.write(batch_export_file.read())
+ inserted_ats_seen.append(last_inserted_at)
+
+ writer = JSONLBatchExportWriter(max_bytes=1, flush_callable=store_in_memory_on_flush)
+
+ record_batch = record_batch.sort_by("_inserted_at")
+ async with writer.open_temporary_file():
+ await writer.write_record_batch(record_batch)
+
+ lines = in_memory_file_obj.readlines()
+ for index, line in enumerate(lines):
+ written_jsonl = json.loads(line)
+
+ single_record_batch = record_batch.slice(offset=index, length=1)
+ expected_jsonl = single_record_batch.to_pylist()[0]
+
+ assert "_inserted_at" not in written_jsonl
+ assert written_jsonl == expected_jsonl
+
+ assert inserted_ats_seen == [record_batch.column("_inserted_at")[-1].as_py()]
+
+
+@pytest.mark.parametrize(
+ "record_batch",
+ TEST_RECORD_BATCHES,
+)
+@pytest.mark.asyncio
+async def test_csv_writer_writes_record_batches(record_batch):
+ """Test record batches are written as valid CSV."""
+ in_memory_file_obj = io.StringIO()
+ inserted_ats_seen = []
+
+ async def store_in_memory_on_flush(
+ batch_export_file, records_since_last_flush, bytes_since_last_flush, last_inserted_at, is_last
+ ):
+ in_memory_file_obj.write(batch_export_file.read().decode("utf-8"))
+ inserted_ats_seen.append(last_inserted_at)
+
+ schema_columns = [column_name for column_name in record_batch.column_names if column_name != "_inserted_at"]
+ writer = CSVBatchExportWriter(max_bytes=1, field_names=schema_columns, flush_callable=store_in_memory_on_flush)
+
+ record_batch = record_batch.sort_by("_inserted_at")
+ async with writer.open_temporary_file():
+ await writer.write_record_batch(record_batch)
+
+ reader = csv.reader(
+ in_memory_file_obj,
+ delimiter=",",
+ quotechar='"',
+ escapechar="\\",
+ quoting=csv.QUOTE_NONE,
+ )
+ for index, written_csv_row in enumerate(reader):
+ single_record_batch = record_batch.slice(offset=index, length=1)
+ expected_csv = single_record_batch.to_pylist()[0]
+
+ assert "_inserted_at" not in written_csv_row
+ assert written_csv_row == expected_csv
+
+ assert inserted_ats_seen == [record_batch.column("_inserted_at")[-1].as_py()]
+
+
+@pytest.mark.parametrize(
+ "record_batch",
+ TEST_RECORD_BATCHES,
+)
+@pytest.mark.asyncio
+async def test_parquet_writer_writes_record_batches(record_batch):
+ """Test record batches are written as valid Parquet."""
+ in_memory_file_obj = io.BytesIO()
+ inserted_ats_seen = []
+
+ async def store_in_memory_on_flush(
+ batch_export_file, records_since_last_flush, bytes_since_last_flush, last_inserted_at, is_last
+ ):
+ in_memory_file_obj.write(batch_export_file.read())
+ inserted_ats_seen.append(last_inserted_at)
+
+ schema_columns = [column_name for column_name in record_batch.column_names if column_name != "_inserted_at"]
+
+ writer = ParquetBatchExportWriter(
+ max_bytes=1,
+ flush_callable=store_in_memory_on_flush,
+ schema=record_batch.select(schema_columns).schema,
+ )
+
+ record_batch = record_batch.sort_by("_inserted_at")
+ async with writer.open_temporary_file():
+ await writer.write_record_batch(record_batch)
+
+ written_parquet = pq.read_table(in_memory_file_obj)
+
+ for index, written_row_as_dict in enumerate(written_parquet.to_pylist()):
+ single_record_batch = record_batch.slice(offset=index, length=1)
+ expected_row_as_dict = single_record_batch.select(schema_columns).to_pylist()[0]
+
+ assert "_inserted_at" not in written_row_as_dict
+ assert written_row_as_dict == expected_row_as_dict
+
+ # NOTE: Parquet gets flushed twice due to the extra flush at the end for footer bytes, so our mock function
+ # will see this value twice.
+ assert inserted_ats_seen == [
+ record_batch.column("_inserted_at")[-1].as_py(),
+ record_batch.column("_inserted_at")[-1].as_py(),
+ ]
+
+
+@pytest.mark.parametrize(
+ "record_batch",
+ TEST_RECORD_BATCHES,
+)
+@pytest.mark.asyncio
+async def test_writing_out_of_scope_of_temporary_file_raises(record_batch):
+ """Test attempting a write out of temporary file scope raises a `ValueError`."""
+
+ async def do_nothing(*args, **kwargs):
+ pass
+
+ schema_columns = [column_name for column_name in record_batch.column_names if column_name != "_inserted_at"]
+ writer = ParquetBatchExportWriter(
+ max_bytes=10,
+ flush_callable=do_nothing,
+ schema=record_batch.select(schema_columns).schema,
+ )
+
+ async with writer.open_temporary_file():
+ pass
+
+ with pytest.raises(ValueError, match="Batch export file is closed"):
+ await writer.write_record_batch(record_batch)
+
+
+@pytest.mark.parametrize(
+ "record_batch",
+ TEST_RECORD_BATCHES,
+)
+@pytest.mark.asyncio
+async def test_flushing_parquet_writer_resets_underlying_file(record_batch):
+ """Test flushing a writer resets underlying file."""
+ flush_counter = 0
+
+ async def track_flushes(*args, **kwargs):
+ nonlocal flush_counter
+ flush_counter += 1
+
+ schema_columns = [column_name for column_name in record_batch.column_names if column_name != "_inserted_at"]
+ writer = ParquetBatchExportWriter(
+ max_bytes=10000000,
+ flush_callable=track_flushes,
+ schema=record_batch.select(schema_columns).schema,
+ )
+
+ async with writer.open_temporary_file():
+ await writer.write_record_batch(record_batch)
+
+ assert writer.batch_export_file.tell() > 0
+ assert writer.bytes_since_last_flush > 0
+ assert writer.bytes_since_last_flush == writer.batch_export_file.bytes_since_last_reset
+ assert writer.records_since_last_flush == record_batch.num_rows
+
+ await writer.flush(dt.datetime.now())
+
+ assert flush_counter == 1
+ assert writer.batch_export_file.tell() == 0
+ assert writer.bytes_since_last_flush == 0
+ assert writer.bytes_since_last_flush == writer.batch_export_file.bytes_since_last_reset
+ assert writer.records_since_last_flush == 0
+
+ assert flush_counter == 2
diff --git a/posthog/warehouse/api/external_data_source.py b/posthog/warehouse/api/external_data_source.py
index 1586f1051379e..6eb06abcad70f 100644
--- a/posthog/warehouse/api/external_data_source.py
+++ b/posthog/warehouse/api/external_data_source.py
@@ -136,6 +136,8 @@ def create(self, request: Request, *args: Any, **kwargs: Any) -> Response:
new_source_model = self._handle_stripe_source(request, *args, **kwargs)
elif source_type == ExternalDataSource.Type.HUBSPOT:
new_source_model = self._handle_hubspot_source(request, *args, **kwargs)
+ elif source_type == ExternalDataSource.Type.ZENDESK:
+ new_source_model = self._handle_zendesk_source(request, *args, **kwargs)
elif source_type == ExternalDataSource.Type.POSTGRES:
try:
new_source_model, table_names = self._handle_postgres_source(request, *args, **kwargs)
@@ -190,6 +192,33 @@ def _handle_stripe_source(self, request: Request, *args: Any, **kwargs: Any) ->
return new_source_model
+ def _handle_zendesk_source(self, request: Request, *args: Any, **kwargs: Any) -> ExternalDataSource:
+ payload = request.data["payload"]
+ api_key = payload.get("api_key")
+ subdomain = payload.get("subdomain")
+ email_address = payload.get("email_address")
+ prefix = request.data.get("prefix", None)
+ source_type = request.data["source_type"]
+
+ # TODO: remove dummy vars
+ new_source_model = ExternalDataSource.objects.create(
+ source_id=str(uuid.uuid4()),
+ connection_id=str(uuid.uuid4()),
+ destination_id=str(uuid.uuid4()),
+ team=self.team,
+ status="Running",
+ source_type=source_type,
+ job_inputs={
+ "zendesk_login_method": "api_key", # We should support the Zendesk OAuth flow in the future, and so with this we can do backwards compatibility
+ "zendesk_api_key": api_key,
+ "zendesk_subdomain": subdomain,
+ "zendesk_email_address": email_address,
+ },
+ prefix=prefix,
+ )
+
+ return new_source_model
+
def _handle_hubspot_source(self, request: Request, *args: Any, **kwargs: Any) -> ExternalDataSource:
payload = request.data["payload"]
code = payload.get("code")
diff --git a/posthog/warehouse/models/external_data_source.py b/posthog/warehouse/models/external_data_source.py
index df668c5abfc54..0a044c0b06315 100644
--- a/posthog/warehouse/models/external_data_source.py
+++ b/posthog/warehouse/models/external_data_source.py
@@ -12,6 +12,7 @@ class Type(models.TextChoices):
STRIPE = "Stripe", "Stripe"
HUBSPOT = "Hubspot", "Hubspot"
POSTGRES = "Postgres", "Postgres"
+ ZENDESK = "Zendesk", "Zendesk"
class Status(models.TextChoices):
RUNNING = "Running", "Running"
diff --git a/posthog/warehouse/models/table.py b/posthog/warehouse/models/table.py
index 91c6f61709d6e..23cc5a7ce9541 100644
--- a/posthog/warehouse/models/table.py
+++ b/posthog/warehouse/models/table.py
@@ -1,4 +1,4 @@
-from typing import Dict
+from typing import Dict, Optional
from django.db import models
from posthog.client import sync_execute
@@ -175,6 +175,17 @@ def hogql_definition(self) -> S3Table:
structure=", ".join(structure),
)
+ def get_clickhouse_column_type(self, column_name: str) -> Optional[str]:
+ clickhouse_type = self.columns.get(column_name, None)
+
+ if isinstance(clickhouse_type, dict) and self.columns[column_name].get("clickhouse"):
+ clickhouse_type = self.columns[column_name].get("clickhouse")
+
+ if clickhouse_type.startswith("Nullable("):
+ clickhouse_type = clickhouse_type.replace("Nullable(", "")[:-1]
+
+ return clickhouse_type
+
def _safe_expose_ch_error(self, err):
err = wrap_query_error(err)
for key, value in ExtractErrors.items():