{displayTitle}
@@ -45,6 +46,17 @@ export function SessionRecordingErrors(): JSX.Element {
},
width: '50%',
},
+ {
+ title: '',
+ render: (_, cluster) => {
+ return (
+
+ )
+ },
+ },
{
title: 'Occurrences',
dataIndex: 'occurrences',
@@ -68,23 +80,40 @@ export function SessionRecordingErrors(): JSX.Element {
title: 'Actions',
render: function Render(_, cluster) {
return (
-
{
- e.preventDefault()
- openSessionPlayer({ id: cluster.sample.session_id })
- }}
- className="p-2 whitespace-nowrap"
- type="primary"
- >
- Watch example
-
+
+ {
+ e.preventDefault()
+ openSessionPlayer({ id: cluster.session_ids[0] })
+ }}
+ className="whitespace-nowrap"
+ type="primary"
+ >
+ Watch example
+
+ {
+ createPlaylist(
+ `Examples of '${parseTitle(cluster.sample)}'`,
+ cluster.session_ids
+ )
+ }}
+ className="whitespace-nowrap"
+ type="secondary"
+ tooltip="Create a playlist of recordings containing this issue"
+ >
+ Create playlist
+
+
)
},
},
]}
dataSource={errors}
- expandable={{ expandedRowRender: (cluster) =>
}}
+ expandable={{
+ expandedRowRender: (cluster) =>
,
+ }}
/>
diff --git a/frontend/src/scenes/session-recordings/errors/sessionRecordingErrorsLogic.ts b/frontend/src/scenes/session-recordings/errors/sessionRecordingErrorsLogic.ts
index 45b887fd33cbb..49de62c7bf5c4 100644
--- a/frontend/src/scenes/session-recordings/errors/sessionRecordingErrorsLogic.ts
+++ b/frontend/src/scenes/session-recordings/errors/sessionRecordingErrorsLogic.ts
@@ -1,13 +1,19 @@
-import { afterMount, kea, path } from 'kea'
+import { actions, afterMount, kea, listeners, path } from 'kea'
import { loaders } from 'kea-loaders'
+import { router } from 'kea-router'
import api from 'lib/api'
+import { urls } from 'scenes/urls'
import { ErrorClusterResponse } from '~/types'
+import { createPlaylist } from '../playlist/playlistUtils'
import type { sessionRecordingErrorsLogicType } from './sessionRecordingErrorsLogicType'
export const sessionRecordingErrorsLogic = kea
([
path(['scenes', 'session-recordings', 'detail', 'sessionRecordingErrorsLogic']),
+ actions({
+ createPlaylist: (name: string, sessionIds: string[]) => ({ name, sessionIds }),
+ }),
loaders(() => ({
errors: [
null as ErrorClusterResponse,
@@ -19,7 +25,19 @@ export const sessionRecordingErrorsLogic = kea(
},
],
})),
+ listeners(() => ({
+ createPlaylist: async ({ name, sessionIds }) => {
+ const playlist = await createPlaylist({ name: name })
+ if (playlist) {
+ const samples = sessionIds.slice(0, 10)
+ await Promise.all(
+ samples.map((sessionId) => api.recordings.addRecordingToPlaylist(playlist.short_id, sessionId))
+ )
+ router.actions.push(urls.replayPlaylist(playlist.short_id))
+ }
+ },
+ })),
afterMount(({ actions }) => {
actions.loadErrorClusters(false)
}),
diff --git a/frontend/src/scenes/session-recordings/file-playback/sessionRecordingFilePlaybackLogic.ts b/frontend/src/scenes/session-recordings/file-playback/sessionRecordingFilePlaybackLogic.ts
index fb01f15b9a0cc..b6f547603114b 100644
--- a/frontend/src/scenes/session-recordings/file-playback/sessionRecordingFilePlaybackLogic.ts
+++ b/frontend/src/scenes/session-recordings/file-playback/sessionRecordingFilePlaybackLogic.ts
@@ -3,7 +3,6 @@ import { eventWithTime } from '@rrweb/types'
import { BuiltLogic, connect, kea, listeners, path, reducers, selectors } from 'kea'
import { loaders } from 'kea-loaders'
import { beforeUnload } from 'kea-router'
-import { FEATURE_FLAGS } from 'lib/constants'
import { dayjs } from 'lib/dayjs'
import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { uuid } from 'lib/utils'
@@ -178,11 +177,7 @@ export const sessionRecordingFilePlaybackLogic = kea void }): J
setTimestampMode(timestampMode === 'absolute' ? 'relative' : 'absolute')}
tooltipPlacement="left"
@@ -191,14 +192,15 @@ export function PlayerInspectorControls({ onClose }: { onClose: () => void }): J
{
- // If the user has syncScrolling on but it is paused due to interacting with the Inspector, we want to resume it
+ // If the user has syncScrolling on, but it is paused due to interacting with the Inspector, we want to resume it
if (syncScroll && syncScrollingPaused) {
setSyncScrollPaused(false)
} else {
- // Otherwise we are just toggling the settting
+ // Otherwise we are just toggling the setting
setSyncScroll(!syncScroll)
}
}}
diff --git a/frontend/src/scenes/session-recordings/player/sessionRecordingDataLogic.ts b/frontend/src/scenes/session-recordings/player/sessionRecordingDataLogic.ts
index 8d630aa10acbb..0aff48444a7e9 100644
--- a/frontend/src/scenes/session-recordings/player/sessionRecordingDataLogic.ts
+++ b/frontend/src/scenes/session-recordings/player/sessionRecordingDataLogic.ts
@@ -67,9 +67,10 @@ function isRecordingSnapshot(x: unknown): x is RecordingSnapshot {
export const parseEncodedSnapshots = async (
items: (RecordingSnapshot | EncodedRecordingSnapshot | string)[],
sessionId: string,
- withMobileTransformer: boolean
+ // this is only kept so that we can export the untransformed data for debugging
+ withMobileTransformer: boolean = true
): Promise => {
- if (!postHogEEModule && withMobileTransformer) {
+ if (!postHogEEModule) {
postHogEEModule = await posthogEE()
}
const lineCount = items.length
@@ -239,11 +240,7 @@ async function processEncodedResponse(
let untransformed: RecordingSnapshot[] | null = null
const transformed = deduplicateSnapshots(
- await parseEncodedSnapshots(
- encodedResponse,
- props.sessionRecordingId,
- !!featureFlags[FEATURE_FLAGS.SESSION_REPLAY_MOBILE]
- ),
+ await parseEncodedSnapshots(encodedResponse, props.sessionRecordingId),
existingData?.snapshots ?? []
)
diff --git a/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx b/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx
index 22dd5881ae378..3c2a9842c0dbc 100644
--- a/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx
+++ b/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylist.tsx
@@ -6,7 +6,6 @@ import clsx from 'clsx'
import { range } from 'd3'
import { BindLogic, useActions, useValues } from 'kea'
import { EmptyMessage } from 'lib/components/EmptyMessage/EmptyMessage'
-import { FlaggedFeature } from 'lib/components/FlaggedFeature'
import { PropertyKeyInfo } from 'lib/components/PropertyKeyInfo'
import { FEATURE_FLAGS } from 'lib/constants'
import { useResizeBreakpoints } from 'lib/hooks/useResizeObserver'
@@ -58,12 +57,10 @@ function UnusableEventsWarning(props: { unusableEventsInFilter: string[] }): JSX
the Web SDK
-
- ,{' '}
-
- the Android SDK
-
-
+ ,{' '}
+
+ the Android SDK
+
)
diff --git a/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylistTroubleshooting.tsx b/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylistTroubleshooting.tsx
index bda13153d9ccb..961b0b54fa246 100644
--- a/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylistTroubleshooting.tsx
+++ b/frontend/src/scenes/session-recordings/playlist/SessionRecordingsPlaylistTroubleshooting.tsx
@@ -1,7 +1,5 @@
import { LemonDivider, Link } from '@posthog/lemon-ui'
import { useActions, useValues } from 'kea'
-import { FlaggedFeature } from 'lib/components/FlaggedFeature'
-import { FEATURE_FLAGS } from 'lib/constants'
import { playerSettingsLogic } from '../player/playerSettingsLogic'
import { sessionRecordingsPlaylistLogic } from './sessionRecordingsPlaylistLogic'
@@ -21,9 +19,7 @@ export const SessionRecordingsPlaylistTroubleshooting = (): JSX.Element => {
-
- All recording sources:
-
+ All recording sources:
{otherRecordings.length > 0 && hideViewedRecordings && (
-
Viewed recordings hidden.{' '}
@@ -42,10 +38,8 @@ export const SessionRecordingsPlaylistTroubleshooting = (): JSX.Element => {
They are outside the retention period
-
-
- Web recordings
-
+
+ Web recordings
-
: null}
type="secondary"
+ status={matchedRecordings.length > 1 ? 'alt' : undefined}
size="small"
>
{matchedRecordings.length > 1 ? `${matchedRecordings.length} recordings` : 'View recording'}
diff --git a/frontend/src/scenes/trends/viz/ActionsHorizontalBar.tsx b/frontend/src/scenes/trends/viz/ActionsHorizontalBar.tsx
index 03743f0c4dd29..ded94e7815490 100644
--- a/frontend/src/scenes/trends/viz/ActionsHorizontalBar.tsx
+++ b/frontend/src/scenes/trends/viz/ActionsHorizontalBar.tsx
@@ -115,6 +115,10 @@ export function ActionsHorizontalBar({ showPersonsModal = true }: ChartParams):
kind: NodeKind.InsightActorsQuery,
source: query.source,
},
+ additionalSelect: {
+ value_at_data_point: 'event_count',
+ matched_recordings: 'matched_recordings',
+ },
})
} else if (selectedUrl) {
openPersonsModal({
diff --git a/frontend/src/scenes/trends/viz/ActionsLineGraph.tsx b/frontend/src/scenes/trends/viz/ActionsLineGraph.tsx
index b930608587e22..b7f3ba8a46d0f 100644
--- a/frontend/src/scenes/trends/viz/ActionsLineGraph.tsx
+++ b/frontend/src/scenes/trends/viz/ActionsLineGraph.tsx
@@ -152,6 +152,10 @@ export function ActionsLineGraph({
breakdown: dataset.breakdown_value,
compare: dataset.compare_label,
},
+ additionalSelect: {
+ value_at_data_point: 'event_count',
+ matched_recordings: 'matched_recordings',
+ },
})
} else {
const datasetUrls = urlsForDatasets(
diff --git a/frontend/src/scenes/trends/viz/ActionsPie.tsx b/frontend/src/scenes/trends/viz/ActionsPie.tsx
index 86804b11e487f..839e06f0d71b6 100644
--- a/frontend/src/scenes/trends/viz/ActionsPie.tsx
+++ b/frontend/src/scenes/trends/viz/ActionsPie.tsx
@@ -118,6 +118,10 @@ export function ActionsPie({
kind: NodeKind.InsightActorsQuery,
source: query.source,
},
+ additionalSelect: {
+ value_at_data_point: 'event_count',
+ matched_recordings: 'matched_recordings',
+ },
})
} else if (selectedUrl) {
openPersonsModal({
diff --git a/frontend/src/toolbar/actions/ActionsListView.tsx b/frontend/src/toolbar/actions/ActionsListView.tsx
index 6054c0bc65241..fb999ed600949 100644
--- a/frontend/src/toolbar/actions/ActionsListView.tsx
+++ b/frontend/src/toolbar/actions/ActionsListView.tsx
@@ -1,7 +1,6 @@
import { Link } from '@posthog/lemon-ui'
import { useActions, useValues } from 'kea'
import { Spinner } from 'lib/lemon-ui/Spinner'
-import { useEffect } from 'react'
import { actionsLogic } from '~/toolbar/actions/actionsLogic'
import { actionsTabLogic } from '~/toolbar/actions/actionsTabLogic'
@@ -13,13 +12,8 @@ interface ActionsListViewProps {
export function ActionsListView({ actions }: ActionsListViewProps): JSX.Element {
const { allActionsLoading, searchTerm } = useValues(actionsLogic)
- const { getActions } = useActions(actionsLogic)
const { selectAction } = useActions(actionsTabLogic)
- useEffect(() => {
- getActions()
- }, [])
-
return (
{actions.length ? (
diff --git a/frontend/src/toolbar/actions/ActionsToolbarMenu.tsx b/frontend/src/toolbar/actions/ActionsToolbarMenu.tsx
index ecbb3800c9260..87ff37b719ee8 100644
--- a/frontend/src/toolbar/actions/ActionsToolbarMenu.tsx
+++ b/frontend/src/toolbar/actions/ActionsToolbarMenu.tsx
@@ -5,6 +5,7 @@ import { LemonButton } from 'lib/lemon-ui/LemonButton'
import { LemonInput } from 'lib/lemon-ui/LemonInput'
import { Link } from 'lib/lemon-ui/Link'
import { Spinner } from 'lib/lemon-ui/Spinner'
+import { useEffect } from 'react'
import { urls } from 'scenes/urls'
import { ActionsEditingToolbarMenu } from '~/toolbar/actions/ActionsEditingToolbarMenu'
@@ -16,13 +17,17 @@ import { toolbarConfigLogic } from '~/toolbar/toolbarConfigLogic'
const ActionsListToolbarMenu = (): JSX.Element => {
const { searchTerm } = useValues(actionsLogic)
- const { setSearchTerm } = useActions(actionsLogic)
+ const { setSearchTerm, getActions } = useActions(actionsLogic)
const { newAction } = useActions(actionsTabLogic)
const { allActions, sortedActions, allActionsLoading } = useValues(actionsLogic)
const { apiURL } = useValues(toolbarConfigLogic)
+ useEffect(() => {
+ getActions()
+ }, [])
+
return (
diff --git a/frontend/src/toolbar/bar/Toolbar.scss b/frontend/src/toolbar/bar/Toolbar.scss
index ff83b7c9d922a..ec98167332fbd 100644
--- a/frontend/src/toolbar/bar/Toolbar.scss
+++ b/frontend/src/toolbar/bar/Toolbar.scss
@@ -144,8 +144,4 @@
transform: var(--toolbar-translate) scale(0);
}
}
-
- &--unauthenticated {
- width: calc(5rem + 1px); // Account for border
- }
}
diff --git a/frontend/src/toolbar/bar/Toolbar.tsx b/frontend/src/toolbar/bar/Toolbar.tsx
index f255ccc26800d..8ed031f904d86 100644
--- a/frontend/src/toolbar/bar/Toolbar.tsx
+++ b/frontend/src/toolbar/bar/Toolbar.tsx
@@ -77,15 +77,18 @@ function MoreMenu(): JSX.Element {
}
maxContentWidth={true}
>
- } title="More options" />
+
+
+
)
}
-export function ToolbarInfoMenu(): JSX.Element {
+export function ToolbarInfoMenu(): JSX.Element | null {
const ref = useRef(null)
const { visibleMenu, isDragging, menuProperties, minimized, isBlurred } = useValues(toolbarLogic)
const { setMenu } = useActions(toolbarLogic)
+ const { isAuthenticated } = useValues(toolbarConfigLogic)
const content = minimized ? null : visibleMenu === 'flags' ? (
@@ -102,6 +105,10 @@ export function ToolbarInfoMenu(): JSX.Element {
return () => setMenu(null)
}, [ref.current])
+ if (!isAuthenticated) {
+ return null
+ }
+
return (
}
onClick={isAuthenticated ? toggleMinimized : authenticate}
title={isAuthenticated ? 'Minimize' : 'Authenticate the PostHog Toolbar'}
titleMinimized={isAuthenticated ? 'Expand the toolbar' : 'Authenticate the PostHog Toolbar'}
- />
+ >
+
+
{isAuthenticated ? (
<>
- } menuId="inspect" />
- } menuId="heatmap" />
- } menuId="actions" />
- } menuId="flags" title="Feature flags" />
+
+
+
+
+
+
+
+
+
+
+
+
- ) : null}
+ ) : (
+
+ Authenticate
+
+ )}
diff --git a/frontend/src/toolbar/bar/ToolbarButton.scss b/frontend/src/toolbar/bar/ToolbarButton.scss
index 0d0bb666fa540..ce480f3fbab35 100644
--- a/frontend/src/toolbar/bar/ToolbarButton.scss
+++ b/frontend/src/toolbar/bar/ToolbarButton.scss
@@ -15,6 +15,8 @@
width: 2rem;
height: 2rem;
min-height: var(--lemon-button-height);
+ margin: 0.25rem;
+ font-weight: 600;
color: var(--muted-alt);
appearance: none !important; // Important as this gets overridden by Ant styles...
cursor: pointer;
@@ -43,4 +45,13 @@
}
}
}
+
+ &--flex {
+ flex-grow: 1;
+ width: auto;
+
+ button {
+ width: 100%;
+ }
+ }
}
diff --git a/frontend/src/toolbar/bar/ToolbarButton.tsx b/frontend/src/toolbar/bar/ToolbarButton.tsx
index add0e5f2580ce..f5dfc755be469 100644
--- a/frontend/src/toolbar/bar/ToolbarButton.tsx
+++ b/frontend/src/toolbar/bar/ToolbarButton.tsx
@@ -10,17 +10,18 @@ import React from 'react'
import { MenuState, toolbarLogic } from './toolbarLogic'
export type ToolbarButtonProps = {
- icon: React.ReactElement | null
+ children: React.ReactNode
onClick?: () => void
title?: string
titleMinimized?: JSX.Element | string
menuId?: MenuState
+ flex?: boolean
}
export const ToolbarButton: FunctionComponent = React.forwardRef<
HTMLDivElement,
ToolbarButtonProps
->(({ icon, title, onClick, titleMinimized, menuId, ...props }, ref): JSX.Element => {
+>(({ children, title, onClick, titleMinimized, menuId, flex, ...props }, ref): JSX.Element => {
const { visibleMenu, minimized, isDragging } = useValues(toolbarLogic)
const { setVisibleMenu } = useActions(toolbarLogic)
@@ -54,9 +55,13 @@ export const ToolbarButton: FunctionComponent = React.forwar
}
const theButton = (
-
+
)
diff --git a/frontend/src/toolbar/flags/flagsToolbarLogic.ts b/frontend/src/toolbar/flags/flagsToolbarLogic.ts
index 60c1f568f45a6..e1f41cabca73c 100644
--- a/frontend/src/toolbar/flags/flagsToolbarLogic.ts
+++ b/frontend/src/toolbar/flags/flagsToolbarLogic.ts
@@ -40,11 +40,6 @@ export const flagsToolbarLogic = kea
([
`/api/projects/@current/feature_flags/my_flags${encodeParams(params, '?')}`
)
- if (response.status >= 400) {
- toolbarConfigLogic.actions.tokenExpired()
- return []
- }
-
breakpoint()
if (!response.ok) {
return []
diff --git a/frontend/src/toolbar/toolbarConfigLogic.ts b/frontend/src/toolbar/toolbarConfigLogic.ts
index 1b4638b8f39f8..853b03bdeea32 100644
--- a/frontend/src/toolbar/toolbarConfigLogic.ts
+++ b/frontend/src/toolbar/toolbarConfigLogic.ts
@@ -119,10 +119,12 @@ export async function toolbarFetch(
})
if (response.status === 403) {
const responseData = await response.json()
- // Do not try to authenticate if the user has no project access altogether
- if (responseData.detail !== "You don't have access to the project.") {
+ if (responseData.detail === "You don't have access to the project.") {
toolbarConfigLogic.actions.authenticate()
}
}
+ if (response.status == 401) {
+ toolbarConfigLogic.actions.tokenExpired()
+ }
return response
}
diff --git a/frontend/src/toolbar/toolbarLogic.ts b/frontend/src/toolbar/toolbarLogic.ts
deleted file mode 100644
index d5183a6734f20..0000000000000
--- a/frontend/src/toolbar/toolbarLogic.ts
+++ /dev/null
@@ -1,87 +0,0 @@
-import { actions, afterMount, kea, listeners, path, props, reducers, selectors } from 'kea'
-import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'
-
-import { actionsTabLogic } from '~/toolbar/actions/actionsTabLogic'
-import { posthog } from '~/toolbar/posthog'
-import { clearSessionToolbarToken } from '~/toolbar/utils'
-import { ToolbarProps } from '~/types'
-
-import type { toolbarLogicType } from './toolbarLogicType'
-
-export const toolbarLogic = kea([
- path(['toolbar', 'toolbarLogic']),
- props({} as ToolbarProps),
-
- actions({
- authenticate: true,
- logout: true,
- tokenExpired: true,
- processUserIntent: true,
- clearUserIntent: true,
- showButton: true,
- hideButton: true,
- }),
-
- reducers(({ props }) => ({
- rawApiURL: [props.apiURL as string],
- rawJsURL: [(props.jsURL || props.apiURL) as string],
- temporaryToken: [props.temporaryToken || null, { logout: () => null, tokenExpired: () => null }],
- actionId: [props.actionId || null, { logout: () => null, clearUserIntent: () => null }],
- userIntent: [props.userIntent || null, { logout: () => null, clearUserIntent: () => null }],
- source: [props.source || null, { logout: () => null }],
- buttonVisible: [true, { showButton: () => true, hideButton: () => false, logout: () => false }],
- dataAttributes: [props.dataAttributes || []],
- posthog: [props.posthog ?? null],
- })),
-
- selectors({
- apiURL: [(s) => [s.rawApiURL], (apiURL) => `${apiURL.endsWith('/') ? apiURL.replace(/\/+$/, '') : apiURL}`],
- jsURL: [
- (s) => [s.rawJsURL, s.apiURL],
- (rawJsURL, apiUrl) =>
- `${rawJsURL ? (rawJsURL.endsWith('/') ? rawJsURL.replace(/\/+$/, '') : rawJsURL) : apiUrl}`,
- ],
- isAuthenticated: [(s) => [s.temporaryToken], (temporaryToken) => !!temporaryToken],
- }),
-
- listeners(({ values, props }) => ({
- authenticate: () => {
- posthog.capture('toolbar authenticate', { is_authenticated: values.isAuthenticated })
- const encodedUrl = encodeURIComponent(window.location.href)
- window.location.href = `${values.apiURL}/authorize_and_redirect/?redirect=${encodedUrl}`
- clearSessionToolbarToken()
- },
- logout: () => {
- posthog.capture('toolbar logout')
- clearSessionToolbarToken()
- },
- tokenExpired: () => {
- posthog.capture('toolbar token expired')
- console.warn('PostHog Toolbar API token expired. Clearing session.')
- if (values.source !== 'localstorage') {
- lemonToast.error('PostHog Toolbar API token expired.')
- }
- clearSessionToolbarToken()
- },
- processUserIntent: () => {
- if (props.userIntent === 'add-action' || props.userIntent === 'edit-action') {
- actionsTabLogic.actions.showButtonActions()
- // the right view will next be opened in `actionsTabLogic` on `getActionsSuccess`
- }
- },
- })),
-
- afterMount(({ props, actions, values }) => {
- if (props.instrument) {
- const distinctId = props.distinctId
- if (distinctId) {
- posthog.identify(distinctId, props.userEmail ? { email: props.userEmail } : {})
- }
- posthog.optIn()
- }
- if (props.userIntent) {
- actions.processUserIntent()
- }
- posthog.capture('toolbar loaded', { is_authenticated: values.isAuthenticated })
- }),
-])
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index 0baf245cf5f39..131598f9a79d2 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -903,8 +903,10 @@ export interface SessionRecordingsResponse {
export type ErrorCluster = {
cluster: number
- sample: { session_id: string; error: string }
+ sample: string
occurrences: number
+ session_ids: string[]
+ sparkline: Record
unique_sessions: number
viewed: number
}
@@ -3542,7 +3544,7 @@ export interface DataWarehouseViewLink {
created_at?: string | null
}
-export type ExternalDataSourceType = 'Stripe' | 'Hubspot' | 'Postgres'
+export type ExternalDataSourceType = 'Stripe' | 'Hubspot' | 'Postgres' | 'Zendesk'
export interface ExternalDataSourceCreatePayload {
source_type: ExternalDataSourceType
@@ -3595,6 +3597,7 @@ export type BatchExportDestinationS3 = {
encryption: string | null
kms_key_id: string | null
endpoint_url: string | null
+ file_format: string
}
}
diff --git a/latest_migrations.manifest b/latest_migrations.manifest
index f232dbc8c186c..f88359530eb78 100644
--- a/latest_migrations.manifest
+++ b/latest_migrations.manifest
@@ -5,7 +5,7 @@ contenttypes: 0002_remove_content_type_name
ee: 0016_rolemembership_organization_member
otp_static: 0002_throttling
otp_totp: 0002_auto_20190420_0723
-posthog: 0397_projects_backfill
+posthog: 0398_alter_externaldatasource_source_type
sessions: 0001_initial
social_django: 0010_uid_db_index
two_factor: 0007_auto_20201201_1019
diff --git a/mypy-baseline.txt b/mypy-baseline.txt
index 781ad2980830b..b8d2d1c94da64 100644
--- a/mypy-baseline.txt
+++ b/mypy-baseline.txt
@@ -1,6 +1,14 @@
posthog/temporal/common/utils.py:0: error: Argument 1 to "abstractclassmethod" has incompatible type "Callable[[HeartbeatDetails, Any], Any]"; expected "Callable[[type[Never], Any], Any]" [arg-type]
posthog/temporal/common/utils.py:0: note: This is likely because "from_activity" has named arguments: "cls". Consider marking them positional-only
posthog/temporal/common/utils.py:0: error: Argument 2 to "__get__" of "classmethod" has incompatible type "type[HeartbeatType]"; expected "type[Never]" [arg-type]
+posthog/temporal/data_imports/pipelines/zendesk/talk_api.py:0: error: Incompatible types in assignment (expression has type "None", variable has type "str") [assignment]
+posthog/temporal/data_imports/pipelines/zendesk/helpers.py:0: error: Argument 1 to "ensure_pendulum_datetime" has incompatible type "DateTime | Date | datetime | date | str | float | int | None"; expected "DateTime | Date | datetime | date | str | float | int" [arg-type]
+posthog/temporal/data_imports/pipelines/zendesk/helpers.py:0: error: Argument 1 to "ensure_pendulum_datetime" has incompatible type "str | None"; expected "DateTime | Date | datetime | date | str | float | int" [arg-type]
+posthog/temporal/data_imports/pipelines/zendesk/helpers.py:0: error: Argument 1 to "ensure_pendulum_datetime" has incompatible type "DateTime | Date | datetime | date | str | float | int | None"; expected "DateTime | Date | datetime | date | str | float | int" [arg-type]
+posthog/temporal/data_imports/pipelines/zendesk/helpers.py:0: error: Argument 1 to "ensure_pendulum_datetime" has incompatible type "str | None"; expected "DateTime | Date | datetime | date | str | float | int" [arg-type]
+posthog/temporal/data_imports/pipelines/zendesk/helpers.py:0: error: Argument 1 to "ensure_pendulum_datetime" has incompatible type "DateTime | Date | datetime | date | str | float | int | None"; expected "DateTime | Date | datetime | date | str | float | int" [arg-type]
+posthog/temporal/data_imports/pipelines/zendesk/helpers.py:0: error: Item "None" of "DateTime | None" has no attribute "int_timestamp" [union-attr]
+posthog/temporal/data_imports/pipelines/zendesk/helpers.py:0: error: Argument 1 to "ensure_pendulum_datetime" has incompatible type "str | None"; expected "DateTime | Date | datetime | date | str | float | int" [arg-type]
posthog/hogql/database/argmax.py:0: error: Argument "chain" to "Field" has incompatible type "list[str]"; expected "list[str | int]" [arg-type]
posthog/hogql/database/argmax.py:0: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance
posthog/hogql/database/argmax.py:0: note: Consider using "Sequence" instead, which is covariant
@@ -77,6 +85,7 @@ posthog/hogql/parser.py:0: error: "None" has no attribute "text" [attr-defined]
posthog/hogql/parser.py:0: error: "None" has no attribute "text" [attr-defined]
posthog/hogql/parser.py:0: error: Statement is unreachable [unreachable]
posthog/hogql/database/schema/person_distinct_ids.py:0: error: Argument 1 to "select_from_person_distinct_ids_table" has incompatible type "dict[str, list[str]]"; expected "dict[str, list[str | int]]" [arg-type]
+posthog/hogql/database/schema/person_distinct_id_overrides.py:0: error: Argument 1 to "select_from_person_distinct_id_overrides_table" has incompatible type "dict[str, list[str]]"; expected "dict[str, list[str | int]]" [arg-type]
posthog/hogql/database/schema/cohort_people.py:0: error: Argument "chain" to "Field" has incompatible type "list[str]"; expected "list[str | int]" [arg-type]
posthog/hogql/database/schema/cohort_people.py:0: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance
posthog/hogql/database/schema/cohort_people.py:0: note: Consider using "Sequence" instead, which is covariant
@@ -97,6 +106,7 @@ posthog/hogql/database/database.py:0: error: "FieldOrTable" has no attribute "fi
posthog/hogql/database/database.py:0: error: "FieldOrTable" has no attribute "fields" [attr-defined]
posthog/hogql/database/database.py:0: error: "FieldOrTable" has no attribute "fields" [attr-defined]
posthog/hogql/database/database.py:0: error: "FieldOrTable" has no attribute "fields" [attr-defined]
+posthog/hogql/database/database.py:0: error: "FieldOrTable" has no attribute "fields" [attr-defined]
posthog/hogql/database/database.py:0: error: Incompatible types (expression has type "Literal['view', 'lazy_table']", TypedDict item "type" has type "Literal['integer', 'float', 'string', 'datetime', 'date', 'boolean', 'array', 'json', 'lazy_table', 'virtual_table', 'field_traverser', 'expression']") [typeddict-item]
posthog/warehouse/models/datawarehouse_saved_query.py:0: error: Argument 1 to "create_hogql_database" has incompatible type "int | None"; expected "int" [arg-type]
posthog/warehouse/models/datawarehouse_saved_query.py:0: error: Incompatible types in assignment (expression has type "Expr", variable has type "SelectQuery | SelectUnionQuery") [assignment]
@@ -243,7 +253,6 @@ posthog/hogql/resolver.py:0: error: Argument 1 to "join" of "str" has incompatib
posthog/temporal/data_imports/external_data_job.py:0: error: Argument "team_id" has incompatible type "int"; expected "str" [arg-type]
posthog/temporal/data_imports/external_data_job.py:0: error: Unused "type: ignore" comment [unused-ignore]
posthog/temporal/data_imports/external_data_job.py:0: error: Argument "team_id" has incompatible type "int"; expected "str" [arg-type]
-posthog/temporal/data_imports/external_data_job.py:0: error: Argument 2 to "DataImportPipeline" has incompatible type "DltSource"; expected "DltResource" [arg-type]
posthog/hogql/transforms/lazy_tables.py:0: error: Incompatible default for argument "context" (default has type "None", argument has type "HogQLContext") [assignment]
posthog/hogql/transforms/lazy_tables.py:0: note: PEP 484 prohibits implicit Optional. Accordingly, mypy has changed its default to no_implicit_optional=True
posthog/hogql/transforms/lazy_tables.py:0: note: Use https://github.com/hauntsaninja/no_implicit_optional to automatically upgrade your codebase
@@ -339,20 +348,12 @@ posthog/hogql_queries/sessions_timeline_query_runner.py:0: error: Statement is u
posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr]
posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_histogram_bin_count" [union-attr]
posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown" [union-attr]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Argument 1 to "parse_expr" has incompatible type "str | float | list[str | float] | Any | None"; expected "str" [arg-type]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown" [union-attr]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Argument 1 to "parse_expr" has incompatible type "str | float | list[str | float] | Any | None"; expected "str" [arg-type]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Statement is unreachable [unreachable]
posthog/hogql_queries/insights/trends/breakdown.py:0: error: Argument "exprs" to "Or" has incompatible type "list[CompareOperation]"; expected "list[Expr]" [arg-type]
posthog/hogql_queries/insights/trends/breakdown.py:0: note: "List" is invariant -- see https://mypy.readthedocs.io/en/stable/common_issues.html#variance
posthog/hogql_queries/insights/trends/breakdown.py:0: note: Consider using "Sequence" instead, which is covariant
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Incompatible types in assignment (expression has type "float", variable has type "int") [assignment]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Incompatible types in assignment (expression has type "float", variable has type "int") [assignment]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment]
-posthog/hogql_queries/insights/trends/breakdown.py:0: error: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment]
+posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr]
+posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown" [union-attr]
+posthog/hogql_queries/insights/trends/breakdown.py:0: error: Argument 1 to "parse_expr" has incompatible type "str | float | list[str | float] | Any | None"; expected "str" [arg-type]
posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown_type" [union-attr]
posthog/hogql_queries/insights/trends/breakdown.py:0: error: Item "None" of "BreakdownFilter | None" has no attribute "breakdown" [union-attr]
posthog/hogql_queries/insights/trends/breakdown.py:0: error: Argument "breakdown_field" to "get_properties_chain" has incompatible type "str | float | list[str | float] | Any | None"; expected "str" [arg-type]
@@ -377,11 +378,11 @@ posthog/hogql_queries/insights/trends/trends_query_runner.py:0: error: Signature
posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: Superclass:
posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: def to_actors_query(self) -> SelectQuery | SelectUnionQuery
posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: Subclass:
-posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: def to_actors_query(self, time_frame: str | int | None, series_index: int, breakdown_value: str | int | None = ..., compare: Compare | None = ...) -> SelectQuery | SelectUnionQuery
+posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: def to_actors_query(self, time_frame: str | None, series_index: int, breakdown_value: str | int | None = ..., compare: Compare | None = ...) -> SelectQuery | SelectUnionQuery
posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: Superclass:
posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: def to_actors_query(self) -> SelectQuery | SelectUnionQuery
posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: Subclass:
-posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: def to_actors_query(self, time_frame: str | int | None, series_index: int, breakdown_value: str | int | None = ..., compare: Compare | None = ...) -> SelectQuery | SelectUnionQuery
+posthog/hogql_queries/insights/trends/trends_query_runner.py:0: note: def to_actors_query(self, time_frame: str | None, series_index: int, breakdown_value: str | int | None = ..., compare: Compare | None = ...) -> SelectQuery | SelectUnionQuery
posthog/hogql_queries/insights/trends/trends_query_runner.py:0: error: Statement is unreachable [unreachable]
posthog/hogql_queries/insights/trends/trends_query_runner.py:0: error: Argument 1 to "_event_property" of "TrendsQueryRunner" has incompatible type "str | float | list[str | float] | None"; expected "str" [arg-type]
posthog/hogql_queries/insights/retention_query_runner.py:0: error: Incompatible types in assignment (expression has type "Expr", variable has type "Call") [assignment]
@@ -542,7 +543,6 @@ posthog/hogql_queries/insights/trends/test/test_aggregation_operations.py:0: err
posthog/hogql_queries/insights/trends/test/test_aggregation_operations.py:0: error: Item "SelectQuery" of "SelectQuery | SelectUnionQuery | Field | Any | None" has no attribute "chain" [union-attr]
posthog/hogql_queries/insights/trends/test/test_aggregation_operations.py:0: error: Item "SelectUnionQuery" of "SelectQuery | SelectUnionQuery | Field | Any | None" has no attribute "chain" [union-attr]
posthog/hogql_queries/insights/trends/test/test_aggregation_operations.py:0: error: Item "None" of "SelectQuery | SelectUnionQuery | Field | Any | None" has no attribute "chain" [union-attr]
-posthog/hogql_queries/insights/test/test_paginators.py:0: error: Argument 2 to "execute_hogql_query" of "HogQLHasMorePaginator" has incompatible type "SelectQuery | SelectUnionQuery"; expected "SelectQuery" [arg-type]
posthog/hogql_queries/insights/test/test_paginators.py:0: error: Value of type "object" is not indexable [index]
posthog/hogql_queries/insights/test/test_paginators.py:0: error: Value of type "object" is not indexable [index]
posthog/hogql_queries/insights/test/test_paginators.py:0: error: Value of type "object" is not indexable [index]
diff --git a/package.json b/package.json
index 0a268fc208fd5..58e9fcdba1d3a 100644
--- a/package.json
+++ b/package.json
@@ -140,11 +140,12 @@
"maplibre-gl": "^3.5.1",
"md5": "^2.3.0",
"monaco-editor": "^0.39.0",
+ "natural-orderby": "^3.0.2",
"papaparse": "^5.4.1",
"pmtiles": "^2.11.0",
"postcss": "^8.4.31",
"postcss-preset-env": "^9.3.0",
- "posthog-js": "1.116.1",
+ "posthog-js": "1.116.5",
"posthog-js-lite": "2.5.0",
"prettier": "^2.8.8",
"prop-types": "^15.7.2",
diff --git a/plugin-server/src/config/config.ts b/plugin-server/src/config/config.ts
index 8e9b50afb9528..def72eea474bb 100644
--- a/plugin-server/src/config/config.ts
+++ b/plugin-server/src/config/config.ts
@@ -163,6 +163,9 @@ export function getDefaultConfig(): PluginsServerConfig {
SESSION_RECORDING_DEBUG_PARTITION: undefined,
SESSION_RECORDING_KAFKA_DEBUG: undefined,
SESSION_RECORDING_MAX_PARALLEL_FLUSHES: 10,
+ SESSION_RECORDING_OVERFLOW_ENABLED: false,
+ SESSION_RECORDING_OVERFLOW_BUCKET_REPLENISH_RATE: 5_000_000, // 5MB/second uncompressed, sustained
+ SESSION_RECORDING_OVERFLOW_BUCKET_CAPACITY: 200_000_000, // 200MB burst
}
}
diff --git a/plugin-server/src/main/ingestion-queues/session-recording/services/overflow-detection.ts b/plugin-server/src/main/ingestion-queues/session-recording/services/overflow-detection.ts
new file mode 100644
index 0000000000000..8b478b781bc95
--- /dev/null
+++ b/plugin-server/src/main/ingestion-queues/session-recording/services/overflow-detection.ts
@@ -0,0 +1,45 @@
+import LRUCache from 'lru-cache'
+import { Gauge } from 'prom-client'
+
+import { Limiter } from '../../../../utils/token-bucket'
+
+export enum OverflowState {
+ Okay,
+ Triggered, // Recently triggered the overflow detection
+ Cooldown, // Already triggered the overflow detection earlier than cooldownSeconds
+}
+
+export const overflowTriggeredGauge = new Gauge({
+ name: 'overflow_detection_triggered_total',
+ help: 'Number of entities that triggered overflow detection.',
+})
+
+/**
+ * OverflowDetection handles consumer-side detection of hot partitions by
+ * accounting for data volumes per entity (a session_id, a distinct_id...).
+ *
+ * The first time that the observed spike crosses the thresholds set via burstCapacity
+ * and replenishRate, observe returns Triggered. Subsequent calls will return Cooldown
+ * until cooldownSeconds is reached.
+ */
+export class OverflowDetection {
+ private limiter: Limiter
+ private triggered: LRUCache
+
+ constructor(burstCapacity: number, replenishRate: number, cooldownSeconds: number) {
+ this.limiter = new Limiter(burstCapacity, replenishRate)
+ this.triggered = new LRUCache({ max: 1_000_000, maxAge: cooldownSeconds * 1000 })
+ }
+
+ public observe(key: string, quantity: number, now?: number): OverflowState {
+ if (this.triggered.has(key)) {
+ return OverflowState.Cooldown
+ }
+ if (this.limiter.consume(key, quantity, now)) {
+ return OverflowState.Okay
+ }
+ this.triggered.set(key, true)
+ overflowTriggeredGauge.inc(1)
+ return OverflowState.Triggered
+ }
+}
diff --git a/plugin-server/src/main/ingestion-queues/session-recording/session-recordings-consumer.ts b/plugin-server/src/main/ingestion-queues/session-recording/session-recordings-consumer.ts
index 30aaab4a023d5..491044652d80f 100644
--- a/plugin-server/src/main/ingestion-queues/session-recording/session-recordings-consumer.ts
+++ b/plugin-server/src/main/ingestion-queues/session-recording/session-recordings-consumer.ts
@@ -20,6 +20,7 @@ import { addSentryBreadcrumbsEventListeners } from '../kafka-metrics'
import { eventDroppedCounter } from '../metrics'
import { ConsoleLogsIngester } from './services/console-logs-ingester'
import { OffsetHighWaterMarker } from './services/offset-high-water-marker'
+import { OverflowDetection } from './services/overflow-detection'
import { RealtimeManager } from './services/realtime-manager'
import { ReplayEventsIngester } from './services/replay-events-ingester'
import { BUCKETS_KB_WRITTEN, SessionManager } from './services/session-manager'
@@ -128,6 +129,7 @@ export class SessionRecordingIngester {
sessionHighWaterMarker: OffsetHighWaterMarker
persistentHighWaterMarker: OffsetHighWaterMarker
realtimeManager: RealtimeManager
+ overflowDetection?: OverflowDetection
replayEventsIngester?: ReplayEventsIngester
consoleLogsIngester?: ConsoleLogsIngester
batchConsumer?: BatchConsumer
@@ -160,6 +162,14 @@ export class SessionRecordingIngester {
this.realtimeManager = new RealtimeManager(this.redisPool, this.config)
+ if (globalServerConfig.SESSION_RECORDING_OVERFLOW_ENABLED) {
+ this.overflowDetection = new OverflowDetection(
+ globalServerConfig.SESSION_RECORDING_OVERFLOW_BUCKET_CAPACITY,
+ globalServerConfig.SESSION_RECORDING_OVERFLOW_BUCKET_REPLENISH_RATE,
+ 24 * 3600 // One day
+ )
+ }
+
// We create a hash of the cluster to use as a unique identifier for the high-water marks
// This enables us to swap clusters without having to worry about resetting the high-water marks
const kafkaClusterIdentifier = crypto.createHash('md5').update(this.config.KAFKA_HOSTS).digest('hex')
@@ -275,6 +285,9 @@ export class SessionRecordingIngester {
return
}
+ // TODO: update Redis if this triggers
+ this.overflowDetection?.observe(key, event.metadata.rawSize, event.metadata.timestamp)
+
if (!this.sessions[key]) {
const { partition, topic } = event.metadata
diff --git a/plugin-server/src/main/ingestion-queues/session-recording/types.ts b/plugin-server/src/main/ingestion-queues/session-recording/types.ts
index 254e3f0897ee7..d61dadda9279e 100644
--- a/plugin-server/src/main/ingestion-queues/session-recording/types.ts
+++ b/plugin-server/src/main/ingestion-queues/session-recording/types.ts
@@ -6,6 +6,7 @@ export type IncomingRecordingMessage = {
metadata: {
topic: string
partition: number
+ rawSize: number
lowOffset: number
highOffset: number
timestamp: number
diff --git a/plugin-server/src/main/ingestion-queues/session-recording/utils.ts b/plugin-server/src/main/ingestion-queues/session-recording/utils.ts
index 4b4345d43b48d..53ce953e5bd92 100644
--- a/plugin-server/src/main/ingestion-queues/session-recording/utils.ts
+++ b/plugin-server/src/main/ingestion-queues/session-recording/utils.ts
@@ -225,6 +225,7 @@ export const parseKafkaMessage = async (
metadata: {
partition: message.partition,
topic: message.topic,
+ rawSize: message.size,
lowOffset: message.offset,
highOffset: message.offset,
timestamp: message.timestamp,
@@ -267,6 +268,7 @@ export const reduceRecordingMessages = (messages: IncomingRecordingMessage[]): I
existingMessage.eventsByWindowId[windowId] = events
}
}
+ existingMessage.metadata.rawSize += clonedMessage.metadata.rawSize
// Update the events ranges
existingMessage.metadata.lowOffset = Math.min(
diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts
index b8eeb5b296a9e..114547cfe605f 100644
--- a/plugin-server/src/types.ts
+++ b/plugin-server/src/types.ts
@@ -230,6 +230,10 @@ export interface PluginsServerConfig {
// a single partition which will output many more log messages to the console
// useful when that partition is lagging unexpectedly
SESSION_RECORDING_DEBUG_PARTITION: string | undefined
+ // overflow detection, updating Redis for capture to move the traffic away
+ SESSION_RECORDING_OVERFLOW_ENABLED: boolean
+ SESSION_RECORDING_OVERFLOW_BUCKET_CAPACITY: number
+ SESSION_RECORDING_OVERFLOW_BUCKET_REPLENISH_RATE: number
// Dedicated infra values
SESSION_RECORDING_KAFKA_HOSTS: string | undefined
diff --git a/plugin-server/tests/main/ingestion-queues/session-recording/__snapshots__/utils.test.ts.snap b/plugin-server/tests/main/ingestion-queues/session-recording/__snapshots__/utils.test.ts.snap
index 9962eb544bc6d..87ca515b22bd6 100644
--- a/plugin-server/tests/main/ingestion-queues/session-recording/__snapshots__/utils.test.ts.snap
+++ b/plugin-server/tests/main/ingestion-queues/session-recording/__snapshots__/utils.test.ts.snap
@@ -33,6 +33,7 @@ Array [
"highOffset": 3,
"lowOffset": 1,
"partition": 1,
+ "rawSize": 12,
"timestamp": 1,
"topic": "the_topic",
},
@@ -59,6 +60,7 @@ Array [
"highOffset": 4,
"lowOffset": 4,
"partition": 1,
+ "rawSize": 30,
"timestamp": 4,
"topic": "the_topic",
},
@@ -85,6 +87,7 @@ Array [
"highOffset": 5,
"lowOffset": 5,
"partition": 1,
+ "rawSize": 31,
"timestamp": 5,
"topic": "the_topic",
},
@@ -130,6 +133,7 @@ Object {
"highOffset": 1,
"lowOffset": 1,
"partition": 1,
+ "rawSize": 42,
"timestamp": 1,
"topic": "the_topic",
},
diff --git a/plugin-server/tests/main/ingestion-queues/session-recording/utils.test.ts b/plugin-server/tests/main/ingestion-queues/session-recording/utils.test.ts
index b8e6dc59284e7..c5a3851486d93 100644
--- a/plugin-server/tests/main/ingestion-queues/session-recording/utils.test.ts
+++ b/plugin-server/tests/main/ingestion-queues/session-recording/utils.test.ts
@@ -57,7 +57,7 @@ describe('session-recording utils', () => {
})
),
timestamp: 1,
- size: 1,
+ size: 42,
topic: 'the_topic',
offset: 1,
partition: 1,
@@ -257,7 +257,7 @@ describe('session-recording utils', () => {
distinct_id: '1',
eventsRange: { start: 1, end: 1 },
eventsByWindowId: { window_1: [{ timestamp: 1, type: 1, data: {} }] },
- metadata: { lowOffset: 1, highOffset: 1, partition: 1, timestamp: 1, topic: 'the_topic' },
+ metadata: { lowOffset: 1, highOffset: 1, partition: 1, timestamp: 1, topic: 'the_topic', rawSize: 5 },
session_id: '1',
team_id: 1,
snapshot_source: null,
@@ -266,7 +266,7 @@ describe('session-recording utils', () => {
distinct_id: '1',
eventsRange: { start: 2, end: 2 },
eventsByWindowId: { window_1: [{ timestamp: 2, type: 2, data: {} }] },
- metadata: { lowOffset: 2, highOffset: 2, partition: 1, timestamp: 2, topic: 'the_topic' },
+ metadata: { lowOffset: 2, highOffset: 2, partition: 1, timestamp: 2, topic: 'the_topic', rawSize: 4 },
session_id: '1',
team_id: 1,
snapshot_source: null,
@@ -276,7 +276,7 @@ describe('session-recording utils', () => {
distinct_id: '1',
eventsRange: { start: 3, end: 3 },
eventsByWindowId: { window_2: [{ timestamp: 3, type: 3, data: {} }] },
- metadata: { lowOffset: 3, highOffset: 3, partition: 1, timestamp: 3, topic: 'the_topic' },
+ metadata: { lowOffset: 3, highOffset: 3, partition: 1, timestamp: 3, topic: 'the_topic', rawSize: 3 },
session_id: '1',
team_id: 1,
snapshot_source: null,
@@ -286,7 +286,7 @@ describe('session-recording utils', () => {
distinct_id: '1',
eventsRange: { start: 4, end: 4 },
eventsByWindowId: { window_1: [{ timestamp: 4, type: 4, data: {} }] },
- metadata: { lowOffset: 4, highOffset: 4, partition: 1, timestamp: 4, topic: 'the_topic' },
+ metadata: { lowOffset: 4, highOffset: 4, partition: 1, timestamp: 4, topic: 'the_topic', rawSize: 30 },
session_id: '1',
team_id: 2,
snapshot_source: null,
@@ -296,7 +296,7 @@ describe('session-recording utils', () => {
distinct_id: '1',
eventsRange: { start: 5, end: 5 },
eventsByWindowId: { window_1: [{ timestamp: 5, type: 5, data: {} }] },
- metadata: { lowOffset: 5, highOffset: 5, partition: 1, timestamp: 5, topic: 'the_topic' },
+ metadata: { lowOffset: 5, highOffset: 5, partition: 1, timestamp: 5, topic: 'the_topic', rawSize: 31 },
session_id: '2',
team_id: 1,
snapshot_source: null,
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index ef81ba7d4c4d9..b6375a3382ae2 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -238,6 +238,9 @@ dependencies:
monaco-editor:
specifier: ^0.39.0
version: 0.39.0
+ natural-orderby:
+ specifier: ^3.0.2
+ version: 3.0.2
papaparse:
specifier: ^5.4.1
version: 5.4.1
@@ -251,8 +254,8 @@ dependencies:
specifier: ^9.3.0
version: 9.3.0(postcss@8.4.31)
posthog-js:
- specifier: 1.116.1
- version: 1.116.1
+ specifier: 1.116.5
+ version: 1.116.5
posthog-js-lite:
specifier: 2.5.0
version: 2.5.0
@@ -6793,7 +6796,7 @@ packages:
'@storybook/csf': 0.1.3
'@storybook/global': 5.0.0
'@storybook/types': 7.6.17
- '@types/qs': 6.9.12
+ '@types/qs': 6.9.14
dequal: 2.0.3
lodash: 4.17.21
memoizerific: 1.11.3
@@ -8195,6 +8198,11 @@ packages:
/@types/qs@6.9.12:
resolution: {integrity: sha512-bZcOkJ6uWrL0Qb2NAWKa7TBU+mJHPzhx9jjLL1KHF+XpzEcR7EXHvjbHlGtR/IsP1vyPrehuS6XqkmaePy//mg==}
+ dev: false
+
+ /@types/qs@6.9.14:
+ resolution: {integrity: sha512-5khscbd3SwWMhFqylJBLQ0zIu7c1K6Vz0uBIt915BI3zV0q1nfjRQD3RqSBcPaO6PHEF4ov/t9y89fSiyThlPA==}
+ dev: true
/@types/query-selector-shadow-dom@1.0.0:
resolution: {integrity: sha512-cTGo8ZxW0WXFDV7gvL/XCq4213t6S/yWaSGqscnXUTNDWqwnsYKegB/VAzQDwzmACoLzIbGbYXdjJOgfPLu7Ig==}
@@ -13625,7 +13633,7 @@ packages:
hogan.js: 3.0.2
htm: 3.1.1
instantsearch-ui-components: 0.3.0
- preact: 10.19.6
+ preact: 10.20.1
qs: 6.9.7
search-insights: 2.13.0
dev: false
@@ -15912,6 +15920,11 @@ packages:
resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==}
dev: true
+ /natural-orderby@3.0.2:
+ resolution: {integrity: sha512-x7ZdOwBxZCEm9MM7+eQCjkrNLrW3rkBKNHVr78zbtqnMGVNlnDi6C/eUEYgxHNrcbu0ymvjzcwIL/6H1iHri9g==}
+ engines: {node: '>=18'}
+ dev: false
+
/needle@3.3.1:
resolution: {integrity: sha512-6k0YULvhpw+RoLNiQCRKOl09Rv1dPLr8hHnVjHqdolKwDrdNyk+Hmrthi4lIGPPz3r39dLx0hsF5s40sZ3Us4Q==}
engines: {node: '>= 4.4.x'}
@@ -17441,19 +17454,19 @@ packages:
resolution: {integrity: sha512-Urvlp0Vu9h3td0BVFWt0QXFJDoOZcaAD83XM9d91NKMKTVPZtfU0ysoxstIf5mw/ce9ZfuMgpWPaagrZI4rmSg==}
dev: false
- /posthog-js@1.116.1:
- resolution: {integrity: sha512-tYKw6K23S3koa2sfX0sylno7jQQ6ET7u1Lw4KqowhciNhS0R5OWTo3HWEJPt64e9IzeWQGcgb9utJIWwrp5D0Q==}
+ /posthog-js@1.116.5:
+ resolution: {integrity: sha512-Dfsy07WGPRqbXNQgfhR5zhcQw4A078xCRX1y0oGmy8xVUmL6QLNNTVKENB4xpo45Ox+88cysi+hEMhd9SfV5Qg==}
dependencies:
fflate: 0.4.8
- preact: 10.19.6
+ preact: 10.20.1
dev: false
/potpack@2.0.0:
resolution: {integrity: sha512-Q+/tYsFU9r7xoOJ+y/ZTtdVQwTWfzjbiXBDMM/JKUux3+QPP02iUuIoeBQ+Ot6oEDlC+/PGjB/5A3K7KKb7hcw==}
dev: false
- /preact@10.19.6:
- resolution: {integrity: sha512-gympg+T2Z1fG1unB8NH29yHJwnEaCH37Z32diPDku316OTnRPeMbiRV9kTrfZpocXjdfnWuFUl/Mj4BHaf6gnw==}
+ /preact@10.20.1:
+ resolution: {integrity: sha512-JIFjgFg9B2qnOoGiYMVBtrcFxHqn+dNXbq76bVmcaHYJFYR4lW67AOcXgAYQQTDYXDOg/kTZrKPNCdRgJ2UJmw==}
dev: false
/prelude-ls@1.2.1:
diff --git a/posthog/admin/admins/user_admin.py b/posthog/admin/admins/user_admin.py
index c1129ef334fa4..9872fad946f30 100644
--- a/posthog/admin/admins/user_admin.py
+++ b/posthog/admin/admins/user_admin.py
@@ -1,12 +1,11 @@
-from django.utils.html import format_html
-
from django.contrib.auth.admin import UserAdmin as DjangoUserAdmin
from django.contrib.auth.forms import UserChangeForm as DjangoUserChangeForm
-from django.contrib.auth.tokens import default_token_generator
+from django.utils.html import format_html
from django.utils.translation import gettext_lazy as _
from posthog.admin.inlines.organization_member_inline import OrganizationMemberInline
from posthog.admin.inlines.totp_device_inline import TOTPDeviceInline
+from posthog.api.authentication import password_reset_token_generator
from posthog.models import User
@@ -18,7 +17,7 @@ def __init__(self, *args, **kwargs):
# we have a link to the password reset page which the _user_ can use themselves.
# This way if some user needs to reset their password and there's a problem with receiving the reset link email,
# an admin can provide that reset link manually – much better than sending a new password in plain text.
- password_reset_token = default_token_generator.make_token(self.instance)
+ password_reset_token = password_reset_token_generator.make_token(self.instance)
self.fields["password"].help_text = (
"Raw passwords are not stored, so there is no way to see this user’s password, but you can send them "
f'this password reset link '
diff --git a/posthog/api/dashboards/dashboard.py b/posthog/api/dashboards/dashboard.py
index 8524ab8618b4b..100e8745b8db1 100644
--- a/posthog/api/dashboards/dashboard.py
+++ b/posthog/api/dashboards/dashboard.py
@@ -7,7 +7,6 @@
from django.utils.timezone import now
from rest_framework import exceptions, serializers, viewsets
from rest_framework.decorators import action
-from rest_framework.exceptions import PermissionDenied
from rest_framework.permissions import SAFE_METHODS, BasePermission
from rest_framework.request import Request
from rest_framework.response import Response
@@ -22,14 +21,12 @@
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api.shared import UserBasicSerializer
from posthog.api.tagged_item import TaggedItemSerializerMixin, TaggedItemViewSetMixin
-from posthog.constants import AvailableFeature
from posthog.event_usage import report_user_action
from posthog.helpers import create_dashboard_from_template
from posthog.helpers.dashboard_templates import create_from_template
from posthog.models import Dashboard, DashboardTile, Insight, Text
from posthog.models.dashboard_templates import DashboardTemplate
from posthog.models.tagged_item import TaggedItem
-from posthog.models.team.team import check_is_feature_available_for_team
from posthog.models.user import User
from posthog.user_permissions import UserPermissionsSerializerMixin
@@ -158,13 +155,6 @@ class Meta:
]
read_only_fields = ["creation_mode", "effective_restriction_level", "is_shared"]
- def validate_description(self, value: str) -> str:
- if value and not check_is_feature_available_for_team(
- self.context["team_id"], AvailableFeature.TEAM_COLLABORATION
- ):
- raise PermissionDenied("You must have paid for dashboard collaboration to set the dashboard description")
- return value
-
def validate_filters(self, value) -> Dict:
if not isinstance(value, dict):
raise serializers.ValidationError("Filters must be a dictionary")
diff --git a/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr b/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr
index 642602f396f8d..9ae54e6e582eb 100644
--- a/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr
+++ b/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr
@@ -2762,6 +2762,24 @@
5 /* ... */) /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/
'''
# ---
+# name: TestDashboard.test_listing_dashboards_is_not_nplus1.57
+ '''
+ SELECT "posthog_sharingconfiguration"."id",
+ "posthog_sharingconfiguration"."team_id",
+ "posthog_sharingconfiguration"."dashboard_id",
+ "posthog_sharingconfiguration"."insight_id",
+ "posthog_sharingconfiguration"."recording_id",
+ "posthog_sharingconfiguration"."created_at",
+ "posthog_sharingconfiguration"."enabled",
+ "posthog_sharingconfiguration"."access_token"
+ FROM "posthog_sharingconfiguration"
+ WHERE "posthog_sharingconfiguration"."dashboard_id" IN (1,
+ 2,
+ 3,
+ 4,
+ 5 /* ... */) /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/
+ '''
+# ---
# name: TestDashboard.test_listing_dashboards_is_not_nplus1.6
'''
SELECT "posthog_team"."id",
@@ -11959,6 +11977,24 @@
5 /* ... */) /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/
'''
# ---
+# name: TestDashboard.test_retrieve_dashboard_list.33
+ '''
+ SELECT "posthog_sharingconfiguration"."id",
+ "posthog_sharingconfiguration"."team_id",
+ "posthog_sharingconfiguration"."dashboard_id",
+ "posthog_sharingconfiguration"."insight_id",
+ "posthog_sharingconfiguration"."recording_id",
+ "posthog_sharingconfiguration"."created_at",
+ "posthog_sharingconfiguration"."enabled",
+ "posthog_sharingconfiguration"."access_token"
+ FROM "posthog_sharingconfiguration"
+ WHERE "posthog_sharingconfiguration"."dashboard_id" IN (1,
+ 2,
+ 3,
+ 4,
+ 5 /* ... */) /*controller='project_dashboards-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/dashboards/%3F%24'*/
+ '''
+# ---
# name: TestDashboard.test_retrieve_dashboard_list.4
'''
SELECT "posthog_dashboardtile"."id"
diff --git a/posthog/batch_exports/service.py b/posthog/batch_exports/service.py
index c26be9a77ed1a..d51dfdb2fbc3c 100644
--- a/posthog/batch_exports/service.py
+++ b/posthog/batch_exports/service.py
@@ -90,6 +90,7 @@ class S3BatchExportInputs:
kms_key_id: str | None = None
batch_export_schema: BatchExportSchema | None = None
endpoint_url: str | None = None
+ file_format: str = "JSONLines"
@dataclass
@@ -439,8 +440,11 @@ def create_batch_export_run(
return run
-def update_batch_export_run_status(
- run_id: UUID, status: str, latest_error: str | None, records_completed: int = 0
+def update_batch_export_run(
+ run_id: UUID,
+ status: str,
+ latest_error: str | None,
+ records_completed: int = 0,
) -> BatchExportRun:
"""Update the status of an BatchExportRun with given id.
@@ -448,7 +452,14 @@ def update_batch_export_run_status(
id: The id of the BatchExportRun to update.
"""
model = BatchExportRun.objects.filter(id=run_id)
- updated = model.update(status=status, latest_error=latest_error, records_completed=records_completed)
+ update_at = dt.datetime.now()
+
+ updated = model.update(
+ status=status,
+ latest_error=latest_error,
+ records_completed=records_completed,
+ last_updated_at=update_at,
+ )
if not updated:
raise ValueError(f"BatchExportRun with id {run_id} not found.")
diff --git a/posthog/errors.py b/posthog/errors.py
index 39f07be762d00..afa0cdd8648e7 100644
--- a/posthog/errors.py
+++ b/posthog/errors.py
@@ -151,7 +151,7 @@ def look_up_error_code_meta(error: ServerException) -> ErrorCodeMeta:
60: ErrorCodeMeta("UNKNOWN_TABLE"),
61: ErrorCodeMeta("ONLY_FILTER_COLUMN_IN_BLOCK"),
62: ErrorCodeMeta("SYNTAX_ERROR"),
- 63: ErrorCodeMeta("UNKNOWN_AGGREGATE_FUNCTION"),
+ 63: ErrorCodeMeta("UNKNOWN_AGGREGATE_FUNCTION", user_safe=True),
64: ErrorCodeMeta("CANNOT_READ_AGGREGATE_FUNCTION_FROM_TEXT"),
65: ErrorCodeMeta("CANNOT_WRITE_AGGREGATE_FUNCTION_AS_TEXT"),
66: ErrorCodeMeta("NOT_A_COLUMN"),
diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py
index a459514f2524f..806226b8f1b9e 100644
--- a/posthog/hogql/ast.py
+++ b/posthog/hogql/ast.py
@@ -46,8 +46,17 @@ def resolve_constant_type(self, context: HogQLContext):
def resolve_database_field(self, context: HogQLContext):
if isinstance(self.type, FieldType):
return self.type.resolve_database_field(context)
+ if isinstance(self.type, PropertyType):
+ return self.type.field_type.resolve_database_field(context)
raise NotImplementedException("FieldAliasType.resolve_database_field not implemented")
+ def resolve_table_type(self, context: HogQLContext):
+ if isinstance(self.type, FieldType):
+ return self.type.table_type
+ if isinstance(self.type, PropertyType):
+ return self.type.field_type.table_type
+ raise NotImplementedException("FieldAliasType.resolve_table_type not implemented")
+
@dataclass(kw_only=True)
class BaseTableType(Type):
@@ -339,6 +348,9 @@ def get_child(self, name: str | int, context: HogQLContext) -> Type:
f'Can not access property "{name}" on field "{self.name}" of type: {type(database_field).__name__}'
)
+ def resolve_table_type(self, context: HogQLContext):
+ return self.table_type
+
@dataclass(kw_only=True)
class PropertyType(Type):
diff --git a/posthog/hogql/base.py b/posthog/hogql/base.py
index fbdafffb2d08c..e8a74025b78be 100644
--- a/posthog/hogql/base.py
+++ b/posthog/hogql/base.py
@@ -32,7 +32,7 @@ def accept(self, visitor):
return visit(self)
if hasattr(visitor, "visit_unknown"):
return visitor.visit_unknown(self)
- raise NotImplementedException(f"Visitor has no method {method_name}")
+ raise NotImplementedException(f"{visitor.__class__.__name__} has no method {method_name}")
@dataclass(kw_only=True)
diff --git a/posthog/hogql/database/__init__.py b/posthog/hogql/database/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/posthog/hogql/database/database.py b/posthog/hogql/database/database.py
index 6909211070e59..afeac3c26a143 100644
--- a/posthog/hogql/database/database.py
+++ b/posthog/hogql/database/database.py
@@ -31,6 +31,11 @@
from posthog.hogql.database.schema.events import EventsTable
from posthog.hogql.database.schema.groups import GroupsTable, RawGroupsTable
from posthog.hogql.database.schema.numbers import NumbersTable
+from posthog.hogql.database.schema.person_distinct_id_overrides import (
+ PersonDistinctIdOverridesTable,
+ RawPersonDistinctIdOverridesTable,
+ join_with_person_distinct_id_overrides_table,
+)
from posthog.hogql.database.schema.person_distinct_ids import (
PersonDistinctIdsTable,
RawPersonDistinctIdsTable,
@@ -53,7 +58,6 @@
from posthog.models.team.team import WeekStartDay
from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode
-
if TYPE_CHECKING:
from posthog.models import Team
@@ -66,6 +70,7 @@ class Database(BaseModel):
groups: GroupsTable = GroupsTable()
persons: PersonsTable = PersonsTable()
person_distinct_ids: PersonDistinctIdsTable = PersonDistinctIdsTable()
+ person_distinct_id_overrides: PersonDistinctIdOverridesTable = PersonDistinctIdOverridesTable()
person_overrides: PersonOverridesTable = PersonOverridesTable()
session_replay_events: SessionReplayEventsTable = SessionReplayEventsTable()
@@ -81,6 +86,7 @@ class Database(BaseModel):
raw_persons: RawPersonsTable = RawPersonsTable()
raw_groups: RawGroupsTable = RawGroupsTable()
raw_cohort_people: RawCohortPeople = RawCohortPeople()
+ raw_person_distinct_id_overrides: RawPersonDistinctIdOverridesTable = RawPersonDistinctIdOverridesTable()
raw_person_overrides: RawPersonOverridesTable = RawPersonOverridesTable()
raw_sessions: RawSessionsTable = RawSessionsTable()
@@ -186,6 +192,24 @@ def create_hogql_database(
database.events.fields["poe"].fields["id"] = database.events.fields["person_id"]
database.events.fields["person"] = FieldTraverser(chain=["poe"])
+ elif modifiers.personsOnEventsMode == PersonsOnEventsMode.v3_enabled:
+ database.events.fields["event_person_id"] = StringDatabaseField(name="person_id")
+ database.events.fields["override"] = LazyJoin(
+ from_field=["distinct_id"], # ???
+ join_table=PersonDistinctIdOverridesTable(),
+ join_function=join_with_person_distinct_id_overrides_table,
+ )
+ database.events.fields["person_id"] = ExpressionField(
+ name="person_id",
+ expr=parse_expr(
+ # NOTE: assumes `join_use_nulls = 0` (the default), as ``override.distinct_id`` is not Nullable
+ "if(not(empty(override.distinct_id)), override.person_id, event_person_id)",
+ start=None,
+ ),
+ )
+ database.events.fields["poe"].fields["id"] = database.events.fields["person_id"]
+ database.events.fields["person"] = FieldTraverser(chain=["poe"])
+
database.persons.fields["$virt_initial_referring_domain_type"] = create_initial_domain_type(
"$virt_initial_referring_domain_type"
)
@@ -209,10 +233,22 @@ def create_hogql_database(
)
if "timestamp" not in tables[warehouse_modifier.table_name].fields.keys():
- tables[warehouse_modifier.table_name].fields["timestamp"] = ExpressionField(
- name="timestamp",
- expr=ast.Call(name="toDateTime", args=[ast.Field(chain=[warehouse_modifier.timestamp_field])]),
- )
+ table_model = DataWarehouseTable.objects.filter(
+ team_id=team.pk, name=warehouse_modifier.table_name
+ ).latest("created_at")
+ timestamp_field_type = table_model.get_clickhouse_column_type(warehouse_modifier.timestamp_field)
+
+ # If field type is none or datetime, we can use the field directly
+ if timestamp_field_type is None or timestamp_field_type.startswith("DateTime"):
+ tables[warehouse_modifier.table_name].fields["timestamp"] = ExpressionField(
+ name="timestamp",
+ expr=ast.Field(chain=[warehouse_modifier.timestamp_field]),
+ )
+ else:
+ tables[warehouse_modifier.table_name].fields["timestamp"] = ExpressionField(
+ name="timestamp",
+ expr=ast.Call(name="toDateTime", args=[ast.Field(chain=[warehouse_modifier.timestamp_field])]),
+ )
# TODO: Need to decide how the distinct_id and person_id fields are going to be handled
if "distinct_id" not in tables[warehouse_modifier.table_name].fields.keys():
diff --git a/posthog/hogql/database/models.py b/posthog/hogql/database/models.py
index 95a00595c6472..d2da7868a7f9c 100644
--- a/posthog/hogql/database/models.py
+++ b/posthog/hogql/database/models.py
@@ -3,7 +3,6 @@
from posthog.hogql.base import Expr
from posthog.hogql.errors import HogQLException, NotImplementedException
-from posthog.schema import HogQLQueryModifiers
if TYPE_CHECKING:
from posthog.hogql.context import HogQLContext
@@ -126,12 +125,14 @@ def resolve_table(self, context: "HogQLContext") -> Table:
class LazyTable(Table):
"""
- A table that is replaced with a subquery returned from `lazy_select(requested_fields: Dict[name, chain], modifiers: HogQLQueryModifiers)`
+ A table that is replaced with a subquery returned from `lazy_select(requested_fields: Dict[name, chain], modifiers: HogQLQueryModifiers, node: SelectQuery)`
"""
model_config = ConfigDict(extra="forbid")
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers) -> Any:
+ def lazy_select(
+ self, requested_fields: Dict[str, List[str | int]], context: "HogQLContext", node: "SelectQuery"
+ ) -> Any:
raise NotImplementedException("LazyTable.lazy_select not overridden")
diff --git a/posthog/hogql/database/schema/__init__.py b/posthog/hogql/database/schema/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/posthog/hogql/database/schema/cohort_people.py b/posthog/hogql/database/schema/cohort_people.py
index 72080419b7355..11723f0194619 100644
--- a/posthog/hogql/database/schema/cohort_people.py
+++ b/posthog/hogql/database/schema/cohort_people.py
@@ -9,7 +9,6 @@
FieldOrTable,
)
from posthog.hogql.database.schema.persons import join_with_persons_table
-from posthog.schema import HogQLQueryModifiers
COHORT_PEOPLE_FIELDS = {
"person_id": StringDatabaseField(name="person_id"),
@@ -67,7 +66,7 @@ def to_printed_hogql(self):
class CohortPeople(LazyTable):
fields: Dict[str, FieldOrTable] = COHORT_PEOPLE_FIELDS
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers):
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node):
return select_from_cohort_people_table(requested_fields)
def to_printed_clickhouse(self, context):
diff --git a/posthog/hogql/database/schema/groups.py b/posthog/hogql/database/schema/groups.py
index bb237d68e8070..3b9de7f08befc 100644
--- a/posthog/hogql/database/schema/groups.py
+++ b/posthog/hogql/database/schema/groups.py
@@ -13,7 +13,6 @@
FieldOrTable,
)
from posthog.hogql.errors import HogQLException
-from posthog.schema import HogQLQueryModifiers
GROUPS_TABLE_FIELDS = {
"index": IntegerDatabaseField(name="group_type_index"),
@@ -83,7 +82,7 @@ def to_printed_hogql(self):
class GroupsTable(LazyTable):
fields: Dict[str, FieldOrTable] = GROUPS_TABLE_FIELDS
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers):
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node):
return select_from_groups_table(requested_fields)
def to_printed_clickhouse(self, context):
diff --git a/posthog/hogql/database/schema/log_entries.py b/posthog/hogql/database/schema/log_entries.py
index c14e90e26da50..9f5dc816ac4b0 100644
--- a/posthog/hogql/database/schema/log_entries.py
+++ b/posthog/hogql/database/schema/log_entries.py
@@ -9,7 +9,6 @@
LazyTable,
FieldOrTable,
)
-from posthog.schema import HogQLQueryModifiers
LOG_ENTRIES_FIELDS: Dict[str, FieldOrTable] = {
"team_id": IntegerDatabaseField(name="team_id"),
@@ -35,7 +34,7 @@ def to_printed_hogql(self):
class ReplayConsoleLogsLogEntriesTable(LazyTable):
fields: Dict[str, FieldOrTable] = LOG_ENTRIES_FIELDS
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers):
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node):
fields: List[ast.Expr] = [ast.Field(chain=["log_entries"] + chain) for name, chain in requested_fields.items()]
return ast.SelectQuery(
@@ -58,7 +57,7 @@ def to_printed_hogql(self):
class BatchExportLogEntriesTable(LazyTable):
fields: Dict[str, FieldOrTable] = LOG_ENTRIES_FIELDS
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers):
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node):
fields: List[ast.Expr] = [ast.Field(chain=["log_entries"] + chain) for name, chain in requested_fields.items()]
return ast.SelectQuery(
diff --git a/posthog/hogql/database/schema/person_distinct_id_overrides.py b/posthog/hogql/database/schema/person_distinct_id_overrides.py
new file mode 100644
index 0000000000000..34df59655c24d
--- /dev/null
+++ b/posthog/hogql/database/schema/person_distinct_id_overrides.py
@@ -0,0 +1,91 @@
+from typing import Dict, List
+from posthog.hogql.ast import SelectQuery
+from posthog.hogql.context import HogQLContext
+
+from posthog.hogql.database.argmax import argmax_select
+from posthog.hogql.database.models import (
+ Table,
+ IntegerDatabaseField,
+ StringDatabaseField,
+ BooleanDatabaseField,
+ LazyJoin,
+ LazyTable,
+ FieldOrTable,
+)
+from posthog.hogql.database.schema.persons import join_with_persons_table
+from posthog.hogql.errors import HogQLException
+
+PERSON_DISTINCT_ID_OVERRIDES_FIELDS = {
+ "team_id": IntegerDatabaseField(name="team_id"),
+ "distinct_id": StringDatabaseField(name="distinct_id"),
+ "person_id": StringDatabaseField(name="person_id"),
+ "person": LazyJoin(
+ from_field=["person_id"],
+ join_table="persons",
+ join_function=join_with_persons_table,
+ ),
+}
+
+
+def select_from_person_distinct_id_overrides_table(requested_fields: Dict[str, List[str | int]]):
+ # Always include "person_id", as it's the key we use to make further joins, and it'd be great if it's available
+ if "person_id" not in requested_fields:
+ requested_fields = {**requested_fields, "person_id": ["person_id"]}
+ return argmax_select(
+ table_name="raw_person_distinct_id_overrides",
+ select_fields=requested_fields,
+ group_fields=["distinct_id"],
+ argmax_field="version",
+ deleted_field="is_deleted",
+ )
+
+
+def join_with_person_distinct_id_overrides_table(
+ from_table: str,
+ to_table: str,
+ requested_fields: Dict[str, List[str]],
+ context: HogQLContext,
+ node: SelectQuery,
+):
+ from posthog.hogql import ast
+
+ if not requested_fields:
+ raise HogQLException("No fields requested from person_distinct_id_overrides")
+ join_expr = ast.JoinExpr(table=select_from_person_distinct_id_overrides_table(requested_fields))
+ join_expr.join_type = "LEFT OUTER JOIN"
+ join_expr.alias = to_table
+ join_expr.constraint = ast.JoinConstraint(
+ expr=ast.CompareOperation(
+ op=ast.CompareOperationOp.Eq,
+ left=ast.Field(chain=[from_table, "distinct_id"]),
+ right=ast.Field(chain=[to_table, "distinct_id"]),
+ )
+ )
+ return join_expr
+
+
+class RawPersonDistinctIdOverridesTable(Table):
+ fields: Dict[str, FieldOrTable] = {
+ **PERSON_DISTINCT_ID_OVERRIDES_FIELDS,
+ "is_deleted": BooleanDatabaseField(name="is_deleted"),
+ "version": IntegerDatabaseField(name="version"),
+ }
+
+ def to_printed_clickhouse(self, context):
+ return "person_distinct_id_overrides"
+
+ def to_printed_hogql(self):
+ return "raw_person_distinct_id_overrides"
+
+
+class PersonDistinctIdOverridesTable(LazyTable):
+ fields: Dict[str, FieldOrTable] = PERSON_DISTINCT_ID_OVERRIDES_FIELDS
+
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context: HogQLContext, node: SelectQuery):
+ return select_from_person_distinct_id_overrides_table(requested_fields)
+
+ def to_printed_clickhouse(self, context):
+ return "person_distinct_id_overrides"
+
+ def to_printed_hogql(self):
+ return "person_distinct_id_overrides"
diff --git a/posthog/hogql/database/schema/person_distinct_ids.py b/posthog/hogql/database/schema/person_distinct_ids.py
index 02144b35fc3d8..3304eccda862e 100644
--- a/posthog/hogql/database/schema/person_distinct_ids.py
+++ b/posthog/hogql/database/schema/person_distinct_ids.py
@@ -14,7 +14,6 @@
)
from posthog.hogql.database.schema.persons import join_with_persons_table
from posthog.hogql.errors import HogQLException
-from posthog.schema import HogQLQueryModifiers
PERSON_DISTINCT_IDS_FIELDS = {
"team_id": IntegerDatabaseField(name="team_id"),
@@ -82,7 +81,7 @@ def to_printed_hogql(self):
class PersonDistinctIdsTable(LazyTable):
fields: Dict[str, FieldOrTable] = PERSON_DISTINCT_IDS_FIELDS
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers):
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node):
return select_from_person_distinct_ids_table(requested_fields)
def to_printed_clickhouse(self, context):
diff --git a/posthog/hogql/database/schema/persons.py b/posthog/hogql/database/schema/persons.py
index a248da56b7307..c7abdd89e14c6 100644
--- a/posthog/hogql/database/schema/persons.py
+++ b/posthog/hogql/database/schema/persons.py
@@ -123,8 +123,8 @@ def to_printed_hogql(self):
class PersonsTable(LazyTable):
fields: Dict[str, FieldOrTable] = PERSONS_FIELDS
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers):
- return select_from_persons_table(requested_fields, modifiers)
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node):
+ return select_from_persons_table(requested_fields, context.modifiers)
def to_printed_clickhouse(self, context):
return "person"
diff --git a/posthog/hogql/database/schema/persons_pdi.py b/posthog/hogql/database/schema/persons_pdi.py
index 9f476f407b4d2..195643b90c08c 100644
--- a/posthog/hogql/database/schema/persons_pdi.py
+++ b/posthog/hogql/database/schema/persons_pdi.py
@@ -10,7 +10,6 @@
FieldOrTable,
)
from posthog.hogql.errors import HogQLException
-from posthog.schema import HogQLQueryModifiers
# :NOTE: We already have person_distinct_ids.py, which most tables link to. This persons_pdi.py is a hack to
@@ -63,7 +62,7 @@ class PersonsPDITable(LazyTable):
"person_id": StringDatabaseField(name="person_id"),
}
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers):
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node):
return persons_pdi_select(requested_fields)
def to_printed_clickhouse(self, context):
diff --git a/posthog/hogql/database/schema/session_replay_events.py b/posthog/hogql/database/schema/session_replay_events.py
index c9d564c7d4588..baaecef89e049 100644
--- a/posthog/hogql/database/schema/session_replay_events.py
+++ b/posthog/hogql/database/schema/session_replay_events.py
@@ -15,7 +15,6 @@
PersonDistinctIdsTable,
join_with_person_distinct_ids_table,
)
-from posthog.schema import HogQLQueryModifiers
RAW_ONLY_FIELDS = ["min_first_timestamp", "max_last_timestamp"]
@@ -115,7 +114,7 @@ class SessionReplayEventsTable(LazyTable):
"first_url": StringDatabaseField(name="first_url"),
}
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers):
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node):
return select_from_session_replay_events_table(requested_fields)
def to_printed_clickhouse(self, context):
diff --git a/posthog/hogql/database/schema/sessions.py b/posthog/hogql/database/schema/sessions.py
index 2a4865798eeb8..770daceaa23c5 100644
--- a/posthog/hogql/database/schema/sessions.py
+++ b/posthog/hogql/database/schema/sessions.py
@@ -1,5 +1,7 @@
from typing import Dict, List, cast
+from posthog.hogql import ast
+from posthog.hogql.context import HogQLContext
from posthog.hogql.database.models import (
StringDatabaseField,
DateTimeDatabaseField,
@@ -11,7 +13,7 @@
LazyTable,
)
from posthog.hogql.database.schema.channel_type import create_channel_type_expr
-from posthog.schema import HogQLQueryModifiers
+from posthog.hogql.database.schema.util.session_where_clause_extractor import SessionMinTimestampWhereClauseExtractor
SESSIONS_COMMON_FIELDS: Dict[str, FieldOrTable] = {
@@ -62,7 +64,9 @@ def avoid_asterisk_fields(self) -> List[str]:
]
-def select_from_sessions_table(requested_fields: Dict[str, List[str | int]]):
+def select_from_sessions_table(
+ requested_fields: Dict[str, List[str | int]], node: ast.SelectQuery, context: HogQLContext
+):
from posthog.hogql import ast
table_name = "raw_sessions"
@@ -134,10 +138,13 @@ def select_from_sessions_table(requested_fields: Dict[str, List[str | int]]):
)
group_by_fields.append(ast.Field(chain=cast(list[str | int], [table_name]) + chain))
+ where = SessionMinTimestampWhereClauseExtractor(context).get_inner_where(node)
+
return ast.SelectQuery(
select=select_fields,
select_from=ast.JoinExpr(table=ast.Field(chain=[table_name])),
group_by=group_by_fields,
+ where=where,
)
@@ -148,8 +155,8 @@ class SessionsTable(LazyTable):
"channel_type": StringDatabaseField(name="channel_type"),
}
- def lazy_select(self, requested_fields: Dict[str, List[str | int]], modifiers: HogQLQueryModifiers):
- return select_from_sessions_table(requested_fields)
+ def lazy_select(self, requested_fields: Dict[str, List[str | int]], context, node: ast.SelectQuery):
+ return select_from_sessions_table(requested_fields, node, context)
def to_printed_clickhouse(self, context):
return "sessions"
diff --git a/posthog/hogql/database/schema/util/__init__.py b/posthog/hogql/database/schema/util/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/posthog/hogql/database/schema/util/session_where_clause_extractor.py b/posthog/hogql/database/schema/util/session_where_clause_extractor.py
new file mode 100644
index 0000000000000..83933bdde8b85
--- /dev/null
+++ b/posthog/hogql/database/schema/util/session_where_clause_extractor.py
@@ -0,0 +1,398 @@
+from dataclasses import dataclass
+from typing import Optional
+
+from posthog.hogql import ast
+from posthog.hogql.ast import CompareOperationOp, ArithmeticOperationOp
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.models import DatabaseField
+
+from posthog.hogql.visitor import clone_expr, CloningVisitor, Visitor
+
+SESSION_BUFFER_DAYS = 3
+
+
+@dataclass
+class SessionMinTimestampWhereClauseExtractor(CloningVisitor):
+ """This class extracts the Where clause from the lazy sessions table, to the clickhouse sessions table.
+
+ The sessions table in Clickhouse is an AggregatingMergeTree, and will have one row per session per day. This means that
+ when we want to query sessions, we need to pre-group these rows, so that we only have one row per session.
+
+ We hide this detail using a lazy table, but to make querying the underlying Clickhouse table faster, we can inline the
+ min_timestamp where conditions from the select on the outer lazy table to the select on the inner real table.
+
+ This class is called on the select query of the lazy table, and will return the where clause that should be applied to
+ the inner table.
+
+ As a query can be unreasonably complex, we only handle simple cases, but this class is designed to fail-safe. If it
+ can't reason about a particular expression, it will just return a constant True, i.e. fetch more rows than necessary.
+
+ This means that we can incrementally add support for more complex queries, without breaking existing queries, by
+ handling more cases.
+
+ Some examples of failing-safe:
+
+ `SELECT * FROM sessions where min_timestamp > '2022-01-01' AND f(session_id)`
+ only the` min_timestamp > '2022-01-01'` part is relevant, so we can ignore the `f(session_id)` part, and it is safe
+ to replace it with a constant True, which collapses the AND to just the `min_timestamp > '2022-01-01'` part.
+
+ `SELECT * FROM sessions where min_timestamp > '2022-01-01' OR f(session_id)`
+ only the` min_timestamp > '2022-01-01'` part is relevant, and turning the `f(session_id)` part into a constant True
+ would collapse the OR to True. In this case we return None as no pre-filtering is possible.
+
+ All min_timestamp comparisons are given a buffer of SESSION_BUFFER_DAYS each side, to ensure that we collect all the
+ relevant rows for each session.
+ """
+
+ context: HogQLContext
+ clear_types: bool = False
+ clear_locations: bool = False
+
+ def get_inner_where(self, parsed_query: ast.SelectQuery) -> Optional[ast.Expr]:
+ if not parsed_query.where:
+ return None
+
+ # visit the where clause
+ where = self.visit(parsed_query.where)
+
+ if isinstance(where, ast.Constant):
+ return None
+
+ return clone_expr(where, clear_types=True, clear_locations=True)
+
+ def visit_compare_operation(self, node: ast.CompareOperation) -> ast.Expr:
+ is_left_constant = is_time_or_interval_constant(node.left)
+ is_right_constant = is_time_or_interval_constant(node.right)
+ is_left_timestamp_field = is_simple_timestamp_field_expression(node.left, self.context)
+ is_right_timestamp_field = is_simple_timestamp_field_expression(node.right, self.context)
+
+ if is_left_constant and is_right_constant:
+ # just ignore this comparison
+ return ast.Constant(value=True)
+
+ # handle the left side being a min_timestamp expression and the right being constant
+ if is_left_timestamp_field and is_right_constant:
+ if node.op == CompareOperationOp.Eq:
+ return ast.And(
+ exprs=[
+ ast.CompareOperation(
+ op=ast.CompareOperationOp.LtEq,
+ left=ast.ArithmeticOperation(
+ op=ast.ArithmeticOperationOp.Sub,
+ left=rewrite_timestamp_field(node.left, self.context),
+ right=ast.Call(name="toIntervalDay", args=[ast.Constant(value=SESSION_BUFFER_DAYS)]),
+ ),
+ right=node.right,
+ ),
+ ast.CompareOperation(
+ op=ast.CompareOperationOp.GtEq,
+ left=ast.ArithmeticOperation(
+ op=ast.ArithmeticOperationOp.Add,
+ left=rewrite_timestamp_field(node.left, self.context),
+ right=ast.Call(name="toIntervalDay", args=[ast.Constant(value=SESSION_BUFFER_DAYS)]),
+ ),
+ right=node.right,
+ ),
+ ]
+ )
+ elif node.op == CompareOperationOp.Gt or node.op == CompareOperationOp.GtEq:
+ return ast.CompareOperation(
+ op=ast.CompareOperationOp.GtEq,
+ left=ast.ArithmeticOperation(
+ op=ast.ArithmeticOperationOp.Add,
+ left=rewrite_timestamp_field(node.left, self.context),
+ right=ast.Call(name="toIntervalDay", args=[ast.Constant(value=SESSION_BUFFER_DAYS)]),
+ ),
+ right=node.right,
+ )
+ elif node.op == CompareOperationOp.Lt or node.op == CompareOperationOp.LtEq:
+ return ast.CompareOperation(
+ op=ast.CompareOperationOp.LtEq,
+ left=ast.ArithmeticOperation(
+ op=ast.ArithmeticOperationOp.Sub,
+ left=rewrite_timestamp_field(node.left, self.context),
+ right=ast.Call(name="toIntervalDay", args=[ast.Constant(value=SESSION_BUFFER_DAYS)]),
+ ),
+ right=node.right,
+ )
+ elif is_right_timestamp_field and is_left_constant:
+ # let's not duplicate the logic above, instead just flip and it and recurse
+ if node.op in [
+ CompareOperationOp.Eq,
+ CompareOperationOp.Lt,
+ CompareOperationOp.LtEq,
+ CompareOperationOp.Gt,
+ CompareOperationOp.GtEq,
+ ]:
+ return self.visit(
+ ast.CompareOperation(
+ op=CompareOperationOp.Eq
+ if node.op == CompareOperationOp.Eq
+ else CompareOperationOp.Lt
+ if node.op == CompareOperationOp.Gt
+ else CompareOperationOp.LtEq
+ if node.op == CompareOperationOp.GtEq
+ else CompareOperationOp.Gt
+ if node.op == CompareOperationOp.Lt
+ else CompareOperationOp.GtEq,
+ left=node.right,
+ right=node.left,
+ )
+ )
+
+ return ast.Constant(value=True)
+
+ def visit_arithmetic_operation(self, node: ast.ArithmeticOperation) -> ast.Expr:
+ # don't even try to handle complex logic
+ return ast.Constant(value=True)
+
+ def visit_not(self, node: ast.Not) -> ast.Expr:
+ return ast.Constant(value=True)
+
+ def visit_call(self, node: ast.Call) -> ast.Expr:
+ if node.name == "and":
+ return self.visit_and(ast.And(exprs=node.args))
+ elif node.name == "or":
+ return self.visit_or(ast.Or(exprs=node.args))
+ return ast.Constant(value=True)
+
+ def visit_field(self, node: ast.Field) -> ast.Expr:
+ return ast.Constant(value=True)
+
+ def visit_constant(self, node: ast.Constant) -> ast.Expr:
+ return ast.Constant(value=True)
+
+ def visit_placeholder(self, node: ast.Placeholder) -> ast.Expr:
+ raise Exception() # this should never happen, as placeholders should be resolved before this runs
+
+ def visit_and(self, node: ast.And) -> ast.Expr:
+ exprs = [self.visit(expr) for expr in node.exprs]
+
+ flattened = []
+ for expr in exprs:
+ if isinstance(expr, ast.And):
+ flattened.extend(expr.exprs)
+ else:
+ flattened.append(expr)
+
+ if any(isinstance(expr, ast.Constant) and expr.value is False for expr in flattened):
+ return ast.Constant(value=False)
+
+ filtered = [expr for expr in flattened if not isinstance(expr, ast.Constant) or expr.value is not True]
+ if len(filtered) == 0:
+ return ast.Constant(value=True)
+ elif len(filtered) == 1:
+ return filtered[0]
+ else:
+ return ast.And(exprs=filtered)
+
+ def visit_or(self, node: ast.Or) -> ast.Expr:
+ exprs = [self.visit(expr) for expr in node.exprs]
+
+ flattened = []
+ for expr in exprs:
+ if isinstance(expr, ast.Or):
+ flattened.extend(expr.exprs)
+ else:
+ flattened.append(expr)
+
+ if any(isinstance(expr, ast.Constant) and expr.value is True for expr in flattened):
+ return ast.Constant(value=True)
+
+ filtered = [expr for expr in flattened if not isinstance(expr, ast.Constant) or expr.value is not False]
+ if len(filtered) == 0:
+ return ast.Constant(value=False)
+ elif len(filtered) == 1:
+ return filtered[0]
+ else:
+ return ast.Or(exprs=filtered)
+
+ def visit_alias(self, node: ast.Alias) -> ast.Expr:
+ return self.visit(node.expr)
+
+
+def is_time_or_interval_constant(expr: ast.Expr) -> bool:
+ return IsTimeOrIntervalConstantVisitor().visit(expr)
+
+
+class IsTimeOrIntervalConstantVisitor(Visitor[bool]):
+ def visit_constant(self, node: ast.Constant) -> bool:
+ return True
+
+ def visit_compare_operation(self, node: ast.CompareOperation) -> bool:
+ return self.visit(node.left) and self.visit(node.right)
+
+ def visit_arithmetic_operation(self, node: ast.ArithmeticOperation) -> bool:
+ return self.visit(node.left) and self.visit(node.right)
+
+ def visit_call(self, node: ast.Call) -> bool:
+ # some functions just return a constant
+ if node.name in ["today", "now"]:
+ return True
+ # some functions return a constant if the first argument is a constant
+ if node.name in [
+ "parseDateTime64BestEffortOrNull",
+ "toDateTime",
+ "toTimeZone",
+ "assumeNotNull",
+ "toIntervalYear",
+ "toIntervalMonth",
+ "toIntervalWeek",
+ "toIntervalDay",
+ "toIntervalHour",
+ "toIntervalMinute",
+ "toIntervalSecond",
+ "toStartOfDay",
+ "toStartOfWeek",
+ "toStartOfMonth",
+ "toStartOfQuarter",
+ "toStartOfYear",
+ ]:
+ return self.visit(node.args[0])
+
+ if node.name in ["minus", "add"]:
+ return all(self.visit(arg) for arg in node.args)
+
+ # otherwise we don't know, so return False
+ return False
+
+ def visit_field(self, node: ast.Field) -> bool:
+ return False
+
+ def visit_and(self, node: ast.And) -> bool:
+ return False
+
+ def visit_or(self, node: ast.Or) -> bool:
+ return False
+
+ def visit_not(self, node: ast.Not) -> bool:
+ return False
+
+ def visit_placeholder(self, node: ast.Placeholder) -> bool:
+ raise Exception()
+
+ def visit_alias(self, node: ast.Alias) -> bool:
+ return self.visit(node.expr)
+
+
+def is_simple_timestamp_field_expression(expr: ast.Expr, context: HogQLContext) -> bool:
+ return IsSimpleTimestampFieldExpressionVisitor(context).visit(expr)
+
+
+@dataclass
+class IsSimpleTimestampFieldExpressionVisitor(Visitor[bool]):
+ context: HogQLContext
+
+ def visit_constant(self, node: ast.Constant) -> bool:
+ return False
+
+ def visit_field(self, node: ast.Field) -> bool:
+ if node.type and isinstance(node.type, ast.FieldType):
+ resolved_field = node.type.resolve_database_field(self.context)
+ if resolved_field and isinstance(resolved_field, DatabaseField) and resolved_field:
+ return resolved_field.name in ["min_timestamp", "timestamp"]
+ # no type information, so just use the name of the field
+ return node.chain[-1] in ["min_timestamp", "timestamp"]
+
+ def visit_arithmetic_operation(self, node: ast.ArithmeticOperation) -> bool:
+ # only allow the min_timestamp field to be used on one side of the arithmetic operation
+ return (
+ self.visit(node.left)
+ and is_time_or_interval_constant(node.right)
+ or (self.visit(node.right) and is_time_or_interval_constant(node.left))
+ )
+
+ def visit_call(self, node: ast.Call) -> bool:
+ # some functions count as a timestamp field expression if their first argument is
+ if node.name in [
+ "parseDateTime64BestEffortOrNull",
+ "toDateTime",
+ "toTimeZone",
+ "assumeNotNull",
+ "toStartOfDay",
+ "toStartOfWeek",
+ "toStartOfMonth",
+ "toStartOfQuarter",
+ "toStartOfYear",
+ ]:
+ return self.visit(node.args[0])
+
+ if node.name in ["minus", "add"]:
+ return self.visit_arithmetic_operation(
+ ast.ArithmeticOperation(
+ op=ArithmeticOperationOp.Sub if node.name == "minus" else ArithmeticOperationOp.Add,
+ left=node.args[0],
+ right=node.args[1],
+ )
+ )
+
+ # otherwise we don't know, so return False
+ return False
+
+ def visit_compare_operation(self, node: ast.CompareOperation) -> bool:
+ return False
+
+ def visit_and(self, node: ast.And) -> bool:
+ return False
+
+ def visit_or(self, node: ast.Or) -> bool:
+ return False
+
+ def visit_not(self, node: ast.Not) -> bool:
+ return False
+
+ def visit_placeholder(self, node: ast.Placeholder) -> bool:
+ raise Exception()
+
+ def visit_alias(self, node: ast.Alias) -> bool:
+ from posthog.hogql.database.schema.events import EventsTable
+ from posthog.hogql.database.schema.sessions import SessionsTable
+
+ if node.type and isinstance(node.type, ast.FieldAliasType):
+ resolved_field = node.type.resolve_database_field(self.context)
+ table_type = node.type.resolve_table_type(self.context)
+ if not table_type:
+ return False
+ return (
+ isinstance(table_type, ast.TableType)
+ and isinstance(table_type.table, EventsTable)
+ and resolved_field.name == "timestamp"
+ ) or (
+ isinstance(table_type, ast.LazyTableType)
+ and isinstance(table_type.table, SessionsTable)
+ and resolved_field.name == "min_timestamp"
+ )
+
+ return self.visit(node.expr)
+
+
+def rewrite_timestamp_field(expr: ast.Expr, context: HogQLContext) -> ast.Expr:
+ return RewriteTimestampFieldVisitor(context).visit(expr)
+
+
+class RewriteTimestampFieldVisitor(CloningVisitor):
+ context: HogQLContext
+
+ def __init__(self, context: HogQLContext, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.context = context
+
+ def visit_field(self, node: ast.Field) -> ast.Field:
+ from posthog.hogql.database.schema.events import EventsTable
+ from posthog.hogql.database.schema.sessions import SessionsTable
+
+ if node.type and isinstance(node.type, ast.FieldType):
+ resolved_field = node.type.resolve_database_field(self.context)
+ table = node.type.resolve_table_type(self.context).table
+ if resolved_field and isinstance(resolved_field, DatabaseField):
+ if (isinstance(table, EventsTable) and resolved_field.name == "timestamp") or (
+ isinstance(table, SessionsTable) and resolved_field.name == "min_timestamp"
+ ):
+ return ast.Field(chain=["raw_sessions", "min_timestamp"])
+ # no type information, so just use the name of the field
+ if node.chain[-1] in ["min_timestamp", "timestamp"]:
+ return ast.Field(chain=["raw_sessions", "min_timestamp"])
+ return node
+
+ def visit_alias(self, node: ast.Alias) -> ast.Expr:
+ return self.visit(node.expr)
diff --git a/posthog/hogql/database/schema/util/test/test_session_where_clause_extractor.py b/posthog/hogql/database/schema/util/test/test_session_where_clause_extractor.py
new file mode 100644
index 0000000000000..bc5324e739ad9
--- /dev/null
+++ b/posthog/hogql/database/schema/util/test/test_session_where_clause_extractor.py
@@ -0,0 +1,284 @@
+from typing import Union, Optional, Dict
+
+from posthog.hogql import ast
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.schema.util.session_where_clause_extractor import SessionMinTimestampWhereClauseExtractor
+from posthog.hogql.modifiers import create_default_modifiers_for_team
+from posthog.hogql.parser import parse_select, parse_expr
+from posthog.hogql.printer import prepare_ast_for_printing, print_prepared_ast
+from posthog.hogql.visitor import clone_expr
+from posthog.test.base import ClickhouseTestMixin, APIBaseTest
+
+
+def f(s: Union[str, ast.Expr, None], placeholders: Optional[dict[str, ast.Expr]] = None) -> Union[ast.Expr, None]:
+ if s is None:
+ return None
+ if isinstance(s, str):
+ expr = parse_expr(s, placeholders=placeholders)
+ else:
+ expr = s
+ return clone_expr(expr, clear_types=True, clear_locations=True)
+
+
+def parse(
+ s: str,
+ placeholders: Optional[Dict[str, ast.Expr]] = None,
+) -> ast.SelectQuery:
+ parsed = parse_select(s, placeholders=placeholders)
+ assert isinstance(parsed, ast.SelectQuery)
+ return parsed
+
+
+class TestSessionTimestampInliner(ClickhouseTestMixin, APIBaseTest):
+ @property
+ def inliner(self):
+ team = self.team
+ modifiers = create_default_modifiers_for_team(team)
+ context = HogQLContext(
+ team_id=team.pk,
+ team=team,
+ enable_select_queries=True,
+ modifiers=modifiers,
+ )
+ return SessionMinTimestampWhereClauseExtractor(context)
+
+ def test_handles_select_with_no_where_claus(self):
+ inner_where = self.inliner.get_inner_where(parse("SELECT * FROM sessions"))
+ assert inner_where is None
+
+ def test_handles_select_with_eq(self):
+ actual = f(self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE min_timestamp = '2021-01-01'")))
+ expected = f(
+ "((raw_sessions.min_timestamp - toIntervalDay(3)) <= '2021-01-01') AND ((raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-01')"
+ )
+ assert expected == actual
+
+ def test_handles_select_with_eq_flipped(self):
+ actual = f(self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE '2021-01-01' = min_timestamp")))
+ expected = f(
+ "((raw_sessions.min_timestamp - toIntervalDay(3)) <= '2021-01-01') AND ((raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-01')"
+ )
+ assert expected == actual
+
+ def test_handles_select_with_simple_gt(self):
+ actual = f(self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE min_timestamp > '2021-01-01'")))
+ expected = f("((raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-01')")
+ assert expected == actual
+
+ def test_handles_select_with_simple_gte(self):
+ actual = f(self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE min_timestamp >= '2021-01-01'")))
+ expected = f("((raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-01')")
+ assert expected == actual
+
+ def test_handles_select_with_simple_lt(self):
+ actual = f(self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE min_timestamp < '2021-01-01'")))
+ expected = f("((raw_sessions.min_timestamp - toIntervalDay(3)) <= '2021-01-01')")
+ assert expected == actual
+
+ def test_handles_select_with_simple_lte(self):
+ actual = f(self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE min_timestamp <= '2021-01-01'")))
+ expected = f("((raw_sessions.min_timestamp - toIntervalDay(3)) <= '2021-01-01')")
+ assert expected == actual
+
+ def test_select_with_placeholder(self):
+ actual = f(
+ self.inliner.get_inner_where(
+ parse(
+ "SELECT * FROM sessions WHERE min_timestamp > {timestamp}",
+ placeholders={"timestamp": ast.Constant(value="2021-01-01")},
+ )
+ )
+ )
+ expected = f("((raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-01')")
+ assert expected == actual
+
+ def test_unrelated_equals(self):
+ actual = self.inliner.get_inner_where(
+ parse("SELECT * FROM sessions WHERE initial_utm_campaign = initial_utm_source")
+ )
+ assert actual is None
+
+ def test_timestamp_and(self):
+ actual = f(
+ self.inliner.get_inner_where(
+ parse("SELECT * FROM sessions WHERE and(min_timestamp >= '2021-01-01', min_timestamp <= '2021-01-03')")
+ )
+ )
+ expected = f(
+ "((raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-01') AND ((raw_sessions.min_timestamp - toIntervalDay(3)) <= '2021-01-03')"
+ )
+ assert expected == actual
+
+ def test_timestamp_or(self):
+ actual = f(
+ self.inliner.get_inner_where(
+ parse("SELECT * FROM sessions WHERE and(min_timestamp <= '2021-01-01', min_timestamp >= '2021-01-03')")
+ )
+ )
+ expected = f(
+ "((raw_sessions.min_timestamp - toIntervalDay(3)) <= '2021-01-01') AND ((raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-03')"
+ )
+ assert expected == actual
+
+ def test_unrelated_function(self):
+ actual = f(self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE like('a', 'b')")))
+ assert actual is None
+
+ def test_timestamp_unrelated_function(self):
+ actual = f(
+ self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE like(toString(min_timestamp), 'b')"))
+ )
+ assert actual is None
+
+ def test_timestamp_unrelated_function_timestamp(self):
+ actual = f(
+ self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE like(toString(min_timestamp), 'b')"))
+ )
+ assert actual is None
+
+ def test_ambiguous_or(self):
+ actual = f(
+ self.inliner.get_inner_where(
+ parse(
+ "SELECT * FROM sessions WHERE or(min_timestamp > '2021-01-03', like(toString(min_timestamp), 'b'))"
+ )
+ )
+ )
+ assert actual is None
+
+ def test_ambiguous_and(self):
+ actual = f(
+ self.inliner.get_inner_where(
+ parse(
+ "SELECT * FROM sessions WHERE and(min_timestamp > '2021-01-03', like(toString(min_timestamp), 'b'))"
+ )
+ )
+ )
+ assert actual == f("(raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-03'")
+
+ def test_join(self):
+ actual = f(
+ self.inliner.get_inner_where(
+ parse(
+ "SELECT * FROM events JOIN sessions ON events.session_id = raw_sessions.session_id WHERE min_timestamp > '2021-01-03'"
+ )
+ )
+ )
+ expected = f("((raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-03')")
+ assert expected == actual
+
+ def test_join_using_events_timestamp_filter(self):
+ actual = f(
+ self.inliner.get_inner_where(
+ parse(
+ "SELECT * FROM events JOIN sessions ON events.session_id = raw_sessions.session_id WHERE timestamp > '2021-01-03'"
+ )
+ )
+ )
+ expected = f("((raw_sessions.min_timestamp + toIntervalDay(3)) >= '2021-01-03')")
+ assert expected == actual
+
+ def test_minus(self):
+ actual = f(self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE min_timestamp >= today() - 2")))
+ expected = f("((raw_sessions.min_timestamp + toIntervalDay(3)) >= (today() - 2))")
+ assert expected == actual
+
+ def test_minus_function(self):
+ actual = f(
+ self.inliner.get_inner_where(parse("SELECT * FROM sessions WHERE min_timestamp >= minus(today() , 2)"))
+ )
+ expected = f("((raw_sessions.min_timestamp + toIntervalDay(3)) >= minus(today(), 2))")
+ assert expected == actual
+
+ def test_real_example(self):
+ actual = f(
+ self.inliner.get_inner_where(
+ parse(
+ "SELECT * FROM events JOIN sessions ON events.session_id = raw_sessions.session_id WHERE event = '$pageview' AND toTimeZone(timestamp, 'US/Pacific') >= toDateTime('2024-03-12 00:00:00', 'US/Pacific') AND toTimeZone(timestamp, 'US/Pacific') <= toDateTime('2024-03-19 23:59:59', 'US/Pacific')"
+ )
+ )
+ )
+ expected = f(
+ "(toTimeZone(raw_sessions.min_timestamp, 'US/Pacific') + toIntervalDay(3)) >= toDateTime('2024-03-12 00:00:00', 'US/Pacific') AND (toTimeZone(raw_sessions.min_timestamp, 'US/Pacific') - toIntervalDay(3)) <= toDateTime('2024-03-19 23:59:59', 'US/Pacific') "
+ )
+ assert expected == actual
+
+ def test_collapse_and(self):
+ actual = f(
+ self.inliner.get_inner_where(
+ parse(
+ "SELECT * FROM sesions WHERE event = '$pageview' AND (TRUE AND (TRUE AND TRUE AND (timestamp >= '2024-03-12' AND TRUE)))"
+ )
+ )
+ )
+ expected = f("(raw_sessions.min_timestamp + toIntervalDay(3)) >= '2024-03-12'")
+ assert expected == actual
+
+
+class TestSessionsQueriesHogQLToClickhouse(ClickhouseTestMixin, APIBaseTest):
+ def print_query(self, query: str) -> str:
+ team = self.team
+ modifiers = create_default_modifiers_for_team(team)
+ context = HogQLContext(
+ team_id=team.pk,
+ team=team,
+ enable_select_queries=True,
+ modifiers=modifiers,
+ )
+ prepared_ast = prepare_ast_for_printing(node=parse(query), context=context, dialect="clickhouse")
+ pretty = print_prepared_ast(prepared_ast, context=context, dialect="clickhouse", pretty=True)
+ return pretty
+
+ def test_select_with_timestamp(self):
+ actual = self.print_query("SELECT session_id FROM sessions WHERE min_timestamp > '2021-01-01'")
+ expected = f"""SELECT
+ sessions.session_id AS session_id
+FROM
+ (SELECT
+ sessions.session_id AS session_id,
+ min(sessions.min_timestamp) AS min_timestamp
+ FROM
+ sessions
+ WHERE
+ and(equals(sessions.team_id, {self.team.id}), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, %(hogql_val_0)s), toIntervalDay(3)), %(hogql_val_1)s), 0))
+ GROUP BY
+ sessions.session_id,
+ sessions.session_id) AS sessions
+WHERE
+ ifNull(greater(toTimeZone(sessions.min_timestamp, %(hogql_val_2)s), %(hogql_val_3)s), 0)
+LIMIT 10000"""
+ assert expected == actual
+
+ def test_join_with_events(self):
+ actual = self.print_query(
+ """
+SELECT
+ sessions.session_id,
+ uniq(uuid)
+FROM events
+JOIN sessions
+ON events.$session_id = sessions.session_id
+WHERE events.timestamp > '2021-01-01'
+GROUP BY sessions.session_id
+"""
+ )
+ expected = f"""SELECT
+ sessions.session_id AS session_id,
+ uniq(events.uuid)
+FROM
+ events
+ JOIN (SELECT
+ sessions.session_id AS session_id
+ FROM
+ sessions
+ WHERE
+ and(equals(sessions.team_id, {self.team.id}), ifNull(greaterOrEquals(plus(toTimeZone(sessions.min_timestamp, %(hogql_val_0)s), toIntervalDay(3)), %(hogql_val_1)s), 0))
+ GROUP BY
+ sessions.session_id,
+ sessions.session_id) AS sessions ON equals(events.`$session_id`, sessions.session_id)
+WHERE
+ and(equals(events.team_id, {self.team.id}), greater(toTimeZone(events.timestamp, %(hogql_val_2)s), %(hogql_val_3)s))
+GROUP BY
+ sessions.session_id
+LIMIT 10000"""
+ assert expected == actual
diff --git a/posthog/hogql/database/test/__snapshots__/test_database.ambr b/posthog/hogql/database/test/__snapshots__/test_database.ambr
index db4dfc8f6df9f..63c2d16ce87aa 100644
--- a/posthog/hogql/database/test/__snapshots__/test_database.ambr
+++ b/posthog/hogql/database/test/__snapshots__/test_database.ambr
@@ -304,6 +304,31 @@
]
}
],
+ "person_distinct_id_overrides": [
+ {
+ "key": "distinct_id",
+ "type": "string"
+ },
+ {
+ "key": "person_id",
+ "type": "string"
+ },
+ {
+ "key": "person",
+ "type": "lazy_table",
+ "table": "persons",
+ "fields": [
+ "id",
+ "created_at",
+ "team_id",
+ "properties",
+ "is_identified",
+ "pdi",
+ "$virt_initial_referring_domain_type",
+ "$virt_initial_channel_type"
+ ]
+ }
+ ],
"person_overrides": [
{
"key": "old_person_id",
@@ -790,6 +815,39 @@
"type": "integer"
}
],
+ "raw_person_distinct_id_overrides": [
+ {
+ "key": "distinct_id",
+ "type": "string"
+ },
+ {
+ "key": "person_id",
+ "type": "string"
+ },
+ {
+ "key": "person",
+ "type": "lazy_table",
+ "table": "persons",
+ "fields": [
+ "id",
+ "created_at",
+ "team_id",
+ "properties",
+ "is_identified",
+ "pdi",
+ "$virt_initial_referring_domain_type",
+ "$virt_initial_channel_type"
+ ]
+ },
+ {
+ "key": "is_deleted",
+ "type": "boolean"
+ },
+ {
+ "key": "version",
+ "type": "integer"
+ }
+ ],
"raw_person_overrides": [
{
"key": "old_person_id",
@@ -1155,6 +1213,31 @@
]
}
],
+ "person_distinct_id_overrides": [
+ {
+ "key": "distinct_id",
+ "type": "string"
+ },
+ {
+ "key": "person_id",
+ "type": "string"
+ },
+ {
+ "key": "person",
+ "type": "lazy_table",
+ "table": "persons",
+ "fields": [
+ "id",
+ "created_at",
+ "team_id",
+ "properties",
+ "is_identified",
+ "pdi",
+ "$virt_initial_referring_domain_type",
+ "$virt_initial_channel_type"
+ ]
+ }
+ ],
"person_overrides": [
{
"key": "old_person_id",
@@ -1641,6 +1724,39 @@
"type": "integer"
}
],
+ "raw_person_distinct_id_overrides": [
+ {
+ "key": "distinct_id",
+ "type": "string"
+ },
+ {
+ "key": "person_id",
+ "type": "string"
+ },
+ {
+ "key": "person",
+ "type": "lazy_table",
+ "table": "persons",
+ "fields": [
+ "id",
+ "created_at",
+ "team_id",
+ "properties",
+ "is_identified",
+ "pdi",
+ "$virt_initial_referring_domain_type",
+ "$virt_initial_channel_type"
+ ]
+ },
+ {
+ "key": "is_deleted",
+ "type": "boolean"
+ },
+ {
+ "key": "version",
+ "type": "integer"
+ }
+ ],
"raw_person_overrides": [
{
"key": "old_person_id",
diff --git a/posthog/hogql/query.py b/posthog/hogql/query.py
index e7bc3f7984205..69b5656020904 100644
--- a/posthog/hogql/query.py
+++ b/posthog/hogql/query.py
@@ -28,6 +28,7 @@
def execute_hogql_query(
query: Union[str, ast.SelectQuery, ast.SelectUnionQuery],
team: Team,
+ *,
query_type: str = "hogql_query",
filters: Optional[HogQLFilters] = None,
placeholders: Optional[Dict[str, ast.Expr]] = None,
diff --git a/posthog/hogql/test/test_bytecode.py b/posthog/hogql/test/test_bytecode.py
index cf0b8113b574d..f7d810700e74a 100644
--- a/posthog/hogql/test/test_bytecode.py
+++ b/posthog/hogql/test/test_bytecode.py
@@ -130,7 +130,7 @@ def test_bytecode_create(self):
def test_bytecode_create_error(self):
with self.assertRaises(NotImplementedException) as e:
to_bytecode("(select 1)")
- self.assertEqual(str(e.exception), "Visitor has no method visit_select_query")
+ self.assertEqual(str(e.exception), "BytecodeBuilder has no method visit_select_query")
with self.assertRaises(NotImplementedException) as e:
to_bytecode("1 in cohort 2")
diff --git a/posthog/hogql/test/test_modifiers.py b/posthog/hogql/test/test_modifiers.py
index eba1f5195ab3d..b2b0ef1e40630 100644
--- a/posthog/hogql/test/test_modifiers.py
+++ b/posthog/hogql/test/test_modifiers.py
@@ -74,6 +74,13 @@ def test_modifiers_persons_on_events_mode_mapping(self):
"events.person_properties AS properties",
"toTimeZone(events.person_created_at, %(hogql_val_1)s) AS created_at",
),
+ (
+ PersonsOnEventsMode.v3_enabled,
+ "events.event AS event",
+ "if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS id",
+ "events.person_properties AS properties",
+ "toTimeZone(events.person_created_at, %(hogql_val_0)s) AS created_at",
+ ),
]
for mode, *expected in test_cases:
diff --git a/posthog/hogql/test/test_visitor.py b/posthog/hogql/test/test_visitor.py
index 8aa6689328fbf..a01193f788d5f 100644
--- a/posthog/hogql/test/test_visitor.py
+++ b/posthog/hogql/test/test_visitor.py
@@ -125,7 +125,7 @@ def visit_arithmetic_operation(self, node: ast.ArithmeticOperation):
with self.assertRaises(HogQLException) as e:
UnknownNotDefinedVisitor().visit(parse_expr("1 + 3 / 'asd2'"))
- self.assertEqual(str(e.exception), "Visitor has no method visit_constant")
+ self.assertEqual(str(e.exception), "UnknownNotDefinedVisitor has no method visit_constant")
def test_hogql_exception_start_end(self):
class EternalVisitor(TraversingVisitor):
diff --git a/posthog/hogql/transforms/lazy_tables.py b/posthog/hogql/transforms/lazy_tables.py
index bdbb322d54397..df8ce6962259c 100644
--- a/posthog/hogql/transforms/lazy_tables.py
+++ b/posthog/hogql/transforms/lazy_tables.py
@@ -309,7 +309,7 @@ def create_override(table_name: str, field_chain: List[str | int]) -> None:
# For all the collected tables, create the subqueries, and add them to the table.
for table_name, table_to_add in tables_to_add.items():
- subquery = table_to_add.lazy_table.lazy_select(table_to_add.fields_accessed, self.context.modifiers)
+ subquery = table_to_add.lazy_table.lazy_select(table_to_add.fields_accessed, self.context, node=node)
subquery = cast(ast.SelectQuery, clone_expr(subquery, clear_locations=True))
subquery = cast(ast.SelectQuery, resolve_types(subquery, self.context, self.dialect, [node.type]))
old_table_type = select_type.tables[table_name]
diff --git a/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr b/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr
index 9ff7f8ee0ab49..e0f5ea847110d 100644
--- a/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr
+++ b/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr
@@ -31,7 +31,7 @@
FROM events LEFT JOIN (
SELECT person_static_cohort.person_id AS cohort_person_id, 1 AS matched, person_static_cohort.cohort_id AS cohort_id
FROM person_static_cohort
- WHERE and(equals(person_static_cohort.team_id, 420), in(person_static_cohort.cohort_id, [12]))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id)
+ WHERE and(equals(person_static_cohort.team_id, 420), in(person_static_cohort.cohort_id, [11]))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id)
WHERE and(equals(events.team_id, 420), 1, ifNull(equals(__in_cohort.matched, 1), 0))
LIMIT 100
SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1
@@ -42,7 +42,7 @@
FROM events LEFT JOIN (
SELECT person_id AS cohort_person_id, 1 AS matched, cohort_id
FROM static_cohort_people
- WHERE in(cohort_id, [12])) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id)
+ WHERE in(cohort_id, [11])) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id)
WHERE and(1, equals(__in_cohort.matched, 1))
LIMIT 100
'''
@@ -55,7 +55,7 @@
FROM events LEFT JOIN (
SELECT person_static_cohort.person_id AS cohort_person_id, 1 AS matched, person_static_cohort.cohort_id AS cohort_id
FROM person_static_cohort
- WHERE and(equals(person_static_cohort.team_id, 420), in(person_static_cohort.cohort_id, [13]))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id)
+ WHERE and(equals(person_static_cohort.team_id, 420), in(person_static_cohort.cohort_id, [12]))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id)
WHERE and(equals(events.team_id, 420), 1, ifNull(equals(__in_cohort.matched, 1), 0))
LIMIT 100
SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1
@@ -66,7 +66,7 @@
FROM events LEFT JOIN (
SELECT person_id AS cohort_person_id, 1 AS matched, cohort_id
FROM static_cohort_people
- WHERE in(cohort_id, [13])) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id)
+ WHERE in(cohort_id, [12])) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id)
WHERE and(1, equals(__in_cohort.matched, 1))
LIMIT 100
'''
diff --git a/posthog/hogql/visitor.py b/posthog/hogql/visitor.py
index c11856169297f..2bf968abf2ab0 100644
--- a/posthog/hogql/visitor.py
+++ b/posthog/hogql/visitor.py
@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Optional, TypeVar, Generic, Any
from posthog.hogql import ast
from posthog.hogql.base import AST, Expr
@@ -14,8 +14,11 @@ def clear_locations(expr: Expr) -> Expr:
return CloningVisitor(clear_locations=True).visit(expr)
-class Visitor(object):
- def visit(self, node: AST):
+T = TypeVar("T")
+
+
+class Visitor(Generic[T]):
+ def visit(self, node: AST) -> T:
if node is None:
return node
@@ -28,7 +31,7 @@ def visit(self, node: AST):
raise e
-class TraversingVisitor(Visitor):
+class TraversingVisitor(Visitor[None]):
"""Visitor that traverses the AST tree without returning anything"""
def visit_expr(self, node: Expr):
@@ -258,7 +261,7 @@ def visit_hogqlx_attribute(self, node: ast.HogQLXAttribute):
self.visit(node.value)
-class CloningVisitor(Visitor):
+class CloningVisitor(Visitor[Any]):
"""Visitor that traverses and clones the AST tree. Clears types."""
def __init__(
diff --git a/posthog/hogql_queries/insights/funnels/base.py b/posthog/hogql_queries/insights/funnels/base.py
index 5d84328d2063d..4e97d79b94534 100644
--- a/posthog/hogql_queries/insights/funnels/base.py
+++ b/posthog/hogql_queries/insights/funnels/base.py
@@ -284,6 +284,7 @@ def _get_breakdown_expr(self) -> ast.Expr:
properties_column = f"group_{breakdownFilter.breakdown_group_type_index}.properties"
return get_breakdown_expr(breakdown, properties_column)
elif breakdownType == "hogql":
+ assert isinstance(breakdown, list)
return ast.Alias(
alias="value",
expr=ast.Array(exprs=[parse_expr(str(value)) for value in breakdown]),
@@ -530,6 +531,7 @@ def _add_breakdown_attribution_subquery(self, inner_query: ast.SelectQuery) -> a
# so just select that. Except for the empty case, where we select the default.
if self._query_has_array_breakdown():
+ assert isinstance(breakdown, list)
default_breakdown_value = f"""[{','.join(["''" for _ in range(len(breakdown or []))])}]"""
# default is [''] when dealing with a single breakdown array, otherwise ['', '', ...., '']
breakdown_selector = parse_expr(
@@ -613,7 +615,7 @@ def _build_step_query(
event_expr = ast.Constant(value=True)
else:
# event
- event_expr = parse_expr(f"event = '{entity.event}'")
+ event_expr = parse_expr("event = {event}", {"event": ast.Constant(value=entity.event)})
if entity.properties is not None and entity.properties != []:
# add property filters
@@ -657,11 +659,15 @@ def _get_funnel_person_step_condition(self) -> ast.Expr:
raise ValueError("Missing both funnelStep and funnelCustomSteps")
if funnelStepBreakdown is not None:
- breakdown_prop_value = funnelStepBreakdown
- if isinstance(breakdown_prop_value, int) and breakdownType != "cohort":
- breakdown_prop_value = str(breakdown_prop_value)
+ if isinstance(funnelStepBreakdown, int) and breakdownType != "cohort":
+ funnelStepBreakdown = str(funnelStepBreakdown)
- conditions.append(parse_expr(f"arrayFlatten(array(prop)) = arrayFlatten(array({breakdown_prop_value}))"))
+ conditions.append(
+ parse_expr(
+ "arrayFlatten(array(prop)) = arrayFlatten(array({funnelStepBreakdown}))",
+ {"funnelStepBreakdown": ast.Constant(value=funnelStepBreakdown)},
+ )
+ )
return ast.And(exprs=conditions)
@@ -830,7 +836,7 @@ def _get_step_times(self, max_steps: int) -> List[ast.Expr]:
for i in range(1, max_steps):
exprs.append(
parse_expr(
- f"if(isNotNull(latest_{i}) AND latest_{i} <= latest_{i-1} + INTERVAL {windowInterval} {windowIntervalUnit}, dateDiff('second', latest_{i - 1}, latest_{i}), NULL) step_{i}_conversion_time"
+ f"if(isNotNull(latest_{i}) AND latest_{i} <= toTimeZone(latest_{i-1}, 'UTC') + INTERVAL {windowInterval} {windowIntervalUnit}, dateDiff('second', latest_{i - 1}, latest_{i}), NULL) step_{i}_conversion_time"
),
)
@@ -898,7 +904,12 @@ def _get_breakdown_prop_expr(self, group_remaining=False) -> List[ast.Expr]:
BreakdownType.group,
]:
breakdown_values = self._get_breakdown_conditions()
- return [parse_expr(f"if(has({breakdown_values}, prop), prop, {other_aggregation}) as prop")]
+ return [
+ parse_expr(
+ f"if(has({{breakdown_values}}, prop), prop, {other_aggregation}) as prop",
+ {"breakdown_values": ast.Constant(value=breakdown_values)},
+ )
+ ]
else:
# Cohorts don't have "Other" aggregation
return [ast.Field(chain=["prop"])]
@@ -959,7 +970,7 @@ def _get_exclusion_condition(self) -> List[ast.Expr]:
to_time = f"latest_{exclusion.funnelToStep}"
exclusion_time = f"exclusion_{exclusion_id}_latest_{exclusion.funnelFromStep}"
condition = parse_expr(
- f"if( {exclusion_time} > {from_time} AND {exclusion_time} < if(isNull({to_time}), {from_time} + INTERVAL {windowInterval} {windowIntervalUnit}, {to_time}), 1, 0)"
+ f"if( {exclusion_time} > {from_time} AND {exclusion_time} < if(isNull({to_time}), toTimeZone({from_time}, 'UTC') + INTERVAL {windowInterval} {windowIntervalUnit}, {to_time}), 1, 0)"
)
conditions.append(condition)
@@ -988,7 +999,11 @@ def _get_sorting_condition(self, curr_index: int, max_steps: int) -> ast.Expr:
duplicate_event = is_equal(series[i], series[i - 1]) or is_superset(series[i], series[i - 1])
conditions.append(parse_expr(f"latest_{i - 1} {'<' if duplicate_event else '<='} latest_{i}"))
- conditions.append(parse_expr(f"latest_{i} <= latest_0 + INTERVAL {windowInterval} {windowIntervalUnit}"))
+ conditions.append(
+ parse_expr(
+ f"latest_{i} <= toTimeZone(latest_0, 'UTC') + INTERVAL {windowInterval} {windowIntervalUnit}"
+ )
+ )
return ast.Call(
name="if",
diff --git a/posthog/hogql_queries/insights/funnels/funnel_correlation_query_runner.py b/posthog/hogql_queries/insights/funnels/funnel_correlation_query_runner.py
index 945a1b7da5cce..bcb362ff3d4f9 100644
--- a/posthog/hogql_queries/insights/funnels/funnel_correlation_query_runner.py
+++ b/posthog/hogql_queries/insights/funnels/funnel_correlation_query_runner.py
@@ -236,6 +236,7 @@ def _calculate(self) -> tuple[List[EventOddsRatio], bool, str, HogQLQueryRespons
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
assert response.results
@@ -761,7 +762,7 @@ def _get_events_join_query(self) -> str:
AND toTimeZone(toDateTime(event.timestamp), 'UTC') > funnel_actors.first_timestamp
AND toTimeZone(toDateTime(event.timestamp), 'UTC') < coalesce(
funnel_actors.final_timestamp,
- funnel_actors.first_timestamp + INTERVAL {windowInterval} {windowIntervalUnit},
+ toTimeZone(funnel_actors.first_timestamp, 'UTC') + INTERVAL {windowInterval} {windowIntervalUnit},
date_to)
-- Ensure that the event is not outside the bounds of the funnel conversion window
diff --git a/posthog/hogql_queries/insights/funnels/funnel_query_context.py b/posthog/hogql_queries/insights/funnels/funnel_query_context.py
index 66a0d28ad3d7f..3b777e3ff8026 100644
--- a/posthog/hogql_queries/insights/funnels/funnel_query_context.py
+++ b/posthog/hogql_queries/insights/funnels/funnel_query_context.py
@@ -25,7 +25,7 @@ class FunnelQueryContext(QueryContext):
interval: IntervalType
- breakdown: List[Union[str, int]] | None
+ breakdown: List[Union[str, int]] | str | int | None
breakdownType: BreakdownType
breakdownAttributionType: BreakdownAttributionType
diff --git a/posthog/hogql_queries/insights/funnels/funnel_trends.py b/posthog/hogql_queries/insights/funnels/funnel_trends.py
index 5c370512a20e8..9d486f1b06196 100644
--- a/posthog/hogql_queries/insights/funnels/funnel_trends.py
+++ b/posthog/hogql_queries/insights/funnels/funnel_trends.py
@@ -203,7 +203,16 @@ def get_query(self) -> ast.SelectQuery:
[
ast.Alias(
alias="breakdown_value",
- expr=ast.Array(exprs=[parse_expr(str(value)) for value in self.breakdown_values]),
+ expr=ast.Array(
+ exprs=[
+ (
+ ast.Array(exprs=[ast.Constant(value=sub_value) for sub_value in value])
+ if isinstance(value, list)
+ else ast.Constant(value=value)
+ )
+ for value in self.breakdown_values
+ ]
+ ),
hidden=False,
)
]
diff --git a/posthog/hogql_queries/insights/funnels/funnel_unordered.py b/posthog/hogql_queries/insights/funnels/funnel_unordered.py
index 09fc322621eac..af3ed18d4f82e 100644
--- a/posthog/hogql_queries/insights/funnels/funnel_unordered.py
+++ b/posthog/hogql_queries/insights/funnels/funnel_unordered.py
@@ -168,7 +168,7 @@ def _get_step_times(self, max_steps: int) -> List[ast.Expr]:
for i in range(1, max_steps):
exprs.append(
parse_expr(
- f"if(isNotNull(conversion_times[{i+1}]) AND conversion_times[{i+1}] <= conversion_times[{i}] + INTERVAL {windowInterval} {windowIntervalUnit}, dateDiff('second', conversion_times[{i}], conversion_times[{i+1}]), NULL) step_{i}_conversion_time"
+ f"if(isNotNull(conversion_times[{i+1}]) AND conversion_times[{i+1}] <= toTimeZone(conversion_times[{i}], 'UTC') + INTERVAL {windowInterval} {windowIntervalUnit}, dateDiff('second', conversion_times[{i}], conversion_times[{i+1}]), NULL) step_{i}_conversion_time"
)
)
# array indices in ClickHouse are 1-based :shrug:
@@ -190,7 +190,7 @@ def get_sorting_condition(self, max_steps: int) -> List[ast.Expr]:
basic_conditions: List[str] = []
for i in range(1, max_steps):
basic_conditions.append(
- f"if(latest_0 < latest_{i} AND latest_{i} <= latest_0 + INTERVAL {windowInterval} {windowIntervalUnit}, 1, 0)"
+ f"if(latest_0 < latest_{i} AND latest_{i} <= toTimeZone(latest_0, 'UTC') + INTERVAL {windowInterval} {windowIntervalUnit}, 1, 0)"
)
if basic_conditions:
@@ -214,7 +214,7 @@ def _get_exclusion_condition(self) -> List[ast.Expr]:
to_time = f"event_times[{exclusion.funnelToStep + 1}]"
exclusion_time = f"exclusion_{exclusion_id}_latest_{exclusion.funnelFromStep}"
condition = parse_expr(
- f"if( {exclusion_time} > {from_time} AND {exclusion_time} < if(isNull({to_time}), {from_time} + INTERVAL {windowInterval} {windowIntervalUnit}, {to_time}), 1, 0)"
+ f"if( {exclusion_time} > {from_time} AND {exclusion_time} < if(isNull({to_time}), toTimeZone({from_time}, 'UTC') + INTERVAL {windowInterval} {windowIntervalUnit}, {to_time}), 1, 0)"
)
conditions.append(condition)
diff --git a/posthog/hogql_queries/insights/funnels/funnels_query_runner.py b/posthog/hogql_queries/insights/funnels/funnels_query_runner.py
index 38e04603f2725..b1ca8dfd6dd54 100644
--- a/posthog/hogql_queries/insights/funnels/funnels_query_runner.py
+++ b/posthog/hogql_queries/insights/funnels/funnels_query_runner.py
@@ -92,6 +92,7 @@ def calculate(self):
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
results = self.funnel_class._format_results(response.results)
diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel.ambr
index 29a1483cafc1c..8379672352c1c 100644
--- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel.ambr
+++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel.ambr
@@ -30,9 +30,9 @@
latest_1 AS latest_1,
step_2 AS step_2,
latest_2 AS latest_2,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalSecond(15))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalSecond(15))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalSecond(15))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalSecond(15))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalSecond(15))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalSecond(15))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalSecond(15))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalSecond(15))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalSecond(15))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalSecond(15))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -128,9 +128,9 @@
latest_1 AS latest_1,
step_2 AS step_2,
latest_2 AS latest_2,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalSecond(15))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalSecond(15))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalSecond(15))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalSecond(15))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalSecond(15))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalSecond(15))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalSecond(15))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalSecond(15))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalSecond(15))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalSecond(15))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -238,9 +238,9 @@
latest_1 AS latest_1,
step_2 AS step_2,
latest_2 AS latest_2,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -341,8 +341,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -413,9 +413,9 @@
latest_1 AS latest_1,
step_2 AS step_2,
latest_2 AS latest_2,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -523,9 +523,9 @@
latest_1 AS latest_1,
step_2 AS step_2,
latest_2 AS latest_2,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -637,9 +637,9 @@
latest_1 AS latest_1,
step_2 AS step_2,
latest_2 AS latest_2,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -751,9 +751,9 @@
latest_1 AS latest_1,
step_2 AS step_2,
latest_2 AS latest_2,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -860,8 +860,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -922,8 +922,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -1009,8 +1009,8 @@
step_1 AS step_1,
latest_1 AS latest_1,
prop AS prop,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
prop AS prop
FROM
(SELECT aggregation_target AS aggregation_target,
@@ -1115,8 +1115,8 @@
step_1 AS step_1,
latest_1 AS latest_1,
prop AS prop,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
prop AS prop
FROM
(SELECT aggregation_target AS aggregation_target,
@@ -1228,8 +1228,8 @@
step_1 AS step_1,
latest_1 AS latest_1,
prop AS prop,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
prop AS prop
FROM
(SELECT aggregation_target AS aggregation_target,
@@ -1343,9 +1343,9 @@
step_2 AS step_2,
latest_2 AS latest_2,
prop AS prop,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
prop AS prop
FROM
(SELECT aggregation_target AS aggregation_target,
@@ -1496,9 +1496,9 @@
step_2 AS step_2,
latest_2 AS latest_2,
prop AS prop,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
prop AS prop
FROM
(SELECT aggregation_target AS aggregation_target,
@@ -1650,9 +1650,9 @@
step_2 AS step_2,
latest_2 AS latest_2,
prop AS prop,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
prop AS prop
FROM
(SELECT aggregation_target AS aggregation_target,
diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_breakdowns_by_current_url.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_breakdowns_by_current_url.ambr
index 656a22a2aff51..a50033fda2f7f 100644
--- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_breakdowns_by_current_url.ambr
+++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_breakdowns_by_current_url.ambr
@@ -50,8 +50,8 @@
step_1 AS step_1,
latest_1 AS latest_1,
prop AS prop,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
prop AS prop
FROM
(SELECT aggregation_target AS aggregation_target,
@@ -156,8 +156,8 @@
step_1 AS step_1,
latest_1 AS latest_1,
prop AS prop,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
prop AS prop
FROM
(SELECT aggregation_target AS aggregation_target,
diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation.ambr
index 5019f0f57f7b7..b5e3e020c1ec0 100644
--- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation.ambr
+++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation.ambr
@@ -41,8 +41,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -74,7 +74,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(event__pdi.person_id, funnel_actors.actor_id)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), notIn(event.event, []))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), notIn(event.event, []))
GROUP BY name
LIMIT 100
UNION ALL
@@ -110,8 +110,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -186,8 +186,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -268,8 +268,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -362,8 +362,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -505,8 +505,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -648,8 +648,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -791,8 +791,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -918,8 +918,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -944,7 +944,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_1`)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), in(event.event, ['positively_related', 'negatively_related'])))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), in(event.event, ['positively_related', 'negatively_related'])))
GROUP BY name
HAVING and(ifNull(greater(plus(success_count, failure_count), 2), 0), ifNull(notIn((prop).1, []), 0))
LIMIT 100
@@ -981,8 +981,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -1052,8 +1052,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -1078,7 +1078,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_1`)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), in(event.event, ['positively_related', 'negatively_related'])))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), in(event.event, ['positively_related', 'negatively_related'])))
GROUP BY name
HAVING and(ifNull(greater(plus(success_count, failure_count), 2), 0), ifNull(notIn((prop).1, []), 0))
LIMIT 100
@@ -1115,8 +1115,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -1181,8 +1181,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -1207,7 +1207,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), notIn(event.event, []))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), notIn(event.event, []))
GROUP BY name
LIMIT 100
UNION ALL
@@ -1243,8 +1243,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -1330,8 +1330,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -1375,7 +1375,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'positively_related'), ifNull(equals(funnel_actors.steps, 2), 0))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'positively_related'), ifNull(equals(funnel_actors.steps, 2), 0))
GROUP BY actor_id
ORDER BY actor_id ASC) AS source ON equals(groups.key, source.actor_id)
ORDER BY source.actor_id ASC
@@ -1441,8 +1441,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -1486,7 +1486,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'positively_related'), ifNull(notEquals(funnel_actors.steps, 2), 1))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'positively_related'), ifNull(notEquals(funnel_actors.steps, 2), 1))
GROUP BY actor_id
ORDER BY actor_id ASC) AS source ON equals(groups.key, source.actor_id)
ORDER BY source.actor_id ASC
@@ -1552,8 +1552,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -1597,7 +1597,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, 2), 0))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, 2), 0))
GROUP BY actor_id
ORDER BY actor_id ASC) AS source ON equals(groups.key, source.actor_id)
ORDER BY source.actor_id ASC
@@ -1663,8 +1663,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -1708,7 +1708,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, 2), 1))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, 2), 1))
GROUP BY actor_id
ORDER BY actor_id ASC) AS source ON equals(groups.key, source.actor_id)
ORDER BY source.actor_id ASC
@@ -1753,8 +1753,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -1787,7 +1787,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), notIn(event.event, []))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), notIn(event.event, []))
GROUP BY name
LIMIT 100
UNION ALL
@@ -1823,8 +1823,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -1918,8 +1918,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -1963,7 +1963,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, 2), 0))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, 2), 0))
GROUP BY actor_id
ORDER BY actor_id ASC) AS source ON equals(groups.key, source.actor_id)
ORDER BY source.actor_id ASC
@@ -2029,8 +2029,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -2074,7 +2074,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, 2), 1))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, 2), 1))
GROUP BY actor_id
ORDER BY actor_id ASC) AS source ON equals(groups.key, source.actor_id)
ORDER BY source.actor_id ASC
@@ -2119,8 +2119,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -2145,7 +2145,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), notIn(event.event, []))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), notIn(event.event, []))
GROUP BY name
LIMIT 100
UNION ALL
@@ -2181,8 +2181,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -2268,8 +2268,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -2313,7 +2313,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'positively_related'), ifNull(equals(funnel_actors.steps, 2), 0))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'positively_related'), ifNull(equals(funnel_actors.steps, 2), 0))
GROUP BY actor_id
ORDER BY actor_id ASC) AS source ON equals(groups.key, source.actor_id)
ORDER BY source.actor_id ASC
@@ -2379,8 +2379,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -2424,7 +2424,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'positively_related'), ifNull(notEquals(funnel_actors.steps, 2), 1))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'positively_related'), ifNull(notEquals(funnel_actors.steps, 2), 1))
GROUP BY actor_id
ORDER BY actor_id ASC) AS source ON equals(groups.key, source.actor_id)
ORDER BY source.actor_id ASC
@@ -2490,8 +2490,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -2535,7 +2535,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, 2), 0))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, 2), 0))
GROUP BY actor_id
ORDER BY actor_id ASC) AS source ON equals(groups.key, source.actor_id)
ORDER BY source.actor_id ASC
@@ -2601,8 +2601,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -2646,7 +2646,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, 2), 1))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, 2), 1))
GROUP BY actor_id
ORDER BY actor_id ASC) AS source ON equals(groups.key, source.actor_id)
ORDER BY source.actor_id ASC
@@ -2691,8 +2691,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -2725,7 +2725,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), notIn(event.event, []))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), notIn(event.event, []))
GROUP BY name
LIMIT 100
UNION ALL
@@ -2761,8 +2761,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -2856,8 +2856,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -2901,7 +2901,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, 2), 0))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, 2), 0))
GROUP BY actor_id
ORDER BY actor_id ASC) AS source ON equals(groups.key, source.actor_id)
ORDER BY source.actor_id ASC
@@ -2967,8 +2967,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -3012,7 +3012,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, 2), 1))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, 2), 1))
GROUP BY actor_id
ORDER BY actor_id ASC) AS source ON equals(groups.key, source.actor_id)
ORDER BY source.actor_id ASC
@@ -3060,8 +3060,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -3134,8 +3134,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -3220,8 +3220,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -3338,8 +3338,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -3456,8 +3456,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -3574,8 +3574,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -3675,8 +3675,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -3749,8 +3749,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -3818,8 +3818,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -3892,8 +3892,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -3978,8 +3978,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -4096,8 +4096,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -4214,8 +4214,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -4332,8 +4332,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -4433,8 +4433,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -4507,8 +4507,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -4576,8 +4576,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -4650,8 +4650,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -4736,8 +4736,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -4854,8 +4854,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -4972,8 +4972,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -5090,8 +5090,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -5191,8 +5191,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -5265,8 +5265,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -5334,8 +5334,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -5408,8 +5408,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -5494,8 +5494,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -5612,8 +5612,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -5730,8 +5730,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -5848,8 +5848,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -5949,8 +5949,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -6023,8 +6023,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -6092,8 +6092,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -6166,8 +6166,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -6252,8 +6252,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -6370,8 +6370,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -6488,8 +6488,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -6606,8 +6606,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -6707,8 +6707,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -6781,8 +6781,8 @@
latest_0 AS latest_0,
step_1 AS step_1,
latest_1 AS latest_1,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlations_persons.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlations_persons.ambr
index fe7404ea9b7a1..62715e7d96b63 100644
--- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlations_persons.ambr
+++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlations_persons.ambr
@@ -63,8 +63,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -115,7 +115,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(event__pdi.person_id, funnel_actors.actor_id)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-08 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-08 23:59:59', 6, 'UTC')))), notIn(event.event, ['$pageview', 'insight analyzed']), equals(event.event, 'insight loaded'), ifNull(equals(funnel_actors.steps, 2), 0))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-08 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-08 23:59:59', 6, 'UTC')))), notIn(event.event, ['$pageview', 'insight analyzed']), equals(event.event, 'insight loaded'), ifNull(equals(funnel_actors.steps, 2), 0))
GROUP BY actor_id
ORDER BY actor_id ASC) AS source ON equals(persons.id, source.actor_id)
ORDER BY persons.id ASC
@@ -213,9 +213,9 @@
uuid_2 AS uuid_2,
`$session_id_2` AS `$session_id_2`,
`$window_id_2` AS `$window_id_2`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
tuple(latest_2, uuid_2, `$session_id_2`, `$window_id_2`) AS step_2_matching_event,
@@ -321,7 +321,7 @@
and isNull(max_steps)))
WHERE ifNull(in(steps, [1, 2, 3]), 0)
ORDER BY aggregation_target ASC) AS funnel_actors ON equals(event__pdi.person_id, funnel_actors.actor_id)
- WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-08 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(funnel_actors.first_timestamp, toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-08 23:59:59', 6, 'UTC')))), notIn(event.event, ['$pageview', 'insight analyzed', 'insight updated']), equals(event.event, 'insight loaded'), ifNull(notEquals(funnel_actors.steps, 3), 1))
+ WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-08 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-08 23:59:59', 6, 'UTC')))), notIn(event.event, ['$pageview', 'insight analyzed', 'insight updated']), equals(event.event, 'insight loaded'), ifNull(notEquals(funnel_actors.steps, 3), 1))
GROUP BY actor_id
ORDER BY actor_id ASC) AS source ON equals(persons.id, source.actor_id)
ORDER BY persons.id ASC
@@ -401,8 +401,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -544,8 +544,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
@@ -687,8 +687,8 @@
uuid_1 AS uuid_1,
`$session_id_1` AS `$session_id_1`,
`$window_id_1` AS `$window_id_1`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
if(isNull(latest_0), tuple(NULL, NULL, NULL, NULL), if(isNull(latest_1), step_0_matching_event, step_1_matching_event)) AS final_matching_event
diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_persons.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_persons.ambr
index a9a6fc5357f72..723165d9d8320 100644
--- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_persons.ambr
+++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_persons.ambr
@@ -52,9 +52,9 @@
uuid_2 AS uuid_2,
`$session_id_2` AS `$session_id_2`,
`$window_id_2` AS `$window_id_2`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
tuple(latest_2, uuid_2, `$session_id_2`, `$window_id_2`) AS step_2_matching_event,
@@ -234,9 +234,9 @@
uuid_2 AS uuid_2,
`$session_id_2` AS `$session_id_2`,
`$window_id_2` AS `$window_id_2`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
tuple(latest_2, uuid_2, `$session_id_2`, `$window_id_2`) AS step_2_matching_event,
@@ -416,9 +416,9 @@
uuid_2 AS uuid_2,
`$session_id_2` AS `$session_id_2`,
`$window_id_2` AS `$window_id_2`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
tuple(latest_2, uuid_2, `$session_id_2`, `$window_id_2`) AS step_2_matching_event,
diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict.ambr
index 5b12c1d8d00e0..927a924d8a884 100644
--- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict.ambr
+++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict.ambr
@@ -50,8 +50,8 @@
step_1 AS step_1,
latest_1 AS latest_1,
prop AS prop,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -155,8 +155,8 @@
step_1 AS step_1,
latest_1 AS latest_1,
prop AS prop,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -267,8 +267,8 @@
step_1 AS step_1,
latest_1 AS latest_1,
prop AS prop,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -381,9 +381,9 @@
step_2 AS step_2,
latest_2 AS latest_2,
prop AS prop,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
prop AS prop
FROM
(SELECT aggregation_target AS aggregation_target,
@@ -534,9 +534,9 @@
step_2 AS step_2,
latest_2 AS latest_2,
prop AS prop,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
prop AS prop
FROM
(SELECT aggregation_target AS aggregation_target,
@@ -688,9 +688,9 @@
step_2 AS step_2,
latest_2 AS latest_2,
prop AS prop,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
prop AS prop
FROM
(SELECT aggregation_target AS aggregation_target,
diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_persons.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_persons.ambr
index 0b2f21460022e..1496e07c65e02 100644
--- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_persons.ambr
+++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_persons.ambr
@@ -52,9 +52,9 @@
uuid_2 AS uuid_2,
`$session_id_2` AS `$session_id_2`,
`$window_id_2` AS `$window_id_2`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
tuple(latest_2, uuid_2, `$session_id_2`, `$window_id_2`) AS step_2_matching_event,
@@ -194,9 +194,9 @@
uuid_2 AS uuid_2,
`$session_id_2` AS `$session_id_2`,
`$window_id_2` AS `$window_id_2`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
tuple(latest_2, uuid_2, `$session_id_2`, `$window_id_2`) AS step_2_matching_event,
@@ -336,9 +336,9 @@
uuid_2 AS uuid_2,
`$session_id_2` AS `$session_id_2`,
`$window_id_2` AS `$window_id_2`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
tuple(latest_2, uuid_2, `$session_id_2`, `$window_id_2`) AS step_2_matching_event,
diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_time_to_convert.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_time_to_convert.ambr
index aae3049d382ec..2463fd084116f 100644
--- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_time_to_convert.ambr
+++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_time_to_convert.ambr
@@ -35,9 +35,9 @@
latest_1 AS latest_1,
step_2 AS step_2,
latest_2 AS latest_2,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(7))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(7))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(7))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -102,7 +102,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(7))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(7))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(7))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2
@@ -135,7 +135,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(7))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(7))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(7))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2
@@ -168,7 +168,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(7))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(7))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(7))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2
@@ -201,7 +201,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(7))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(7))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(7))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2
@@ -249,9 +249,9 @@
latest_1 AS latest_1,
step_2 AS step_2,
latest_2 AS latest_2,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(7))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(7))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(7))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -316,7 +316,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(7))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(7))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(7))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2
@@ -349,7 +349,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(7))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(7))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(7))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2
@@ -383,7 +383,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(7))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(7))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(7))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(7))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(7))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2
@@ -450,9 +450,9 @@
latest_1 AS latest_1,
step_2 AS step_2,
latest_2 AS latest_2,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -498,7 +498,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -526,7 +526,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -554,7 +554,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -582,7 +582,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -625,9 +625,9 @@
latest_1 AS latest_1,
step_2 AS step_2,
latest_2 AS latest_2,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -673,7 +673,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -701,7 +701,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -730,7 +730,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps, if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -793,10 +793,10 @@
step_2 AS step_2,
latest_2 AS latest_2,
arraySort([latest_0, latest_1, latest_2]) AS event_times,
- arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
+ arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
arraySort([latest_0, latest_1, latest_2]) AS conversion_times,
- if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time,
- if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -836,10 +836,10 @@
step_2 AS step_2,
latest_2 AS latest_2,
arraySort([latest_0, latest_1, latest_2]) AS event_times,
- arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
+ arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
arraySort([latest_0, latest_1, latest_2]) AS conversion_times,
- if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time,
- if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -879,10 +879,10 @@
step_2 AS step_2,
latest_2 AS latest_2,
arraySort([latest_0, latest_1, latest_2]) AS event_times,
- arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
+ arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
arraySort([latest_0, latest_1, latest_2]) AS conversion_times,
- if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time,
- if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -928,7 +928,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -944,7 +944,7 @@
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-07 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-13 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))))
WHERE ifNull(equals(step_0, 1), 0)
- UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -960,7 +960,7 @@
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-07 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-13 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))))
WHERE ifNull(equals(step_0, 1), 0)
- UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -988,7 +988,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -1004,7 +1004,7 @@
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-07 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-13 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))))
WHERE ifNull(equals(step_0, 1), 0)
- UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -1020,7 +1020,7 @@
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-07 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-13 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))))
WHERE ifNull(equals(step_0, 1), 0)
- UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -1048,7 +1048,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -1064,7 +1064,7 @@
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-07 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-13 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))))
WHERE ifNull(equals(step_0, 1), 0)
- UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -1080,7 +1080,7 @@
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-07 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-13 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))))
WHERE ifNull(equals(step_0, 1), 0)
- UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -1108,7 +1108,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -1124,7 +1124,7 @@
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-07 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-13 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))))
WHERE ifNull(equals(step_0, 1), 0)
- UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -1140,7 +1140,7 @@
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-07 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-13 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))))
WHERE ifNull(equals(step_0, 1), 0)
- UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -1184,10 +1184,10 @@
step_2 AS step_2,
latest_2 AS latest_2,
arraySort([latest_0, latest_1, latest_2]) AS event_times,
- arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
+ arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
arraySort([latest_0, latest_1, latest_2]) AS conversion_times,
- if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time,
- if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -1227,10 +1227,10 @@
step_2 AS step_2,
latest_2 AS latest_2,
arraySort([latest_0, latest_1, latest_2]) AS event_times,
- arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
+ arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
arraySort([latest_0, latest_1, latest_2]) AS conversion_times,
- if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time,
- if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -1270,10 +1270,10 @@
step_2 AS step_2,
latest_2 AS latest_2,
arraySort([latest_0, latest_1, latest_2]) AS event_times,
- arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
+ arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
arraySort([latest_0, latest_1, latest_2]) AS conversion_times,
- if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time,
- if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -1319,7 +1319,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -1335,7 +1335,7 @@
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-07 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-13 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))))
WHERE ifNull(equals(step_0, 1), 0)
- UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -1351,7 +1351,7 @@
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-07 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-13 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))))
WHERE ifNull(equals(step_0, 1), 0)
- UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -1379,7 +1379,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -1395,7 +1395,7 @@
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-07 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-13 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))))
WHERE ifNull(equals(step_0, 1), 0)
- UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -1411,7 +1411,7 @@
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-07 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-13 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))))
WHERE ifNull(equals(step_0, 1), 0)
- UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -1440,7 +1440,7 @@
FROM
(SELECT aggregation_target AS aggregation_target, steps AS steps, max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, step_1_conversion_time AS step_1_conversion_time, step_2_conversion_time AS step_2_conversion_time
FROM
- (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ (SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -1456,7 +1456,7 @@
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-07 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-13 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))))
WHERE ifNull(equals(step_0, 1), 0)
- UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
@@ -1472,7 +1472,7 @@
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-07 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-06-13 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))))
WHERE ifNull(equals(step_0, 1), 0)
- UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ UNION ALL SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, latest_1 AS latest_1, step_2 AS step_2, latest_2 AS latest_2, arraySort([latest_0, latest_1, latest_2]) AS event_times, arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps, arraySort([latest_0, latest_1, latest_2]) AS conversion_times, if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time, if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target, timestamp AS timestamp, step_0 AS step_0, latest_0 AS latest_0, step_1 AS step_1, min(latest_1) OVER (PARTITION BY aggregation_target
ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, step_2 AS step_2, min(latest_2) OVER (PARTITION BY aggregation_target
diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends.ambr
index 5fa91548e3037..c86902d5df182 100644
--- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends.ambr
+++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends.ambr
@@ -22,9 +22,9 @@
latest_1 AS latest_1,
step_2 AS step_2,
latest_2 AS latest_2,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -109,9 +109,9 @@
latest_1 AS latest_1,
step_2 AS step_2,
latest_2 AS latest_2,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -196,9 +196,9 @@
latest_1 AS latest_1,
step_2 AS step_2,
latest_2 AS latest_2,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_persons.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_persons.ambr
index 9dcb793d30329..32518bdbf9bbc 100644
--- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_persons.ambr
+++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_persons.ambr
@@ -39,9 +39,9 @@
uuid_2 AS uuid_2,
`$session_id_2` AS `$session_id_2`,
`$window_id_2` AS `$window_id_2`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
tuple(latest_2, uuid_2, `$session_id_2`, `$window_id_2`) AS step_2_matching_event,
@@ -207,9 +207,9 @@
uuid_2 AS uuid_2,
`$session_id_2` AS `$session_id_2`,
`$window_id_2` AS `$window_id_2`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
tuple(latest_2, uuid_2, `$session_id_2`, `$window_id_2`) AS step_2_matching_event,
@@ -375,9 +375,9 @@
uuid_2 AS uuid_2,
`$session_id_2` AS `$session_id_2`,
`$window_id_2` AS `$window_id_2`,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
tuple(latest_0, uuid_0, `$session_id_0`, `$window_id_0`) AS step_0_matching_event,
tuple(latest_1, uuid_1, `$session_id_1`, `$window_id_1`) AS step_1_matching_event,
tuple(latest_2, uuid_2, `$session_id_2`, `$window_id_2`) AS step_2_matching_event,
diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_unordered.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_unordered.ambr
index 214583b03f081..52269964088e1 100644
--- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_unordered.ambr
+++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_unordered.ambr
@@ -49,9 +49,9 @@
latest_1 AS latest_1,
prop AS prop,
arraySort([latest_0, latest_1]) AS event_times,
- arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
+ arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
arraySort([latest_0, latest_1]) AS conversion_times,
- if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time
+ if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -101,9 +101,9 @@
latest_1 AS latest_1,
prop AS prop,
arraySort([latest_0, latest_1]) AS event_times,
- arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
+ arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
arraySort([latest_0, latest_1]) AS conversion_times,
- if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time
+ if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -206,9 +206,9 @@
latest_1 AS latest_1,
prop AS prop,
arraySort([latest_0, latest_1]) AS event_times,
- arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
+ arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
arraySort([latest_0, latest_1]) AS conversion_times,
- if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time
+ if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -265,9 +265,9 @@
latest_1 AS latest_1,
prop AS prop,
arraySort([latest_0, latest_1]) AS event_times,
- arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
+ arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
arraySort([latest_0, latest_1]) AS conversion_times,
- if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time
+ if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -377,9 +377,9 @@
latest_1 AS latest_1,
prop AS prop,
arraySort([latest_0, latest_1]) AS event_times,
- arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
+ arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
arraySort([latest_0, latest_1]) AS conversion_times,
- if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time
+ if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -429,9 +429,9 @@
latest_1 AS latest_1,
prop AS prop,
arraySort([latest_0, latest_1]) AS event_times,
- arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
+ arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
arraySort([latest_0, latest_1]) AS conversion_times,
- if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time
+ if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -544,9 +544,9 @@
step_2 AS step_2,
latest_2 AS latest_2,
prop AS prop,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
prop AS prop
FROM
(SELECT aggregation_target AS aggregation_target,
@@ -697,9 +697,9 @@
step_2 AS step_2,
latest_2 AS latest_2,
prop AS prop,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
prop AS prop
FROM
(SELECT aggregation_target AS aggregation_target,
@@ -851,9 +851,9 @@
step_2 AS step_2,
latest_2 AS latest_2,
prop AS prop,
- if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 2, 1)) AS steps,
- if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
- if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(latest_1, toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
+ if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps,
+ if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time,
prop AS prop
FROM
(SELECT aggregation_target AS aggregation_target,
diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_unordered_persons.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_unordered_persons.ambr
index 6f308a54388e3..5ced3fb2b9766 100644
--- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_unordered_persons.ambr
+++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_unordered_persons.ambr
@@ -45,10 +45,10 @@
`$session_id_2` AS `$session_id_2`,
`$window_id_2` AS `$window_id_2`,
arraySort([latest_0, latest_1, latest_2]) AS event_times,
- arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
+ arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
arraySort([latest_0, latest_1, latest_2]) AS conversion_times,
- if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time,
- if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -122,10 +122,10 @@
`$session_id_2` AS `$session_id_2`,
`$window_id_2` AS `$window_id_2`,
arraySort([latest_0, latest_1, latest_2]) AS event_times,
- arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
+ arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
arraySort([latest_0, latest_1, latest_2]) AS conversion_times,
- if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time,
- if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
@@ -199,10 +199,10 @@
`$session_id_2` AS `$session_id_2`,
`$window_id_2` AS `$window_id_2`,
arraySort([latest_0, latest_1, latest_2]) AS event_times,
- arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(latest_0, toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
+ arraySum([if(and(ifNull(less(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), if(and(ifNull(less(latest_0, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 1, 0), 1]) AS steps,
arraySort([latest_0, latest_1, latest_2]) AS conversion_times,
- if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(conversion_times[1], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time,
- if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(conversion_times[2], toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
+ if(and(isNotNull(conversion_times[2]), ifNull(lessOrEquals(conversion_times[2], plus(toTimeZone(conversion_times[1], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[1], conversion_times[2]), NULL) AS step_1_conversion_time,
+ if(and(isNotNull(conversion_times[3]), ifNull(lessOrEquals(conversion_times[3], plus(toTimeZone(conversion_times[2], 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', conversion_times[2], conversion_times[3]), NULL) AS step_2_conversion_time
FROM
(SELECT aggregation_target AS aggregation_target,
timestamp AS timestamp,
diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel.py b/posthog/hogql_queries/insights/funnels/test/test_funnel.py
index d132927863542..89382bebfb994 100644
--- a/posthog/hogql_queries/insights/funnels/test/test_funnel.py
+++ b/posthog/hogql_queries/insights/funnels/test/test_funnel.py
@@ -18,7 +18,14 @@
from posthog.models.group_type_mapping import GroupTypeMapping
from posthog.models.property_definition import PropertyDefinition
from posthog.queries.funnels import ClickhouseFunnelActors
-from posthog.schema import ActorsQuery, EventsNode, FunnelsActorsQuery, FunnelsQuery
+from posthog.schema import (
+ ActorsQuery,
+ BreakdownFilter,
+ DateRange,
+ EventsNode,
+ FunnelsActorsQuery,
+ FunnelsQuery,
+)
from posthog.test.base import (
APIBaseTest,
BaseTest,
@@ -3523,6 +3530,125 @@ def test_funnel_aggregation_with_groups_with_cohort_filtering(self):
self.assertEqual(results[1]["name"], "paid")
self.assertEqual(results[1]["count"], 1)
+ def test_funnel_window_ignores_dst_transition(self):
+ _create_person(
+ distinct_ids=[f"user_1"],
+ team=self.team,
+ )
+
+ events_by_person = {
+ "user_1": [
+ {
+ "event": "$pageview",
+ "timestamp": datetime(2024, 3, 1, 15, 10), # 1st March 15:10
+ },
+ {
+ "event": "user signed up",
+ "timestamp": datetime(
+ 2024, 3, 15, 14, 27
+ ), # 15th March 14:27 (within 14 day conversion window that ends at 15:10)
+ },
+ ],
+ }
+ journeys_for(events_by_person, self.team)
+
+ filters = {
+ "events": [
+ {"id": "$pageview", "type": "events", "order": 0},
+ {"id": "user signed up", "type": "events", "order": 1},
+ ],
+ "insight": INSIGHT_FUNNELS,
+ "date_from": "2024-02-17",
+ "date_to": "2024-03-18",
+ }
+
+ query = cast(FunnelsQuery, filter_to_query(filters))
+ results = FunnelsQueryRunner(query=query, team=self.team).calculate().results
+
+ self.assertEqual(results[1]["name"], "user signed up")
+ self.assertEqual(results[1]["count"], 1)
+ self.assertEqual(results[1]["average_conversion_time"], 1_207_020)
+ self.assertEqual(results[1]["median_conversion_time"], 1_207_020)
+
+ # there is a PST -> PDT transition on 10th of March
+ self.team.timezone = "US/Pacific"
+ self.team.save()
+
+ query = cast(FunnelsQuery, filter_to_query(filters))
+ results = FunnelsQueryRunner(query=query, team=self.team).calculate().results
+
+ # we still should have the user here, as the conversion window should not be affected by DST
+ self.assertEqual(results[1]["name"], "user signed up")
+ self.assertEqual(results[1]["count"], 1)
+ self.assertEqual(results[1]["average_conversion_time"], 1_207_020)
+ self.assertEqual(results[1]["median_conversion_time"], 1_207_020)
+
+ def test_parses_breakdowns_correctly(self):
+ _create_person(
+ distinct_ids=[f"user_1"],
+ team=self.team,
+ )
+
+ events_by_person = {
+ "user_1": [
+ {
+ "event": "$pageview",
+ "timestamp": datetime(2024, 3, 22, 13, 46),
+ "properties": {"utm_medium": "test''123"},
+ },
+ {
+ "event": "$pageview",
+ "timestamp": datetime(2024, 3, 22, 13, 47),
+ "properties": {"utm_medium": "test''123"},
+ },
+ ],
+ }
+ journeys_for(events_by_person, self.team)
+
+ query = FunnelsQuery(
+ series=[EventsNode(event="$pageview"), EventsNode(event="$pageview")],
+ dateRange=DateRange(
+ date_from="2024-03-22",
+ date_to="2024-03-22",
+ ),
+ breakdownFilter=BreakdownFilter(breakdown="utm_medium"),
+ )
+ results = FunnelsQueryRunner(query=query, team=self.team).calculate().results
+
+ self.assertEqual(results[0][1]["breakdown_value"], ["test'123"])
+ self.assertEqual(results[0][1]["count"], 1)
+
+ def test_funnel_parses_event_names_correctly(self):
+ _create_person(
+ distinct_ids=[f"user_1"],
+ team=self.team,
+ )
+
+ events_by_person = {
+ "user_1": [
+ {
+ "event": "test''1",
+ "timestamp": datetime(2024, 3, 22, 13, 46),
+ },
+ {
+ "event": "test''2",
+ "timestamp": datetime(2024, 3, 22, 13, 47),
+ },
+ ],
+ }
+ journeys_for(events_by_person, self.team)
+
+ query = FunnelsQuery(
+ series=[EventsNode(event="test'1"), EventsNode()],
+ dateRange=DateRange(
+ date_from="2024-03-22",
+ date_to="2024-03-22",
+ ),
+ )
+ results = FunnelsQueryRunner(query=query, team=self.team).calculate().results
+
+ self.assertEqual(results[0]["count"], 1)
+
return TestGetFunnel
@@ -3550,8 +3676,8 @@ def test_smoke(self):
latest_0,
step_1,
latest_1,
- if(and(less(latest_0, latest_1), lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14)))), 2, 1) AS steps,
- if(and(isNotNull(latest_1), lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14)))), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(less(latest_0, latest_1), lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14)))), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14)))), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT
aggregation_target,
@@ -3610,8 +3736,8 @@ def test_smoke(self):
latest_0,
step_1,
latest_1,
- if(and(less(latest_0, latest_1), lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14)))), 2, 1) AS steps,
- if(and(isNotNull(latest_1), lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14)))), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(less(latest_0, latest_1), lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14)))), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14)))), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT
aggregation_target,
@@ -3681,8 +3807,8 @@ def test_smoke(self):
latest_0,
step_1,
latest_1,
- if(and(less(latest_0, latest_1), lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14)))), 2, 1) AS steps,
- if(and(isNotNull(latest_1), lessOrEquals(latest_1, plus(latest_0, toIntervalDay(14)))), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
+ if(and(less(latest_0, latest_1), lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14)))), 2, 1) AS steps,
+ if(and(isNotNull(latest_1), lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14)))), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time
FROM
(SELECT
aggregation_target,
diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_persons.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_persons.py
index 4c342d2f2926c..dec7bdd933b3e 100644
--- a/posthog/hogql_queries/insights/funnels/test/test_funnel_persons.py
+++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_persons.py
@@ -626,3 +626,45 @@ def test_funnel_person_recordings(self):
}
],
)
+
+ def test_parses_step_breakdown_correctly(self):
+ person1 = _create_person(
+ distinct_ids=["person1"],
+ team_id=self.team.pk,
+ properties={"$country": "PL"},
+ )
+ journeys_for(
+ {
+ "person1": [
+ {
+ "event": "sign up",
+ "timestamp": datetime(2020, 1, 1, 12),
+ "properties": {"$browser": "test''123"},
+ },
+ {
+ "event": "play movie",
+ "timestamp": datetime(2020, 1, 1, 13),
+ "properties": {"$browser": "test''123"},
+ },
+ ],
+ },
+ self.team,
+ create_people=False,
+ )
+
+ filters = {
+ "insight": INSIGHT_FUNNELS,
+ "date_from": "2020-01-01",
+ "date_to": "2020-01-08",
+ "interval": "day",
+ "funnel_window_days": 7,
+ "events": [
+ {"id": "sign up", "order": 0},
+ {"id": "play movie", "order": 1},
+ ],
+ "breakdown_type": "event",
+ "breakdown": "$browser",
+ }
+
+ results = get_actors(filters, self.team, funnelStep=1, funnelStepBreakdown=["test'123"])
+ self.assertCountEqual([results[0][0]], [person1.uuid])
diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py
index 6ca333b036f14..f9c7b107074de 100644
--- a/posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py
+++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py
@@ -1387,3 +1387,43 @@ def test_trend_for_hour_based_conversion_window(self):
results = FunnelsQueryRunner(query=query, team=self.team, just_summarize=True).calculate().results
conversion_rates = [row["conversion_rate"] for row in results]
self.assertEqual(conversion_rates, [50.0, 0.0, 0.0, 0.0, 0.0, 0.0])
+
+ def test_parses_breakdown_correctly(self):
+ journeys_for(
+ {
+ "user_one": [
+ {
+ "event": "step one",
+ "timestamp": datetime(2021, 5, 1),
+ "properties": {"$browser": "test''123"},
+ },
+ {
+ "event": "step two",
+ "timestamp": datetime(2021, 5, 3),
+ "properties": {"$browser": "test''123"},
+ },
+ ],
+ },
+ self.team,
+ )
+
+ filters = {
+ "insight": INSIGHT_FUNNELS,
+ "funnel_viz_type": "trends",
+ "display": TRENDS_LINEAR,
+ "interval": "day",
+ "date_from": "2021-05-01 00:00:00",
+ "date_to": "2021-05-13 23:59:59",
+ "funnel_window_days": 7,
+ "events": [
+ {"id": "step one", "order": 0},
+ {"id": "step two", "order": 1},
+ ],
+ "breakdown_type": "event",
+ "breakdown": "$browser",
+ }
+
+ query = cast(FunnelsQuery, filter_to_query(filters))
+ results = FunnelsQueryRunner(query=query, team=self.team).calculate().results
+
+ self.assertEqual(len(results), 1)
diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_unordered.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_unordered.py
index 36e5d87f39e49..49e495e69222d 100644
--- a/posthog/hogql_queries/insights/funnels/test/test_funnel_unordered.py
+++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_unordered.py
@@ -1620,3 +1620,57 @@ def test_funnel_unordered_entity_filters(self):
self.assertEqual(results[0]["count"], 1)
self.assertEqual(results[1]["count"], 1)
+
+ def test_funnel_window_ignores_dst_transition(self):
+ _create_person(
+ distinct_ids=[f"user_1"],
+ team=self.team,
+ )
+
+ events_by_person = {
+ "user_1": [
+ {
+ "event": "$pageview",
+ "timestamp": datetime(2024, 3, 1, 15, 10), # 1st March 15:10
+ },
+ {
+ "event": "user signed up",
+ "timestamp": datetime(
+ 2024, 3, 15, 14, 27
+ ), # 15th March 14:27 (within 14 day conversion window that ends at 15:10)
+ },
+ ],
+ }
+ journeys_for(events_by_person, self.team)
+
+ filters = {
+ "events": [
+ {"id": "$pageview", "type": "events", "order": 0},
+ {"id": "user signed up", "type": "events", "order": 1},
+ ],
+ "insight": INSIGHT_FUNNELS,
+ "funnel_order_type": "unordered",
+ "date_from": "2024-02-17",
+ "date_to": "2024-03-18",
+ }
+
+ query = cast(FunnelsQuery, filter_to_query(filters))
+ results = FunnelsQueryRunner(query=query, team=self.team).calculate().results
+
+ self.assertEqual(results[1]["name"], "Completed 2 steps")
+ self.assertEqual(results[1]["count"], 1)
+ self.assertEqual(results[1]["average_conversion_time"], 1_207_020)
+ self.assertEqual(results[1]["median_conversion_time"], 1_207_020)
+
+ # there is a PST -> PDT transition on 10th of March
+ self.team.timezone = "US/Pacific"
+ self.team.save()
+
+ query = cast(FunnelsQuery, filter_to_query(filters))
+ results = FunnelsQueryRunner(query=query, team=self.team).calculate().results
+
+ # we still should have the user here, as the conversion window should not be affected by DST
+ self.assertEqual(results[1]["name"], "Completed 2 steps")
+ self.assertEqual(results[1]["count"], 1)
+ self.assertEqual(results[1]["average_conversion_time"], 1_207_020)
+ self.assertEqual(results[1]["median_conversion_time"], 1_207_020)
diff --git a/posthog/hogql_queries/insights/funnels/utils.py b/posthog/hogql_queries/insights/funnels/utils.py
index 47c1487e5fbcc..cdccce0251a33 100644
--- a/posthog/hogql_queries/insights/funnels/utils.py
+++ b/posthog/hogql_queries/insights/funnels/utils.py
@@ -61,23 +61,26 @@ def funnel_window_interval_unit_to_sql(
def get_breakdown_expr(
- breakdown: List[str | int] | None, properties_column: str, normalize_url: bool | None = False
+ breakdowns: List[str | int] | str | int, properties_column: str, normalize_url: bool | None = False
) -> ast.Expr:
- if isinstance(breakdown, str) or isinstance(breakdown, int) or breakdown is None:
- return parse_expr(f"ifNull({properties_column}.\"{breakdown}\", '')")
+ if isinstance(breakdowns, str) or isinstance(breakdowns, int) or breakdowns is None:
+ return ast.Call(
+ name="ifNull", args=[ast.Field(chain=[*properties_column.split("."), breakdowns]), ast.Constant(value="")]
+ )
else:
exprs = []
- for b in breakdown:
- expr = parse_expr(normalize_url_breakdown(f"ifNull({properties_column}.\"{b}\", '')", normalize_url))
+ for breakdown in breakdowns:
+ expr: ast.Expr = ast.Call(
+ name="ifNull",
+ args=[ast.Field(chain=[*properties_column.split("."), breakdown]), ast.Constant(value="")],
+ )
+ if normalize_url:
+ regex = "[\\\\/?#]*$"
+ expr = parse_expr(
+ f"if( empty( replaceRegexpOne({{breakdown_value}}, '{regex}', '') ), '/', replaceRegexpOne({{breakdown_value}}, '{regex}', ''))",
+ {"breakdown_value": expr},
+ )
exprs.append(expr)
expression = ast.Array(exprs=exprs)
return expression
-
-
-def normalize_url_breakdown(breakdown_value, breakdown_normalize_url: bool | None):
- if breakdown_normalize_url:
- regex = "[\\\\/?#]*$"
- return f"if( empty( replaceRegexpOne({breakdown_value}, '{regex}', '') ), '/', replaceRegexpOne({breakdown_value}, '{regex}', ''))"
-
- return breakdown_value
diff --git a/posthog/hogql_queries/insights/insight_actors_query_runner.py b/posthog/hogql_queries/insights/insight_actors_query_runner.py
index 782dd5b054a0e..d58f36cb6f7ee 100644
--- a/posthog/hogql_queries/insights/insight_actors_query_runner.py
+++ b/posthog/hogql_queries/insights/insight_actors_query_runner.py
@@ -1,5 +1,5 @@
from datetime import timedelta
-from typing import cast
+from typing import cast, Optional
from posthog.hogql import ast
from posthog.hogql.query import execute_hogql_query
@@ -37,7 +37,7 @@ def to_query(self) -> ast.SelectQuery | ast.SelectUnionQuery:
trends_runner = cast(TrendsQueryRunner, self.source_runner)
query = cast(InsightActorsQuery, self.query)
return trends_runner.to_actors_query(
- time_frame=query.day,
+ time_frame=cast(Optional[str], query.day), # Other runner accept day as int, but not this one
series_index=query.series or 0,
breakdown_value=query.breakdown,
compare=query.compare,
@@ -102,6 +102,7 @@ def calculate(self) -> HogQLQueryResponse:
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
def _is_stale(self, cached_result_package):
diff --git a/posthog/hogql_queries/insights/lifecycle_query_runner.py b/posthog/hogql_queries/insights/lifecycle_query_runner.py
index 24bbe36f1c6bf..ea883eec542bc 100644
--- a/posthog/hogql_queries/insights/lifecycle_query_runner.py
+++ b/posthog/hogql_queries/insights/lifecycle_query_runner.py
@@ -126,7 +126,7 @@ def to_actors_query(
def to_actors_query_options(self) -> InsightActorsQueryOptionsResponse:
return InsightActorsQueryOptionsResponse(
- day=[{"label": day, "value": day} for day in self.query_date_range.all_values()],
+ day=[{"label": format_label_date(value), "value": value} for value in self.query_date_range.all_values()],
status=[
{
"label": "Dormant",
@@ -157,6 +157,7 @@ def calculate(self) -> LifecycleQueryResponse:
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
# TODO: can we move the data conversion part into the query as well? It would make it easier to swap
diff --git a/posthog/hogql_queries/insights/paginators.py b/posthog/hogql_queries/insights/paginators.py
index 6dbdb1543b929..0dfda79ced617 100644
--- a/posthog/hogql_queries/insights/paginators.py
+++ b/posthog/hogql_queries/insights/paginators.py
@@ -54,8 +54,9 @@ def trim_results(self) -> list[Any]:
def execute_hogql_query(
self,
- query_type: str,
query: ast.SelectQuery,
+ *,
+ query_type: str,
**kwargs,
) -> HogQLQueryResponse:
self.response = cast(
diff --git a/posthog/hogql_queries/insights/paths_query_runner.py b/posthog/hogql_queries/insights/paths_query_runner.py
index c10a5a2320207..c454feb8e56ac 100644
--- a/posthog/hogql_queries/insights/paths_query_runner.py
+++ b/posthog/hogql_queries/insights/paths_query_runner.py
@@ -725,6 +725,7 @@ def calculate(self) -> PathsQueryResponse:
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
response.results = self.validate_results(response.results)
diff --git a/posthog/hogql_queries/insights/retention_query_runner.py b/posthog/hogql_queries/insights/retention_query_runner.py
index 221cb976757d2..3ac2c5b4b5462 100644
--- a/posthog/hogql_queries/insights/retention_query_runner.py
+++ b/posthog/hogql_queries/insights/retention_query_runner.py
@@ -313,6 +313,7 @@ def calculate(self) -> RetentionQueryResponse:
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
result_dict = {
diff --git a/posthog/hogql_queries/insights/stickiness_query_runner.py b/posthog/hogql_queries/insights/stickiness_query_runner.py
index d0b4b65c67f9b..184e3c0af02df 100644
--- a/posthog/hogql_queries/insights/stickiness_query_runner.py
+++ b/posthog/hogql_queries/insights/stickiness_query_runner.py
@@ -212,6 +212,7 @@ def calculate(self):
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
if response.timings is not None:
diff --git a/posthog/hogql_queries/insights/test/test_paginators.py b/posthog/hogql_queries/insights/test/test_paginators.py
index ac83efb45b353..6698115c46535 100644
--- a/posthog/hogql_queries/insights/test/test_paginators.py
+++ b/posthog/hogql_queries/insights/test/test_paginators.py
@@ -1,3 +1,5 @@
+from typing import cast
+from posthog.hogql.ast import SelectQuery
from posthog.hogql.constants import (
LimitContext,
get_default_limit_for_context,
@@ -136,8 +138,8 @@ def test_response_params_consistency(self):
"""Test consistency of response_params method."""
paginator = HogQLHasMorePaginator(limit=5, offset=10)
paginator.response = paginator.execute_hogql_query(
- "test_query",
- parse_select("SELECT * FROM persons"),
+ cast(SelectQuery, parse_select("SELECT * FROM persons")),
+ query_type="test_query",
team=self.team,
)
params = paginator.response_params()
diff --git a/posthog/hogql_queries/insights/test/test_retention_query_runner.py b/posthog/hogql_queries/insights/test/test_retention_query_runner.py
index 30edb32102f76..04c108dd779f1 100644
--- a/posthog/hogql_queries/insights/test/test_retention_query_runner.py
+++ b/posthog/hogql_queries/insights/test/test_retention_query_runner.py
@@ -1,3 +1,5 @@
+from typing import Optional
+from unittest.mock import MagicMock, patch
import uuid
from datetime import datetime
@@ -6,11 +8,14 @@
from django.test import override_settings
from rest_framework import status
+from posthog.clickhouse.client.execute import sync_execute
from posthog.constants import (
RETENTION_FIRST_TIME,
TREND_FILTER_TYPE_ACTIONS,
TREND_FILTER_TYPE_EVENTS,
)
+from posthog.hogql.constants import LimitContext
+from posthog.hogql.query import INCREASED_MAX_EXECUTION_TIME
from posthog.hogql_queries.insights.retention_query_runner import RetentionQueryRunner
from posthog.hogql_queries.actors_query_runner import ActorsQueryRunner
from posthog.models import Action, ActionStep
@@ -1685,10 +1690,10 @@ def test_day_interval_sampled(self):
class TestClickhouseRetentionGroupAggregation(ClickhouseTestMixin, APIBaseTest):
- def run_query(self, query):
+ def run_query(self, query, *, limit_context: Optional[LimitContext] = None):
if not query.get("retentionFilter"):
query["retentionFilter"] = {}
- runner = RetentionQueryRunner(team=self.team, query=query)
+ runner = RetentionQueryRunner(team=self.team, query=query, limit_context=limit_context)
return runner.calculate().model_dump()["results"]
def run_actors_query(self, interval, query, select=None, actor="person"):
@@ -1920,3 +1925,10 @@ def test_groups_aggregating_person_on_events(self):
[1],
],
)
+
+ @patch("posthog.hogql.query.sync_execute", wraps=sync_execute)
+ def test_limit_is_context_aware(self, mock_sync_execute: MagicMock):
+ self.run_query(query={}, limit_context=LimitContext.QUERY_ASYNC)
+
+ mock_sync_execute.assert_called_once()
+ self.assertIn(f" max_execution_time={INCREASED_MAX_EXECUTION_TIME},", mock_sync_execute.call_args[0][0])
diff --git a/posthog/hogql_queries/insights/test/test_stickiness_query_runner.py b/posthog/hogql_queries/insights/test/test_stickiness_query_runner.py
index 3de1fb6ce865e..6e25827e6ecba 100644
--- a/posthog/hogql_queries/insights/test/test_stickiness_query_runner.py
+++ b/posthog/hogql_queries/insights/test/test_stickiness_query_runner.py
@@ -1,8 +1,12 @@
from dataclasses import dataclass
from typing import Dict, List, Optional, Union
+from unittest.mock import MagicMock, patch
from django.test import override_settings
from freezegun import freeze_time
+from posthog.clickhouse.client.execute import sync_execute
+from posthog.hogql.constants import LimitContext
+from posthog.hogql.query import INCREASED_MAX_EXECUTION_TIME
from posthog.hogql_queries.insights.stickiness_query_runner import StickinessQueryRunner
from posthog.models.action.action import Action
from posthog.models.action_step import ActionStep
@@ -197,6 +201,7 @@ def _run_query(
properties: Optional[StickinessProperties] = None,
filters: Optional[StickinessFilter] = None,
filter_test_accounts: Optional[bool] = False,
+ limit_context: Optional[LimitContext] = None,
):
query_series: List[EventsNode | ActionsNode] = [EventsNode(event="$pageview")] if series is None else series
query_date_from = date_from or self.default_date_from
@@ -211,7 +216,7 @@ def _run_query(
stickinessFilter=filters,
filterTestAccounts=filter_test_accounts,
)
- return StickinessQueryRunner(team=self.team, query=query).calculate()
+ return StickinessQueryRunner(team=self.team, query=query, limit_context=limit_context).calculate()
def test_stickiness_runs(self):
self._create_test_events()
@@ -580,3 +585,10 @@ def test_hogql_aggregations(self):
1,
0,
]
+
+ @patch("posthog.hogql.query.sync_execute", wraps=sync_execute)
+ def test_limit_is_context_aware(self, mock_sync_execute: MagicMock):
+ self._run_query(limit_context=LimitContext.QUERY_ASYNC)
+
+ mock_sync_execute.assert_called_once()
+ self.assertIn(f" max_execution_time={INCREASED_MAX_EXECUTION_TIME},", mock_sync_execute.call_args[0][0])
diff --git a/posthog/hogql_queries/insights/trends/breakdown.py b/posthog/hogql_queries/insights/trends/breakdown.py
index 45a3a8421e8d8..bde2cd807b6a7 100644
--- a/posthog/hogql_queries/insights/trends/breakdown.py
+++ b/posthog/hogql_queries/insights/trends/breakdown.py
@@ -3,9 +3,7 @@
from posthog.hogql.parser import parse_expr
from posthog.hogql.timings import HogQLTimings
from posthog.hogql_queries.insights.trends.breakdown_values import (
- BREAKDOWN_NULL_NUMERIC_LABEL,
BREAKDOWN_NULL_STRING_LABEL,
- BREAKDOWN_OTHER_NUMERIC_LABEL,
BREAKDOWN_OTHER_STRING_LABEL,
BreakdownValues,
)
@@ -19,6 +17,10 @@
from posthog.schema import ActionsNode, EventsNode, DataWarehouseNode, HogQLQueryModifiers, InCohortVia, TrendsQuery
+def hogql_to_string(expr: ast.Expr) -> ast.Call:
+ return ast.Call(name="toString", args=[expr])
+
+
class Breakdown:
query: TrendsQuery
team: Team
@@ -27,7 +29,7 @@ class Breakdown:
timings: HogQLTimings
modifiers: HogQLQueryModifiers
events_filter: ast.Expr
- breakdown_values_override: Optional[List[str | int | float]]
+ breakdown_values_override: Optional[List[str]]
def __init__(
self,
@@ -38,7 +40,7 @@ def __init__(
timings: HogQLTimings,
modifiers: HogQLQueryModifiers,
events_filter: ast.Expr,
- breakdown_values_override: Optional[List[str | int | float]] = None,
+ breakdown_values_override: Optional[List[str]] = None,
):
self.team = team
self.query = query
@@ -70,19 +72,15 @@ def placeholders(self) -> Dict[str, ast.Expr]:
return {"cross_join_breakdown_values": ast.Alias(alias="breakdown_value", expr=values)}
- def column_expr(self) -> ast.Expr:
+ def column_expr(self) -> ast.Alias:
if self.is_histogram_breakdown:
return ast.Alias(alias="breakdown_value", expr=self._get_breakdown_histogram_multi_if())
- elif self.query.breakdownFilter.breakdown_type == "hogql":
- return ast.Alias(
- alias="breakdown_value",
- expr=parse_expr(self.query.breakdownFilter.breakdown),
- )
- elif self.query.breakdownFilter.breakdown_type == "cohort":
+
+ if self.query.breakdownFilter.breakdown_type == "cohort":
if self.modifiers.inCohortVia == InCohortVia.leftjoin_conjoined:
return ast.Alias(
alias="breakdown_value",
- expr=ast.Field(chain=["__in_cohort", "cohort_id"]),
+ expr=hogql_to_string(ast.Field(chain=["__in_cohort", "cohort_id"])),
)
cohort_breakdown = (
@@ -90,19 +88,9 @@ def column_expr(self) -> ast.Expr:
)
return ast.Alias(
alias="breakdown_value",
- expr=ast.Constant(value=cohort_breakdown),
- )
-
- if self.query.breakdownFilter.breakdown_type == "hogql":
- return ast.Alias(
- alias="breakdown_value",
- expr=parse_expr(self.query.breakdownFilter.breakdown),
+ expr=hogql_to_string(ast.Constant(value=cohort_breakdown)),
)
- # If there's no breakdown values
- if len(self._breakdown_values) == 1 and self._breakdown_values[0] is None:
- return ast.Alias(alias="breakdown_value", expr=ast.Field(chain=self._properties_chain))
-
return ast.Alias(alias="breakdown_value", expr=self._get_breakdown_transform_func)
def events_where_filter(self) -> ast.Expr | None:
@@ -148,15 +136,14 @@ def events_where_filter(self) -> ast.Expr | None:
else:
left = ast.Field(chain=self._properties_chain)
+ if not self.is_histogram_breakdown:
+ left = hogql_to_string(left)
+
compare_ops = []
for _value in self._breakdown_values:
- value: Optional[str | int | float] = _value
+ value: Optional[str] = str(_value) # non-cohorts are always strings
# If the value is one of the "other" values, then use the `transform()` func
- if (
- value == BREAKDOWN_OTHER_STRING_LABEL
- or value == BREAKDOWN_OTHER_NUMERIC_LABEL
- or value == float(BREAKDOWN_OTHER_NUMERIC_LABEL)
- ):
+ if value == BREAKDOWN_OTHER_STRING_LABEL:
transform_func = self._get_breakdown_transform_func
compare_ops.append(
ast.CompareOperation(
@@ -164,11 +151,7 @@ def events_where_filter(self) -> ast.Expr | None:
)
)
else:
- if (
- value == BREAKDOWN_NULL_STRING_LABEL
- or value == BREAKDOWN_NULL_NUMERIC_LABEL
- or value == float(BREAKDOWN_NULL_NUMERIC_LABEL)
- ):
+ if value == BREAKDOWN_NULL_STRING_LABEL:
value = None
compare_ops.append(
@@ -184,30 +167,25 @@ def events_where_filter(self) -> ast.Expr | None:
@cached_property
def _get_breakdown_transform_func(self) -> ast.Call:
- values = self._breakdown_values
- all_values_are_ints_or_none = all(isinstance(value, int) or value is None for value in values)
- all_values_are_floats_or_none = all(isinstance(value, float) or value is None for value in values)
-
- if all_values_are_ints_or_none:
- breakdown_other_value = BREAKDOWN_OTHER_NUMERIC_LABEL
- breakdown_null_value = BREAKDOWN_NULL_NUMERIC_LABEL
- elif all_values_are_floats_or_none:
- breakdown_other_value = float(BREAKDOWN_OTHER_NUMERIC_LABEL)
- breakdown_null_value = float(BREAKDOWN_NULL_NUMERIC_LABEL)
- else:
- breakdown_other_value = BREAKDOWN_OTHER_STRING_LABEL
- breakdown_null_value = BREAKDOWN_NULL_STRING_LABEL
+ if self.query.breakdownFilter.breakdown_type == "hogql":
+ return self._get_breakdown_values_transform(parse_expr(self.query.breakdownFilter.breakdown))
+ return self._get_breakdown_values_transform(ast.Field(chain=self._properties_chain))
+ def _get_breakdown_values_transform(self, node: ast.Expr) -> ast.Call:
+ breakdown_values = self._breakdown_values_ast
return ast.Call(
name="transform",
args=[
ast.Call(
name="ifNull",
- args=[ast.Field(chain=self._properties_chain), ast.Constant(value=breakdown_null_value)],
+ args=[
+ hogql_to_string(node),
+ ast.Constant(value=BREAKDOWN_NULL_STRING_LABEL),
+ ],
),
- self._breakdown_values_ast,
- self._breakdown_values_ast,
- ast.Constant(value=breakdown_other_value),
+ breakdown_values,
+ breakdown_values,
+ ast.Constant(value=BREAKDOWN_OTHER_STRING_LABEL),
],
)
@@ -220,15 +198,21 @@ def _breakdown_buckets_ast(self) -> ast.Array:
return ast.Array(exprs=list(map(lambda v: ast.Constant(value=v), values)))
- @cached_property
+ @property
def _breakdown_values_ast(self) -> ast.Array:
- return ast.Array(exprs=[ast.Constant(value=v) for v in self._breakdown_values])
+ exprs: list[ast.Expr] = []
+ for value in self._breakdown_values:
+ if isinstance(value, str):
+ exprs.append(ast.Constant(value=value))
+ else:
+ exprs.append(hogql_to_string(ast.Constant(value=value)))
+ return ast.Array(exprs=exprs)
@cached_property
- def _all_breakdown_values(self) -> List[str | int | float | None]:
+ def _all_breakdown_values(self) -> List[str | int | None]:
# Used in the actors query
if self.breakdown_values_override is not None:
- return cast(List[str | int | float | None], self.breakdown_values_override)
+ return cast(List[str | int | None], self.breakdown_values_override)
if self.query.breakdownFilter is None:
return []
@@ -243,25 +227,12 @@ def _all_breakdown_values(self) -> List[str | int | float | None]:
query_date_range=self.query_date_range,
modifiers=self.modifiers,
)
- return cast(List[str | int | float | None], breakdown.get_breakdown_values())
+ return cast(List[str | int | None], breakdown.get_breakdown_values())
@cached_property
- def _breakdown_values(self) -> List[str | int | float]:
- values = self._all_breakdown_values
- if len(values) == 0 or all(value is None for value in values):
- return []
-
- if None in values:
- all_values_are_ints_or_none = all(isinstance(value, int) or value is None for value in values)
- all_values_are_floats_or_none = all(isinstance(value, float) or value is None for value in values)
-
- if all_values_are_ints_or_none:
- values = [v if v is not None else BREAKDOWN_NULL_NUMERIC_LABEL for v in values]
- elif all_values_are_floats_or_none:
- values = [v if v is not None else float(BREAKDOWN_NULL_NUMERIC_LABEL) for v in values]
- else:
- values = [v if v is not None else BREAKDOWN_NULL_STRING_LABEL for v in values]
- return cast(List[str | int | float], values)
+ def _breakdown_values(self) -> List[str | int]:
+ values = [BREAKDOWN_NULL_STRING_LABEL if v is None else v for v in self._all_breakdown_values]
+ return cast(List[str | int], values)
@cached_property
def has_breakdown_values(self) -> bool:
diff --git a/posthog/hogql_queries/insights/trends/breakdown_values.py b/posthog/hogql_queries/insights/trends/breakdown_values.py
index 7b1522d5f25c5..d9ab11891f210 100644
--- a/posthog/hogql_queries/insights/trends/breakdown_values.py
+++ b/posthog/hogql_queries/insights/trends/breakdown_values.py
@@ -97,6 +97,9 @@ def get_breakdown_values(self) -> List[str | int]:
),
)
+ if not self.histogram_bin_count:
+ select_field.expr = ast.Call(name="toString", args=[select_field.expr])
+
if self.chart_display_type == ChartDisplayType.WorldMap:
breakdown_limit = BREAKDOWN_VALUES_LIMIT_FOR_COUNTRIES
else:
@@ -211,23 +214,9 @@ def get_breakdown_values(self) -> List[str | int]:
# Add "other" value if "other" is not hidden and we're not bucketing numeric values
if self.hide_other_aggregation is not True and self.histogram_bin_count is None:
- all_values_are_ints_or_none = all(isinstance(value, int) or value is None for value in values)
- all_values_are_floats_or_none = all(isinstance(value, float) or value is None for value in values)
- all_values_are_string_or_none = all(isinstance(value, str) or value is None for value in values)
-
- if all_values_are_string_or_none:
- values = [BREAKDOWN_NULL_STRING_LABEL if value in (None, "") else value for value in values]
- if needs_other:
- values.insert(0, BREAKDOWN_OTHER_STRING_LABEL)
- elif all_values_are_ints_or_none or all_values_are_floats_or_none:
- if all_values_are_ints_or_none:
- values = [BREAKDOWN_NULL_NUMERIC_LABEL if value is None else value for value in values]
- if needs_other:
- values.insert(0, BREAKDOWN_OTHER_NUMERIC_LABEL)
- else:
- values = [float(BREAKDOWN_NULL_NUMERIC_LABEL) if value is None else value for value in values]
- if needs_other:
- values.insert(0, float(BREAKDOWN_OTHER_NUMERIC_LABEL))
+ values = [BREAKDOWN_NULL_STRING_LABEL if value in (None, "") else value for value in values]
+ if needs_other:
+ values = [BREAKDOWN_OTHER_STRING_LABEL] + values
if len(values) == 0:
values.insert(0, None)
diff --git a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr
index f6eb3748afb2b..e2ec22fb9fb1c 100644
--- a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr
+++ b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr
@@ -187,7 +187,7 @@
# ---
# name: TestTrends.test_breakdown_by_group_props_person_on_events
'''
- SELECT e__group_0.properties___industry AS value,
+ SELECT toString(e__group_0.properties___industry) AS value,
count(e.uuid) AS count
FROM events AS e
LEFT JOIN
@@ -210,7 +210,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -231,7 +231,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.uuid) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(e__group_0.properties___industry, '$$_posthog_breakdown_null_$$'), ['finance', 'technology'], ['finance', 'technology'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(e__group_0.properties___industry), '$$_posthog_breakdown_null_$$'), ['finance', 'technology'], ['finance', 'technology'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
LEFT JOIN
(SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), groups._timestamp) AS properties___industry,
@@ -241,7 +241,7 @@
WHERE and(equals(groups.team_id, 2), ifNull(equals(index, 0), 0))
GROUP BY groups.group_type_index,
groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(e__group_0.properties___industry, 'finance'), 0), ifNull(equals(e__group_0.properties___industry, 'technology'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(toString(e__group_0.properties___industry), 'finance'), 0), ifNull(equals(toString(e__group_0.properties___industry), 'technology'), 0)))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -287,7 +287,7 @@
# ---
# name: TestTrends.test_breakdown_by_group_props_with_person_filter_person_on_events
'''
- SELECT e__group_0.properties___industry AS value,
+ SELECT toString(e__group_0.properties___industry) AS value,
count(e.uuid) AS count
FROM events AS e
LEFT JOIN
@@ -310,7 +310,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -331,7 +331,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.uuid) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(e__group_0.properties___industry, '$$_posthog_breakdown_null_$$'), ['finance'], ['finance'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(e__group_0.properties___industry), '$$_posthog_breakdown_null_$$'), ['finance'], ['finance'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
LEFT JOIN
(SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), groups._timestamp) AS properties___industry,
@@ -341,7 +341,7 @@
WHERE and(equals(groups.team_id, 2), ifNull(equals(index, 0), 0))
GROUP BY groups.group_type_index,
groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, 'key'), ''), 'null'), '^"|"$', ''), 'value'), 0), ifNull(equals(e__group_0.properties___industry, 'finance'), 0))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, 'key'), ''), 'null'), '^"|"$', ''), 'value'), 0), ifNull(equals(toString(e__group_0.properties___industry), 'finance'), 0))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -356,7 +356,7 @@
# ---
# name: TestTrends.test_breakdown_filtering_with_properties_in_new_format
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Firefox'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Windows'), 0)), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0)))
@@ -371,7 +371,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -392,9 +392,9 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.uuid) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['second url'], ['second url'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['second url'], ['second url'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Firefox'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Windows'), 0)), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'second url'), 0))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Firefox'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Windows'), 0)), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '')), 'second url'), 0))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -409,7 +409,7 @@
# ---
# name: TestTrends.test_breakdown_filtering_with_properties_in_new_format.2
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Firefox'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Windows'), 0)), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0)))
@@ -423,24 +423,38 @@
# name: TestTrends.test_breakdown_filtering_with_properties_in_new_format.3
'''
SELECT groupArray(day_start) AS date,
- groupArray(count) AS total
+ groupArray(count) AS total,
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
- day_start AS day_start
+ day_start AS day_start,
+ breakdown_value AS breakdown_value
FROM
(SELECT 0 AS total,
- minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start
- FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), 0)) AS numbers
- UNION ALL SELECT 0 AS total,
- toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC'))) AS day_start
+ ticks.day_start AS day_start,
+ sec.breakdown_value AS breakdown_value
+ FROM
+ (SELECT minus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), toIntervalDay(numbers.number)) AS day_start
+ FROM numbers(coalesce(dateDiff('day', assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), 0)) AS numbers
+ UNION ALL SELECT toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC'))) AS day_start) AS ticks
+ CROSS JOIN
+ (SELECT breakdown_value
+ FROM
+ (SELECT ['$$_posthog_breakdown_null_$$'] AS breakdown_value) ARRAY
+ JOIN breakdown_value AS breakdown_value) AS sec
+ ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.uuid) AS total,
- toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start
+ toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$'], ['$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Firefox'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Windows'), 0)), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0))
- GROUP BY day_start)
- GROUP BY day_start
- ORDER BY day_start ASC)
- ORDER BY sum(count) DESC
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-22 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', ''), 'Firefox'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Windows'), 0)), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''))))
+ GROUP BY day_start,
+ breakdown_value)
+ GROUP BY day_start,
+ breakdown_value
+ ORDER BY day_start ASC, breakdown_value ASC)
+ GROUP BY breakdown_value
+ ORDER BY sum(count) DESC, breakdown_value ASC
LIMIT 10000 SETTINGS readonly=2,
max_execution_time=60,
allow_experimental_object_type=1
@@ -448,7 +462,7 @@
# ---
# name: TestTrends.test_breakdown_weekly_active_users_aggregated
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')) AS value,
count(DISTINCT e__pdi.person_id) AS count
FROM events AS e
INNER JOIN
@@ -480,7 +494,7 @@
CROSS JOIN
(SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp,
e__pdi.person_id AS actor_id,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['val', 'bor'], ['val', 'bor'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['val', 'bor'], ['val', 'bor'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -489,7 +503,7 @@
WHERE equals(person_distinct_id2.team_id, 2)
GROUP BY person_distinct_id2.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
- WHERE and(equals(e.team_id, 2), and(equals(e.event, '$pageview'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'bor'), 0))), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 00:00:00', 6, 'UTC')), toIntervalDay(7))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC'))), 0))
+ WHERE and(equals(e.team_id, 2), and(equals(e.event, '$pageview'), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), 'val'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), 'bor'), 0))), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 00:00:00', 6, 'UTC')), toIntervalDay(7))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC'))), 0))
GROUP BY timestamp, actor_id,
breakdown_value) AS e
WHERE and(ifNull(lessOrEquals(e.timestamp, plus(d.timestamp, toIntervalDay(1))), 0), ifNull(greater(e.timestamp, minus(d.timestamp, toIntervalDay(6))), 0))
@@ -506,7 +520,7 @@
# ---
# name: TestTrends.test_breakdown_weekly_active_users_aggregated_materialized
'''
- SELECT nullIf(nullIf(e.mat_key, ''), 'null') AS value,
+ SELECT toString(nullIf(nullIf(e.mat_key, ''), 'null')) AS value,
count(DISTINCT e__pdi.person_id) AS count
FROM events AS e
INNER JOIN
@@ -538,7 +552,7 @@
CROSS JOIN
(SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp,
e__pdi.person_id AS actor_id,
- transform(ifNull(nullIf(nullIf(e.mat_key, ''), 'null'), '$$_posthog_breakdown_null_$$'), ['val', 'bor'], ['val', 'bor'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(nullIf(nullIf(e.mat_key, ''), 'null')), '$$_posthog_breakdown_null_$$'), ['val', 'bor'], ['val', 'bor'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -547,7 +561,7 @@
WHERE equals(person_distinct_id2.team_id, 2)
GROUP BY person_distinct_id2.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
- WHERE and(equals(e.team_id, 2), and(equals(e.event, '$pageview'), or(ifNull(equals(nullIf(nullIf(e.mat_key, ''), 'null'), 'val'), 0), ifNull(equals(nullIf(nullIf(e.mat_key, ''), 'null'), 'bor'), 0))), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 00:00:00', 6, 'UTC')), toIntervalDay(7))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC'))), 0))
+ WHERE and(equals(e.team_id, 2), and(equals(e.event, '$pageview'), or(ifNull(equals(toString(nullIf(nullIf(e.mat_key, ''), 'null')), 'val'), 0), ifNull(equals(toString(nullIf(nullIf(e.mat_key, ''), 'null')), 'bor'), 0))), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 00:00:00', 6, 'UTC')), toIntervalDay(7))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-11 23:59:59', 6, 'UTC'))), 0))
GROUP BY timestamp, actor_id,
breakdown_value) AS e
WHERE and(ifNull(lessOrEquals(e.timestamp, plus(d.timestamp, toIntervalDay(1))), 0), ifNull(greater(e.timestamp, minus(d.timestamp, toIntervalDay(6))), 0))
@@ -584,7 +598,7 @@
# ---
# name: TestTrends.test_breakdown_weekly_active_users_daily_based_on_action.2
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')) AS value,
count(DISTINCT e__pdi.person_id) AS count
FROM events AS e
INNER JOIN
@@ -622,7 +636,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -654,7 +668,7 @@
CROSS JOIN
(SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp,
e__pdi.person_id AS actor_id,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['val'], ['val'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['val'], ['val'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -679,7 +693,7 @@
FROM cohortpeople
WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 2))
GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version
- HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0))), 0))), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0)), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), toIntervalDay(7))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), 0))
+ HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0))), 0))), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), 'val'), 0)), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), toIntervalDay(7))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), 0))
GROUP BY timestamp, actor_id,
breakdown_value) AS e
WHERE and(ifNull(lessOrEquals(e.timestamp, plus(d.timestamp, toIntervalDay(1))), 0), ifNull(greater(e.timestamp, minus(d.timestamp, toIntervalDay(6))), 0))
@@ -699,7 +713,7 @@
# ---
# name: TestTrends.test_breakdown_with_filter_groups_person_on_events
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
LEFT JOIN
@@ -722,7 +736,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -743,7 +757,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.uuid) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['uh', 'oh'], ['uh', 'oh'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['uh', 'oh'], ['uh', 'oh'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
LEFT JOIN
(SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), groups._timestamp) AS properties___industry,
@@ -753,7 +767,7 @@
WHERE and(equals(groups.team_id, 2), ifNull(equals(index, 0), 0))
GROUP BY groups.group_type_index,
groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(e__group_0.properties___industry, 'finance'), 0), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'uh'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'oh'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(e__group_0.properties___industry, 'finance'), 0), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), 'uh'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), 'oh'), 0)))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -782,7 +796,7 @@
# ---
# name: TestTrends.test_breakdown_with_filter_groups_person_on_events_v2.1
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
LEFT JOIN
@@ -805,7 +819,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -826,7 +840,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT ifNull(nullIf(e__override.override_person_id, '00000000-0000-0000-0000-000000000000'), e.person_id)) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['uh', 'oh'], ['uh', 'oh'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['uh', 'oh'], ['uh', 'oh'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
LEFT OUTER JOIN
(SELECT argMax(person_overrides.override_person_id, person_overrides.version) AS override_person_id,
@@ -842,7 +856,7 @@
WHERE and(equals(groups.team_id, 2), ifNull(equals(index, 0), 0))
GROUP BY groups.group_type_index,
groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(e__group_0.properties___industry, 'finance'), 0), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'uh'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'oh'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-12 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(e__group_0.properties___industry, 'finance'), 0), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), 'uh'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', '')), 'oh'), 0)))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -857,7 +871,7 @@
# ---
# name: TestTrends.test_dau_with_breakdown_filtering_with_sampling
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')))
@@ -872,7 +886,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -893,7 +907,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT e__pdi.person_id) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['other_value', '$$_posthog_breakdown_null_$$', 'value'], ['other_value', '$$_posthog_breakdown_null_$$', 'value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['other_value', '$$_posthog_breakdown_null_$$', 'value'], ['other_value', '$$_posthog_breakdown_null_$$', 'value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1.0
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -902,7 +916,7 @@
WHERE equals(person_distinct_id2.team_id, 2)
GROUP BY person_distinct_id2.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'other_value'), 0), isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'other_value'), 0), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''))), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value'), 0)))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -917,7 +931,7 @@
# ---
# name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.2
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')))
@@ -932,7 +946,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -953,7 +967,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT e__pdi.person_id) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['other_value', '$$_posthog_breakdown_null_$$', 'value'], ['other_value', '$$_posthog_breakdown_null_$$', 'value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['other_value', '$$_posthog_breakdown_null_$$', 'value'], ['other_value', '$$_posthog_breakdown_null_$$', 'value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1.0
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -962,7 +976,7 @@
WHERE equals(person_distinct_id2.team_id, 2)
GROUP BY person_distinct_id2.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'other_value'), 0), isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'other_value'), 0), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''))), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value'), 0)))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -1242,7 +1256,7 @@
# ---
# name: TestTrends.test_mau_with_breakdown_filtering_and_prop_filter
'''
- SELECT e__pdi__person.`properties___$some_prop` AS value,
+ SELECT toString(e__pdi__person.`properties___$some_prop`) AS value,
count(DISTINCT e__pdi.person_id) AS count
FROM events AS e
INNER JOIN
@@ -1276,7 +1290,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -1308,7 +1322,7 @@
CROSS JOIN
(SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp,
e__pdi.person_id AS actor_id,
- transform(ifNull(e__pdi__person.`properties___$some_prop`, '$$_posthog_breakdown_null_$$'), ['some_val2', 'some_val'], ['some_val2', 'some_val'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(e__pdi__person.`properties___$some_prop`), '$$_posthog_breakdown_null_$$'), ['some_val2', 'some_val'], ['some_val2', 'some_val'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -1329,7 +1343,7 @@
WHERE equals(person.team_id, 2)
GROUP BY person.id
HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id)
- WHERE and(equals(e.team_id, 2), and(equals(e.event, 'sign up'), ifNull(equals(e__pdi__person.properties___filter_prop, 'filter_val'), 0), or(ifNull(equals(e__pdi__person.`properties___$some_prop`, 'some_val2'), 0), ifNull(equals(e__pdi__person.`properties___$some_prop`, 'some_val'), 0))), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), toIntervalDay(30))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0))
+ WHERE and(equals(e.team_id, 2), and(equals(e.event, 'sign up'), ifNull(equals(e__pdi__person.properties___filter_prop, 'filter_val'), 0), or(ifNull(equals(toString(e__pdi__person.`properties___$some_prop`), 'some_val2'), 0), ifNull(equals(toString(e__pdi__person.`properties___$some_prop`), 'some_val'), 0))), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), toIntervalDay(30))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0))
GROUP BY timestamp, actor_id,
breakdown_value) AS e
WHERE and(ifNull(lessOrEquals(e.timestamp, plus(d.timestamp, toIntervalDay(1))), 0), ifNull(greater(e.timestamp, minus(d.timestamp, toIntervalDay(29))), 0))
@@ -1349,7 +1363,7 @@
# ---
# name: TestTrends.test_mau_with_breakdown_filtering_and_prop_filter_poe_v2
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$some_prop'), ''), 'null'), '^"|"$', '')) AS value,
count(DISTINCT ifNull(nullIf(e__override.override_person_id, '00000000-0000-0000-0000-000000000000'), e.person_id)) AS count
FROM events AS e
LEFT OUTER JOIN
@@ -1370,7 +1384,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -1402,7 +1416,7 @@
CROSS JOIN
(SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp,
ifNull(nullIf(e__override.override_person_id, '00000000-0000-0000-0000-000000000000'), e.person_id) AS actor_id,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$some_prop'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['some_val2', 'some_val'], ['some_val2', 'some_val'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$some_prop'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['some_val2', 'some_val'], ['some_val2', 'some_val'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
LEFT OUTER JOIN
(SELECT argMax(person_overrides.override_person_id, person_overrides.version) AS override_person_id,
@@ -1410,7 +1424,7 @@
FROM person_overrides
WHERE equals(person_overrides.team_id, 2)
GROUP BY person_overrides.old_person_id) AS e__override ON equals(e.person_id, e__override.old_person_id)
- WHERE and(equals(e.team_id, 2), and(equals(e.event, 'sign up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, 'filter_prop'), ''), 'null'), '^"|"$', ''), 'filter_val'), 0), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$some_prop'), ''), 'null'), '^"|"$', ''), 'some_val2'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$some_prop'), ''), 'null'), '^"|"$', ''), 'some_val'), 0))), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), toIntervalDay(30))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0))
+ WHERE and(equals(e.team_id, 2), and(equals(e.event, 'sign up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, 'filter_prop'), ''), 'null'), '^"|"$', ''), 'filter_val'), 0), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$some_prop'), ''), 'null'), '^"|"$', '')), 'some_val2'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.person_properties, '$some_prop'), ''), 'null'), '^"|"$', '')), 'some_val'), 0))), ifNull(greaterOrEquals(timestamp, minus(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), toIntervalDay(30))), 0), ifNull(lessOrEquals(timestamp, assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), 0))
GROUP BY timestamp, actor_id,
breakdown_value) AS e
WHERE and(ifNull(lessOrEquals(e.timestamp, plus(d.timestamp, toIntervalDay(1))), 0), ifNull(greater(e.timestamp, minus(d.timestamp, toIntervalDay(29))), 0))
@@ -1476,7 +1490,7 @@
# ---
# name: TestTrends.test_person_filtering_in_cohort_in_action.2
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
INNER JOIN
@@ -1503,7 +1517,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -1524,7 +1538,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.uuid) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -1538,7 +1552,7 @@
FROM cohortpeople
WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 2))
GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version
- HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0))), 0)), or(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'other_value'), 0)))
+ HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0))), 0)), or(isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''))), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'other_value'), 0)))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -1573,7 +1587,7 @@
# ---
# name: TestTrends.test_person_filtering_in_cohort_in_action_poe_v2.2
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
LEFT OUTER JOIN
@@ -1599,7 +1613,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -1620,7 +1634,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.uuid) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
LEFT OUTER JOIN
(SELECT argMax(person_overrides.override_person_id, person_overrides.version) AS override_person_id,
@@ -1633,7 +1647,7 @@
FROM cohortpeople
WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 2))
GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version
- HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0))), 0)), or(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'other_value'), 0)))
+ HAVING ifNull(greater(sum(cohortpeople.sign), 0), 0))), 0)), or(isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''))), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'other_value'), 0)))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -2217,7 +2231,7 @@
# ---
# name: TestTrends.test_timezones_daily.4
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')))
@@ -2232,7 +2246,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -2253,7 +2267,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT e__pdi.person_id) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['Mac'], ['Mac'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['Mac'], ['Mac'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -2262,7 +2276,7 @@
WHERE equals(person_distinct_id2.team_id, 2)
GROUP BY person_distinct_id2.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')), 'Mac'), 0))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -2408,7 +2422,7 @@
# ---
# name: TestTrends.test_timezones_daily_minus_utc.4
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'America/Phoenix')))), lessOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'America/Phoenix')))), lessOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix'))), equals(e.event, 'sign up')))
@@ -2423,7 +2437,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -2444,7 +2458,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT e__pdi.person_id) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'America/Phoenix')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['Mac'], ['Mac'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['Mac'], ['Mac'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -2453,7 +2467,7 @@
WHERE equals(person_distinct_id2.team_id, 2)
GROUP BY person_distinct_id2.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'America/Phoenix')))), lessOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix'))), equals(e.event, 'sign up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'America/Phoenix')))), lessOrEquals(toTimeZone(e.timestamp, 'America/Phoenix'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'America/Phoenix'))), equals(e.event, 'sign up'), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')), 'Mac'), 0))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -2599,7 +2613,7 @@
# ---
# name: TestTrends.test_timezones_daily_plus_utc.4
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'Asia/Tokyo')))), lessOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'Asia/Tokyo')))), lessOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo'))), equals(e.event, 'sign up')))
@@ -2614,7 +2628,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -2635,7 +2649,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT e__pdi.person_id) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'Asia/Tokyo')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['Mac'], ['Mac'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['Mac'], ['Mac'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -2644,7 +2658,7 @@
WHERE equals(person_distinct_id2.team_id, 2)
GROUP BY person_distinct_id2.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'Asia/Tokyo')))), lessOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo'))), equals(e.event, 'sign up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', ''), 'Mac'), 0))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-29 00:00:00', 6, 'Asia/Tokyo')))), lessOrEquals(toTimeZone(e.timestamp, 'Asia/Tokyo'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-05 23:59:59', 6, 'Asia/Tokyo'))), equals(e.event, 'sign up'), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$os'), ''), 'null'), '^"|"$', '')), 'Mac'), 0))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -2992,7 +3006,7 @@
# ---
# name: TestTrends.test_trend_breakdown_user_props_with_filter_with_partial_property_pushdowns
'''
- SELECT e__pdi__person.properties___email AS value,
+ SELECT toString(e__pdi__person.properties___email) AS value,
count(e.uuid) AS count
FROM events AS e
INNER JOIN
@@ -3027,7 +3041,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -3048,7 +3062,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.uuid) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(e__pdi__person.properties___email, '$$_posthog_breakdown_null_$$'), ['test2@posthog.com', 'test@gmail.com', 'test5@posthog.com', 'test4@posthog.com', 'test3@posthog.com'], ['test2@posthog.com', 'test@gmail.com', 'test5@posthog.com', 'test4@posthog.com', 'test3@posthog.com'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(e__pdi__person.properties___email), '$$_posthog_breakdown_null_$$'), ['test2@posthog.com', 'test@gmail.com', 'test5@posthog.com', 'test4@posthog.com', 'test3@posthog.com'], ['test2@posthog.com', 'test@gmail.com', 'test5@posthog.com', 'test4@posthog.com', 'test3@posthog.com'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id,
@@ -3070,7 +3084,7 @@
WHERE equals(person.team_id, 2)
GROUP BY person.id
HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(or(ifNull(notILike(e__pdi__person.properties___email, '%@posthog.com%'), 1), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0)), or(ifNull(equals(e__pdi__person.`properties___$os`, 'android'), 0), ifNull(equals(e__pdi__person.`properties___$browser`, 'safari'), 0))), or(ifNull(equals(e__pdi__person.properties___email, 'test2@posthog.com'), 0), ifNull(equals(e__pdi__person.properties___email, 'test@gmail.com'), 0), ifNull(equals(e__pdi__person.properties___email, 'test5@posthog.com'), 0), ifNull(equals(e__pdi__person.properties___email, 'test4@posthog.com'), 0), ifNull(equals(e__pdi__person.properties___email, 'test3@posthog.com'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(or(ifNull(notILike(e__pdi__person.properties___email, '%@posthog.com%'), 1), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0)), or(ifNull(equals(e__pdi__person.`properties___$os`, 'android'), 0), ifNull(equals(e__pdi__person.`properties___$browser`, 'safari'), 0))), or(ifNull(equals(toString(e__pdi__person.properties___email), 'test2@posthog.com'), 0), ifNull(equals(toString(e__pdi__person.properties___email), 'test@gmail.com'), 0), ifNull(equals(toString(e__pdi__person.properties___email), 'test5@posthog.com'), 0), ifNull(equals(toString(e__pdi__person.properties___email), 'test4@posthog.com'), 0), ifNull(equals(toString(e__pdi__person.properties___email), 'test3@posthog.com'), 0)))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -3085,7 +3099,7 @@
# ---
# name: TestTrends.test_trend_breakdown_user_props_with_filter_with_partial_property_pushdowns.2
'''
- SELECT e__pdi__person.properties___email AS value,
+ SELECT toString(e__pdi__person.properties___email) AS value,
count(e.uuid) AS count
FROM events AS e
INNER JOIN
@@ -3120,7 +3134,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -3141,7 +3155,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.uuid) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(e__pdi__person.properties___email, '$$_posthog_breakdown_null_$$'), ['test2@posthog.com'], ['test2@posthog.com'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(e__pdi__person.properties___email), '$$_posthog_breakdown_null_$$'), ['test2@posthog.com'], ['test2@posthog.com'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id,
@@ -3163,7 +3177,7 @@
WHERE equals(person.team_id, 2)
GROUP BY person.id
HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(ifNull(equals(e__pdi__person.`properties___$os`, 'android'), 0), ifNull(equals(e__pdi__person.`properties___$browser`, 'chrome'), 0)), and(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0), ifNull(ilike(e__pdi__person.properties___email, '%@posthog.com%'), 0)), ifNull(equals(e__pdi__person.properties___email, 'test2@posthog.com'), 0))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-07-01 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), and(ifNull(equals(e__pdi__person.`properties___$os`, 'android'), 0), ifNull(equals(e__pdi__person.`properties___$browser`, 'chrome'), 0)), and(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0), ifNull(ilike(e__pdi__person.properties___email, '%@posthog.com%'), 0)), ifNull(equals(toString(e__pdi__person.properties___email), 'test2@posthog.com'), 0))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -3248,7 +3262,7 @@
# ---
# name: TestTrends.test_trends_aggregate_by_distinct_id.2
'''
- SELECT e__pdi__person.`properties___$some_prop` AS value,
+ SELECT toString(e__pdi__person.`properties___$some_prop`) AS value,
count(e.uuid) AS count
FROM events AS e
INNER JOIN
@@ -3281,7 +3295,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -3302,7 +3316,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT e.distinct_id) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(e__pdi__person.`properties___$some_prop`, '$$_posthog_breakdown_null_$$'), ['some_val', '$$_posthog_breakdown_null_$$'], ['some_val', '$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(e__pdi__person.`properties___$some_prop`), '$$_posthog_breakdown_null_$$'), ['some_val', '$$_posthog_breakdown_null_$$'], ['some_val', '$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id,
@@ -3322,7 +3336,7 @@
WHERE equals(person.team_id, 2)
GROUP BY person.id
HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(e__pdi__person.`properties___$some_prop`, 'some_val'), 0), isNull(e__pdi__person.`properties___$some_prop`)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(toString(e__pdi__person.`properties___$some_prop`), 'some_val'), 0), isNull(toString(e__pdi__person.`properties___$some_prop`))))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -3415,7 +3429,7 @@
# ---
# name: TestTrends.test_trends_aggregate_by_distinct_id.6
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_prop'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_prop'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')))
@@ -3430,7 +3444,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -3451,9 +3465,9 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT e.distinct_id) AS total,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_prop'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$'], ['$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_prop'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$'], ['$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_prop'), ''), 'null'), '^"|"$', '')))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-24 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-31 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_prop'), ''), 'null'), '^"|"$', ''))))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -3520,7 +3534,7 @@
# ---
# name: TestTrends.test_trends_breakdown_cumulative
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')))
@@ -3535,7 +3549,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT day_start AS day_start,
sum(count) OVER (PARTITION BY breakdown_value
@@ -3561,7 +3575,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT e__pdi.person_id) AS total,
min(toStartOfDay(toTimeZone(e.timestamp, 'UTC'))) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -3570,7 +3584,7 @@
WHERE equals(person_distinct_id2.team_id, 2)
GROUP BY person_distinct_id2.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'other_value'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''))), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'other_value'), 0)))
GROUP BY e__pdi.person_id,
breakdown_value)
GROUP BY day_start,
@@ -3585,7 +3599,7 @@
# ---
# name: TestTrends.test_trends_breakdown_cumulative_poe_v2
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC')))), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')))
@@ -3600,7 +3614,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT day_start AS day_start,
sum(count) OVER (PARTITION BY breakdown_value
@@ -3626,7 +3640,7 @@
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(DISTINCT ifNull(nullIf(e__override.override_person_id, '00000000-0000-0000-0000-000000000000'), e.person_id)) AS total,
min(toStartOfDay(toTimeZone(e.timestamp, 'UTC'))) AS day_start,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], ['$$_posthog_breakdown_null_$$', 'value', 'other_value'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
LEFT OUTER JOIN
(SELECT argMax(person_overrides.override_person_id, person_overrides.version) AS override_person_id,
@@ -3634,7 +3648,7 @@
FROM person_overrides
WHERE equals(person_overrides.team_id, 2)
GROUP BY person_overrides.old_person_id) AS e__override ON equals(e.person_id, e__override.old_person_id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'other_value'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''))), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'other_value'), 0)))
GROUP BY ifNull(nullIf(e__override.override_person_id, '00000000-0000-0000-0000-000000000000'), e.person_id),
breakdown_value)
GROUP BY day_start,
@@ -3649,7 +3663,7 @@
# ---
# name: TestTrends.test_trends_breakdown_with_session_property_single_aggregate_math_and_breakdown
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
max(e__session.duration) AS count
FROM events AS e
INNER JOIN
@@ -3672,7 +3686,7 @@
breakdown_value AS breakdown_value
FROM
(SELECT any(e__session.duration) AS session_duration,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['value2', 'value1', '$$_posthog_breakdown_null_$$'], ['value2', 'value1', '$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['value2', 'value1', '$$_posthog_breakdown_null_$$'], ['value2', 'value1', '$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT events.`$session_id` AS id,
@@ -3680,7 +3694,7 @@
FROM events
WHERE and(equals(events.team_id, 2), greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), ifNull(notEquals(id, ''), 1))
GROUP BY id) AS e__session ON equals(e.`$session_id`, e__session.id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value2'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value1'), 0), isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''))))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value2'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value1'), 0), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')))))
GROUP BY e__session.id,
breakdown_value)
GROUP BY breakdown_value
@@ -3691,7 +3705,7 @@
# ---
# name: TestTrends.test_trends_breakdown_with_session_property_single_aggregate_math_and_breakdown.2
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
max(e__session.duration) AS count
FROM events AS e
INNER JOIN
@@ -3714,7 +3728,7 @@
breakdown_value AS breakdown_value
FROM
(SELECT any(e__session.duration) AS session_duration,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['value2', 'value1', '$$_posthog_breakdown_null_$$'], ['value2', 'value1', '$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['value2', 'value1', '$$_posthog_breakdown_null_$$'], ['value2', 'value1', '$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT events.`$session_id` AS id,
@@ -3722,7 +3736,7 @@
FROM events
WHERE and(equals(events.team_id, 2), greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), ifNull(notEquals(id, ''), 1))
GROUP BY id) AS e__session ON equals(e.`$session_id`, e__session.id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value2'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value1'), 0), isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''))))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value2'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value1'), 0), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')))))
GROUP BY e__session.id,
breakdown_value)
GROUP BY breakdown_value
@@ -3854,7 +3868,7 @@
# ---
# name: TestTrends.test_trends_count_per_user_average_aggregated_with_event_property_breakdown_with_sampling
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', '')) AS value,
count(e.uuid) AS count
FROM events AS e
WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC')))), equals(e.event, 'viewed video'))
@@ -3874,7 +3888,7 @@
breakdown_value AS breakdown_value
FROM
(SELECT count(e.uuid) AS total,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['red', 'blue', '$$_posthog_breakdown_null_$$'], ['red', 'blue', '$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['red', 'blue', '$$_posthog_breakdown_null_$$'], ['red', 'blue', '$$_posthog_breakdown_null_$$'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1.0
INNER JOIN
(SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
@@ -3883,7 +3897,7 @@
WHERE equals(person_distinct_id2.team_id, 2)
GROUP BY person_distinct_id2.distinct_id
HAVING ifNull(equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0), 0)) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id)
- WHERE and(equals(e.team_id, 2), and(equals(e.event, 'viewed video'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', ''), 'red'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', ''), 'blue'), 0), isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', '')))), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), toIntervalDay(0))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC'))))
+ WHERE and(equals(e.team_id, 2), and(equals(e.event, 'viewed video'), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', '')), 'red'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', '')), 'blue'), 0), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'color'), ''), 'null'), '^"|"$', ''))))), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), minus(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC')), toIntervalDay(0))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-07 23:59:59', 6, 'UTC'))))
GROUP BY e__pdi.person_id,
breakdown_value)
GROUP BY breakdown_value)
@@ -4098,7 +4112,7 @@
# ---
# name: TestTrends.test_trends_person_breakdown_with_session_property_single_aggregate_math_and_breakdown
'''
- SELECT e__pdi__person.`properties___$some_prop` AS value,
+ SELECT toString(e__pdi__person.`properties___$some_prop`) AS value,
max(e__session.duration) AS count
FROM events AS e
INNER JOIN
@@ -4139,7 +4153,7 @@
breakdown_value AS breakdown_value
FROM
(SELECT any(e__session.duration) AS session_duration,
- transform(ifNull(e__pdi__person.`properties___$some_prop`, '$$_posthog_breakdown_null_$$'), ['some_val', 'another_val'], ['some_val', 'another_val'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ transform(ifNull(toString(e__pdi__person.`properties___$some_prop`), '$$_posthog_breakdown_null_$$'), ['some_val', 'another_val'], ['some_val', 'another_val'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM events AS e SAMPLE 1
INNER JOIN
(SELECT events.`$session_id` AS id,
@@ -4165,7 +4179,7 @@
WHERE equals(person.team_id, 2)
GROUP BY person.id
HAVING ifNull(equals(argMax(person.is_deleted, person.version), 0), 0))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(e__pdi__person.`properties___$some_prop`, 'some_val'), 0), ifNull(equals(e__pdi__person.`properties___$some_prop`, 'another_val'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(toString(e__pdi__person.`properties___$some_prop`), 'some_val'), 0), ifNull(equals(toString(e__pdi__person.`properties___$some_prop`), 'another_val'), 0)))
GROUP BY e__session.id,
breakdown_value)
GROUP BY breakdown_value
@@ -4316,7 +4330,7 @@
# ---
# name: TestTrends.test_trends_with_session_property_total_volume_math_with_breakdowns
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
max(e__session.duration) AS count
FROM events AS e
INNER JOIN
@@ -4337,7 +4351,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -4361,7 +4375,7 @@
breakdown_value AS breakdown_value
FROM
(SELECT any(e__session.duration) AS session_duration,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['value2', 'value1'], ['value2', 'value1'], '$$_posthog_breakdown_other_$$') AS breakdown_value,
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['value2', 'value1'], ['value2', 'value1'], '$$_posthog_breakdown_other_$$') AS breakdown_value,
toStartOfWeek(toTimeZone(e.timestamp, 'UTC'), 0) AS day_start
FROM events AS e SAMPLE 1
INNER JOIN
@@ -4370,7 +4384,7 @@
FROM events
WHERE and(equals(events.team_id, 2), greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), ifNull(notEquals(id, ''), 1))
GROUP BY id) AS e__session ON equals(e.`$session_id`, e__session.id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value2'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value1'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')), 0)), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value2'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value1'), 0)))
GROUP BY day_start,
e__session.id,
breakdown_value,
@@ -4389,7 +4403,7 @@
# ---
# name: TestTrends.test_trends_with_session_property_total_volume_math_with_breakdowns.2
'''
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '') AS value,
+ SELECT toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')) AS value,
max(e__session.duration) AS count
FROM events AS e
INNER JOIN
@@ -4410,7 +4424,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -4434,7 +4448,7 @@
breakdown_value AS breakdown_value
FROM
(SELECT any(e__session.duration) AS session_duration,
- transform(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), '$$_posthog_breakdown_null_$$'), ['value2', 'value1'], ['value2', 'value1'], '$$_posthog_breakdown_other_$$') AS breakdown_value,
+ transform(ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), '$$_posthog_breakdown_null_$$'), ['value2', 'value1'], ['value2', 'value1'], '$$_posthog_breakdown_other_$$') AS breakdown_value,
toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start
FROM events AS e SAMPLE 1
INNER JOIN
@@ -4443,7 +4457,7 @@
FROM events
WHERE and(equals(events.team_id, 2), greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), ifNull(notEquals(id, ''), 1))
GROUP BY id) AS e__session ON equals(e.`$session_id`, e__session.id)
- WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value2'), 0), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', ''), 'value1'), 0)))
+ WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up'), or(ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value2'), 0), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), 'value1'), 0)))
GROUP BY day_start,
e__session.id,
breakdown_value,
diff --git a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends_data_warehouse_query.ambr b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends_data_warehouse_query.ambr
index db9e8e1d45000..bd7142030fe3a 100644
--- a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends_data_warehouse_query.ambr
+++ b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends_data_warehouse_query.ambr
@@ -1,10 +1,10 @@
# serializer version: 1
# name: TestTrendsDataWarehouseQuery.test_trends_breakdown
'''
- SELECT e.prop_1 AS value,
+ SELECT toString(e.prop_1) AS value,
count(e.id) AS count
FROM s3('http://host.docker.internal:19000/posthog/test_storage_bucket-posthog.hogql.datawarehouse.trendquery/*.parquet', 'object_storage_root_user', 'object_storage_root_password', 'Parquet', 'id String, prop_1 String, prop_2 String, created DateTime64(3, \'UTC\')') AS e
- WHERE and(and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0)), and(ifNull(greaterOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0)))
+ WHERE and(and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0)), and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0)))
GROUP BY value
ORDER BY count DESC, value DESC
LIMIT 26 SETTINGS readonly=2,
@@ -16,7 +16,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -36,10 +36,10 @@
JOIN breakdown_value AS breakdown_value) AS sec
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.id) AS total,
- toStartOfDay(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC')) AS day_start,
- transform(ifNull(e.prop_1, '$$_posthog_breakdown_null_$$'), ['d', 'c', 'b', 'a'], ['d', 'c', 'b', 'a'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ toStartOfDay(toTimeZone(e.created, 'UTC')) AS day_start,
+ transform(ifNull(toString(e.prop_1), '$$_posthog_breakdown_null_$$'), ['d', 'c', 'b', 'a'], ['d', 'c', 'b', 'a'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM s3('http://host.docker.internal:19000/posthog/test_storage_bucket-posthog.hogql.datawarehouse.trendquery/*.parquet', 'object_storage_root_user', 'object_storage_root_password', 'Parquet', 'id String, prop_1 String, prop_2 String, created DateTime64(3, \'UTC\')') AS e
- WHERE and(ifNull(greaterOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), or(equals(e.prop_1, 'd'), equals(e.prop_1, 'c'), equals(e.prop_1, 'b'), equals(e.prop_1, 'a')))
+ WHERE and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), or(ifNull(equals(toString(e.prop_1), 'd'), 0), ifNull(equals(toString(e.prop_1), 'c'), 0), ifNull(equals(toString(e.prop_1), 'b'), 0), ifNull(equals(toString(e.prop_1), 'a'), 0)))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -54,10 +54,10 @@
# ---
# name: TestTrendsDataWarehouseQuery.test_trends_breakdown_with_property
'''
- SELECT e.prop_1 AS value,
+ SELECT toString(e.prop_1) AS value,
count(e.id) AS count
FROM s3('http://host.docker.internal:19000/posthog/test_storage_bucket-posthog.hogql.datawarehouse.trendquery/*.parquet', 'object_storage_root_user', 'object_storage_root_password', 'Parquet', 'id String, prop_1 String, prop_2 String, created DateTime64(3, \'UTC\')') AS e
- WHERE and(and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0)), and(ifNull(greaterOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), equals(e.prop_1, 'a')))
+ WHERE and(and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0)), and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), equals(e.prop_1, 'a')))
GROUP BY value
ORDER BY count DESC, value DESC
LIMIT 26 SETTINGS readonly=2,
@@ -69,7 +69,7 @@
'''
SELECT groupArray(day_start) AS date,
groupArray(count) AS total,
- ifNull(toString(breakdown_value), '') AS breakdown_value
+ ifNull(toString(breakdown_value), '$$_posthog_breakdown_null_$$') AS breakdown_value
FROM
(SELECT sum(total) AS count,
day_start AS day_start,
@@ -89,10 +89,10 @@
JOIN breakdown_value AS breakdown_value) AS sec
ORDER BY sec.breakdown_value ASC, day_start ASC
UNION ALL SELECT count(e.id) AS total,
- toStartOfDay(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC')) AS day_start,
- transform(ifNull(e.prop_1, '$$_posthog_breakdown_null_$$'), ['a'], ['a'], '$$_posthog_breakdown_other_$$') AS breakdown_value
+ toStartOfDay(toTimeZone(e.created, 'UTC')) AS day_start,
+ transform(ifNull(toString(e.prop_1), '$$_posthog_breakdown_null_$$'), ['a'], ['a'], '$$_posthog_breakdown_other_$$') AS breakdown_value
FROM s3('http://host.docker.internal:19000/posthog/test_storage_bucket-posthog.hogql.datawarehouse.trendquery/*.parquet', 'object_storage_root_user', 'object_storage_root_password', 'Parquet', 'id String, prop_1 String, prop_2 String, created DateTime64(3, \'UTC\')') AS e
- WHERE and(ifNull(greaterOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), equals(e.prop_1, 'a'), equals(e.prop_1, 'a'))
+ WHERE and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), equals(e.prop_1, 'a'), ifNull(equals(toString(e.prop_1), 'a'), 0))
GROUP BY day_start,
breakdown_value)
GROUP BY day_start,
@@ -119,9 +119,9 @@
UNION ALL SELECT 0 AS total,
toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC'))) AS day_start
UNION ALL SELECT count(e.id) AS total,
- toStartOfDay(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC')) AS day_start
+ toStartOfDay(toTimeZone(e.created, 'UTC')) AS day_start
FROM s3('http://host.docker.internal:19000/posthog/test_storage_bucket-posthog.hogql.datawarehouse.trendquery/*.parquet', 'object_storage_root_user', 'object_storage_root_password', 'Parquet', 'id String, prop_1 String, prop_2 String, created DateTime64(3, \'UTC\')') AS e
- WHERE and(ifNull(greaterOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0))
+ WHERE and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0))
GROUP BY day_start)
GROUP BY day_start
ORDER BY day_start ASC)
@@ -145,9 +145,9 @@
UNION ALL SELECT 0 AS total,
toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC'))) AS day_start
UNION ALL SELECT count(e.id) AS total,
- toStartOfDay(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC')) AS day_start
+ toStartOfDay(toTimeZone(e.created, 'UTC')) AS day_start
FROM s3('http://host.docker.internal:19000/posthog/test_storage_bucket-posthog.hogql.datawarehouse.trendquery/*.parquet', 'object_storage_root_user', 'object_storage_root_password', 'Parquet', 'id String, prop_1 String, prop_2 String, created DateTime64(3, \'UTC\')') AS e
- WHERE and(ifNull(greaterOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), equals(e.prop_1, 'a'))
+ WHERE and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), equals(e.prop_1, 'a'))
GROUP BY day_start)
GROUP BY day_start
ORDER BY day_start ASC)
@@ -171,9 +171,9 @@
UNION ALL SELECT 0 AS total,
toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC'))) AS day_start
UNION ALL SELECT count(e.id) AS total,
- toStartOfDay(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC')) AS day_start
+ toStartOfDay(toTimeZone(e.created, 'UTC')) AS day_start
FROM s3('http://host.docker.internal:19000/posthog/test_storage_bucket-posthog.hogql.datawarehouse.trendquery/*.parquet', 'object_storage_root_user', 'object_storage_root_password', 'Parquet', 'id String, prop_1 String, prop_2 String, created DateTime64(3, \'UTC\')') AS e
- WHERE and(ifNull(greaterOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toDateTime(toTimeZone(e.created, 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), equals(e.prop_1, 'a'))
+ WHERE and(ifNull(greaterOrEquals(toTimeZone(e.created, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-01 00:00:00', 6, 'UTC')))), 0), ifNull(lessOrEquals(toTimeZone(e.created, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-01-07 23:59:59', 6, 'UTC'))), 0), equals(e.prop_1, 'a'))
GROUP BY day_start)
GROUP BY day_start
ORDER BY day_start ASC)
diff --git a/posthog/hogql_queries/insights/trends/test/test_trends.py b/posthog/hogql_queries/insights/trends/test/test_trends.py
index 1ac54e16de629..9e885fbadcc1d 100644
--- a/posthog/hogql_queries/insights/trends/test/test_trends.py
+++ b/posthog/hogql_queries/insights/trends/test/test_trends.py
@@ -5180,7 +5180,9 @@ def test_breakdown_filtering_with_properties_in_new_format(self):
)
response = sorted(response, key=lambda x: x["label"])
- self.assertEqual(len(response), 0)
+ self.assertEqual(len(response), 1)
+ self.assertEqual(response[0]["label"], "$$_posthog_breakdown_null_$$")
+ self.assertEqual(response[0]["count"], 0)
@also_test_with_person_on_events_v2
@snapshot_clickhouse_queries
diff --git a/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py b/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py
index 104e232a01406..6bb41b19c79cf 100644
--- a/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py
+++ b/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py
@@ -1,11 +1,15 @@
+import zoneinfo
from dataclasses import dataclass
+from datetime import datetime
from typing import Dict, List, Optional
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
from django.test import override_settings
from freezegun import freeze_time
+from posthog.clickhouse.client.execute import sync_execute
from posthog.hogql import ast
-from posthog.hogql.constants import MAX_SELECT_RETURNED_ROWS
+from posthog.hogql.constants import MAX_SELECT_RETURNED_ROWS, LimitContext
from posthog.hogql.modifiers import create_default_modifiers_for_team
+from posthog.hogql.query import INCREASED_MAX_EXECUTION_TIME
from posthog.hogql_queries.insights.trends.trends_query_runner import TrendsQueryRunner
from posthog.models.cohort.cohort import Cohort
from posthog.models.property_definition import PropertyDefinition
@@ -175,6 +179,7 @@ def _create_query_runner(
breakdown: Optional[BreakdownFilter] = None,
filter_test_accounts: Optional[bool] = None,
hogql_modifiers: Optional[HogQLQueryModifiers] = None,
+ limit_context: Optional[LimitContext] = None,
) -> TrendsQueryRunner:
query_series: List[EventsNode | ActionsNode] = [EventsNode(event="$pageview")] if series is None else series
query = TrendsQuery(
@@ -185,7 +190,7 @@ def _create_query_runner(
breakdownFilter=breakdown,
filterTestAccounts=filter_test_accounts,
)
- return TrendsQueryRunner(team=self.team, query=query, modifiers=hogql_modifiers)
+ return TrendsQueryRunner(team=self.team, query=query, modifiers=hogql_modifiers, limit_context=limit_context)
def _run_trends_query(
self,
@@ -195,8 +200,10 @@ def _run_trends_query(
series: Optional[List[EventsNode | ActionsNode]],
trends_filters: Optional[TrendsFilter] = None,
breakdown: Optional[BreakdownFilter] = None,
+ *,
filter_test_accounts: Optional[bool] = None,
hogql_modifiers: Optional[HogQLQueryModifiers] = None,
+ limit_context: Optional[LimitContext] = None,
):
return self._create_query_runner(
date_from=date_from,
@@ -207,6 +214,7 @@ def _run_trends_query(
breakdown=breakdown,
filter_test_accounts=filter_test_accounts,
hogql_modifiers=hogql_modifiers,
+ limit_context=limit_context,
).calculate()
def test_trends_query_label(self):
@@ -364,6 +372,19 @@ def test_trends_query_formula(self):
self.assertEqual("Formula (A+B)", response.results[0]["label"])
self.assertEqual([1, 0, 2, 4, 4, 0, 2, 1, 1, 0, 1], response.results[0]["data"])
+ def test_trends_query_formula_breakdown_no_data(self):
+ self._create_test_events()
+
+ response = self._run_trends_query(
+ self.default_date_from,
+ self.default_date_to,
+ IntervalType.day,
+ [EventsNode(event="$pageviewxxx"), EventsNode(event="$pageleavexxx")],
+ TrendsFilter(formula="A+B"),
+ BreakdownFilter(breakdown_type=BreakdownType.person, breakdown="$browser"),
+ )
+ self.assertEqual([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], response.results[0]["data"])
+
def test_trends_query_formula_aggregate(self):
self._create_test_events()
@@ -695,16 +716,7 @@ def test_trends_breakdowns_multiple_hogql(self):
breakdown_labels = [result["breakdown_value"] for result in response.results]
assert len(response.results) == 8
- assert breakdown_labels == [
- "Chrome",
- "Firefox",
- "Edge",
- "Safari",
- "Chrome",
- "Edge",
- "Firefox",
- "Safari",
- ]
+ assert breakdown_labels == ["Chrome", "Firefox", "Edge", "Safari", "Chrome", "Edge", "Firefox", "Safari"]
assert response.results[0]["label"] == f"$pageview - Chrome"
assert response.results[1]["label"] == f"$pageview - Firefox"
assert response.results[2]["label"] == f"$pageview - Edge"
@@ -804,6 +816,7 @@ def test_trends_breakdown_and_aggregation_query_orchestration(self):
10,
0,
]
+
assert response.results[1]["data"] == [
20,
0,
@@ -1114,6 +1127,38 @@ def test_breakdown_values_limit(self):
)
self.assertEqual(len(response.results), 11)
+ def test_breakdown_values_unknown_property(self):
+ # same as above test, just without creating the property definition
+ for value in list(range(30)):
+ _create_event(
+ team=self.team,
+ event="$pageview",
+ distinct_id=f"person_{value}",
+ timestamp="2020-01-11T12:00:00Z",
+ properties={"breakdown_value": f"{value}"},
+ )
+
+ response = self._run_trends_query(
+ "2020-01-09",
+ "2020-01-20",
+ IntervalType.day,
+ [EventsNode(event="$pageview")],
+ TrendsFilter(display=ChartDisplayType.ActionsLineGraph),
+ BreakdownFilter(breakdown="breakdown_value", breakdown_type=BreakdownType.event),
+ )
+
+ self.assertEqual(len(response.results), 26)
+
+ response = self._run_trends_query(
+ "2020-01-09",
+ "2020-01-20",
+ IntervalType.day,
+ [EventsNode(event="$pageview")],
+ TrendsFilter(display=ChartDisplayType.ActionsLineGraph),
+ BreakdownFilter(breakdown="breakdown_value", breakdown_type=BreakdownType.event, breakdown_limit=10),
+ )
+ self.assertEqual(len(response.results), 11)
+
def test_breakdown_values_world_map_limit(self):
PropertyDefinition.objects.create(team=self.team, name="breakdown_value", property_type="String")
@@ -1435,18 +1480,18 @@ def test_to_actors_query_options(self):
response = runner.to_actors_query_options()
assert response.day == [
- DayItem(label="2020-01-09", value="2020-01-09"),
- DayItem(label="2020-01-10", value="2020-01-10"),
- DayItem(label="2020-01-11", value="2020-01-11"),
- DayItem(label="2020-01-12", value="2020-01-12"),
- DayItem(label="2020-01-13", value="2020-01-13"),
- DayItem(label="2020-01-14", value="2020-01-14"),
- DayItem(label="2020-01-15", value="2020-01-15"),
- DayItem(label="2020-01-16", value="2020-01-16"),
- DayItem(label="2020-01-17", value="2020-01-17"),
- DayItem(label="2020-01-18", value="2020-01-18"),
- DayItem(label="2020-01-19", value="2020-01-19"),
- DayItem(label="2020-01-20", value="2020-01-20"),
+ DayItem(label="9-Jan-2020", value=datetime(2020, 1, 9, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="10-Jan-2020", value=datetime(2020, 1, 10, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="11-Jan-2020", value=datetime(2020, 1, 11, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="12-Jan-2020", value=datetime(2020, 1, 12, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="13-Jan-2020", value=datetime(2020, 1, 13, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="14-Jan-2020", value=datetime(2020, 1, 14, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="15-Jan-2020", value=datetime(2020, 1, 15, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="16-Jan-2020", value=datetime(2020, 1, 16, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="17-Jan-2020", value=datetime(2020, 1, 17, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="18-Jan-2020", value=datetime(2020, 1, 18, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="19-Jan-2020", value=datetime(2020, 1, 19, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="20-Jan-2020", value=datetime(2020, 1, 20, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
]
assert response.breakdown is None
@@ -1470,18 +1515,18 @@ def test_to_actors_query_options_compare(self):
response = runner.to_actors_query_options()
assert response.day == [
- DayItem(label="2020-01-09", value="2020-01-09"),
- DayItem(label="2020-01-10", value="2020-01-10"),
- DayItem(label="2020-01-11", value="2020-01-11"),
- DayItem(label="2020-01-12", value="2020-01-12"),
- DayItem(label="2020-01-13", value="2020-01-13"),
- DayItem(label="2020-01-14", value="2020-01-14"),
- DayItem(label="2020-01-15", value="2020-01-15"),
- DayItem(label="2020-01-16", value="2020-01-16"),
- DayItem(label="2020-01-17", value="2020-01-17"),
- DayItem(label="2020-01-18", value="2020-01-18"),
- DayItem(label="2020-01-19", value="2020-01-19"),
- DayItem(label="2020-01-20", value="2020-01-20"),
+ DayItem(label="9-Jan-2020", value=datetime(2020, 1, 9, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="10-Jan-2020", value=datetime(2020, 1, 10, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="11-Jan-2020", value=datetime(2020, 1, 11, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="12-Jan-2020", value=datetime(2020, 1, 12, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="13-Jan-2020", value=datetime(2020, 1, 13, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="14-Jan-2020", value=datetime(2020, 1, 14, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="15-Jan-2020", value=datetime(2020, 1, 15, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="16-Jan-2020", value=datetime(2020, 1, 16, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="17-Jan-2020", value=datetime(2020, 1, 17, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="18-Jan-2020", value=datetime(2020, 1, 18, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="19-Jan-2020", value=datetime(2020, 1, 19, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
+ DayItem(label="20-Jan-2020", value=datetime(2020, 1, 20, 0, 0, tzinfo=zoneinfo.ZoneInfo(key="UTC"))),
]
assert response.breakdown is None
@@ -1555,9 +1600,8 @@ def test_to_actors_query_options_breakdowns_boolean(self):
assert response.series == [InsightActorsQuerySeries(label="$pageview", value=0)]
assert response.breakdown == [
- # BreakdownItem(label="Other", value="$$_posthog_breakdown_other_$$"), # TODO: Add when "Other" works
- BreakdownItem(label="true", value=1),
- BreakdownItem(label="false", value=0),
+ BreakdownItem(label="true", value="true"),
+ BreakdownItem(label="false", value="false"),
]
def test_to_actors_query_options_breakdowns_histogram(self):
@@ -1649,3 +1693,16 @@ def test_to_actors_query_options_breakdowns_hogql(self):
BreakdownItem(label="Safari", value="Safari"),
BreakdownItem(label="Edge", value="Edge"),
]
+
+ @patch("posthog.hogql.query.sync_execute", wraps=sync_execute)
+ def test_limit_is_context_aware(self, mock_sync_execute: MagicMock):
+ self._run_trends_query(
+ "2020-01-09",
+ "2020-01-20",
+ IntervalType.day,
+ [EventsNode(event="$pageview")],
+ limit_context=LimitContext.QUERY_ASYNC,
+ )
+
+ mock_sync_execute.assert_called_once()
+ self.assertIn(f" max_execution_time={INCREASED_MAX_EXECUTION_TIME},", mock_sync_execute.call_args[0][0])
diff --git a/posthog/hogql_queries/insights/trends/trends_query_builder.py b/posthog/hogql_queries/insights/trends/trends_query_builder.py
index ed5d867b48b75..a911e4bf8302a 100644
--- a/posthog/hogql_queries/insights/trends/trends_query_builder.py
+++ b/posthog/hogql_queries/insights/trends/trends_query_builder.py
@@ -14,6 +14,7 @@
from posthog.models.action.action import Action
from posthog.models.filters.mixins.utils import cached_property
from posthog.models.team.team import Team
+from posthog.queries.trends.breakdown import BREAKDOWN_NULL_STRING_LABEL
from posthog.schema import (
ActionsNode,
DataWarehouseNode,
@@ -68,14 +69,18 @@ def build_query(self) -> ast.SelectQuery | ast.SelectUnionQuery:
return full_query
def build_actors_query(
- self, time_frame: Optional[str | int] = None, breakdown_filter: Optional[str | int] = None
+ self, time_frame: Optional[str] = None, breakdown_filter: Optional[str] = None
) -> ast.SelectQuery | ast.SelectUnionQuery:
breakdown = self._breakdown(is_actors_query=True, breakdown_values_override=breakdown_filter)
return parse_select(
"""
- SELECT DISTINCT actor_id
+ SELECT
+ actor_id,
+ count() as event_count,
+ groupUniqArray(100)((timestamp, uuid, $session_id, $window_id)) as matching_events
FROM {subquery}
+ GROUP BY actor_id
""",
placeholders={
"subquery": self._get_events_subquery(
@@ -165,7 +170,7 @@ def _get_events_subquery(
is_actors_query: bool,
breakdown: Breakdown,
breakdown_values_override: Optional[str | int] = None,
- actors_query_time_frame: Optional[str | int] = None,
+ actors_query_time_frame: Optional[str] = None,
) -> ast.SelectQuery:
day_start = ast.Alias(
alias="day_start",
@@ -182,31 +187,16 @@ def _get_events_subquery(
actors_query_time_frame=actors_query_time_frame,
)
- default_query = cast(
- ast.SelectQuery,
- parse_select(
- """
- SELECT
- {aggregation_operation} AS total
- FROM {table} AS e
- WHERE {events_filter}
- """
- if isinstance(self.series, DataWarehouseNode)
- else """
- SELECT
- {aggregation_operation} AS total
- FROM {table} AS e
- SAMPLE {sample}
- WHERE {events_filter}
- """,
- placeholders={
- "table": self._table_expr,
- "events_filter": events_filter,
- "aggregation_operation": self._aggregation_operation.select_aggregation(),
- "sample": self._sample_value(),
- },
- ),
+ default_query = ast.SelectQuery(
+ select=[ast.Alias(alias="total", expr=self._aggregation_operation.select_aggregation())],
+ select_from=ast.JoinExpr(table=self._table_expr, alias="e"),
+ where=events_filter,
)
+ if not isinstance(self.series, DataWarehouseNode):
+ assert default_query.select_from is not None
+ default_query.select_from.sample = ast.SampleExpr(
+ sample_value=self._sample_value(),
+ )
default_query.group_by = []
@@ -225,8 +215,13 @@ def _get_events_subquery(
# TODO: Move this logic into the below branches when working on adding breakdown support for the person modal
if is_actors_query:
- default_query.select = [ast.Alias(alias="actor_id", expr=self._aggregation_operation.actor_id())]
- default_query.distinct = True
+ default_query.select = [
+ ast.Alias(alias="actor_id", expr=self._aggregation_operation.actor_id()),
+ ast.Field(chain=["e", "timestamp"]),
+ ast.Field(chain=["e", "uuid"]),
+ ast.Field(chain=["e", "$session_id"]),
+ ast.Field(chain=["e", "$window_id"]),
+ ]
default_query.group_by = []
# No breakdowns and no complex series aggregation
@@ -298,7 +293,8 @@ def _get_events_subquery(
# Just breakdowns
elif breakdown.enabled:
if not is_actors_query:
- default_query.select.append(breakdown.column_expr())
+ breakdown_expr = breakdown.column_expr()
+ default_query.select.append(breakdown_expr)
default_query.group_by.append(ast.Field(chain=["breakdown_value"]))
# Just session duration math property
elif self._aggregation_operation.aggregating_on_session_duration():
@@ -375,7 +371,7 @@ def _outer_select_query(self, breakdown: Breakdown, inner_query: ast.SelectQuery
name="ifNull",
args=[
ast.Call(name="toString", args=[ast.Field(chain=["breakdown_value"])]),
- ast.Constant(value=""),
+ ast.Constant(value=BREAKDOWN_NULL_STRING_LABEL),
],
),
)
@@ -454,20 +450,27 @@ def _events_filter(
breakdown: Breakdown | None,
ignore_breakdowns: bool = False,
breakdown_values_override: Optional[str | int] = None,
- actors_query_time_frame: Optional[str | int] = None,
+ actors_query_time_frame: Optional[str] = None,
) -> ast.Expr:
series = self.series
filters: List[ast.Expr] = []
# Dates
if is_actors_query and actors_query_time_frame is not None:
- to_start_of_time_frame = f"toStartOf{self.query_date_range.interval_name.capitalize()}"
- filters.append(
- ast.CompareOperation(
- left=ast.Call(name=to_start_of_time_frame, args=[ast.Field(chain=["timestamp"])]),
- op=ast.CompareOperationOp.Eq,
- right=ast.Call(name="toDateTime", args=[ast.Constant(value=actors_query_time_frame)]),
- )
+ actors_from, actors_to = self.query_date_range.interval_bounds_from_str(actors_query_time_frame)
+ filters.extend(
+ [
+ ast.CompareOperation(
+ left=ast.Field(chain=["timestamp"]),
+ op=ast.CompareOperationOp.GtEq,
+ right=ast.Constant(value=actors_from),
+ ),
+ ast.CompareOperation(
+ left=ast.Field(chain=["timestamp"]),
+ op=ast.CompareOperationOp.Lt,
+ right=ast.Constant(value=actors_to),
+ ),
+ ]
)
elif not self._aggregation_operation.requires_query_orchestration():
filters.extend(
@@ -564,7 +567,7 @@ def session_duration_math_property_wrapper(self, default_query: ast.SelectQuery)
query.group_by = []
return query
- def _breakdown(self, is_actors_query: bool, breakdown_values_override: Optional[str | int] = None):
+ def _breakdown(self, is_actors_query: bool, breakdown_values_override: Optional[str] = None):
return Breakdown(
team=self.team,
query=self.query,
diff --git a/posthog/hogql_queries/insights/trends/trends_query_runner.py b/posthog/hogql_queries/insights/trends/trends_query_runner.py
index b9acb5c37d000..67e160084e68e 100644
--- a/posthog/hogql_queries/insights/trends/trends_query_runner.py
+++ b/posthog/hogql_queries/insights/trends/trends_query_runner.py
@@ -23,9 +23,7 @@
from posthog.hogql.query import execute_hogql_query
from posthog.hogql.timings import HogQLTimings
from posthog.hogql_queries.insights.trends.breakdown_values import (
- BREAKDOWN_NULL_NUMERIC_LABEL,
BREAKDOWN_NULL_STRING_LABEL,
- BREAKDOWN_OTHER_NUMERIC_LABEL,
BREAKDOWN_OTHER_STRING_LABEL,
)
from posthog.hogql_queries.insights.trends.display import TrendsDisplay
@@ -142,7 +140,7 @@ def to_queries(self) -> List[ast.SelectQuery | ast.SelectUnionQuery]:
def to_actors_query(
self,
- time_frame: Optional[str | int],
+ time_frame: Optional[str],
series_index: int,
breakdown_value: Optional[str | int] = None,
compare: Optional[Compare] = None,
@@ -175,7 +173,7 @@ def to_actors_query(
modifiers=self.modifiers,
)
- query = query_builder.build_actors_query(time_frame=time_frame, breakdown_filter=breakdown_value)
+ query = query_builder.build_actors_query(time_frame=time_frame, breakdown_filter=str(breakdown_value))
return query
@@ -185,7 +183,13 @@ def to_actors_query_options(self) -> InsightActorsQueryOptionsResponse:
res_compare: List[CompareItem] | None = None
# Days
- res_days: List[DayItem] = [DayItem(label=day, value=day) for day in self.query_date_range.all_values()]
+ res_days: list[DayItem] = [
+ DayItem(
+ label=format_label_date(value, self.query_date_range.interval_name),
+ value=value,
+ )
+ for value in self.query_date_range.all_values()
+ ]
# Series
for index, series in enumerate(self.query.series):
@@ -240,14 +244,10 @@ def to_actors_query_options(self) -> InsightActorsQueryOptionsResponse:
cohort_name = "all users" if str(value) == "0" else Cohort.objects.get(pk=value).name
label = cohort_name
value = value
- elif value == BREAKDOWN_OTHER_STRING_LABEL or value == BREAKDOWN_OTHER_NUMERIC_LABEL:
- # label = "Other"
- # value = BREAKDOWN_OTHER_STRING_LABEL
- continue # TODO: Add support for "other" breakdowns
- elif value == BREAKDOWN_NULL_STRING_LABEL or value == BREAKDOWN_NULL_NUMERIC_LABEL:
- # label = "Null"
- # value = BREAKDOWN_NULL_STRING_LABEL
- continue # TODO: Add support for "null" breakdowns
+ elif value == BREAKDOWN_OTHER_STRING_LABEL:
+ label = "Other (Groups all remaining values)"
+ elif value == BREAKDOWN_NULL_STRING_LABEL:
+ label = "None (No value)"
elif is_boolean_breakdown:
label = self._convert_boolean(value)
else:
@@ -292,6 +292,7 @@ def run(index: int, query: ast.SelectQuery | ast.SelectUnionQuery, is_parallel:
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
timings_matrix[index] = response.timings
@@ -500,18 +501,6 @@ def get_value(name: str, val: Any):
series_object["breakdown_value"] = remapped_label
- # If the breakdown value is the numeric "other", then set it to the string version
- if (
- remapped_label == BREAKDOWN_OTHER_NUMERIC_LABEL
- or remapped_label == str(BREAKDOWN_OTHER_NUMERIC_LABEL)
- or remapped_label == float(BREAKDOWN_OTHER_NUMERIC_LABEL)
- ):
- series_object["breakdown_value"] = BREAKDOWN_OTHER_STRING_LABEL
- if real_series_count > 1 or self._is_breakdown_field_boolean():
- series_object["label"] = "{} - {}".format(series_label or "All events", "Other")
- else:
- series_object["label"] = "Other"
-
res.append(series_object)
return res
@@ -666,25 +655,27 @@ def apply_formula(self, formula: str, results: List[Dict[str, Any]]) -> List[Dic
res.append(new_result)
return res
- if self._trends_display.should_aggregate_values():
- series_data = list(map(lambda s: [s["aggregated_value"]], results))
- new_series_data = FormulaAST(series_data).call(formula)
+ if len(results) > 0:
+ if self._trends_display.should_aggregate_values():
+ series_data = list(map(lambda s: [s["aggregated_value"]], results))
+ new_series_data = FormulaAST(series_data).call(formula)
- new_result = results[0]
- new_result["aggregated_value"] = float(sum(new_series_data))
- new_result["data"] = None
- new_result["count"] = 0
- new_result["label"] = f"Formula ({formula})"
- else:
- series_data = list(map(lambda s: s["data"], results))
- new_series_data = FormulaAST(series_data).call(formula)
+ new_result = results[0]
+ new_result["aggregated_value"] = float(sum(new_series_data))
+ new_result["data"] = None
+ new_result["count"] = 0
+ new_result["label"] = f"Formula ({formula})"
+ else:
+ series_data = list(map(lambda s: s["data"], results))
+ new_series_data = FormulaAST(series_data).call(formula)
- new_result = results[0]
- new_result["data"] = new_series_data
- new_result["count"] = float(sum(new_series_data))
- new_result["label"] = f"Formula ({formula})"
+ new_result = results[0]
+ new_result["data"] = new_series_data
+ new_result["count"] = float(sum(new_series_data))
+ new_result["label"] = f"Formula ({formula})"
- return [new_result]
+ return [new_result]
+ return []
def _is_breakdown_field_boolean(self):
if not self.query.breakdownFilter or not self.query.breakdownFilter.breakdown_type:
@@ -709,7 +700,7 @@ def _is_breakdown_field_boolean(self):
if not table_model:
raise ValueError(f"Table {series.table_name} not found")
- field_type = dict(table_model.columns)[self.query.breakdownFilter.breakdown]
+ field_type = dict(table_model.columns)[self.query.breakdownFilter.breakdown]["clickhouse"]
if field_type.startswith("Nullable("):
field_type = field_type.replace("Nullable(", "")[:-1]
@@ -741,13 +732,19 @@ def _event_property(
field: str,
field_type: PropertyDefinition.Type,
group_type_index: Optional[int],
- ):
- return PropertyDefinition.objects.get(
- name=field,
- team=self.team,
- type=field_type,
- group_type_index=group_type_index if field_type == PropertyDefinition.Type.GROUP else None,
- ).property_type
+ ) -> str:
+ try:
+ return (
+ PropertyDefinition.objects.get(
+ name=field,
+ team=self.team,
+ type=field_type,
+ group_type_index=group_type_index if field_type == PropertyDefinition.Type.GROUP else None,
+ ).property_type
+ or "String"
+ )
+ except PropertyDefinition.DoesNotExist:
+ return "String"
# TODO: Move this to posthog/hogql_queries/legacy_compatibility/query_to_filter.py
def _query_to_filter(self) -> Dict[str, Any]:
diff --git a/posthog/hogql_queries/sessions_timeline_query_runner.py b/posthog/hogql_queries/sessions_timeline_query_runner.py
index d920ec7cf94fd..cda9433d63efa 100644
--- a/posthog/hogql_queries/sessions_timeline_query_runner.py
+++ b/posthog/hogql_queries/sessions_timeline_query_runner.py
@@ -135,6 +135,7 @@ def calculate(self) -> SessionsTimelineQueryResponse:
query_type="SessionsTimelineQuery",
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
assert query_result.results is not None
timeline_entries_map: Dict[str, TimelineEntry] = {}
diff --git a/posthog/hogql_queries/utils/query_date_range.py b/posthog/hogql_queries/utils/query_date_range.py
index f2e5cef3d82a3..b6386ac85f4ed 100644
--- a/posthog/hogql_queries/utils/query_date_range.py
+++ b/posthog/hogql_queries/utils/query_date_range.py
@@ -1,9 +1,10 @@
import re
from datetime import datetime, timedelta
from functools import cached_property
-from typing import Literal, Optional, Dict, List
+from typing import Literal, Optional, Dict
from zoneinfo import ZoneInfo
+from dateutil.parser import parse
from dateutil.relativedelta import relativedelta
from posthog.hogql.errors import HogQLException
@@ -116,36 +117,38 @@ def interval_type(self) -> IntervalType:
def interval_name(self) -> Literal["hour", "day", "week", "month"]:
return self.interval_type.name
- def all_values(self) -> List[str]:
- start: datetime = self.date_from()
- end: datetime = self.date_to()
- interval = self.interval_name
-
- if interval == "hour":
- start = start.replace(minute=0, second=0, microsecond=0)
- elif interval == "day":
- start = start.replace(hour=0, minute=0, second=0, microsecond=0)
- elif interval == "week":
+ def align_with_interval(self, start: datetime) -> datetime:
+ if self.interval_name == "hour":
+ return start.replace(minute=0, second=0, microsecond=0)
+ elif self.interval_name == "day":
+ return start.replace(hour=0, minute=0, second=0, microsecond=0)
+ elif self.interval_name == "week":
start = start.replace(hour=0, minute=0, second=0, microsecond=0)
week_start_alignment_days = start.isoweekday() % 7
if self._team.week_start_day == WeekStartDay.MONDAY:
week_start_alignment_days = start.weekday()
start -= timedelta(days=week_start_alignment_days)
- elif interval == "month":
- start = start.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
+ return start
+ elif self.interval_name == "month":
+ return start.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
+
+ def interval_relativedelta(self) -> relativedelta:
+ return relativedelta(
+ days=1 if self.interval_name == "day" else 0,
+ weeks=1 if self.interval_name == "week" else 0,
+ months=1 if self.interval_name == "month" else 0,
+ hours=1 if self.interval_name == "hour" else 0,
+ )
- values: List[str] = []
+ def all_values(self) -> list[datetime]:
+ start = self.align_with_interval(self.date_from())
+ end: datetime = self.date_to()
+ delta = self.interval_relativedelta()
+
+ values: list[datetime] = []
while start <= end:
- if interval == "hour":
- values.append(start.strftime("%Y-%m-%d %H:%M:%S"))
- else:
- values.append(start.strftime("%Y-%m-%d"))
- start += relativedelta(
- days=1 if interval == "day" else 0,
- weeks=1 if interval == "week" else 0,
- months=1 if interval == "month" else 0,
- hours=1 if interval == "hour" else 0,
- )
+ values.append(start)
+ start += delta
return values
def date_to_as_hogql(self) -> ast.Expr:
@@ -257,6 +260,11 @@ def to_placeholders(self) -> Dict[str, ast.Expr]:
else self.date_from_as_hogql(),
}
+ def interval_bounds_from_str(self, time_frame: str) -> tuple[datetime, datetime]:
+ date_from = parse(time_frame, tzinfos={None: self._team.timezone_info})
+ date_to = date_from + self.interval_relativedelta()
+ return date_from, date_to
+
class QueryDateRangeWithIntervals(QueryDateRange):
def __init__(
diff --git a/posthog/hogql_queries/utils/test/test_query_date_range.py b/posthog/hogql_queries/utils/test/test_query_date_range.py
index fd38ef700e137..f377e06880bbe 100644
--- a/posthog/hogql_queries/utils/test/test_query_date_range.py
+++ b/posthog/hogql_queries/utils/test/test_query_date_range.py
@@ -61,32 +61,47 @@ def test_all_values(self):
QueryDateRange(
team=self.team, date_range=DateRange(date_from="-20h"), interval=IntervalType.day, now=now
).all_values(),
- ["2021-08-24", "2021-08-25"],
+ [parser.isoparse("2021-08-24T00:00:00Z"), parser.isoparse("2021-08-25T00:00:00Z")],
)
self.assertEqual(
QueryDateRange(
team=self.team, date_range=DateRange(date_from="-20d"), interval=IntervalType.week, now=now
).all_values(),
- ["2021-08-01", "2021-08-08", "2021-08-15", "2021-08-22"],
+ [
+ parser.isoparse("2021-08-01T00:00:00Z"),
+ parser.isoparse("2021-08-08T00:00:00Z"),
+ parser.isoparse("2021-08-15T00:00:00Z"),
+ parser.isoparse("2021-08-22T00:00:00Z"),
+ ],
)
self.team.week_start_day = WeekStartDay.MONDAY
self.assertEqual(
QueryDateRange(
team=self.team, date_range=DateRange(date_from="-20d"), interval=IntervalType.week, now=now
).all_values(),
- ["2021-08-02", "2021-08-09", "2021-08-16", "2021-08-23"],
+ [
+ parser.isoparse("2021-08-02T00:00:00Z"),
+ parser.isoparse("2021-08-09T00:00:00Z"),
+ parser.isoparse("2021-08-16T00:00:00Z"),
+ parser.isoparse("2021-08-23T00:00:00Z"),
+ ],
)
self.assertEqual(
QueryDateRange(
team=self.team, date_range=DateRange(date_from="-50d"), interval=IntervalType.month, now=now
).all_values(),
- ["2021-07-01", "2021-08-01"],
+ [parser.isoparse("2021-07-01T00:00:00Z"), parser.isoparse("2021-08-01T00:00:00Z")],
)
self.assertEqual(
QueryDateRange(
team=self.team, date_range=DateRange(date_from="-3h"), interval=IntervalType.hour, now=now
).all_values(),
- ["2021-08-24 21:00:00", "2021-08-24 22:00:00", "2021-08-24 23:00:00", "2021-08-25 00:00:00"],
+ [
+ parser.isoparse("2021-08-24T21:00:00Z"),
+ parser.isoparse("2021-08-24T22:00:00Z"),
+ parser.isoparse("2021-08-24T23:00:00Z"),
+ parser.isoparse("2021-08-25T00:00:00Z"),
+ ],
)
diff --git a/posthog/hogql_queries/web_analytics/test/test_web_overview.py b/posthog/hogql_queries/web_analytics/test/test_web_overview.py
index 63a26ffea9233..dcafe660fc72d 100644
--- a/posthog/hogql_queries/web_analytics/test/test_web_overview.py
+++ b/posthog/hogql_queries/web_analytics/test/test_web_overview.py
@@ -1,6 +1,11 @@
+from typing import Optional
+from unittest.mock import MagicMock, patch
from freezegun import freeze_time
from parameterized import parameterized
+from posthog.clickhouse.client.execute import sync_execute
+from posthog.hogql.constants import LimitContext
+from posthog.hogql.query import INCREASED_MAX_EXECUTION_TIME
from posthog.hogql_queries.web_analytics.web_overview import WebOverviewQueryRunner
from posthog.schema import WebOverviewQuery, DateRange
from posthog.test.base import (
@@ -36,14 +41,21 @@ def _create_events(self, data, event="$pageview"):
)
return person_result
- def _run_web_overview_query(self, date_from, date_to, use_sessions_table=False, compare=True):
+ def _run_web_overview_query(
+ self,
+ date_from: str,
+ date_to: str,
+ use_sessions_table: bool = False,
+ compare: bool = True,
+ limit_context: Optional[LimitContext] = None,
+ ):
query = WebOverviewQuery(
dateRange=DateRange(date_from=date_from, date_to=date_to),
properties=[],
compare=compare,
useSessionsTable=use_sessions_table,
)
- runner = WebOverviewQueryRunner(team=self.team, query=query)
+ runner = WebOverviewQueryRunner(team=self.team, query=query, limit_context=limit_context)
return runner.calculate()
@parameterized.expand([(True,), (False,)])
@@ -185,3 +197,10 @@ def test_correctly_counts_pageviews_in_long_running_session(self, use_sessions_t
sessions = results[2]
self.assertEqual(1, sessions.value)
+
+ @patch("posthog.hogql.query.sync_execute", wraps=sync_execute)
+ def test_limit_is_context_aware(self, mock_sync_execute: MagicMock):
+ self._run_web_overview_query("2023-12-01", "2023-12-03", limit_context=LimitContext.QUERY_ASYNC)
+
+ mock_sync_execute.assert_called_once()
+ self.assertIn(f" max_execution_time={INCREASED_MAX_EXECUTION_TIME},", mock_sync_execute.call_args[0][0])
diff --git a/posthog/hogql_queries/web_analytics/top_clicks.py b/posthog/hogql_queries/web_analytics/top_clicks.py
index 3218e68975f7a..192d7b279b704 100644
--- a/posthog/hogql_queries/web_analytics/top_clicks.py
+++ b/posthog/hogql_queries/web_analytics/top_clicks.py
@@ -51,6 +51,7 @@ def calculate(self):
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
return WebTopClicksQueryResponse(
diff --git a/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py b/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py
index da4f98edcbf32..12ef703271c51 100644
--- a/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py
+++ b/posthog/hogql_queries/web_analytics/web_analytics_query_runner.py
@@ -211,6 +211,7 @@ def _get_or_calculate_sample_ratio(self) -> SamplingRate:
query=event_count,
team=self.team,
timings=self.timings,
+ limit_context=self.limit_context,
)
if not response.results or not response.results[0] or not response.results[0][0]:
diff --git a/posthog/hogql_queries/web_analytics/web_overview.py b/posthog/hogql_queries/web_analytics/web_overview.py
index 38388315c8f0b..2da015a60ac4e 100644
--- a/posthog/hogql_queries/web_analytics/web_overview.py
+++ b/posthog/hogql_queries/web_analytics/web_overview.py
@@ -285,6 +285,7 @@ def calculate(self):
team=self.team,
timings=self.timings,
modifiers=self.modifiers,
+ limit_context=self.limit_context,
)
assert response.results
diff --git a/posthog/management/commands/backfill_sessions_table.py b/posthog/management/commands/backfill_sessions_table.py
index 798a501eb5b60..c01f4b6159749 100644
--- a/posthog/management/commands/backfill_sessions_table.py
+++ b/posthog/management/commands/backfill_sessions_table.py
@@ -16,6 +16,10 @@
TARGET_TABLE = "sessions"
+SETTINGS = {
+ "max_execution_time": 3600 # 1 hour
+}
+
@dataclass
class BackfillQuery:
@@ -26,6 +30,7 @@ class BackfillQuery:
def execute(
self,
dry_run: bool = True,
+ print_counts: bool = True,
) -> None:
def source_column(column_name: str) -> str:
return get_property_string_expr(
@@ -108,13 +113,14 @@ def select_query(select_date: Optional[datetime] = None) -> str:
"""
# print the count of entries in the main sessions table
- count_query = f"SELECT count(), uniq(session_id) FROM {TARGET_TABLE}"
- [(sessions_row_count, sessions_event_count)] = sync_execute(count_query)
- logger.info(f"{sessions_row_count} rows and {sessions_event_count} unique session_ids in sessions table")
+ if print_counts:
+ count_query = f"SELECT count(), uniq(session_id) FROM {TARGET_TABLE}"
+ [(sessions_row_count, sessions_event_count)] = sync_execute(count_query, settings=SETTINGS)
+ logger.info(f"{sessions_row_count} rows and {sessions_event_count} unique session_ids in sessions table")
if dry_run:
count_query = f"SELECT count(), uniq(session_id) FROM ({select_query()})"
- [(events_count, sessions_count)] = sync_execute(count_query)
+ [(events_count, sessions_count)] = sync_execute(count_query, settings=SETTINGS)
logger.info(f"{events_count} events and {sessions_count} sessions to backfill for")
logger.info(f"The first select query would be:\n{select_query(self.start_date)}")
return
@@ -125,12 +131,14 @@ def select_query(select_date: Optional[datetime] = None) -> str:
sync_execute(
query=f"""INSERT INTO writable_sessions {select_query(select_date=date)} SETTINGS max_execution_time=3600""",
workload=Workload.OFFLINE if self.use_offline_workload else Workload.DEFAULT,
+ settings=SETTINGS,
)
# print the count of entries in the main sessions table
- count_query = f"SELECT count(), uniq(session_id) FROM {TARGET_TABLE}"
- [(sessions_row_count, sessions_event_count)] = sync_execute(count_query)
- logger.info(f"{sessions_row_count} rows and {sessions_event_count} unique session_ids in sessions table")
+ if print_counts:
+ count_query = f"SELECT count(), uniq(session_id) FROM {TARGET_TABLE}"
+ [(sessions_row_count, sessions_event_count)] = sync_execute(count_query, settings=SETTINGS)
+ logger.info(f"{sessions_row_count} rows and {sessions_event_count} unique session_ids in sessions table")
class Command(BaseCommand):
@@ -149,11 +157,25 @@ def add_arguments(self, parser):
parser.add_argument(
"--use-offline-workload", action="store_true", help="actually execute INSERT queries (default is dry-run)"
)
+ parser.add_argument(
+ "--print-counts", action="store_true", help="print events and session count beforehand and afterwards"
+ )
- def handle(self, *, live_run: bool, start_date: str, end_date: str, use_offline_workload: bool, **options):
+ def handle(
+ self,
+ *,
+ live_run: bool,
+ start_date: str,
+ end_date: str,
+ use_offline_workload: bool,
+ print_counts: bool,
+ **options,
+ ):
logger.setLevel(logging.INFO)
start_datetime = datetime.strptime(start_date, "%Y-%m-%d")
end_datetime = datetime.strptime(end_date, "%Y-%m-%d")
- BackfillQuery(start_datetime, end_datetime, use_offline_workload).execute(dry_run=not live_run)
+ BackfillQuery(start_datetime, end_datetime, use_offline_workload).execute(
+ dry_run=not live_run, print_counts=print_counts
+ )
diff --git a/posthog/migrations/0398_alter_externaldatasource_source_type.py b/posthog/migrations/0398_alter_externaldatasource_source_type.py
new file mode 100644
index 0000000000000..af95cd44eef98
--- /dev/null
+++ b/posthog/migrations/0398_alter_externaldatasource_source_type.py
@@ -0,0 +1,25 @@
+# Generated by Django 4.1.13 on 2024-03-21 13:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("posthog", "0397_projects_backfill"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="externaldatasource",
+ name="source_type",
+ field=models.CharField(
+ choices=[
+ ("Stripe", "Stripe"),
+ ("Hubspot", "Hubspot"),
+ ("Postgres", "Postgres"),
+ ("Zendesk", "Zendesk"),
+ ],
+ max_length=128,
+ ),
+ ),
+ ]
diff --git a/posthog/schema.py b/posthog/schema.py
index dc77da163db17..9d83587351683 100644
--- a/posthog/schema.py
+++ b/posthog/schema.py
@@ -180,6 +180,10 @@ class DateRange(BaseModel):
date_to: Optional[str] = None
+class DatetimeDay(RootModel[AwareDatetime]):
+ root: AwareDatetime
+
+
class Day(RootModel[int]):
root: int
@@ -418,6 +422,7 @@ class PersonsOnEventsMode(str, Enum):
v1_enabled = "v1_enabled"
v1_mixed = "v1_mixed"
v2_enabled = "v2_enabled"
+ v3_enabled = "v3_enabled"
class HogQLQueryModifiers(BaseModel):
@@ -457,7 +462,7 @@ class DayItem(BaseModel):
extra="forbid",
)
label: str
- value: Union[str, int]
+ value: Union[str, AwareDatetime, int]
class IntervalItem(BaseModel):
diff --git a/posthog/temporal/batch_exports/batch_exports.py b/posthog/temporal/batch_exports/batch_exports.py
index c776e1f245ef3..88cf9e32f274f 100644
--- a/posthog/temporal/batch_exports/batch_exports.py
+++ b/posthog/temporal/batch_exports/batch_exports.py
@@ -1,15 +1,10 @@
import collections.abc
-import csv
import dataclasses
import datetime as dt
-import gzip
-import tempfile
import typing
import uuid
from string import Template
-import brotli
-import orjson
import pyarrow as pa
from asgiref.sync import sync_to_async
from django.conf import settings
@@ -22,7 +17,7 @@
create_batch_export_backfill,
create_batch_export_run,
update_batch_export_backfill_status,
- update_batch_export_run_status,
+ update_batch_export_run,
)
from posthog.temporal.batch_exports.metrics import (
get_export_finished_metric,
@@ -286,202 +281,6 @@ def get_data_interval(interval: str, data_interval_end: str | None) -> tuple[dt.
return (data_interval_start_dt, data_interval_end_dt)
-def json_dumps_bytes(d) -> bytes:
- return orjson.dumps(d, default=str)
-
-
-class BatchExportTemporaryFile:
- """A TemporaryFile used to as an intermediate step while exporting data.
-
- This class does not implement the file-like interface but rather passes any calls
- to the underlying tempfile.NamedTemporaryFile. We do override 'write' methods
- to allow tracking bytes and records.
- """
-
- def __init__(
- self,
- mode: str = "w+b",
- buffering=-1,
- compression: str | None = None,
- encoding: str | None = None,
- newline: str | None = None,
- suffix: str | None = None,
- prefix: str | None = None,
- dir: str | None = None,
- *,
- errors: str | None = None,
- ):
- self._file = tempfile.NamedTemporaryFile(
- mode=mode,
- encoding=encoding,
- newline=newline,
- buffering=buffering,
- suffix=suffix,
- prefix=prefix,
- dir=dir,
- errors=errors,
- )
- self.compression = compression
- self.bytes_total = 0
- self.records_total = 0
- self.bytes_since_last_reset = 0
- self.records_since_last_reset = 0
- self._brotli_compressor = None
-
- def __getattr__(self, name):
- """Pass get attr to underlying tempfile.NamedTemporaryFile."""
- return self._file.__getattr__(name)
-
- def __enter__(self):
- """Context-manager protocol enter method."""
- self._file.__enter__()
- return self
-
- def __exit__(self, exc, value, tb):
- """Context-manager protocol exit method."""
- return self._file.__exit__(exc, value, tb)
-
- def __iter__(self):
- yield from self._file
-
- @property
- def brotli_compressor(self):
- if self._brotli_compressor is None:
- self._brotli_compressor = brotli.Compressor()
- return self._brotli_compressor
-
- def compress(self, content: bytes | str) -> bytes:
- if isinstance(content, str):
- encoded = content.encode("utf-8")
- else:
- encoded = content
-
- match self.compression:
- case "gzip":
- return gzip.compress(encoded)
- case "brotli":
- self.brotli_compressor.process(encoded)
- return self.brotli_compressor.flush()
- case None:
- return encoded
- case _:
- raise ValueError(f"Unsupported compression: '{self.compression}'")
-
- def write(self, content: bytes | str):
- """Write bytes to underlying file keeping track of how many bytes were written."""
- compressed_content = self.compress(content)
-
- if "b" in self.mode:
- result = self._file.write(compressed_content)
- else:
- result = self._file.write(compressed_content.decode("utf-8"))
-
- self.bytes_total += result
- self.bytes_since_last_reset += result
-
- return result
-
- def write_record_as_bytes(self, record: bytes):
- result = self.write(record)
-
- self.records_total += 1
- self.records_since_last_reset += 1
-
- return result
-
- def write_records_to_jsonl(self, records):
- """Write records to a temporary file as JSONL."""
- if len(records) == 1:
- jsonl_dump = orjson.dumps(records[0], option=orjson.OPT_APPEND_NEWLINE, default=str)
- else:
- jsonl_dump = b"\n".join(map(json_dumps_bytes, records))
-
- result = self.write(jsonl_dump)
-
- self.records_total += len(records)
- self.records_since_last_reset += len(records)
-
- return result
-
- def write_records_to_csv(
- self,
- records,
- fieldnames: None | collections.abc.Sequence[str] = None,
- extrasaction: typing.Literal["raise", "ignore"] = "ignore",
- delimiter: str = ",",
- quotechar: str = '"',
- escapechar: str | None = "\\",
- lineterminator: str = "\n",
- quoting=csv.QUOTE_NONE,
- ):
- """Write records to a temporary file as CSV."""
- if len(records) == 0:
- return
-
- if fieldnames is None:
- fieldnames = list(records[0].keys())
-
- writer = csv.DictWriter(
- self,
- fieldnames=fieldnames,
- extrasaction=extrasaction,
- delimiter=delimiter,
- quotechar=quotechar,
- escapechar=escapechar,
- quoting=quoting,
- lineterminator=lineterminator,
- )
- writer.writerows(records)
-
- self.records_total += len(records)
- self.records_since_last_reset += len(records)
-
- def write_records_to_tsv(
- self,
- records,
- fieldnames: None | list[str] = None,
- extrasaction: typing.Literal["raise", "ignore"] = "ignore",
- quotechar: str = '"',
- escapechar: str | None = "\\",
- lineterminator: str = "\n",
- quoting=csv.QUOTE_NONE,
- ):
- """Write records to a temporary file as TSV."""
- return self.write_records_to_csv(
- records,
- fieldnames=fieldnames,
- extrasaction=extrasaction,
- delimiter="\t",
- quotechar=quotechar,
- escapechar=escapechar,
- quoting=quoting,
- lineterminator=lineterminator,
- )
-
- def rewind(self):
- """Rewind the file before reading it."""
- if self.compression == "brotli":
- result = self._file.write(self.brotli_compressor.finish())
-
- self.bytes_total += result
- self.bytes_since_last_reset += result
-
- self._brotli_compressor = None
-
- self._file.seek(0)
-
- def reset(self):
- """Reset underlying file by truncating it.
-
- Also resets the tracker attributes for bytes and records since last reset.
- """
- self._file.seek(0)
- self._file.truncate()
-
- self.bytes_since_last_reset = 0
- self.records_since_last_reset = 0
-
-
@dataclasses.dataclass
class CreateBatchExportRunInputs:
"""Inputs to the create_export_run activity.
@@ -542,7 +341,7 @@ async def update_export_run_status(inputs: UpdateBatchExportRunStatusInputs) ->
"""Activity that updates the status of an BatchExportRun."""
logger = await bind_temporal_worker_logger(team_id=inputs.team_id)
- batch_export_run = await sync_to_async(update_batch_export_run_status)(
+ batch_export_run = await sync_to_async(update_batch_export_run)(
run_id=uuid.UUID(inputs.id),
status=inputs.status,
latest_error=inputs.latest_error,
diff --git a/posthog/temporal/batch_exports/bigquery_batch_export.py b/posthog/temporal/batch_exports/bigquery_batch_export.py
index a0469de79bb9e..b754a7add16b4 100644
--- a/posthog/temporal/batch_exports/bigquery_batch_export.py
+++ b/posthog/temporal/batch_exports/bigquery_batch_export.py
@@ -15,7 +15,6 @@
from posthog.batch_exports.service import BatchExportField, BatchExportSchema, BigQueryBatchExportInputs
from posthog.temporal.batch_exports.base import PostHogWorkflow
from posthog.temporal.batch_exports.batch_exports import (
- BatchExportTemporaryFile,
CreateBatchExportRunInputs,
UpdateBatchExportRunStatusInputs,
create_export_run,
@@ -29,6 +28,9 @@
get_bytes_exported_metric,
get_rows_exported_metric,
)
+from posthog.temporal.batch_exports.temporary_file import (
+ BatchExportTemporaryFile,
+)
from posthog.temporal.batch_exports.utils import peek_first_and_rewind
from posthog.temporal.common.clickhouse import get_client
from posthog.temporal.common.logger import bind_temporal_worker_logger
diff --git a/posthog/temporal/batch_exports/http_batch_export.py b/posthog/temporal/batch_exports/http_batch_export.py
index 8aca65c80ff38..2866d50c99876 100644
--- a/posthog/temporal/batch_exports/http_batch_export.py
+++ b/posthog/temporal/batch_exports/http_batch_export.py
@@ -13,7 +13,6 @@
from posthog.models import BatchExportRun
from posthog.temporal.batch_exports.base import PostHogWorkflow
from posthog.temporal.batch_exports.batch_exports import (
- BatchExportTemporaryFile,
CreateBatchExportRunInputs,
UpdateBatchExportRunStatusInputs,
create_export_run,
@@ -21,12 +20,15 @@
get_data_interval,
get_rows_count,
iter_records,
- json_dumps_bytes,
)
from posthog.temporal.batch_exports.metrics import (
get_bytes_exported_metric,
get_rows_exported_metric,
)
+from posthog.temporal.batch_exports.temporary_file import (
+ BatchExportTemporaryFile,
+ json_dumps_bytes,
+)
from posthog.temporal.common.clickhouse import get_client
from posthog.temporal.common.logger import bind_temporal_worker_logger
diff --git a/posthog/temporal/batch_exports/postgres_batch_export.py b/posthog/temporal/batch_exports/postgres_batch_export.py
index 5dbfc6faa4acf..98969ee78de79 100644
--- a/posthog/temporal/batch_exports/postgres_batch_export.py
+++ b/posthog/temporal/batch_exports/postgres_batch_export.py
@@ -17,7 +17,6 @@
from posthog.batch_exports.service import BatchExportField, BatchExportSchema, PostgresBatchExportInputs
from posthog.temporal.batch_exports.base import PostHogWorkflow
from posthog.temporal.batch_exports.batch_exports import (
- BatchExportTemporaryFile,
CreateBatchExportRunInputs,
UpdateBatchExportRunStatusInputs,
create_export_run,
@@ -31,6 +30,9 @@
get_bytes_exported_metric,
get_rows_exported_metric,
)
+from posthog.temporal.batch_exports.temporary_file import (
+ BatchExportTemporaryFile,
+)
from posthog.temporal.batch_exports.utils import peek_first_and_rewind
from posthog.temporal.common.clickhouse import get_client
from posthog.temporal.common.logger import bind_temporal_worker_logger
diff --git a/posthog/temporal/batch_exports/s3_batch_export.py b/posthog/temporal/batch_exports/s3_batch_export.py
index 4d99cbeffd7c3..e83fe3f12915d 100644
--- a/posthog/temporal/batch_exports/s3_batch_export.py
+++ b/posthog/temporal/batch_exports/s3_batch_export.py
@@ -1,4 +1,5 @@
import asyncio
+import collections.abc
import contextlib
import datetime as dt
import io
@@ -8,6 +9,8 @@
from dataclasses import dataclass
import aioboto3
+import orjson
+import pyarrow as pa
from django.conf import settings
from temporalio import activity, workflow
from temporalio.common import RetryPolicy
@@ -16,7 +19,6 @@
from posthog.batch_exports.service import BatchExportField, BatchExportSchema, S3BatchExportInputs
from posthog.temporal.batch_exports.base import PostHogWorkflow
from posthog.temporal.batch_exports.batch_exports import (
- BatchExportTemporaryFile,
CreateBatchExportRunInputs,
UpdateBatchExportRunStatusInputs,
create_export_run,
@@ -30,6 +32,15 @@
get_bytes_exported_metric,
get_rows_exported_metric,
)
+from posthog.temporal.batch_exports.temporary_file import (
+ BatchExportTemporaryFile,
+ BatchExportWriter,
+ FlushCallable,
+ JSONLBatchExportWriter,
+ ParquetBatchExportWriter,
+ UnsupportedFileFormatError,
+)
+from posthog.temporal.batch_exports.utils import peek_first_and_rewind
from posthog.temporal.common.clickhouse import get_client
from posthog.temporal.common.logger import bind_temporal_worker_logger
@@ -50,19 +61,31 @@ def get_allowed_template_variables(inputs) -> dict[str, str]:
}
+FILE_FORMAT_EXTENSIONS = {
+ "Parquet": "parquet",
+ "JSONLines": "jsonl",
+}
+
+COMPRESSION_EXTENSIONS = {
+ "gzip": "gz",
+ "snappy": "sz",
+ "brotli": "br",
+ "ztsd": "zst",
+ "lz4": "lz4",
+}
+
+
def get_s3_key(inputs) -> str:
"""Return an S3 key given S3InsertInputs."""
template_variables = get_allowed_template_variables(inputs)
key_prefix = inputs.prefix.format(**template_variables)
+ file_extension = FILE_FORMAT_EXTENSIONS[inputs.file_format]
base_file_name = f"{inputs.data_interval_start}-{inputs.data_interval_end}"
- match inputs.compression:
- case "gzip":
- file_name = base_file_name + ".jsonl.gz"
- case "brotli":
- file_name = base_file_name + ".jsonl.br"
- case _:
- file_name = base_file_name + ".jsonl"
+ if inputs.compression is not None:
+ file_name = base_file_name + f".{file_extension}.{COMPRESSION_EXTENSIONS[inputs.compression]}"
+ else:
+ file_name = base_file_name + f".{file_extension}"
key = posixpath.join(key_prefix, file_name)
@@ -311,6 +334,8 @@ class S3InsertInputs:
kms_key_id: str | None = None
batch_export_schema: BatchExportSchema | None = None
endpoint_url: str | None = None
+ # TODO: In Python 3.11, this could be a enum.StrEnum.
+ file_format: str = "JSONLines"
async def initialize_and_resume_multipart_upload(inputs: S3InsertInputs) -> tuple[S3MultiPartUpload, str]:
@@ -451,7 +476,7 @@ async def insert_into_s3_activity(inputs: S3InsertInputs) -> int:
last_uploaded_part_timestamp: str | None = None
- async def worker_shutdown_handler():
+ async def worker_shutdown_handler() -> None:
"""Handle the Worker shutting down by heart-beating our latest status."""
await activity.wait_for_worker_shutdown()
logger.warn(
@@ -466,50 +491,147 @@ async def worker_shutdown_handler():
asyncio.create_task(worker_shutdown_handler())
- record = None
-
async with s3_upload as s3_upload:
- with BatchExportTemporaryFile(compression=inputs.compression) as local_results_file:
+
+ async def flush_to_s3(
+ local_results_file,
+ records_since_last_flush: int,
+ bytes_since_last_flush: int,
+ last_inserted_at: dt.datetime,
+ last: bool,
+ ):
+ nonlocal last_uploaded_part_timestamp
+
+ logger.debug(
+ "Uploading %s part %s containing %s records with size %s bytes",
+ "last " if last else "",
+ s3_upload.part_number + 1,
+ records_since_last_flush,
+ bytes_since_last_flush,
+ )
+
+ await s3_upload.upload_part(local_results_file)
+ rows_exported.add(records_since_last_flush)
+ bytes_exported.add(bytes_since_last_flush)
+
+ last_uploaded_part_timestamp = str(last_inserted_at)
+ activity.heartbeat(last_uploaded_part_timestamp, s3_upload.to_state())
+
+ first_record_batch, record_iterator = peek_first_and_rewind(record_iterator)
+ first_record_batch = cast_record_batch_json_columns(first_record_batch)
+ column_names = first_record_batch.column_names
+ column_names.pop(column_names.index("_inserted_at"))
+
+ schema = pa.schema(
+ # NOTE: For some reason, some batches set non-nullable fields as non-nullable, whereas other
+ # record batches have them as nullable.
+ # Until we figure it out, we set all fields to nullable. There are some fields we know
+ # are not nullable, but I'm opting for the more flexible option until we out why schemas differ
+ # between batches.
+ [field.with_nullable(True) for field in first_record_batch.select(column_names).schema]
+ )
+
+ writer = get_batch_export_writer(
+ inputs,
+ flush_callable=flush_to_s3,
+ max_bytes=settings.BATCH_EXPORT_S3_UPLOAD_CHUNK_SIZE_BYTES,
+ schema=schema,
+ )
+
+ async with writer.open_temporary_file():
rows_exported = get_rows_exported_metric()
bytes_exported = get_bytes_exported_metric()
- async def flush_to_s3(last_uploaded_part_timestamp: str, last=False):
- logger.debug(
- "Uploading %s part %s containing %s records with size %s bytes",
- "last " if last else "",
- s3_upload.part_number + 1,
- local_results_file.records_since_last_reset,
- local_results_file.bytes_since_last_reset,
- )
+ for record_batch in record_iterator:
+ record_batch = cast_record_batch_json_columns(record_batch)
- await s3_upload.upload_part(local_results_file)
- rows_exported.add(local_results_file.records_since_last_reset)
- bytes_exported.add(local_results_file.bytes_since_last_reset)
+ await writer.write_record_batch(record_batch)
- activity.heartbeat(last_uploaded_part_timestamp, s3_upload.to_state())
+ await s3_upload.complete()
- for record_batch in record_iterator:
- for record in record_batch.to_pylist():
- for json_column in ("properties", "person_properties", "set", "set_once"):
- if (json_str := record.get(json_column, None)) is not None:
- record[json_column] = json.loads(json_str)
+ return writer.records_total
- inserted_at = record.pop("_inserted_at")
- local_results_file.write_records_to_jsonl([record])
+def get_batch_export_writer(
+ inputs: S3InsertInputs, flush_callable: FlushCallable, max_bytes: int, schema: pa.Schema | None = None
+) -> BatchExportWriter:
+ """Return the `BatchExportWriter` corresponding to configured `file_format`.
- if local_results_file.tell() > settings.BATCH_EXPORT_S3_UPLOAD_CHUNK_SIZE_BYTES:
- last_uploaded_part_timestamp = str(inserted_at)
- await flush_to_s3(last_uploaded_part_timestamp)
- local_results_file.reset()
+ Raises:
+ UnsupportedFileFormatError: If no writer exists for given `file_format`.
+ """
+ writer: BatchExportWriter
- if local_results_file.tell() > 0 and record is not None:
- last_uploaded_part_timestamp = str(inserted_at)
- await flush_to_s3(last_uploaded_part_timestamp, last=True)
+ if inputs.file_format == "Parquet":
+ writer = ParquetBatchExportWriter(
+ max_bytes=max_bytes,
+ flush_callable=flush_callable,
+ compression=inputs.compression,
+ schema=schema,
+ )
+ elif inputs.file_format == "JSONLines":
+ writer = JSONLBatchExportWriter(
+ max_bytes=settings.BATCH_EXPORT_S3_UPLOAD_CHUNK_SIZE_BYTES,
+ flush_callable=flush_callable,
+ compression=inputs.compression,
+ )
+ else:
+ raise UnsupportedFileFormatError(inputs.file_format, "S3")
- await s3_upload.complete()
+ return writer
+
+
+def cast_record_batch_json_columns(
+ record_batch: pa.RecordBatch,
+ json_columns: collections.abc.Sequence = ("properties", "person_properties", "set", "set_once"),
+) -> pa.RecordBatch:
+ """Cast json_columns in record_batch to JsonType.
+
+ We return a new RecordBatch with any json_columns replaced by fields casted to JsonType.
+ Casting is not copying the underlying array buffers, so memory usage does not increase when creating
+ the new array or the new record batch.
+ """
+ column_names = set(record_batch.column_names)
+ intersection = column_names & set(json_columns)
+
+ casted_arrays = []
+ for array in record_batch.select(intersection):
+ if pa.types.is_string(array.type):
+ casted_array = array.cast(JsonType())
+ casted_arrays.append(casted_array)
+
+ remaining_column_names = list(column_names - intersection)
+ return pa.RecordBatch.from_arrays(
+ record_batch.select(remaining_column_names).columns + casted_arrays,
+ names=remaining_column_names + list(intersection),
+ )
+
+
+class JsonScalar(pa.ExtensionScalar):
+ """Represents a JSON binary string."""
+
+ def as_py(self) -> dict | None:
+ if self.value:
+ return orjson.loads(self.value.as_py().encode("utf-8"))
+ else:
+ return None
+
+
+class JsonType(pa.ExtensionType):
+ """Type for JSON binary strings."""
+
+ def __init__(self):
+ super().__init__(pa.string(), "json")
+
+ def __arrow_ext_serialize__(self):
+ return b""
+
+ @classmethod
+ def __arrow_ext_deserialize__(self, storage_type, serialized):
+ return JsonType()
- return local_results_file.records_total
+ def __arrow_ext_scalar_class__(self):
+ return JsonScalar
@workflow.defn(name="s3-export")
@@ -572,6 +694,7 @@ async def run(self, inputs: S3BatchExportInputs):
encryption=inputs.encryption,
kms_key_id=inputs.kms_key_id,
batch_export_schema=inputs.batch_export_schema,
+ file_format=inputs.file_format,
)
await execute_batch_export_insert_activity(
diff --git a/posthog/temporal/batch_exports/snowflake_batch_export.py b/posthog/temporal/batch_exports/snowflake_batch_export.py
index be94eca89a799..9053f3e1006ad 100644
--- a/posthog/temporal/batch_exports/snowflake_batch_export.py
+++ b/posthog/temporal/batch_exports/snowflake_batch_export.py
@@ -18,7 +18,6 @@
from posthog.batch_exports.service import BatchExportField, BatchExportSchema, SnowflakeBatchExportInputs
from posthog.temporal.batch_exports.base import PostHogWorkflow
from posthog.temporal.batch_exports.batch_exports import (
- BatchExportTemporaryFile,
CreateBatchExportRunInputs,
UpdateBatchExportRunStatusInputs,
create_export_run,
@@ -32,6 +31,9 @@
get_bytes_exported_metric,
get_rows_exported_metric,
)
+from posthog.temporal.batch_exports.temporary_file import (
+ BatchExportTemporaryFile,
+)
from posthog.temporal.batch_exports.utils import peek_first_and_rewind
from posthog.temporal.common.clickhouse import get_client
from posthog.temporal.common.logger import bind_temporal_worker_logger
diff --git a/posthog/temporal/batch_exports/temporary_file.py b/posthog/temporal/batch_exports/temporary_file.py
new file mode 100644
index 0000000000000..f955f45553727
--- /dev/null
+++ b/posthog/temporal/batch_exports/temporary_file.py
@@ -0,0 +1,528 @@
+"""This module contains a temporary file to stage data in batch exports."""
+import abc
+import collections.abc
+import contextlib
+import csv
+import datetime as dt
+import gzip
+import tempfile
+import typing
+
+import brotli
+import orjson
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+
+def json_dumps_bytes(d) -> bytes:
+ return orjson.dumps(d, default=str)
+
+
+class BatchExportTemporaryFile:
+ """A TemporaryFile used to as an intermediate step while exporting data.
+
+ This class does not implement the file-like interface but rather passes any calls
+ to the underlying tempfile.NamedTemporaryFile. We do override 'write' methods
+ to allow tracking bytes and records.
+ """
+
+ def __init__(
+ self,
+ mode: str = "w+b",
+ buffering=-1,
+ compression: str | None = None,
+ encoding: str | None = None,
+ newline: str | None = None,
+ suffix: str | None = None,
+ prefix: str | None = None,
+ dir: str | None = None,
+ *,
+ errors: str | None = None,
+ ):
+ self._file = tempfile.NamedTemporaryFile(
+ mode=mode,
+ encoding=encoding,
+ newline=newline,
+ buffering=buffering,
+ suffix=suffix,
+ prefix=prefix,
+ dir=dir,
+ errors=errors,
+ )
+ self.compression = compression
+ self.bytes_total = 0
+ self.records_total = 0
+ self.bytes_since_last_reset = 0
+ self.records_since_last_reset = 0
+ self._brotli_compressor = None
+
+ def __getattr__(self, name):
+ """Pass get attr to underlying tempfile.NamedTemporaryFile."""
+ return self._file.__getattr__(name)
+
+ def __enter__(self):
+ """Context-manager protocol enter method."""
+ self._file.__enter__()
+ return self
+
+ def __exit__(self, exc, value, tb):
+ """Context-manager protocol exit method."""
+ return self._file.__exit__(exc, value, tb)
+
+ def __iter__(self):
+ yield from self._file
+
+ @property
+ def brotli_compressor(self):
+ if self._brotli_compressor is None:
+ self._brotli_compressor = brotli.Compressor()
+ return self._brotli_compressor
+
+ def finish_brotli_compressor(self):
+ """Flush remaining brotli bytes."""
+ # TODO: Move compression out of `BatchExportTemporaryFile` to a standard class for all writers.
+ if self.compression != "brotli":
+ raise ValueError(f"Compression is '{self.compression}', not 'brotli'")
+
+ result = self._file.write(self.brotli_compressor.finish())
+ self.bytes_total += result
+ self.bytes_since_last_reset += result
+ self._brotli_compressor = None
+
+ def compress(self, content: bytes | str) -> bytes:
+ if isinstance(content, str):
+ encoded = content.encode("utf-8")
+ else:
+ encoded = content
+
+ match self.compression:
+ case "gzip":
+ return gzip.compress(encoded)
+ case "brotli":
+ self.brotli_compressor.process(encoded)
+ return self.brotli_compressor.flush()
+ case None:
+ return encoded
+ case _:
+ raise ValueError(f"Unsupported compression: '{self.compression}'")
+
+ def write(self, content: bytes | str):
+ """Write bytes to underlying file keeping track of how many bytes were written."""
+ compressed_content = self.compress(content)
+
+ if "b" in self.mode:
+ result = self._file.write(compressed_content)
+ else:
+ result = self._file.write(compressed_content.decode("utf-8"))
+
+ self.bytes_total += result
+ self.bytes_since_last_reset += result
+
+ return result
+
+ def write_record_as_bytes(self, record: bytes):
+ result = self.write(record)
+
+ self.records_total += 1
+ self.records_since_last_reset += 1
+
+ return result
+
+ def write_records_to_jsonl(self, records):
+ """Write records to a temporary file as JSONL."""
+ if len(records) == 1:
+ jsonl_dump = orjson.dumps(records[0], option=orjson.OPT_APPEND_NEWLINE, default=str)
+ else:
+ jsonl_dump = b"\n".join(map(json_dumps_bytes, records))
+
+ result = self.write(jsonl_dump)
+
+ self.records_total += len(records)
+ self.records_since_last_reset += len(records)
+
+ return result
+
+ def write_records_to_csv(
+ self,
+ records,
+ fieldnames: None | collections.abc.Sequence[str] = None,
+ extrasaction: typing.Literal["raise", "ignore"] = "ignore",
+ delimiter: str = ",",
+ quotechar: str = '"',
+ escapechar: str | None = "\\",
+ lineterminator: str = "\n",
+ quoting=csv.QUOTE_NONE,
+ ):
+ """Write records to a temporary file as CSV."""
+ if len(records) == 0:
+ return
+
+ if fieldnames is None:
+ fieldnames = list(records[0].keys())
+
+ writer = csv.DictWriter(
+ self,
+ fieldnames=fieldnames,
+ extrasaction=extrasaction,
+ delimiter=delimiter,
+ quotechar=quotechar,
+ escapechar=escapechar,
+ quoting=quoting,
+ lineterminator=lineterminator,
+ )
+ writer.writerows(records)
+
+ self.records_total += len(records)
+ self.records_since_last_reset += len(records)
+
+ def write_records_to_tsv(
+ self,
+ records,
+ fieldnames: None | list[str] = None,
+ extrasaction: typing.Literal["raise", "ignore"] = "ignore",
+ quotechar: str = '"',
+ escapechar: str | None = "\\",
+ lineterminator: str = "\n",
+ quoting=csv.QUOTE_NONE,
+ ):
+ """Write records to a temporary file as TSV."""
+ return self.write_records_to_csv(
+ records,
+ fieldnames=fieldnames,
+ extrasaction=extrasaction,
+ delimiter="\t",
+ quotechar=quotechar,
+ escapechar=escapechar,
+ quoting=quoting,
+ lineterminator=lineterminator,
+ )
+
+ def rewind(self):
+ """Rewind the file before reading it."""
+ self._file.seek(0)
+
+ def reset(self):
+ """Reset underlying file by truncating it.
+
+ Also resets the tracker attributes for bytes and records since last reset.
+ """
+ self._file.seek(0)
+ self._file.truncate()
+
+ self.bytes_since_last_reset = 0
+ self.records_since_last_reset = 0
+
+
+LastInsertedAt = dt.datetime
+IsLast = bool
+RecordsSinceLastFlush = int
+BytesSinceLastFlush = int
+FlushCallable = collections.abc.Callable[
+ [BatchExportTemporaryFile, RecordsSinceLastFlush, BytesSinceLastFlush, LastInsertedAt, IsLast],
+ collections.abc.Awaitable[None],
+]
+
+
+class UnsupportedFileFormatError(Exception):
+ """Raised when a writer for an unsupported file format is requested."""
+
+ def __init__(self, file_format: str, destination: str):
+ super().__init__(f"{file_format} is not a supported format for {destination} batch exports.")
+
+
+class BatchExportWriter(abc.ABC):
+ """A temporary file writer to be used by batch export workflows.
+
+ Subclasses should define `_write_record_batch` with the particular intricacies
+ of the format they are writing as.
+
+ Actual writing calls are passed to the underlying `batch_export_file`.
+
+ Attributes:
+ _batch_export_file: The temporary file we are writing to.
+ max_bytes: Flush the temporary file with the provided `flush_callable`
+ upon reaching or surpassing this threshold. Keep in mind we write on a RecordBatch
+ per RecordBatch basis, which means the threshold will be surpassed by at most the
+ size of a RecordBatch before a flush occurs.
+ flush_callable: A callback to flush the temporary file when `max_bytes` is reached.
+ The temporary file will be reset after calling `flush_callable`. When calling
+ `flush_callable` the following positional arguments will be passed: The temporary file
+ that must be flushed, the number of records since the last flush, the number of bytes
+ since the last flush, the latest recorded `_inserted_at`, and a `bool` indicating if
+ this is the last flush (when exiting the context manager).
+ file_kwargs: Optional keyword arguments passed when initializing `_batch_export_file`.
+ last_inserted_at: Latest `_inserted_at` written. This attribute leaks some implementation
+ details, as we are assuming assume `_inserted_at` is present, as it's added to all
+ batch export queries.
+ records_total: The total number of records (not RecordBatches!) written.
+ records_since_last_flush: The number of records written since last flush.
+ bytes_total: The total number of bytes written.
+ bytes_since_last_flush: The number of bytes written since last flush.
+ """
+
+ def __init__(
+ self,
+ flush_callable: FlushCallable,
+ max_bytes: int,
+ file_kwargs: collections.abc.Mapping[str, typing.Any] | None = None,
+ ):
+ self.flush_callable = flush_callable
+ self.max_bytes = max_bytes
+ self.file_kwargs: collections.abc.Mapping[str, typing.Any] = file_kwargs or {}
+
+ self._batch_export_file: BatchExportTemporaryFile | None = None
+ self.reset_writer_tracking()
+
+ def reset_writer_tracking(self):
+ """Reset this writer's tracking state."""
+ self.last_inserted_at: dt.datetime | None = None
+ self.records_total = 0
+ self.records_since_last_flush = 0
+ self.bytes_total = 0
+ self.bytes_since_last_flush = 0
+
+ @contextlib.asynccontextmanager
+ async def open_temporary_file(self):
+ """Explicitly open the temporary file this writer is writing to.
+
+ The underlying `BatchExportTemporaryFile` is only accessible within this context manager. This helps
+ us separate the lifetime of the underlying temporary file from the writer: The writer may still be
+ accessed even after the temporary file is closed, while on the other hand we ensure the file and all
+ its data is flushed and not leaked outside the context. Any relevant tracking information is copied
+ to the writer.
+ """
+ self.reset_writer_tracking()
+
+ with BatchExportTemporaryFile(**self.file_kwargs) as temp_file:
+ self._batch_export_file = temp_file
+
+ try:
+ yield
+ finally:
+ self.track_bytes_written(temp_file)
+
+ if self.last_inserted_at is not None and self.bytes_since_last_flush > 0:
+ # `bytes_since_last_flush` should be 0 unless:
+ # 1. The last batch wasn't flushed as it didn't reach `max_bytes`.
+ # 2. The last batch was flushed but there was another write after the last call to
+ # `write_record_batch`. For example, footer bytes.
+ await self.flush(self.last_inserted_at, is_last=True)
+
+ self._batch_export_file = None
+
+ @property
+ def batch_export_file(self):
+ """Property for underlying temporary file.
+
+ Raises:
+ ValueError: if attempting to access the temporary file before it has been opened.
+ """
+ if self._batch_export_file is None:
+ raise ValueError("Batch export file is closed. Did you forget to call 'open_temporary_file'?")
+ return self._batch_export_file
+
+ @abc.abstractmethod
+ def _write_record_batch(self, record_batch: pa.RecordBatch) -> None:
+ """Write a record batch to the underlying `BatchExportTemporaryFile`.
+
+ Subclasses must override this to provide the actual implementation according to the supported
+ file format.
+ """
+ pass
+
+ def track_records_written(self, record_batch: pa.RecordBatch) -> None:
+ """Update this writer's state with the number of records in `record_batch`."""
+ self.records_total += record_batch.num_rows
+ self.records_since_last_flush += record_batch.num_rows
+
+ def track_bytes_written(self, batch_export_file: BatchExportTemporaryFile) -> None:
+ """Update this writer's state with the bytes in `batch_export_file`."""
+ self.bytes_total = batch_export_file.bytes_total
+ self.bytes_since_last_flush = batch_export_file.bytes_since_last_reset
+
+ async def write_record_batch(self, record_batch: pa.RecordBatch) -> None:
+ """Issue a record batch write tracking progress and flushing if required."""
+ record_batch = record_batch.sort_by("_inserted_at")
+ last_inserted_at = record_batch.column("_inserted_at")[-1].as_py()
+
+ column_names = record_batch.column_names
+ column_names.pop(column_names.index("_inserted_at"))
+
+ self._write_record_batch(record_batch.select(column_names))
+
+ self.last_inserted_at = last_inserted_at
+ self.track_records_written(record_batch)
+ self.track_bytes_written(self.batch_export_file)
+
+ if self.bytes_since_last_flush >= self.max_bytes:
+ await self.flush(last_inserted_at)
+
+ async def flush(self, last_inserted_at: dt.datetime, is_last: bool = False) -> None:
+ """Call the provided `flush_callable` and reset underlying file.
+
+ The underlying batch export temporary file will be reset after calling `flush_callable`.
+ """
+ if is_last is True and self.batch_export_file.compression == "brotli":
+ self.batch_export_file.finish_brotli_compressor()
+
+ self.batch_export_file.seek(0)
+
+ await self.flush_callable(
+ self.batch_export_file,
+ self.records_since_last_flush,
+ self.bytes_since_last_flush,
+ last_inserted_at,
+ is_last,
+ )
+ self.batch_export_file.reset()
+
+ self.records_since_last_flush = 0
+ self.bytes_since_last_flush = 0
+
+
+class JSONLBatchExportWriter(BatchExportWriter):
+ """A `BatchExportWriter` for JSONLines format.
+
+ Attributes:
+ default: The default function to use to cast non-serializable Python objects to serializable objects.
+ By default, non-serializable objects will be cast to string via `str()`.
+ """
+
+ def __init__(
+ self,
+ max_bytes: int,
+ flush_callable: FlushCallable,
+ compression: None | str = None,
+ default: typing.Callable = str,
+ ):
+ super().__init__(
+ max_bytes=max_bytes,
+ flush_callable=flush_callable,
+ file_kwargs={"compression": compression},
+ )
+
+ self.default = default
+
+ def write(self, content: bytes) -> int:
+ """Write a single row of JSONL."""
+ n = self.batch_export_file.write(orjson.dumps(content, default=str) + b"\n")
+ return n
+
+ def _write_record_batch(self, record_batch: pa.RecordBatch) -> None:
+ """Write records to a temporary file as JSONL."""
+ for record in record_batch.to_pylist():
+ self.write(record)
+
+
+class CSVBatchExportWriter(BatchExportWriter):
+ """A `BatchExportWriter` for CSV format."""
+
+ def __init__(
+ self,
+ max_bytes: int,
+ flush_callable: FlushCallable,
+ field_names: collections.abc.Sequence[str],
+ extras_action: typing.Literal["raise", "ignore"] = "ignore",
+ delimiter: str = ",",
+ quote_char: str = '"',
+ escape_char: str | None = "\\",
+ line_terminator: str = "\n",
+ quoting=csv.QUOTE_NONE,
+ compression: str | None = None,
+ ):
+ super().__init__(
+ max_bytes=max_bytes,
+ flush_callable=flush_callable,
+ file_kwargs={"compression": compression},
+ )
+ self.field_names = field_names
+ self.extras_action: typing.Literal["raise", "ignore"] = extras_action
+ self.delimiter = delimiter
+ self.quote_char = quote_char
+ self.escape_char = escape_char
+ self.line_terminator = line_terminator
+ self.quoting = quoting
+
+ self._csv_writer: csv.DictWriter | None = None
+
+ @property
+ def csv_writer(self) -> csv.DictWriter:
+ if self._csv_writer is None:
+ self._csv_writer = csv.DictWriter(
+ self.batch_export_file,
+ fieldnames=self.field_names,
+ extrasaction=self.extras_action,
+ delimiter=self.delimiter,
+ quotechar=self.quote_char,
+ escapechar=self.escape_char,
+ quoting=self.quoting,
+ lineterminator=self.line_terminator,
+ )
+
+ return self._csv_writer
+
+ def _write_record_batch(self, record_batch: pa.RecordBatch) -> None:
+ """Write records to a temporary file as CSV."""
+ self.csv_writer.writerows(record_batch.to_pylist())
+
+
+class ParquetBatchExportWriter(BatchExportWriter):
+ """A `BatchExportWriter` for Apache Parquet format.
+
+ We utilize and wrap a `pyarrow.parquet.ParquetWriter` to do the actual writing. We default to their
+ defaults for most parameters; however this class could be extended with more attributes to pass along
+ to `pyarrow.parquet.ParquetWriter`.
+
+ See the pyarrow docs for more details on what parameters can the writer be configured with:
+ https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetWriter.html
+
+ In contrast to other writers, instead of us handling compression we let `pyarrow.parquet.ParquetWriter`
+ handle it, so `BatchExportTemporaryFile` is always initialized with `compression=None`.
+
+ Attributes:
+ schema: The schema used by the Parquet file. Should match the schema of written RecordBatches.
+ compression: Compression codec passed to underlying `pyarrow.parquet.ParquetWriter`.
+ """
+
+ def __init__(
+ self,
+ max_bytes: int,
+ flush_callable: FlushCallable,
+ schema: pa.Schema,
+ compression: str | None = "snappy",
+ ):
+ super().__init__(
+ max_bytes=max_bytes,
+ flush_callable=flush_callable,
+ file_kwargs={"compression": None}, # ParquetWriter handles compression
+ )
+ self.schema = schema
+ self.compression = compression
+
+ self._parquet_writer: pq.ParquetWriter | None = None
+
+ @property
+ def parquet_writer(self) -> pq.ParquetWriter:
+ if self._parquet_writer is None:
+ self._parquet_writer = pq.ParquetWriter(
+ self.batch_export_file,
+ schema=self.schema,
+ compression="none" if self.compression is None else self.compression,
+ )
+ return self._parquet_writer
+
+ @contextlib.asynccontextmanager
+ async def open_temporary_file(self):
+ """Ensure underlying Parquet writer is closed before flushing and closing temporary file."""
+ async with super().open_temporary_file():
+ try:
+ yield
+ finally:
+ if self._parquet_writer is not None:
+ self._parquet_writer.writer.close()
+ self._parquet_writer = None
+
+ def _write_record_batch(self, record_batch: pa.RecordBatch) -> None:
+ """Write records to a temporary file as Parquet."""
+
+ self.parquet_writer.write_batch(record_batch.select(self.parquet_writer.schema.names))
diff --git a/posthog/temporal/common/utils.py b/posthog/temporal/common/utils.py
index 1b61a356dc898..022c8270d7748 100644
--- a/posthog/temporal/common/utils.py
+++ b/posthog/temporal/common/utils.py
@@ -119,10 +119,9 @@ async def should_resume_from_activity_heartbeat(
heartbeat_details = heartbeat_type.from_activity(activity)
except EmptyHeartbeatError:
- # We don't log this as a warning/error because it's the expected exception when heartbeat is empty.
+ # We don't log this as it's the expected exception when heartbeat is empty.
heartbeat_details = None
received = False
- logger.debug("Did not receive details from previous activity execution")
except NotEnoughHeartbeatValuesError:
heartbeat_details = None
diff --git a/posthog/temporal/data_imports/external_data_job.py b/posthog/temporal/data_imports/external_data_job.py
index db99eeb1de315..bf78c99e9d9e0 100644
--- a/posthog/temporal/data_imports/external_data_job.py
+++ b/posthog/temporal/data_imports/external_data_job.py
@@ -10,6 +10,7 @@
# TODO: remove dependency
from posthog.temporal.batch_exports.base import PostHogWorkflow
+from posthog.temporal.data_imports.pipelines.zendesk.credentials import ZendeskCredentialsToken
from posthog.warehouse.data_load.source_templates import create_warehouse_templates_for_source
from posthog.warehouse.data_load.validate_schema import validate_schema_and_update_table
@@ -220,7 +221,20 @@ async def run_external_data_job(inputs: ExternalDataJobInputs) -> TSchemaTables:
schema=schema,
table_names=endpoints,
)
+ elif model.pipeline.source_type == ExternalDataSource.Type.ZENDESK:
+ from posthog.temporal.data_imports.pipelines.zendesk.helpers import zendesk_support
+ credentials = ZendeskCredentialsToken()
+ credentials.token = model.pipeline.job_inputs.get("zendesk_api_key")
+ credentials.subdomain = model.pipeline.job_inputs.get("zendesk_subdomain")
+ credentials.email = model.pipeline.job_inputs.get("zendesk_email_address")
+
+ data_support = zendesk_support(credentials=credentials, endpoints=tuple(endpoints), team_id=inputs.team_id)
+ # Uncomment to support zendesk chat and talk
+ # data_chat = zendesk_chat()
+ # data_talk = zendesk_talk()
+
+ source = data_support
else:
raise ValueError(f"Source type {model.pipeline.source_type} not supported")
diff --git a/posthog/temporal/data_imports/pipelines/pipeline.py b/posthog/temporal/data_imports/pipelines/pipeline.py
index 5297f2e39ac29..6a922d2d96a67 100644
--- a/posthog/temporal/data_imports/pipelines/pipeline.py
+++ b/posthog/temporal/data_imports/pipelines/pipeline.py
@@ -1,4 +1,5 @@
from dataclasses import dataclass
+from typing import Literal
from uuid import UUID
import dlt
@@ -9,7 +10,7 @@
import os
from posthog.settings.base_variables import TEST
from structlog.typing import FilteringBoundLogger
-from dlt.sources import DltResource
+from dlt.sources import DltSource
@dataclass
@@ -23,9 +24,9 @@ class PipelineInputs:
class DataImportPipeline:
- loader_file_format = "parquet"
+ loader_file_format: Literal["parquet"] = "parquet"
- def __init__(self, inputs: PipelineInputs, source: DltResource, logger: FilteringBoundLogger):
+ def __init__(self, inputs: PipelineInputs, source: DltSource, logger: FilteringBoundLogger):
self.inputs = inputs
self.logger = logger
self.source = source
@@ -47,6 +48,7 @@ def _get_destination(self):
credentials = {
"aws_access_key_id": settings.AIRBYTE_BUCKET_KEY,
"aws_secret_access_key": settings.AIRBYTE_BUCKET_SECRET,
+ "region_name": settings.AIRBYTE_BUCKET_REGION,
}
return dlt.destinations.filesystem(
diff --git a/posthog/temporal/data_imports/pipelines/schemas.py b/posthog/temporal/data_imports/pipelines/schemas.py
index 371f8087b7966..1caea1364899a 100644
--- a/posthog/temporal/data_imports/pipelines/schemas.py
+++ b/posthog/temporal/data_imports/pipelines/schemas.py
@@ -1,3 +1,4 @@
+from posthog.temporal.data_imports.pipelines.zendesk.settings import BASE_ENDPOINTS, SUPPORT_ENDPOINTS
from posthog.warehouse.models import ExternalDataSource
from posthog.temporal.data_imports.pipelines.stripe.settings import ENDPOINTS as STRIPE_ENDPOINTS
from posthog.temporal.data_imports.pipelines.hubspot.settings import ENDPOINTS as HUBSPOT_ENDPOINTS
@@ -5,5 +6,8 @@
PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING = {
ExternalDataSource.Type.STRIPE: STRIPE_ENDPOINTS,
ExternalDataSource.Type.HUBSPOT: HUBSPOT_ENDPOINTS,
+ ExternalDataSource.Type.ZENDESK: tuple(
+ list(BASE_ENDPOINTS) + [resource for resource, endpoint_url, data_key, cursor_paginated in SUPPORT_ENDPOINTS]
+ ),
ExternalDataSource.Type.POSTGRES: (),
}
diff --git a/posthog/temporal/data_imports/pipelines/zendesk/api_helpers.py b/posthog/temporal/data_imports/pipelines/zendesk/api_helpers.py
new file mode 100644
index 0000000000000..c6e4eb4809ee7
--- /dev/null
+++ b/posthog/temporal/data_imports/pipelines/zendesk/api_helpers.py
@@ -0,0 +1,103 @@
+from typing import Optional, TypedDict, Dict
+
+from dlt.common import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import DictStrAny, DictStrStr, TDataItem
+
+
+class TCustomFieldInfo(TypedDict):
+ title: str
+ options: DictStrStr
+
+
+def _parse_date_or_none(value: Optional[str]) -> Optional[pendulum.DateTime]:
+ if not value:
+ return None
+ return ensure_pendulum_datetime(value)
+
+
+def process_ticket(
+ ticket: DictStrAny,
+ custom_fields: Dict[str, TCustomFieldInfo],
+ pivot_custom_fields: bool = True,
+) -> DictStrAny:
+ """
+ Helper function that processes a ticket object and returns a dictionary of ticket data.
+
+ Args:
+ ticket: The ticket dict object returned by a Zendesk API call.
+ custom_fields: A dictionary containing all the custom fields available for tickets.
+ pivot_custom_fields: A boolean indicating whether to pivot all custom fields or not.
+ Defaults to True.
+
+ Returns:
+ DictStrAny: A dictionary containing cleaned data about a ticket.
+ """
+ # Commented out due to how slow this processing code is, and how often it'd break the pipeline.
+ # to be revisited on whether we want/need this pre-processing and figure out the best way to do it.
+
+ # pivot custom field if indicated as such
+ # get custom fields
+ # pivoted_fields = set()
+ # for custom_field in ticket.get("custom_fields", []):
+ # if pivot_custom_fields:
+ # cus_field_id = str(custom_field["id"])
+ # field = custom_fields.get(cus_field_id, None)
+ # if field is None:
+ # logger.warning(
+ # "Custom field with ID %s does not exist in fields state. It may have been created after the pipeline run started.",
+ # cus_field_id,
+ # )
+ # custom_field["ticket_id"] = ticket["id"]
+ # continue
+
+ # pivoted_fields.add(cus_field_id)
+ # field_name = field["title"]
+ # current_value = custom_field["value"]
+ # options = field["options"]
+ # # Map dropdown values to labels
+ # if not current_value or not options:
+ # ticket[field_name] = current_value
+ # elif isinstance(current_value, list): # Multiple choice field has a list of values
+ # ticket[field_name] = [options.get(key, key) for key in current_value]
+ # else:
+ # ticket[field_name] = options.get(current_value)
+ # else:
+ # custom_field["ticket_id"] = ticket["id"]
+ # # delete fields that are not needed for pivoting
+ # if pivot_custom_fields:
+ # ticket["custom_fields"] = [f for f in ticket.get("custom_fields", []) if str(f["id"]) not in pivoted_fields]
+ # if not ticket.get("custom_fields"):
+ # del ticket["custom_fields"]
+ # del ticket["fields"]
+
+ # modify dates to return datetime objects instead
+ ticket["updated_at"] = _parse_date_or_none(ticket["updated_at"])
+ ticket["created_at"] = _parse_date_or_none(ticket["created_at"])
+ ticket["due_at"] = _parse_date_or_none(ticket["due_at"])
+ return ticket
+
+
+def process_ticket_field(field: DictStrAny, custom_fields_state: Dict[str, TCustomFieldInfo]) -> TDataItem:
+ """Update custom field mapping in dlt state for the given field."""
+ # grab id and update state dict
+ # if the id is new, add a new key to indicate that this is the initial value for title
+ # New dropdown options are added to existing field but existing options are not changed
+ return_dict = field.copy()
+ field_id = str(field["id"])
+
+ options = field.get("custom_field_options", [])
+ new_options = {o["value"]: o["name"] for o in options}
+ existing_field = custom_fields_state.get(field_id)
+ if existing_field:
+ existing_options = existing_field["options"]
+ if return_options := return_dict.get("custom_field_options"):
+ for item in return_options:
+ item["name"] = existing_options.get(item["value"], item["name"])
+ for key, value in new_options.items():
+ if key not in existing_options:
+ existing_options[key] = value
+ else:
+ custom_fields_state[field_id] = dict(title=field["title"], options=new_options)
+ return_dict["initial_title"] = field["title"]
+ return return_dict
diff --git a/posthog/temporal/data_imports/pipelines/zendesk/credentials.py b/posthog/temporal/data_imports/pipelines/zendesk/credentials.py
new file mode 100644
index 0000000000000..1f8110ae9b911
--- /dev/null
+++ b/posthog/temporal/data_imports/pipelines/zendesk/credentials.py
@@ -0,0 +1,49 @@
+"""
+This module handles how credentials are read in dlt sources
+"""
+from typing import ClassVar, List, Union
+from dlt.common.configuration import configspec
+from dlt.common.configuration.specs import CredentialsConfiguration
+from dlt.common.typing import TSecretValue
+
+
+@configspec
+class ZendeskCredentialsBase(CredentialsConfiguration):
+ """
+ The Base version of all the ZendeskCredential classes.
+ """
+
+ subdomain: str
+ __config_gen_annotations__: ClassVar[List[str]] = []
+
+
+@configspec
+class ZendeskCredentialsEmailPass(ZendeskCredentialsBase):
+ """
+ This class is used to store credentials for Email + Password Authentication
+ """
+
+ email: str
+ password: TSecretValue
+
+
+@configspec
+class ZendeskCredentialsOAuth(ZendeskCredentialsBase):
+ """
+ This class is used to store credentials for OAuth Token Authentication
+ """
+
+ oauth_token: TSecretValue
+
+
+@configspec
+class ZendeskCredentialsToken(ZendeskCredentialsBase):
+ """
+ This class is used to store credentials for Token Authentication
+ """
+
+ email: str
+ token: TSecretValue
+
+
+TZendeskCredentials = Union[ZendeskCredentialsEmailPass, ZendeskCredentialsToken, ZendeskCredentialsOAuth]
diff --git a/posthog/temporal/data_imports/pipelines/zendesk/helpers.py b/posthog/temporal/data_imports/pipelines/zendesk/helpers.py
new file mode 100644
index 0000000000000..a3e0328c8ab28
--- /dev/null
+++ b/posthog/temporal/data_imports/pipelines/zendesk/helpers.py
@@ -0,0 +1,444 @@
+from typing import Iterator, List, Optional, Iterable, Tuple
+from itertools import chain
+
+import dlt
+from dlt.common import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TDataItem, TAnyDateTime, TDataItems
+from dlt.sources import DltResource
+
+from .api_helpers import process_ticket, process_ticket_field
+from .talk_api import PaginationType, ZendeskAPIClient
+from .credentials import TZendeskCredentials, ZendeskCredentialsOAuth
+
+from .settings import (
+ DEFAULT_START_DATE,
+ CUSTOM_FIELDS_STATE_KEY,
+ SUPPORT_ENDPOINTS,
+ TALK_ENDPOINTS,
+ INCREMENTAL_TALK_ENDPOINTS,
+ SUPPORT_EXTRA_ENDPOINTS,
+)
+
+
+@dlt.source(max_table_nesting=0)
+def zendesk_talk(
+ credentials: TZendeskCredentials = dlt.secrets.value,
+ start_date: Optional[TAnyDateTime] = DEFAULT_START_DATE,
+ end_date: Optional[TAnyDateTime] = None,
+) -> Iterable[DltResource]:
+ """
+ Retrieves data from Zendesk Talk for phone calls and voicemails.
+
+ `start_date` argument can be used on its own or together with `end_date`. When both are provided
+ data is limited to items updated in that time range.
+ The range is "half-open", meaning elements equal and higher than `start_date` and elements lower than `end_date` are included.
+ All resources opt-in to use Airflow scheduler if run as Airflow task
+
+ Args:
+ credentials: The credentials for authentication. Defaults to the value in the `dlt.secrets` object.
+ start_date: The start time of the range for which to load. Defaults to January 1st 2000.
+ end_date: The end time of the range for which to load data.
+ If end time is not provided, the incremental loading will be enabled and after initial run, only new data will be retrieved
+ Yields:
+ DltResource: Data resources from Zendesk Talk.
+ """
+
+ # use the credentials to authenticate with the ZendeskClient
+ zendesk_client = ZendeskAPIClient(credentials)
+ start_date_obj = ensure_pendulum_datetime(start_date)
+ end_date_obj = ensure_pendulum_datetime(end_date) if end_date else None
+
+ # regular endpoints
+ for key, talk_endpoint, item_name, cursor_paginated in TALK_ENDPOINTS:
+ yield dlt.resource(
+ talk_resource(
+ zendesk_client,
+ key,
+ item_name or talk_endpoint,
+ PaginationType.CURSOR if cursor_paginated else PaginationType.OFFSET,
+ ),
+ name=key,
+ write_disposition="replace",
+ )
+
+ # adding incremental endpoints
+ for key, talk_incremental_endpoint in INCREMENTAL_TALK_ENDPOINTS.items():
+ yield dlt.resource(
+ talk_incremental_resource,
+ name=f"{key}_incremental",
+ primary_key="id",
+ write_disposition="merge",
+ )(
+ zendesk_client=zendesk_client,
+ talk_endpoint_name=key,
+ talk_endpoint=talk_incremental_endpoint,
+ updated_at=dlt.sources.incremental[str](
+ "updated_at",
+ initial_value=start_date_obj.isoformat(),
+ end_value=end_date_obj.isoformat() if end_date_obj else None,
+ allow_external_schedulers=True,
+ ),
+ )
+
+
+def talk_resource(
+ zendesk_client: ZendeskAPIClient,
+ talk_endpoint_name: str,
+ talk_endpoint: str,
+ pagination_type: PaginationType,
+) -> Iterator[TDataItem]:
+ """
+ Loads data from a Zendesk Talk endpoint.
+
+ Args:
+ zendesk_client: An instance of ZendeskAPIClient for making API calls to Zendesk Talk.
+ talk_endpoint_name: The name of the talk_endpoint.
+ talk_endpoint: The actual URL ending of the endpoint.
+ pagination: Type of pagination type used by endpoint
+
+ Yields:
+ TDataItem: Dictionary containing the data from the endpoint.
+ """
+ # send query and process it
+ yield from zendesk_client.get_pages(talk_endpoint, talk_endpoint_name, pagination_type)
+
+
+def talk_incremental_resource(
+ zendesk_client: ZendeskAPIClient,
+ talk_endpoint_name: str,
+ talk_endpoint: str,
+ updated_at: dlt.sources.incremental[str],
+) -> Iterator[TDataItem]:
+ """
+ Loads data from a Zendesk Talk endpoint with incremental loading.
+
+ Args:
+ zendesk_client: An instance of ZendeskAPIClient for making API calls to Zendesk Talk.
+ talk_endpoint_name: The name of the talk_endpoint.
+ talk_endpoint: The actual URL ending of the endpoint.
+ updated_at: Source for the last updated timestamp.
+
+ Yields:
+ TDataItem: Dictionary containing the data from the endpoint.
+ """
+ # send the request and process it
+ for page in zendesk_client.get_pages(
+ talk_endpoint,
+ talk_endpoint_name,
+ PaginationType.START_TIME,
+ params={"start_time": ensure_pendulum_datetime(updated_at.last_value).int_timestamp},
+ ):
+ yield page
+ if updated_at.end_out_of_range:
+ return
+
+
+@dlt.source(max_table_nesting=0)
+def zendesk_chat(
+ credentials: ZendeskCredentialsOAuth = dlt.secrets.value,
+ start_date: Optional[TAnyDateTime] = DEFAULT_START_DATE,
+ end_date: Optional[TAnyDateTime] = None,
+) -> Iterable[DltResource]:
+ """
+ Retrieves data from Zendesk Chat for chat interactions.
+
+ `start_date` argument can be used on its own or together with `end_date`. When both are provided
+ data is limited to items updated in that time range.
+ The range is "half-open", meaning elements equal and higher than `start_date` and elements lower than `end_date` are included.
+ All resources opt-in to use Airflow scheduler if run as Airflow task
+
+ Args:
+ credentials: The credentials for authentication. Defaults to the value in the `dlt.secrets` object.
+ start_date: The start time of the range for which to load. Defaults to January 1st 2000.
+ end_date: The end time of the range for which to load data.
+ If end time is not provided, the incremental loading will be enabled and after initial run, only new data will be retrieved
+
+ Yields:
+ DltResource: Data resources from Zendesk Chat.
+ """
+
+ # Authenticate
+ zendesk_client = ZendeskAPIClient(credentials, url_prefix="https://www.zopim.com")
+ start_date_obj = ensure_pendulum_datetime(start_date)
+ end_date_obj = ensure_pendulum_datetime(end_date) if end_date else None
+
+ yield dlt.resource(chats_table_resource, name="chats", write_disposition="merge")(
+ zendesk_client,
+ dlt.sources.incremental[str](
+ "update_timestamp|updated_timestamp",
+ initial_value=start_date_obj.isoformat(),
+ end_value=end_date_obj.isoformat() if end_date_obj else None,
+ allow_external_schedulers=True,
+ ),
+ )
+
+
+def chats_table_resource(
+ zendesk_client: ZendeskAPIClient,
+ update_timestamp: dlt.sources.incremental[str],
+) -> Iterator[TDataItems]:
+ """
+ Resource for Chats
+
+ Args:
+ zendesk_client: The Zendesk API client instance, used to make calls to Zendesk API.
+ update_timestamp: Incremental source specifying the timestamp for incremental loading.
+
+ Yields:
+ dict: A dictionary representing each row of data.
+ """
+ chat_pages = zendesk_client.get_pages(
+ "/api/v2/incremental/chats",
+ "chats",
+ PaginationType.START_TIME,
+ params={
+ "start_time": ensure_pendulum_datetime(update_timestamp.last_value).int_timestamp,
+ "fields": "chats(*)",
+ },
+ )
+ for page in chat_pages:
+ yield page
+
+ if update_timestamp.end_out_of_range:
+ return
+
+
+@dlt.source(max_table_nesting=0)
+def zendesk_support(
+ team_id: int,
+ credentials: TZendeskCredentials = dlt.secrets.value,
+ endpoints: Tuple[str, ...] = (),
+ pivot_ticket_fields: bool = True,
+ start_date: Optional[TAnyDateTime] = DEFAULT_START_DATE,
+ end_date: Optional[TAnyDateTime] = None,
+) -> Iterable[DltResource]:
+ """
+ Retrieves data from Zendesk Support for tickets, users, brands, organizations, and groups.
+
+ `start_date` argument can be used on its own or together with `end_date`. When both are provided
+ data is limited to items updated in that time range.
+ The range is "half-open", meaning elements equal and higher than `start_date` and elements lower than `end_date` are included.
+ All resources opt-in to use Airflow scheduler if run as Airflow task
+
+ Args:
+ credentials: The credentials for authentication. Defaults to the value in the `dlt.secrets` object.
+ load_all: Whether to load extra resources for the API. Defaults to True.
+ pivot_ticket_fields: Whether to pivot the custom fields in tickets. Defaults to True.
+ start_date: The start time of the range for which to load. Defaults to January 1st 2000.
+ end_date: The end time of the range for which to load data.
+ If end time is not provided, the incremental loading will be enabled and after initial run, only new data will be retrieved
+
+ Returns:
+ Sequence[DltResource]: Multiple dlt resources.
+ """
+
+ start_date_obj = ensure_pendulum_datetime(start_date)
+ end_date_obj = ensure_pendulum_datetime(end_date) if end_date else None
+
+ start_date_ts = start_date_obj.int_timestamp
+ start_date_iso_str = start_date_obj.isoformat()
+ end_date_ts: Optional[int] = None
+ end_date_iso_str: Optional[str] = None
+ if end_date_obj:
+ end_date_ts = end_date_obj.int_timestamp
+ end_date_iso_str = end_date_obj.isoformat()
+
+ @dlt.resource(name="ticket_events", primary_key="id", write_disposition="append")
+ def ticket_events(
+ zendesk_client: ZendeskAPIClient,
+ timestamp: dlt.sources.incremental[int] = dlt.sources.incremental( # noqa: B008
+ "timestamp",
+ initial_value=start_date_ts,
+ end_value=end_date_ts,
+ allow_external_schedulers=True,
+ ),
+ ) -> Iterator[TDataItem]:
+ # URL For ticket events
+ # 'https://d3v-dlthub.zendesk.com/api/v2/incremental/ticket_events.json?start_time=946684800'
+ event_pages = zendesk_client.get_pages(
+ "/api/v2/incremental/ticket_events.json",
+ "ticket_events",
+ PaginationType.STREAM,
+ params={"start_time": timestamp.last_value},
+ )
+ for page in event_pages:
+ yield page
+ if timestamp.end_out_of_range:
+ return
+
+ @dlt.resource(
+ name="tickets",
+ primary_key="id",
+ write_disposition="merge",
+ columns={
+ "tags": {"data_type": "complex"},
+ "custom_fields": {"data_type": "complex"},
+ },
+ )
+ def ticket_table(
+ zendesk_client: ZendeskAPIClient,
+ pivot_fields: bool = True,
+ updated_at: dlt.sources.incremental[pendulum.DateTime] = dlt.sources.incremental( # noqa: B008
+ "updated_at",
+ initial_value=start_date_obj,
+ end_value=end_date_obj,
+ allow_external_schedulers=True,
+ ),
+ ) -> Iterator[TDataItem]:
+ """
+ Resource for tickets table. Uses DLT state to handle column renaming of custom fields to prevent changing the names of said columns.
+ This resource uses pagination, loading and side loading to make API calls more efficient.
+
+ Args:
+ zendesk_client: The Zendesk API client instance, used to make calls to Zendesk API.
+ pivot_fields: Indicates whether to pivot the custom fields in tickets. Defaults to True.
+ per_page: The number of Ticket objects to load per page. Defaults to 1000.
+ updated_at: Incremental source for the 'updated_at' column.
+ Defaults to dlt.sources.incremental("updated_at", initial_value=start_date).
+
+ Yields:
+ TDataItem: Dictionary containing the ticket data.
+ """
+ # grab the custom fields from dlt state if any
+ if pivot_fields:
+ load_ticket_fields_state(zendesk_client)
+ fields_dict = dlt.current.source_state().setdefault(CUSTOM_FIELDS_STATE_KEY, {})
+ # include_objects = ["users", "groups", "organisation", "brands"]
+ ticket_pages = zendesk_client.get_pages(
+ "/api/v2/incremental/tickets",
+ "tickets",
+ PaginationType.STREAM,
+ params={"start_time": updated_at.last_value.int_timestamp},
+ )
+ for page in ticket_pages:
+ yield [process_ticket(ticket, fields_dict, pivot_custom_fields=pivot_fields) for ticket in page]
+
+ # stop loading when using end_value and end is reached
+ if updated_at.end_out_of_range:
+ return
+
+ @dlt.resource(name="ticket_metric_events", primary_key="id", write_disposition="append")
+ def ticket_metric_table(
+ zendesk_client: ZendeskAPIClient,
+ time: dlt.sources.incremental[str] = dlt.sources.incremental( # noqa: B008
+ "time",
+ initial_value=start_date_iso_str,
+ end_value=end_date_iso_str,
+ allow_external_schedulers=True,
+ ),
+ ) -> Iterator[TDataItem]:
+ """
+ Resource for ticket metric events table. Returns all the ticket metric events from the starting date,
+ with the default starting date being January 1st of the current year.
+
+ Args:
+ zendesk_client: The Zendesk API client instance, used to make calls to Zendesk API.
+ time: Incremental source for the 'time' column,
+ indicating the starting date for retrieving ticket metric events.
+ Defaults to dlt.sources.incremental("time", initial_value=start_date_iso_str).
+
+ Yields:
+ TDataItem: Dictionary containing the ticket metric event data.
+ """
+ # "https://example.zendesk.com/api/v2/incremental/ticket_metric_events?start_time=1332034771"
+ metric_event_pages = zendesk_client.get_pages(
+ "/api/v2/incremental/ticket_metric_events",
+ "ticket_metric_events",
+ PaginationType.CURSOR,
+ params={
+ "start_time": ensure_pendulum_datetime(time.last_value).int_timestamp,
+ },
+ )
+ for page in metric_event_pages:
+ yield page
+
+ if time.end_out_of_range:
+ return
+
+ def ticket_fields_table(zendesk_client: ZendeskAPIClient) -> Iterator[TDataItem]:
+ """
+ Loads ticket fields data from Zendesk API.
+
+ Args:
+ zendesk_client: The Zendesk API client instance, used to make calls to Zendesk API.
+
+ Yields:
+ TDataItem: Dictionary containing the ticket fields data.
+ """
+ # get dlt state
+ ticket_custom_fields = dlt.current.source_state().setdefault(CUSTOM_FIELDS_STATE_KEY, {})
+ # get all custom fields and update state if needed, otherwise just load dicts into tables
+ all_fields = list(
+ chain.from_iterable(
+ zendesk_client.get_pages("/api/v2/ticket_fields.json", "ticket_fields", PaginationType.OFFSET)
+ )
+ )
+ # all_fields = zendesk_client.ticket_fields()
+ for field in all_fields:
+ yield process_ticket_field(field, ticket_custom_fields)
+
+ def load_ticket_fields_state(
+ zendesk_client: ZendeskAPIClient,
+ ) -> None:
+ for _ in ticket_fields_table(zendesk_client):
+ pass
+
+ ticket_fields_resource = dlt.resource(name="ticket_fields", write_disposition="replace")(ticket_fields_table)
+
+ # Authenticate
+ zendesk_client = ZendeskAPIClient(credentials)
+
+ all_endpoints = SUPPORT_ENDPOINTS + SUPPORT_EXTRA_ENDPOINTS
+ resource_list: List[DltResource] = []
+
+ for endpoint in endpoints:
+ # loading base tables
+ if endpoint == "ticket_fields":
+ resource_list.append(ticket_fields_resource(zendesk_client=zendesk_client))
+ elif endpoint == "ticket_events":
+ resource_list.append(ticket_events(zendesk_client=zendesk_client))
+ elif endpoint == "tickets":
+ resource_list.append(ticket_table(zendesk_client=zendesk_client, pivot_fields=pivot_ticket_fields))
+ elif endpoint == "ticket_metric_events":
+ resource_list.append(ticket_metric_table(zendesk_client=zendesk_client))
+ else:
+ # other tables to be loaded
+ for resource, endpoint_url, data_key, cursor_paginated in all_endpoints:
+ if endpoint == resource:
+ resource_list.append(
+ dlt.resource(
+ basic_resource(zendesk_client, endpoint_url, data_key or resource, cursor_paginated),
+ name=resource,
+ write_disposition="replace",
+ )
+ )
+ break
+
+ return resource_list
+
+
+def basic_resource(
+ zendesk_client: ZendeskAPIClient,
+ endpoint_url: str,
+ data_key: str,
+ cursor_paginated: bool,
+) -> Iterator[TDataItem]:
+ """
+ Basic loader for most endpoints offered by Zenpy. Supports pagination. Expects to be called as a DLT Resource.
+
+ Args:
+ zendesk_client: The Zendesk API client instance, used to make calls to Zendesk API.
+ resource: The Zenpy endpoint to retrieve data from, usually directly linked to a Zendesk API endpoint.
+ cursor_paginated: Tells to use CURSOR pagination or OFFSET/no pagination
+
+ Yields:
+ TDataItem: Dictionary containing the resource data.
+ """
+
+ pages = zendesk_client.get_pages(
+ endpoint_url,
+ data_key,
+ PaginationType.CURSOR if cursor_paginated else PaginationType.OFFSET,
+ )
+ yield from pages
diff --git a/posthog/temporal/data_imports/pipelines/zendesk/settings.py b/posthog/temporal/data_imports/pipelines/zendesk/settings.py
new file mode 100644
index 0000000000000..aa44df7c20297
--- /dev/null
+++ b/posthog/temporal/data_imports/pipelines/zendesk/settings.py
@@ -0,0 +1,73 @@
+"""Zendesk source settings and constants"""
+
+from dlt.common import pendulum
+
+DEFAULT_START_DATE = pendulum.datetime(year=2000, month=1, day=1)
+PAGE_SIZE = 100
+INCREMENTAL_PAGE_SIZE = 1000
+
+
+CUSTOM_FIELDS_STATE_KEY = "ticket_custom_fields_v2"
+
+# Resources that will always get pulled
+BASE_ENDPOINTS = ["ticket_fields", "ticket_events", "tickets", "ticket_metric_events"]
+
+# Tuples of (Resource name, endpoint URL, data_key, supports pagination)
+# data_key is the key which data list is nested under in responses
+# if the data key is None it is assumed to be the same as the resource name
+# The last element of the tuple says if endpoint supports cursor pagination
+SUPPORT_ENDPOINTS = [
+ ("users", "/api/v2/users.json", "users", True),
+ ("sla_policies", "/api/v2/slas/policies.json", None, False),
+ ("groups", "/api/v2/groups.json", None, True),
+ ("organizations", "/api/v2/organizations.json", None, True),
+ ("brands", "/api/v2/brands.json", None, True),
+]
+
+SUPPORT_EXTRA_ENDPOINTS = [
+ ("activities", "/api/v2/activities.json", None, True),
+ ("automations", "/api/v2/automations.json", None, True),
+ ("custom_agent_roles", "/api/v2/custom_roles.json", "custom_roles", False),
+ ("dynamic_content", "/api/v2/dynamic_content/items.json", "items", True),
+ ("group_memberships", "/api/v2/group_memberships.json", None, True),
+ ("job_status", "/api/v2/job_statuses.json", "job_statuses", True),
+ ("macros", "/api/v2/macros.json", None, True),
+ ("organization_fields", "/api/v2/organization_fields.json", None, True),
+ ("organization_memberships", "/api/v2/organization_memberships.json", None, True),
+ ("recipient_addresses", "/api/v2/recipient_addresses.json", None, True),
+ ("requests", "/api/v2/requests.json", None, True),
+ ("satisfaction_ratings", "/api/v2/satisfaction_ratings.json", None, True),
+ ("sharing_agreements", "/api/v2/sharing_agreements.json", None, False),
+ ("skips", "/api/v2/skips.json", None, True),
+ ("suspended_tickets", "/api/v2/suspended_tickets.json", None, True),
+ ("targets", "/api/v2/targets.json", None, False),
+ ("ticket_forms", "/api/v2/ticket_forms.json", None, False),
+ ("ticket_metrics", "/api/v2/ticket_metrics.json", None, True),
+ ("triggers", "/api/v2/triggers.json", None, True),
+ ("user_fields", "/api/v2/user_fields.json", None, True),
+ ("views", "/api/v2/views.json", None, True),
+ ("tags", "/api/v2/tags.json", None, True),
+]
+
+TALK_ENDPOINTS = [
+ ("calls", "/api/v2/channels/voice/calls", None, False),
+ ("addresses", "/api/v2/channels/voice/addresses", None, False),
+ ("greeting_categories", "/api/v2/channels/voice/greeting_categories", None, False),
+ ("greetings", "/api/v2/channels/voice/greetings", None, False),
+ ("ivrs", "/api/v2/channels/voice/ivr", None, False),
+ ("phone_numbers", "/api/v2/channels/voice/phone_numbers", None, False),
+ ("settings", "/api/v2/channels/voice/settings", None, False),
+ ("lines", "/api/v2/channels/voice/lines", None, False),
+ ("agents_activity", "/api/v2/channels/voice/stats/agents_activity", None, False),
+ (
+ "current_queue_activity",
+ "/api/v2/channels/voice/stats/current_queue_activity",
+ None,
+ False,
+ ),
+]
+
+INCREMENTAL_TALK_ENDPOINTS = {
+ "calls": "/api/v2/channels/voice/stats/incremental/calls.json",
+ "legs": "/api/v2/channels/voice/stats/incremental/legs.json",
+}
diff --git a/posthog/temporal/data_imports/pipelines/zendesk/talk_api.py b/posthog/temporal/data_imports/pipelines/zendesk/talk_api.py
new file mode 100644
index 0000000000000..5db9a28eafc74
--- /dev/null
+++ b/posthog/temporal/data_imports/pipelines/zendesk/talk_api.py
@@ -0,0 +1,114 @@
+from enum import Enum
+from typing import Dict, Iterator, Optional, Tuple, Any
+from dlt.common.typing import DictStrStr, TDataItems, TSecretValue
+from dlt.sources.helpers.requests import client
+
+from . import settings
+from .credentials import (
+ ZendeskCredentialsEmailPass,
+ ZendeskCredentialsOAuth,
+ ZendeskCredentialsToken,
+ TZendeskCredentials,
+)
+
+
+class PaginationType(Enum):
+ OFFSET = 0
+ CURSOR = 1
+ STREAM = 2
+ START_TIME = 3
+
+
+class ZendeskAPIClient:
+ """
+ API client used to make requests to Zendesk talk, support and chat API
+ """
+
+ subdomain: str = ""
+ url: str = ""
+ headers: Optional[DictStrStr]
+ auth: Optional[Tuple[str, TSecretValue]]
+
+ def __init__(self, credentials: TZendeskCredentials, url_prefix: Optional[str] = None) -> None:
+ """
+ Initializer for the API client which is then used to make API calls to the ZendeskAPI
+
+ Args:
+ credentials: ZendeskCredentials object which contains the necessary credentials to authenticate to ZendeskAPI
+ """
+ # oauth token is the preferred way to authenticate, followed by api token and then email + password combo
+ # fill headers and auth for every possibility of credentials given, raise error if credentials are of incorrect type
+ if isinstance(credentials, ZendeskCredentialsOAuth):
+ self.headers = {"Authorization": f"Bearer {credentials.oauth_token}"}
+ self.auth = None
+ elif isinstance(credentials, ZendeskCredentialsToken):
+ self.headers = None
+ self.auth = (f"{credentials.email}/token", credentials.token)
+ elif isinstance(credentials, ZendeskCredentialsEmailPass):
+ self.auth = (credentials.email, credentials.password)
+ self.headers = None
+ else:
+ raise TypeError(
+ "Wrong credentials type provided to ZendeskAPIClient. The credentials need to be of type: ZendeskCredentialsOAuth, ZendeskCredentialsToken or ZendeskCredentialsEmailPass"
+ )
+
+ # If url_prefix is set it overrides the default API URL (e.g. chat api uses zopim.com domain)
+ if url_prefix:
+ self.url = url_prefix
+ else:
+ self.subdomain = credentials.subdomain
+ self.url = f"https://{self.subdomain}.zendesk.com"
+
+ def get_pages(
+ self,
+ endpoint: str,
+ data_point_name: str,
+ pagination: PaginationType,
+ params: Optional[Dict[str, Any]] = None,
+ ) -> Iterator[TDataItems]:
+ """
+ Makes a request to a paginated endpoint and returns a generator of data items per page.
+
+ Args:
+ endpoint: The url to the endpoint, e.g. /api/v2/calls
+ data_point_name: The key which data items are nested under in the response object (e.g. calls)
+ params: Optional dict of query params to include in the request
+ pagination: Type of pagination type used by endpoint
+
+ Returns:
+ Generator of pages, each page is a list of dict data items
+ """
+
+ # update the page size to enable cursor pagination
+ params = params or {}
+ if pagination == PaginationType.CURSOR:
+ params["page[size]"] = settings.PAGE_SIZE
+ elif pagination == PaginationType.STREAM:
+ params["per_page"] = settings.INCREMENTAL_PAGE_SIZE
+ elif pagination == PaginationType.START_TIME:
+ params["limit"] = settings.INCREMENTAL_PAGE_SIZE
+
+ # make request and keep looping until there is no next page
+ get_url = f"{self.url}{endpoint}"
+ while get_url:
+ response = client.get(get_url, headers=self.headers, auth=self.auth, params=params)
+ response.raise_for_status()
+ response_json = response.json()
+ result = response_json[data_point_name]
+ yield result
+
+ get_url = None
+ if pagination == PaginationType.CURSOR:
+ if response_json["meta"]["has_more"]:
+ get_url = response_json["links"]["next"]
+ elif pagination == PaginationType.OFFSET:
+ get_url = response_json.get("next_page", None)
+ elif pagination == PaginationType.STREAM:
+ # See https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/#json-format
+ if not response_json["end_of_stream"]:
+ get_url = response_json["next_page"]
+ elif pagination == PaginationType.START_TIME:
+ if response_json["count"] > 0:
+ get_url = response_json["next_page"]
+
+ params = {}
diff --git a/posthog/temporal/tests/batch_exports/test_batch_exports.py b/posthog/temporal/tests/batch_exports/test_batch_exports.py
index 0afbfcabb71cb..756c07e442e4f 100644
--- a/posthog/temporal/tests/batch_exports/test_batch_exports.py
+++ b/posthog/temporal/tests/batch_exports/test_batch_exports.py
@@ -1,6 +1,4 @@
-import csv
import datetime as dt
-import io
import json
import operator
from random import randint
@@ -9,11 +7,9 @@
from django.test import override_settings
from posthog.temporal.batch_exports.batch_exports import (
- BatchExportTemporaryFile,
get_data_interval,
get_rows_count,
iter_records,
- json_dumps_bytes,
)
from posthog.temporal.tests.utils.events import generate_test_events_in_clickhouse
@@ -558,181 +554,3 @@ def test_get_data_interval(interval, data_interval_end, expected):
"""Test get_data_interval returns the expected data interval tuple."""
result = get_data_interval(interval, data_interval_end)
assert result == expected
-
-
-@pytest.mark.parametrize(
- "to_write",
- [
- (b"",),
- (b"", b""),
- (b"12345",),
- (b"12345", b"12345"),
- (b"abbcccddddeeeee",),
- (b"abbcccddddeeeee", b"abbcccddddeeeee"),
- ],
-)
-def test_batch_export_temporary_file_tracks_bytes(to_write):
- """Test the bytes written by BatchExportTemporaryFile match expected."""
- with BatchExportTemporaryFile() as be_file:
- for content in to_write:
- be_file.write(content)
-
- assert be_file.bytes_total == sum(len(content) for content in to_write)
- assert be_file.bytes_since_last_reset == sum(len(content) for content in to_write)
-
- be_file.reset()
-
- assert be_file.bytes_total == sum(len(content) for content in to_write)
- assert be_file.bytes_since_last_reset == 0
-
-
-TEST_RECORDS = [
- [],
- [
- {"id": "record-1", "property": "value", "property_int": 1},
- {"id": "record-2", "property": "another-value", "property_int": 2},
- {
- "id": "record-3",
- "property": {"id": "nested-record", "property": "nested-value"},
- "property_int": 3,
- },
- ],
-]
-
-
-@pytest.mark.parametrize(
- "records",
- TEST_RECORDS,
-)
-def test_batch_export_temporary_file_write_records_to_jsonl(records):
- """Test JSONL records written by BatchExportTemporaryFile match expected."""
- jsonl_dump = b"\n".join(map(json_dumps_bytes, records))
-
- with BatchExportTemporaryFile() as be_file:
- be_file.write_records_to_jsonl(records)
-
- assert be_file.bytes_total == len(jsonl_dump)
- assert be_file.bytes_since_last_reset == len(jsonl_dump)
- assert be_file.records_total == len(records)
- assert be_file.records_since_last_reset == len(records)
-
- be_file.seek(0)
- lines = be_file.readlines()
- assert len(lines) == len(records)
-
- for line_index, jsonl_record in enumerate(lines):
- json_loaded = json.loads(jsonl_record)
- assert json_loaded == records[line_index]
-
- be_file.reset()
-
- assert be_file.bytes_total == len(jsonl_dump)
- assert be_file.bytes_since_last_reset == 0
- assert be_file.records_total == len(records)
- assert be_file.records_since_last_reset == 0
-
-
-@pytest.mark.parametrize(
- "records",
- TEST_RECORDS,
-)
-def test_batch_export_temporary_file_write_records_to_csv(records):
- """Test CSV written by BatchExportTemporaryFile match expected."""
- in_memory_file_obj = io.StringIO()
- writer = csv.DictWriter(
- in_memory_file_obj,
- fieldnames=records[0].keys() if len(records) > 0 else [],
- delimiter=",",
- quotechar='"',
- escapechar="\\",
- lineterminator="\n",
- quoting=csv.QUOTE_NONE,
- )
- writer.writerows(records)
-
- with BatchExportTemporaryFile(mode="w+") as be_file:
- be_file.write_records_to_csv(records)
-
- assert be_file.bytes_total == in_memory_file_obj.tell()
- assert be_file.bytes_since_last_reset == in_memory_file_obj.tell()
- assert be_file.records_total == len(records)
- assert be_file.records_since_last_reset == len(records)
-
- be_file.seek(0)
- reader = csv.reader(
- be_file._file,
- delimiter=",",
- quotechar='"',
- escapechar="\\",
- quoting=csv.QUOTE_NONE,
- )
-
- rows = [row for row in reader]
- assert len(rows) == len(records)
-
- for row_index, csv_record in enumerate(rows):
- for value_index, value in enumerate(records[row_index].values()):
- # Everything returned by csv.reader is a str.
- # This means type information is lost when writing to CSV
- # but this just a limitation of the format.
- assert csv_record[value_index] == str(value)
-
- be_file.reset()
-
- assert be_file.bytes_total == in_memory_file_obj.tell()
- assert be_file.bytes_since_last_reset == 0
- assert be_file.records_total == len(records)
- assert be_file.records_since_last_reset == 0
-
-
-@pytest.mark.parametrize(
- "records",
- TEST_RECORDS,
-)
-def test_batch_export_temporary_file_write_records_to_tsv(records):
- """Test TSV written by BatchExportTemporaryFile match expected."""
- in_memory_file_obj = io.StringIO()
- writer = csv.DictWriter(
- in_memory_file_obj,
- fieldnames=records[0].keys() if len(records) > 0 else [],
- delimiter="\t",
- quotechar='"',
- escapechar="\\",
- lineterminator="\n",
- quoting=csv.QUOTE_NONE,
- )
- writer.writerows(records)
-
- with BatchExportTemporaryFile(mode="w+") as be_file:
- be_file.write_records_to_tsv(records)
-
- assert be_file.bytes_total == in_memory_file_obj.tell()
- assert be_file.bytes_since_last_reset == in_memory_file_obj.tell()
- assert be_file.records_total == len(records)
- assert be_file.records_since_last_reset == len(records)
-
- be_file.seek(0)
- reader = csv.reader(
- be_file._file,
- delimiter="\t",
- quotechar='"',
- escapechar="\\",
- quoting=csv.QUOTE_NONE,
- )
-
- rows = [row for row in reader]
- assert len(rows) == len(records)
-
- for row_index, csv_record in enumerate(rows):
- for value_index, value in enumerate(records[row_index].values()):
- # Everything returned by csv.reader is a str.
- # This means type information is lost when writing to CSV
- # but this just a limitation of the format.
- assert csv_record[value_index] == str(value)
-
- be_file.reset()
-
- assert be_file.bytes_total == in_memory_file_obj.tell()
- assert be_file.bytes_since_last_reset == 0
- assert be_file.records_total == len(records)
- assert be_file.records_since_last_reset == 0
diff --git a/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py b/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py
index e04e345d11245..e6583d049e2a8 100644
--- a/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py
+++ b/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py
@@ -10,10 +10,12 @@
import aioboto3
import botocore.exceptions
import brotli
+import pyarrow.parquet as pq
import pytest
import pytest_asyncio
from django.conf import settings
from django.test import override_settings
+from pyarrow import fs
from temporalio import activity
from temporalio.client import WorkflowFailureError
from temporalio.common import RetryPolicy
@@ -27,6 +29,7 @@
update_export_run_status,
)
from posthog.temporal.batch_exports.s3_batch_export import (
+ FILE_FORMAT_EXTENSIONS,
HeartbeatDetails,
S3BatchExportInputs,
S3BatchExportWorkflow,
@@ -107,6 +110,15 @@ def s3_key_prefix():
return f"posthog-events-{str(uuid4())}"
+@pytest.fixture
+def file_format(request) -> str:
+ """S3 file format."""
+ try:
+ return request.param
+ except AttributeError:
+ return f"JSONLines"
+
+
async def delete_all_from_s3(minio_client, bucket_name: str, key_prefix: str):
"""Delete all objects in bucket_name under key_prefix."""
response = await minio_client.list_objects_v2(Bucket=bucket_name, Prefix=key_prefix)
@@ -138,6 +150,61 @@ async def minio_client(bucket_name):
await minio_client.delete_bucket(Bucket=bucket_name)
+async def read_parquet_from_s3(bucket_name: str, key: str, json_columns) -> list:
+ async with aioboto3.Session().client("sts") as sts:
+ try:
+ await sts.get_caller_identity()
+ except botocore.exceptions.NoCredentialsError:
+ s3 = fs.S3FileSystem(
+ access_key="object_storage_root_user",
+ secret_key="object_storage_root_password",
+ endpoint_override=settings.OBJECT_STORAGE_ENDPOINT,
+ )
+
+ else:
+ if os.getenv("S3_TEST_BUCKET") is not None:
+ s3 = fs.S3FileSystem()
+ else:
+ s3 = fs.S3FileSystem(
+ access_key="object_storage_root_user",
+ secret_key="object_storage_root_password",
+ endpoint_override=settings.OBJECT_STORAGE_ENDPOINT,
+ )
+
+ table = pq.read_table(f"{bucket_name}/{key}", filesystem=s3)
+
+ parquet_data = []
+ for batch in table.to_batches():
+ for record in batch.to_pylist():
+ casted_record = {}
+ for k, v in record.items():
+ if isinstance(v, dt.datetime):
+ # We read data from clickhouse as string, but parquet already casts them as dates.
+ # To facilitate comparison, we isoformat the dates.
+ casted_record[k] = v.isoformat()
+ elif k in json_columns and v is not None:
+ # Parquet doesn't have a variable map type, so JSON fields are just strings.
+ casted_record[k] = json.loads(v)
+ else:
+ casted_record[k] = v
+ parquet_data.append(casted_record)
+
+ return parquet_data
+
+
+def read_s3_data_as_json(data: bytes, compression: str | None) -> list:
+ match compression:
+ case "gzip":
+ data = gzip.decompress(data)
+ case "brotli":
+ data = brotli.decompress(data)
+ case _:
+ pass
+
+ json_data = [json.loads(line) for line in data.decode("utf-8").split("\n") if line]
+ return json_data
+
+
async def assert_clickhouse_records_in_s3(
s3_compatible_client,
clickhouse_client: ClickHouseClient,
@@ -150,6 +217,7 @@ async def assert_clickhouse_records_in_s3(
include_events: list[str] | None = None,
batch_export_schema: BatchExportSchema | None = None,
compression: str | None = None,
+ file_format: str = "JSONLines",
):
"""Assert ClickHouse records are written to JSON in key_prefix in S3 bucket_name.
@@ -175,28 +243,24 @@ async def assert_clickhouse_records_in_s3(
# Get the object.
key = objects["Contents"][0].get("Key")
assert key
- s3_object = await s3_compatible_client.get_object(Bucket=bucket_name, Key=key)
- data = await s3_object["Body"].read()
- # Check that the data is correct.
- match compression:
- case "gzip":
- data = gzip.decompress(data)
- case "brotli":
- data = brotli.decompress(data)
- case _:
- pass
+ json_columns = ("properties", "person_properties", "set", "set_once")
- json_data = [json.loads(line) for line in data.decode("utf-8").split("\n") if line]
- # Pull out the fields we inserted only
+ if file_format == "Parquet":
+ s3_data = await read_parquet_from_s3(bucket_name, key, json_columns)
+
+ elif file_format == "JSONLines":
+ s3_object = await s3_compatible_client.get_object(Bucket=bucket_name, Key=key)
+ data = await s3_object["Body"].read()
+ s3_data = read_s3_data_as_json(data, compression)
+ else:
+ raise ValueError(f"Unsupported file format: {file_format}")
if batch_export_schema is not None:
schema_column_names = [field["alias"] for field in batch_export_schema["fields"]]
else:
schema_column_names = [field["alias"] for field in s3_default_fields()]
- json_columns = ("properties", "person_properties", "set", "set_once")
-
expected_records = []
for record_batch in iter_records(
client=clickhouse_client,
@@ -225,9 +289,9 @@ async def assert_clickhouse_records_in_s3(
expected_records.append(expected_record)
- assert len(json_data) == len(expected_records)
- assert json_data[0] == expected_records[0]
- assert json_data == expected_records
+ assert len(s3_data) == len(expected_records)
+ assert s3_data[0] == expected_records[0]
+ assert s3_data == expected_records
TEST_S3_SCHEMAS: list[BatchExportSchema | None] = [
@@ -255,6 +319,7 @@ async def assert_clickhouse_records_in_s3(
@pytest.mark.parametrize("compression", [None, "gzip", "brotli"], indirect=True)
@pytest.mark.parametrize("exclude_events", [None, ["test-exclude"]], indirect=True)
@pytest.mark.parametrize("batch_export_schema", TEST_S3_SCHEMAS)
+@pytest.mark.parametrize("file_format", FILE_FORMAT_EXTENSIONS.keys())
async def test_insert_into_s3_activity_puts_data_into_s3(
clickhouse_client,
bucket_name,
@@ -262,6 +327,7 @@ async def test_insert_into_s3_activity_puts_data_into_s3(
activity_environment,
compression,
exclude_events,
+ file_format,
batch_export_schema: BatchExportSchema | None,
):
"""Test that the insert_into_s3_activity function ends up with data into S3.
@@ -339,12 +405,15 @@ async def test_insert_into_s3_activity_puts_data_into_s3(
compression=compression,
exclude_events=exclude_events,
batch_export_schema=batch_export_schema,
+ file_format=file_format,
)
with override_settings(
BATCH_EXPORT_S3_UPLOAD_CHUNK_SIZE_BYTES=5 * 1024**2
): # 5MB, the minimum for Multipart uploads
- await activity_environment.run(insert_into_s3_activity, insert_inputs)
+ records_total = await activity_environment.run(insert_into_s3_activity, insert_inputs)
+
+ assert records_total == 10005
await assert_clickhouse_records_in_s3(
s3_compatible_client=minio_client,
@@ -358,6 +427,7 @@ async def test_insert_into_s3_activity_puts_data_into_s3(
exclude_events=exclude_events,
include_events=None,
compression=compression,
+ file_format=file_format,
)
@@ -371,6 +441,7 @@ async def s3_batch_export(
exclude_events,
temporal_client,
encryption,
+ file_format,
):
destination_data = {
"type": "S3",
@@ -385,6 +456,7 @@ async def s3_batch_export(
"exclude_events": exclude_events,
"encryption": encryption,
"kms_key_id": os.getenv("S3_TEST_KMS_KEY_ID") if encryption == "aws:kms" else None,
+ "file_format": file_format,
},
}
@@ -410,6 +482,7 @@ async def s3_batch_export(
@pytest.mark.parametrize("compression", [None, "gzip", "brotli"], indirect=True)
@pytest.mark.parametrize("exclude_events", [None, ["test-exclude"]], indirect=True)
@pytest.mark.parametrize("batch_export_schema", TEST_S3_SCHEMAS)
+@pytest.mark.parametrize("file_format", FILE_FORMAT_EXTENSIONS.keys(), indirect=True)
async def test_s3_export_workflow_with_minio_bucket(
clickhouse_client,
minio_client,
@@ -421,6 +494,7 @@ async def test_s3_export_workflow_with_minio_bucket(
exclude_events,
s3_key_prefix,
batch_export_schema,
+ file_format,
):
"""Test S3BatchExport Workflow end-to-end by using a local MinIO bucket instead of S3.
@@ -508,6 +582,7 @@ async def test_s3_export_workflow_with_minio_bucket(
batch_export_schema=batch_export_schema,
exclude_events=exclude_events,
compression=compression,
+ file_format=file_format,
)
@@ -537,6 +612,7 @@ async def s3_client(bucket_name, s3_key_prefix):
@pytest.mark.parametrize("encryption", [None, "AES256", "aws:kms"], indirect=True)
@pytest.mark.parametrize("bucket_name", [os.getenv("S3_TEST_BUCKET")], indirect=True)
@pytest.mark.parametrize("batch_export_schema", TEST_S3_SCHEMAS)
+@pytest.mark.parametrize("file_format", FILE_FORMAT_EXTENSIONS.keys(), indirect=True)
async def test_s3_export_workflow_with_s3_bucket(
s3_client,
clickhouse_client,
@@ -549,6 +625,7 @@ async def test_s3_export_workflow_with_s3_bucket(
exclude_events,
ateam,
batch_export_schema,
+ file_format,
):
"""Test S3 Export Workflow end-to-end by using an S3 bucket.
@@ -646,6 +723,7 @@ async def test_s3_export_workflow_with_s3_bucket(
exclude_events=exclude_events,
include_events=None,
compression=compression,
+ file_format=file_format,
)
@@ -1206,6 +1284,49 @@ async def never_finish_activity(_: S3InsertInputs) -> str:
),
"nested/prefix/2023-01-01 00:00:00-2023-01-01 01:00:00.jsonl.br",
),
+ (
+ S3InsertInputs(
+ prefix="/nested/prefix/",
+ data_interval_start="2023-01-01 00:00:00",
+ data_interval_end="2023-01-01 01:00:00",
+ file_format="Parquet",
+ compression="snappy",
+ **base_inputs, # type: ignore
+ ),
+ "nested/prefix/2023-01-01 00:00:00-2023-01-01 01:00:00.parquet.sz",
+ ),
+ (
+ S3InsertInputs(
+ prefix="/nested/prefix/",
+ data_interval_start="2023-01-01 00:00:00",
+ data_interval_end="2023-01-01 01:00:00",
+ file_format="Parquet",
+ **base_inputs, # type: ignore
+ ),
+ "nested/prefix/2023-01-01 00:00:00-2023-01-01 01:00:00.parquet",
+ ),
+ (
+ S3InsertInputs(
+ prefix="/nested/prefix/",
+ data_interval_start="2023-01-01 00:00:00",
+ data_interval_end="2023-01-01 01:00:00",
+ compression="gzip",
+ file_format="Parquet",
+ **base_inputs, # type: ignore
+ ),
+ "nested/prefix/2023-01-01 00:00:00-2023-01-01 01:00:00.parquet.gz",
+ ),
+ (
+ S3InsertInputs(
+ prefix="/nested/prefix/",
+ data_interval_start="2023-01-01 00:00:00",
+ data_interval_end="2023-01-01 01:00:00",
+ compression="brotli",
+ file_format="Parquet",
+ **base_inputs, # type: ignore
+ ),
+ "nested/prefix/2023-01-01 00:00:00-2023-01-01 01:00:00.parquet.br",
+ ),
],
)
def test_get_s3_key(inputs, expected):
@@ -1271,7 +1392,7 @@ def assert_heartbeat_details(*details):
endpoint_url=settings.OBJECT_STORAGE_ENDPOINT,
)
- with override_settings(BATCH_EXPORT_S3_UPLOAD_CHUNK_SIZE_BYTES=5 * 1024**2):
+ with override_settings(BATCH_EXPORT_S3_UPLOAD_CHUNK_SIZE_BYTES=1, CLICKHOUSE_MAX_BLOCK_SIZE_DEFAULT=1):
await activity_environment.run(insert_into_s3_activity, insert_inputs)
# This checks that the assert_heartbeat_details function was actually called.
diff --git a/posthog/temporal/tests/batch_exports/test_temporary_file.py b/posthog/temporal/tests/batch_exports/test_temporary_file.py
new file mode 100644
index 0000000000000..4fd7e69c0c12f
--- /dev/null
+++ b/posthog/temporal/tests/batch_exports/test_temporary_file.py
@@ -0,0 +1,389 @@
+import csv
+import datetime as dt
+import io
+import json
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pytest
+
+from posthog.temporal.batch_exports.temporary_file import (
+ BatchExportTemporaryFile,
+ CSVBatchExportWriter,
+ JSONLBatchExportWriter,
+ ParquetBatchExportWriter,
+ json_dumps_bytes,
+)
+
+
+@pytest.mark.parametrize(
+ "to_write",
+ [
+ (b"",),
+ (b"", b""),
+ (b"12345",),
+ (b"12345", b"12345"),
+ (b"abbcccddddeeeee",),
+ (b"abbcccddddeeeee", b"abbcccddddeeeee"),
+ ],
+)
+def test_batch_export_temporary_file_tracks_bytes(to_write):
+ """Test the bytes written by BatchExportTemporaryFile match expected."""
+ with BatchExportTemporaryFile() as be_file:
+ for content in to_write:
+ be_file.write(content)
+
+ assert be_file.bytes_total == sum(len(content) for content in to_write)
+ assert be_file.bytes_since_last_reset == sum(len(content) for content in to_write)
+
+ be_file.reset()
+
+ assert be_file.bytes_total == sum(len(content) for content in to_write)
+ assert be_file.bytes_since_last_reset == 0
+
+
+TEST_RECORDS = [
+ [],
+ [
+ {"id": "record-1", "property": "value", "property_int": 1},
+ {"id": "record-2", "property": "another-value", "property_int": 2},
+ {
+ "id": "record-3",
+ "property": {"id": "nested-record", "property": "nested-value"},
+ "property_int": 3,
+ },
+ ],
+]
+
+
+@pytest.mark.parametrize(
+ "records",
+ TEST_RECORDS,
+)
+def test_batch_export_temporary_file_write_records_to_jsonl(records):
+ """Test JSONL records written by BatchExportTemporaryFile match expected."""
+ jsonl_dump = b"\n".join(map(json_dumps_bytes, records))
+
+ with BatchExportTemporaryFile() as be_file:
+ be_file.write_records_to_jsonl(records)
+
+ assert be_file.bytes_total == len(jsonl_dump)
+ assert be_file.bytes_since_last_reset == len(jsonl_dump)
+ assert be_file.records_total == len(records)
+ assert be_file.records_since_last_reset == len(records)
+
+ be_file.seek(0)
+ lines = be_file.readlines()
+ assert len(lines) == len(records)
+
+ for line_index, jsonl_record in enumerate(lines):
+ json_loaded = json.loads(jsonl_record)
+ assert json_loaded == records[line_index]
+
+ be_file.reset()
+
+ assert be_file.bytes_total == len(jsonl_dump)
+ assert be_file.bytes_since_last_reset == 0
+ assert be_file.records_total == len(records)
+ assert be_file.records_since_last_reset == 0
+
+
+@pytest.mark.parametrize(
+ "records",
+ TEST_RECORDS,
+)
+def test_batch_export_temporary_file_write_records_to_csv(records):
+ """Test CSV written by BatchExportTemporaryFile match expected."""
+ in_memory_file_obj = io.StringIO()
+ writer = csv.DictWriter(
+ in_memory_file_obj,
+ fieldnames=records[0].keys() if len(records) > 0 else [],
+ delimiter=",",
+ quotechar='"',
+ escapechar="\\",
+ lineterminator="\n",
+ quoting=csv.QUOTE_NONE,
+ )
+ writer.writerows(records)
+
+ with BatchExportTemporaryFile(mode="w+") as be_file:
+ be_file.write_records_to_csv(records)
+
+ assert be_file.bytes_total == in_memory_file_obj.tell()
+ assert be_file.bytes_since_last_reset == in_memory_file_obj.tell()
+ assert be_file.records_total == len(records)
+ assert be_file.records_since_last_reset == len(records)
+
+ be_file.seek(0)
+ reader = csv.reader(
+ be_file._file,
+ delimiter=",",
+ quotechar='"',
+ escapechar="\\",
+ quoting=csv.QUOTE_NONE,
+ )
+
+ rows = [row for row in reader]
+ assert len(rows) == len(records)
+
+ for row_index, csv_record in enumerate(rows):
+ for value_index, value in enumerate(records[row_index].values()):
+ # Everything returned by csv.reader is a str.
+ # This means type information is lost when writing to CSV
+ # but this just a limitation of the format.
+ assert csv_record[value_index] == str(value)
+
+ be_file.reset()
+
+ assert be_file.bytes_total == in_memory_file_obj.tell()
+ assert be_file.bytes_since_last_reset == 0
+ assert be_file.records_total == len(records)
+ assert be_file.records_since_last_reset == 0
+
+
+@pytest.mark.parametrize(
+ "records",
+ TEST_RECORDS,
+)
+def test_batch_export_temporary_file_write_records_to_tsv(records):
+ """Test TSV written by BatchExportTemporaryFile match expected."""
+ in_memory_file_obj = io.StringIO()
+ writer = csv.DictWriter(
+ in_memory_file_obj,
+ fieldnames=records[0].keys() if len(records) > 0 else [],
+ delimiter="\t",
+ quotechar='"',
+ escapechar="\\",
+ lineterminator="\n",
+ quoting=csv.QUOTE_NONE,
+ )
+ writer.writerows(records)
+
+ with BatchExportTemporaryFile(mode="w+") as be_file:
+ be_file.write_records_to_tsv(records)
+
+ assert be_file.bytes_total == in_memory_file_obj.tell()
+ assert be_file.bytes_since_last_reset == in_memory_file_obj.tell()
+ assert be_file.records_total == len(records)
+ assert be_file.records_since_last_reset == len(records)
+
+ be_file.seek(0)
+ reader = csv.reader(
+ be_file._file,
+ delimiter="\t",
+ quotechar='"',
+ escapechar="\\",
+ quoting=csv.QUOTE_NONE,
+ )
+
+ rows = [row for row in reader]
+ assert len(rows) == len(records)
+
+ for row_index, csv_record in enumerate(rows):
+ for value_index, value in enumerate(records[row_index].values()):
+ # Everything returned by csv.reader is a str.
+ # This means type information is lost when writing to CSV
+ # but this just a limitation of the format.
+ assert csv_record[value_index] == str(value)
+
+ be_file.reset()
+
+ assert be_file.bytes_total == in_memory_file_obj.tell()
+ assert be_file.bytes_since_last_reset == 0
+ assert be_file.records_total == len(records)
+ assert be_file.records_since_last_reset == 0
+
+
+TEST_RECORD_BATCHES = [
+ pa.RecordBatch.from_pydict(
+ {
+ "event": pa.array(["test-event-0", "test-event-1", "test-event-2"]),
+ "properties": pa.array(['{"prop_0": 1, "prop_1": 2}', "{}", "null"]),
+ "_inserted_at": pa.array([0, 1, 2]),
+ }
+ )
+]
+
+
+@pytest.mark.parametrize(
+ "record_batch",
+ TEST_RECORD_BATCHES,
+)
+@pytest.mark.asyncio
+async def test_jsonl_writer_writes_record_batches(record_batch):
+ """Test record batches are written as valid JSONL."""
+ in_memory_file_obj = io.BytesIO()
+ inserted_ats_seen = []
+
+ async def store_in_memory_on_flush(
+ batch_export_file, records_since_last_flush, bytes_since_last_flush, last_inserted_at, is_last
+ ):
+ in_memory_file_obj.write(batch_export_file.read())
+ inserted_ats_seen.append(last_inserted_at)
+
+ writer = JSONLBatchExportWriter(max_bytes=1, flush_callable=store_in_memory_on_flush)
+
+ record_batch = record_batch.sort_by("_inserted_at")
+ async with writer.open_temporary_file():
+ await writer.write_record_batch(record_batch)
+
+ lines = in_memory_file_obj.readlines()
+ for index, line in enumerate(lines):
+ written_jsonl = json.loads(line)
+
+ single_record_batch = record_batch.slice(offset=index, length=1)
+ expected_jsonl = single_record_batch.to_pylist()[0]
+
+ assert "_inserted_at" not in written_jsonl
+ assert written_jsonl == expected_jsonl
+
+ assert inserted_ats_seen == [record_batch.column("_inserted_at")[-1].as_py()]
+
+
+@pytest.mark.parametrize(
+ "record_batch",
+ TEST_RECORD_BATCHES,
+)
+@pytest.mark.asyncio
+async def test_csv_writer_writes_record_batches(record_batch):
+ """Test record batches are written as valid CSV."""
+ in_memory_file_obj = io.StringIO()
+ inserted_ats_seen = []
+
+ async def store_in_memory_on_flush(
+ batch_export_file, records_since_last_flush, bytes_since_last_flush, last_inserted_at, is_last
+ ):
+ in_memory_file_obj.write(batch_export_file.read().decode("utf-8"))
+ inserted_ats_seen.append(last_inserted_at)
+
+ schema_columns = [column_name for column_name in record_batch.column_names if column_name != "_inserted_at"]
+ writer = CSVBatchExportWriter(max_bytes=1, field_names=schema_columns, flush_callable=store_in_memory_on_flush)
+
+ record_batch = record_batch.sort_by("_inserted_at")
+ async with writer.open_temporary_file():
+ await writer.write_record_batch(record_batch)
+
+ reader = csv.reader(
+ in_memory_file_obj,
+ delimiter=",",
+ quotechar='"',
+ escapechar="\\",
+ quoting=csv.QUOTE_NONE,
+ )
+ for index, written_csv_row in enumerate(reader):
+ single_record_batch = record_batch.slice(offset=index, length=1)
+ expected_csv = single_record_batch.to_pylist()[0]
+
+ assert "_inserted_at" not in written_csv_row
+ assert written_csv_row == expected_csv
+
+ assert inserted_ats_seen == [record_batch.column("_inserted_at")[-1].as_py()]
+
+
+@pytest.mark.parametrize(
+ "record_batch",
+ TEST_RECORD_BATCHES,
+)
+@pytest.mark.asyncio
+async def test_parquet_writer_writes_record_batches(record_batch):
+ """Test record batches are written as valid Parquet."""
+ in_memory_file_obj = io.BytesIO()
+ inserted_ats_seen = []
+
+ async def store_in_memory_on_flush(
+ batch_export_file, records_since_last_flush, bytes_since_last_flush, last_inserted_at, is_last
+ ):
+ in_memory_file_obj.write(batch_export_file.read())
+ inserted_ats_seen.append(last_inserted_at)
+
+ schema_columns = [column_name for column_name in record_batch.column_names if column_name != "_inserted_at"]
+
+ writer = ParquetBatchExportWriter(
+ max_bytes=1,
+ flush_callable=store_in_memory_on_flush,
+ schema=record_batch.select(schema_columns).schema,
+ )
+
+ record_batch = record_batch.sort_by("_inserted_at")
+ async with writer.open_temporary_file():
+ await writer.write_record_batch(record_batch)
+
+ written_parquet = pq.read_table(in_memory_file_obj)
+
+ for index, written_row_as_dict in enumerate(written_parquet.to_pylist()):
+ single_record_batch = record_batch.slice(offset=index, length=1)
+ expected_row_as_dict = single_record_batch.select(schema_columns).to_pylist()[0]
+
+ assert "_inserted_at" not in written_row_as_dict
+ assert written_row_as_dict == expected_row_as_dict
+
+ # NOTE: Parquet gets flushed twice due to the extra flush at the end for footer bytes, so our mock function
+ # will see this value twice.
+ assert inserted_ats_seen == [
+ record_batch.column("_inserted_at")[-1].as_py(),
+ record_batch.column("_inserted_at")[-1].as_py(),
+ ]
+
+
+@pytest.mark.parametrize(
+ "record_batch",
+ TEST_RECORD_BATCHES,
+)
+@pytest.mark.asyncio
+async def test_writing_out_of_scope_of_temporary_file_raises(record_batch):
+ """Test attempting a write out of temporary file scope raises a `ValueError`."""
+
+ async def do_nothing(*args, **kwargs):
+ pass
+
+ schema_columns = [column_name for column_name in record_batch.column_names if column_name != "_inserted_at"]
+ writer = ParquetBatchExportWriter(
+ max_bytes=10,
+ flush_callable=do_nothing,
+ schema=record_batch.select(schema_columns).schema,
+ )
+
+ async with writer.open_temporary_file():
+ pass
+
+ with pytest.raises(ValueError, match="Batch export file is closed"):
+ await writer.write_record_batch(record_batch)
+
+
+@pytest.mark.parametrize(
+ "record_batch",
+ TEST_RECORD_BATCHES,
+)
+@pytest.mark.asyncio
+async def test_flushing_parquet_writer_resets_underlying_file(record_batch):
+ """Test flushing a writer resets underlying file."""
+ flush_counter = 0
+
+ async def track_flushes(*args, **kwargs):
+ nonlocal flush_counter
+ flush_counter += 1
+
+ schema_columns = [column_name for column_name in record_batch.column_names if column_name != "_inserted_at"]
+ writer = ParquetBatchExportWriter(
+ max_bytes=10000000,
+ flush_callable=track_flushes,
+ schema=record_batch.select(schema_columns).schema,
+ )
+
+ async with writer.open_temporary_file():
+ await writer.write_record_batch(record_batch)
+
+ assert writer.batch_export_file.tell() > 0
+ assert writer.bytes_since_last_flush > 0
+ assert writer.bytes_since_last_flush == writer.batch_export_file.bytes_since_last_reset
+ assert writer.records_since_last_flush == record_batch.num_rows
+
+ await writer.flush(dt.datetime.now())
+
+ assert flush_counter == 1
+ assert writer.batch_export_file.tell() == 0
+ assert writer.bytes_since_last_flush == 0
+ assert writer.bytes_since_last_flush == writer.batch_export_file.bytes_since_last_reset
+ assert writer.records_since_last_flush == 0
+
+ assert flush_counter == 2
diff --git a/posthog/warehouse/api/external_data_source.py b/posthog/warehouse/api/external_data_source.py
index 1586f1051379e..6eb06abcad70f 100644
--- a/posthog/warehouse/api/external_data_source.py
+++ b/posthog/warehouse/api/external_data_source.py
@@ -136,6 +136,8 @@ def create(self, request: Request, *args: Any, **kwargs: Any) -> Response:
new_source_model = self._handle_stripe_source(request, *args, **kwargs)
elif source_type == ExternalDataSource.Type.HUBSPOT:
new_source_model = self._handle_hubspot_source(request, *args, **kwargs)
+ elif source_type == ExternalDataSource.Type.ZENDESK:
+ new_source_model = self._handle_zendesk_source(request, *args, **kwargs)
elif source_type == ExternalDataSource.Type.POSTGRES:
try:
new_source_model, table_names = self._handle_postgres_source(request, *args, **kwargs)
@@ -190,6 +192,33 @@ def _handle_stripe_source(self, request: Request, *args: Any, **kwargs: Any) ->
return new_source_model
+ def _handle_zendesk_source(self, request: Request, *args: Any, **kwargs: Any) -> ExternalDataSource:
+ payload = request.data["payload"]
+ api_key = payload.get("api_key")
+ subdomain = payload.get("subdomain")
+ email_address = payload.get("email_address")
+ prefix = request.data.get("prefix", None)
+ source_type = request.data["source_type"]
+
+ # TODO: remove dummy vars
+ new_source_model = ExternalDataSource.objects.create(
+ source_id=str(uuid.uuid4()),
+ connection_id=str(uuid.uuid4()),
+ destination_id=str(uuid.uuid4()),
+ team=self.team,
+ status="Running",
+ source_type=source_type,
+ job_inputs={
+ "zendesk_login_method": "api_key", # We should support the Zendesk OAuth flow in the future, and so with this we can do backwards compatibility
+ "zendesk_api_key": api_key,
+ "zendesk_subdomain": subdomain,
+ "zendesk_email_address": email_address,
+ },
+ prefix=prefix,
+ )
+
+ return new_source_model
+
def _handle_hubspot_source(self, request: Request, *args: Any, **kwargs: Any) -> ExternalDataSource:
payload = request.data["payload"]
code = payload.get("code")
diff --git a/posthog/warehouse/models/external_data_source.py b/posthog/warehouse/models/external_data_source.py
index df668c5abfc54..0a044c0b06315 100644
--- a/posthog/warehouse/models/external_data_source.py
+++ b/posthog/warehouse/models/external_data_source.py
@@ -12,6 +12,7 @@ class Type(models.TextChoices):
STRIPE = "Stripe", "Stripe"
HUBSPOT = "Hubspot", "Hubspot"
POSTGRES = "Postgres", "Postgres"
+ ZENDESK = "Zendesk", "Zendesk"
class Status(models.TextChoices):
RUNNING = "Running", "Running"
diff --git a/posthog/warehouse/models/table.py b/posthog/warehouse/models/table.py
index 91c6f61709d6e..23cc5a7ce9541 100644
--- a/posthog/warehouse/models/table.py
+++ b/posthog/warehouse/models/table.py
@@ -1,4 +1,4 @@
-from typing import Dict
+from typing import Dict, Optional
from django.db import models
from posthog.client import sync_execute
@@ -175,6 +175,17 @@ def hogql_definition(self) -> S3Table:
structure=", ".join(structure),
)
+ def get_clickhouse_column_type(self, column_name: str) -> Optional[str]:
+ clickhouse_type = self.columns.get(column_name, None)
+
+ if isinstance(clickhouse_type, dict) and self.columns[column_name].get("clickhouse"):
+ clickhouse_type = self.columns[column_name].get("clickhouse")
+
+ if clickhouse_type.startswith("Nullable("):
+ clickhouse_type = clickhouse_type.replace("Nullable(", "")[:-1]
+
+ return clickhouse_type
+
def _safe_expose_ch_error(self, err):
err = wrap_query_error(err)
for key, value in ExtractErrors.items():
diff --git a/unit.json b/unit.json
index 72e3d2f03edb6..0b8de8774edf1 100644
--- a/unit.json
+++ b/unit.json
@@ -6,7 +6,7 @@
},
"listeners": {
"*:8000": {
- "pass": "routes/posthog"
+ "pass": "applications/posthog"
},
"*:8001": {
"pass": "routes/metrics"
@@ -16,21 +16,6 @@
}
},
"routes": {
- "posthog": [
- {
- "match": {
- "uri": ["/_health", "/_readyz", "/_livez"]
- },
- "action": {
- "pass": "applications/posthog-health"
- }
- },
- {
- "action": {
- "pass": "applications/posthog"
- }
- }
- ],
"metrics": [
{
"match": {
@@ -53,17 +38,6 @@
]
},
"applications": {
- "posthog-health": {
- "type": "python 3.10",
- "processes": 1,
- "working_directory": "/code",
- "path": ".",
- "module": "posthog.wsgi",
- "user": "nobody",
- "limits": {
- "requests": 5000
- }
- },
"posthog": {
"type": "python 3.10",
"processes": 4,