diff --git a/.github/actions/run-backend-tests/action.yml b/.github/actions/run-backend-tests/action.yml index cd632a491c2ab..bde3554bb4e20 100644 --- a/.github/actions/run-backend-tests/action.yml +++ b/.github/actions/run-backend-tests/action.yml @@ -210,7 +210,7 @@ runs: - name: Upload updated timing data as artifacts uses: actions/upload-artifact@v4 - if: ${{ inputs.person-on-events != 'true' && inputs.clickhouse-server-image == 'clickhouse/clickhouse-server:23.12.6.19-alpine' }} + if: ${{ inputs.person-on-events != 'true' && inputs.clickhouse-server-image == 'clickhouse/clickhouse-server:23.12.6.19' }} with: name: timing_data-${{ inputs.segment }}-${{ inputs.group }} path: .test_durations diff --git a/.github/workflows/ci-backend-update-test-timing.yml b/.github/workflows/ci-backend-update-test-timing.yml index 2d722584cc95e..e37df8defe9cb 100644 --- a/.github/workflows/ci-backend-update-test-timing.yml +++ b/.github/workflows/ci-backend-update-test-timing.yml @@ -29,13 +29,13 @@ jobs: group: 1 token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }} python-version: '3.11.9' - clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.6.19-alpine' + clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.6.19' segment: 'FOSS' person-on-events: false - name: Upload updated timing data as artifacts uses: actions/upload-artifact@v4 - if: ${{ inputs.person-on-events != 'true' && inputs.clickhouse-server-image == 'clickhouse/clickhouse-server:23.12.6.19-alpine' }} + if: ${{ inputs.person-on-events != 'true' && inputs.clickhouse-server-image == 'clickhouse/clickhouse-server:23.12.6.19' }} with: name: timing_data-${{ inputs.segment }}-${{ inputs.group }} path: .test_durations diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml index cb53aefe20655..56dae66425a29 100644 --- a/.github/workflows/ci-backend.yml +++ b/.github/workflows/ci-backend.yml @@ -240,7 +240,7 @@ jobs: fail-fast: false matrix: python-version: ['3.11.9'] - clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.6.19-alpine'] + clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.6.19'] segment: ['Core'] person-on-events: [false, true] # :NOTE: Keep concurrency and groups in sync @@ -249,7 +249,7 @@ jobs: include: - segment: 'Temporal' person-on-events: false - clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.6.19-alpine' + clickhouse-server-image: 'clickhouse/clickhouse-server:23.12.6.19' python-version: '3.11.9' concurrency: 1 group: 1 @@ -320,7 +320,7 @@ jobs: strategy: fail-fast: false matrix: - clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.6.19-alpine'] + clickhouse-server-image: ['clickhouse/clickhouse-server:23.12.6.19'] if: needs.changes.outputs.backend == 'true' runs-on: ubuntu-latest steps: diff --git a/docker-compose.base.yml b/docker-compose.base.yml index 4b17a35977555..137416e40422c 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -79,7 +79,7 @@ services: # Note: please keep the default version in sync across # `posthog` and the `charts-clickhouse` repos # - image: ${CLICKHOUSE_SERVER_IMAGE:-clickhouse/clickhouse-server:23.12.6.19-alpine} + image: ${CLICKHOUSE_SERVER_IMAGE:-clickhouse/clickhouse-server:23.12.6.19} restart: on-failure zookeeper: diff --git a/docker/clickhouse/docker-entrypoint-initdb.d/init-db.sh b/docker/clickhouse/docker-entrypoint-initdb.d/init-db.sh index 4141e3345d05b..1128061c84482 100755 --- a/docker/clickhouse/docker-entrypoint-initdb.d/init-db.sh +++ b/docker/clickhouse/docker-entrypoint-initdb.d/init-db.sh @@ -1,5 +1,11 @@ #!/bin/bash set -e -apk add python3 +apt-get update +# Necessary because clickhouse runs on Ubuntu 20, which has an old glibc and an old python default +# Can remove when we upgrade clickhouse, as the new images run on Ubuntu 22 +apt-get -y install python3.9 +ln -s /usr/bin/python3.9 /usr/bin/python3 +wget http://launchpadlibrarian.net/588931980/libc6_2.35-0ubuntu3_amd64.deb +dpkg --auto-deconfigure -i libc6_2.35-0ubuntu3_amd64.deb cp -r /idl/* /var/lib/clickhouse/format_schemas/ diff --git a/docker/clickhouse/user_defined_function.xml b/docker/clickhouse/user_defined_function.xml index 9617d2495ced1..9b58b380066e6 100644 --- a/docker/clickhouse/user_defined_function.xml +++ b/docker/clickhouse/user_defined_function.xml @@ -2,7 +2,7 @@ executable aggregate_funnel - Array(Tuple(Int8, Nullable(String), Array(Float64))) + Array(Tuple(Int8, Nullable(String), Array(Float64), Array(Array(UUID)))) result UInt8 @@ -25,17 +25,18 @@ prop_vals - Array(Tuple(Nullable(Float64), Nullable(String), Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, Nullable(String), Array(Int8))) value JSONEachRow - aggregate_funnel.py + aggregate_funnel + 600 executable aggregate_funnel_cohort - Array(Tuple(Int8, UInt64, Array(Float64))) + Array(Tuple(Int8, UInt64, Array(Float64), Array(Array(UUID)))) result UInt8 @@ -58,17 +59,18 @@ prop_vals - Array(Tuple(Nullable(Float64), UInt64, Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, UInt64, Array(Int8))) value JSONEachRow - aggregate_funnel_cohort.py + aggregate_funnel + 600 executable aggregate_funnel_array - Array(Tuple(Int8, Array(String), Array(Float64))) + Array(Tuple(Int8, Array(String), Array(Float64), Array(Array(UUID)))) result UInt8 @@ -91,11 +93,12 @@ prop_vals - Array(Tuple(Nullable(Float64), Array(String), Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, Array(String), Array(Int8))) value JSONEachRow - aggregate_funnel_array.py + aggregate_funnel + 600 @@ -124,11 +127,12 @@ prop_vals - Array(Tuple(Nullable(Float64), Nullable(String), Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, Nullable(String), Array(Int8))) value JSONEachRow aggregate_funnel_test.py + 600 @@ -170,6 +174,7 @@ JSONEachRow aggregate_funnel_trends.py + 600 @@ -208,6 +213,7 @@ JSONEachRow aggregate_funnel_array_trends.py + 600 @@ -246,6 +252,7 @@ JSONEachRow aggregate_funnel_cohort_trends.py + 600 @@ -283,5 +290,6 @@ JSONEachRow aggregate_funnel_array_trends_test.py + 600 \ No newline at end of file diff --git a/funnel-udf/.gitignore b/funnel-udf/.gitignore new file mode 100644 index 0000000000000..ea8c4bf7f35f6 --- /dev/null +++ b/funnel-udf/.gitignore @@ -0,0 +1 @@ +/target diff --git a/funnel-udf/Cargo.lock b/funnel-udf/Cargo.lock new file mode 100644 index 0000000000000..c58e632f4bfac --- /dev/null +++ b/funnel-udf/Cargo.lock @@ -0,0 +1,152 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "funnels" +version = "0.1.0" +dependencies = [ + "itertools", + "serde", + "serde_json", + "uuid", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "libc" +version = "0.2.159" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "serde" +version = "1.0.210" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.210" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "2.0.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" + +[[package]] +name = "uuid" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +dependencies = [ + "getrandom", + "serde", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" diff --git a/funnel-udf/Cargo.toml b/funnel-udf/Cargo.toml new file mode 100644 index 0000000000000..b5bdd57d5e67d --- /dev/null +++ b/funnel-udf/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "funnels" +version = "0.1.0" +edition = "2021" + +[dependencies] +serde = { version = "1.0.104", features = ["derive"] } +serde_json = "1.0.48" +itertools = "0.11" +uuid = { version = "1.10.0", features = ["v4", "serde"] } + + +[profile.release] +lto = true diff --git a/funnel-udf/src/main.rs b/funnel-udf/src/main.rs new file mode 100644 index 0000000000000..78775a9dd35b3 --- /dev/null +++ b/funnel-udf/src/main.rs @@ -0,0 +1,256 @@ +use serde::{Deserialize, Serialize}; +use serde_json::json; +use std::io::{self, BufRead, Write}; +use std::iter::repeat; +use itertools::Itertools; +use uuid::Uuid; + +#[derive(Clone, PartialEq, Deserialize, Serialize)] +#[serde(untagged)] +enum PropVal { + String(String), + Vec(Vec), + Int(u32), +} + +#[derive(Clone, Deserialize)] +struct EnteredTimestamp { + timestamp: f64, + timings: Vec, + uuids: Vec, +} + +#[derive(Clone, Deserialize)] +struct Event { + timestamp: f64, + uuid: Uuid, + breakdown: PropVal, + steps: Vec, +} + +#[derive(Deserialize)] +struct Args { + num_steps: i32, + conversion_window_limit: u64, + breakdown_attribution_type: String, + funnel_order_type: String, + prop_vals: Vec, + value: Vec, +} + +#[derive(Serialize)] +struct Result(i32, PropVal, Vec, Vec>); + +const MAX_REPLAY_EVENTS: usize = 10; + +#[inline(always)] +fn parse_args(line: &str) -> Args { + serde_json::from_str(line).expect("Invalid JSON input") +} + +#[inline(always)] +fn calculate_funnel_from_user_events( + num_steps: i32, + conversion_window_limit_seconds: u64, + breakdown_attribution_type: &str, + funnel_order_type: &str, + prop_vals: Vec, + events: Vec, +) -> Vec { + let default_entered_timestamp = EnteredTimestamp { + timestamp: 0.0, + timings: vec![], + uuids: vec![], + }; + let breakdown_step = if breakdown_attribution_type.starts_with("step_") { + breakdown_attribution_type[5..].parse::().ok() + } else { + None + }; + + let mut results: Vec = Vec::with_capacity(prop_vals.len()); + + for prop_val in prop_vals { + let mut max_step = (0, default_entered_timestamp.clone()); + let mut entered_timestamp = vec![default_entered_timestamp.clone(); (num_steps + 1) as usize]; + let mut event_uuids: Vec> = repeat(Vec::new()).take(num_steps as usize).collect(); + let mut add_max_step = true; + + let filtered_events = events.iter() + .filter(|e| { + if breakdown_attribution_type == "all_events" { + e.breakdown == prop_val + } else { + true + } + }) + .group_by(|e| e.timestamp); + + for (timestamp, events_with_same_timestamp) in &filtered_events { + let events_with_same_timestamp: Vec<_> = events_with_same_timestamp.collect(); + entered_timestamp[0] = EnteredTimestamp { + timestamp, + timings: vec![], + uuids: vec![], + }; + + if events_with_same_timestamp.len() == 1 { + if !process_event( + &events_with_same_timestamp[0], + &mut entered_timestamp, + &prop_val, + &mut event_uuids, + conversion_window_limit_seconds, + funnel_order_type, + &mut max_step, + breakdown_step, + &mut results, + ) { + add_max_step = false; + break; + } + } else { + // Handle permutations for events with the same timestamp + let mut entered_timestamps: Vec<_> = vec![]; + for perm in events_with_same_timestamp.iter().permutations(events_with_same_timestamp.len()) { + entered_timestamps.push(entered_timestamp.clone()); + for event in perm { + if !process_event( + &event, + &mut entered_timestamps.last_mut().unwrap(), + &prop_val, + &mut event_uuids, + conversion_window_limit_seconds, + funnel_order_type, + &mut max_step, + breakdown_step, + &mut results, + ) { + add_max_step = false; + break; + } + } + } + for i in 0..entered_timestamp.len() { + entered_timestamp[i] = entered_timestamps.iter().max_by_key(|x| x[i].timestamp as i32).unwrap()[i].clone(); + } + } + + if entered_timestamp[num_steps as usize].timestamp > 0.0 { + break; + } + } + + if add_max_step { + let final_index = max_step.0; + let final_value = &max_step.1; + + for i in 0..final_index { + //if event_uuids[i].len() >= MAX_REPLAY_EVENTS && !event_uuids[i].contains(&final_value.uuids[i]) { + // Always put the actual event uuids first, we use it to extract timestamps + // This might create duplicates, but that's fine (we can remove it in clickhouse) + event_uuids[i].insert(0, final_value.uuids[i].clone()); + } + results.push(Result( + final_index as i32 - 1, + prop_val, + final_value.timings.windows(2).map(|w| w[1] - w[0]).collect(), + event_uuids + )) + } + } + results +} + +#[inline(always)] +fn process_event( + event: &Event, + entered_timestamp: &mut Vec, + prop_val: &PropVal, + event_uuids: &mut Vec>, + conversion_window_limit_seconds: u64, + funnel_order_type: &str, + max_step: &mut (usize, EnteredTimestamp), + breakdown_step: Option, + results: &mut Vec +) -> bool { + + for step in event.steps.iter().rev() { + let mut exclusion = false; + let step = (if *step < 0 { + exclusion = true; + -*step + } else { + *step + }) as usize; + + let in_match_window = (event.timestamp - entered_timestamp[step - 1].timestamp) <= conversion_window_limit_seconds as f64; + let already_reached_this_step = entered_timestamp[step].timestamp == entered_timestamp[step - 1].timestamp + && entered_timestamp[step].timestamp != 0.0; + + if in_match_window && !already_reached_this_step { + if exclusion { + results.push(Result ( -1, prop_val.clone(), vec![], vec![] )); + return false; + } + let is_unmatched_step_attribution = breakdown_step.map( |breakdown_step| step == breakdown_step - 1 ).unwrap_or(false) && *prop_val != event.breakdown; + if !is_unmatched_step_attribution { + entered_timestamp[step] = EnteredTimestamp { + timestamp: entered_timestamp[step - 1].timestamp, + timings: { + let mut timings = entered_timestamp[step - 1].timings.clone(); + timings.push(event.timestamp); + timings + }, + uuids: { + let mut uuids = entered_timestamp[step - 1].uuids.clone(); + uuids.push(event.uuid); + uuids + }, + }; + if event_uuids[step - 1].len() < MAX_REPLAY_EVENTS - 1 { + event_uuids[step - 1].push(event.uuid); + } + } + if step > max_step.0 { + *max_step = (step, entered_timestamp[step].clone()); + } + } + } + + if funnel_order_type == "strict" { + for i in 1..entered_timestamp.len() { + if !event.steps.contains(&(i as i32)) { + entered_timestamp[i] = EnteredTimestamp { + timestamp: 0.0, + timings: vec![], + uuids: vec![], + }; + } + } + } + + true +} + +fn main() { + let stdin = io::stdin(); + let mut stdout = io::stdout(); + + for line in stdin.lock().lines() { + if let Ok(line) = line { + let args = parse_args(&line); + let result = calculate_funnel_from_user_events( + args.num_steps, + args.conversion_window_limit, + &args.breakdown_attribution_type, + &args.funnel_order_type, + args.prop_vals, + args.value + ); + let output = json!({ "result": result }); + writeln!(stdout, "{}", output).unwrap(); + stdout.flush().unwrap(); + } + } +} \ No newline at end of file diff --git a/posthog/hogql_queries/insights/funnels/base.py b/posthog/hogql_queries/insights/funnels/base.py index d5757225246f4..8066ada3c642f 100644 --- a/posthog/hogql_queries/insights/funnels/base.py +++ b/posthog/hogql_queries/insights/funnels/base.py @@ -435,31 +435,26 @@ def _get_inner_event_query_for_udf( extra_fields.append(prop) funnel_events_query = FunnelEventQuery( - context=self.context, - extra_fields=[*self._extra_event_fields, *extra_fields], - extra_event_properties=self._extra_event_properties, + context=self.context, extra_fields=[*self.extra_event_fields_and_properties, *extra_fields] ).to_query( skip_entity_filter=skip_entity_filter, ) - # funnel_events_query, params = FunnelEventQuery( - # extra_fields=[*self._extra_event_fields, *extra_fields], - # extra_event_properties=self._extra_event_properties, - # ).get_query(entities_to_use, entity_name, skip_entity_filter=skip_entity_filter) all_step_cols: list[ast.Expr] = [] all_exclusions: list[list[FunnelExclusionEventsNode | FunnelExclusionActionsNode]] = [] for index, entity in enumerate(entities_to_use): - step_cols = self._get_step_col(entity, index, entity_name) + step_cols = self._get_step_col(entity, index, entity_name, for_udf=True) all_step_cols.extend(step_cols) all_exclusions.append([]) - for excluded_entity in funnelsFilter.exclusions or []: - for i in range(excluded_entity.funnelFromStep + 1, excluded_entity.funnelToStep + 1): - all_exclusions[i].append(excluded_entity) + if funnelsFilter.exclusions: + for excluded_entity in funnelsFilter.exclusions: + for i in range(excluded_entity.funnelFromStep + 1, excluded_entity.funnelToStep + 1): + all_exclusions[i].append(excluded_entity) - for index, exclusions in enumerate(all_exclusions): - exclusion_col_expr = self._get_exclusions_col(exclusions, index, entity_name) - all_step_cols.append(exclusion_col_expr) + for index, exclusions in enumerate(all_exclusions): + exclusion_col_expr = self._get_exclusions_col(exclusions, index, entity_name) + all_step_cols.append(exclusion_col_expr) breakdown_select_prop = self._get_breakdown_select_prop() @@ -491,7 +486,6 @@ def _get_exclusions_col( ) -> ast.Expr: if not exclusions: return parse_expr(f"0 as exclusion_{index}") - conditions = [self._build_step_query(exclusion, index, entity_name, "") for exclusion in exclusions] return parse_expr( f"if({{condition}}, 1, 0) as exclusion_{index}", placeholders={"condition": ast.Or(exprs=conditions)} @@ -664,10 +658,10 @@ def _get_step_col( parse_expr(f"if({step_prefix}step_{index} = 1, timestamp, null) as {step_prefix}latest_{index}") ) - for field in self.extra_event_fields_and_properties: - step_cols.append( - parse_expr(f'if({step_prefix}step_{index} = 1, "{field}", null) as "{step_prefix}{field}_{index}"') - ) + for field in self.extra_event_fields_and_properties: + step_cols.append( + parse_expr(f'if({step_prefix}step_{index} = 1, "{field}", null) as "{step_prefix}{field}_{index}"') + ) return step_cols @@ -758,12 +752,15 @@ def _get_funnel_person_step_condition(self) -> ast.Expr: return ast.And(exprs=conditions) - def _get_funnel_person_step_events(self) -> list[ast.Expr]: - if ( + def _include_matched_events(self): + return ( hasattr(self.context, "actorsQuery") and self.context.actorsQuery is not None and self.context.actorsQuery.includeRecordings - ): + ) + + def _get_funnel_person_step_events(self) -> list[ast.Expr]: + if self._include_matched_events(): if self.context.includeFinalMatchingEvents: # Always returns the user's final step of the funnel return [parse_expr("final_matching_events as matching_events")] diff --git a/posthog/hogql_queries/insights/funnels/funnel_correlation_query_runner.py b/posthog/hogql_queries/insights/funnels/funnel_correlation_query_runner.py index e37ee95ce6b48..e67e00d9d04a1 100644 --- a/posthog/hogql_queries/insights/funnels/funnel_correlation_query_runner.py +++ b/posthog/hogql_queries/insights/funnels/funnel_correlation_query_runner.py @@ -4,6 +4,7 @@ from posthog.constants import AUTOCAPTURE_EVENT from posthog.hogql.parser import parse_select from posthog.hogql.property import property_to_expr +from posthog.hogql_queries.insights.funnels import FunnelUDF from posthog.hogql_queries.insights.funnels.funnel_event_query import FunnelEventQuery from posthog.hogql_queries.insights.funnels.funnel_persons import FunnelActors from posthog.hogql_queries.insights.funnels.funnel_strict_persons import FunnelStrictActors @@ -19,7 +20,11 @@ from posthog.hogql.query import execute_hogql_query from posthog.hogql.timings import HogQLTimings from posthog.hogql_queries.insights.funnels.funnel_query_context import FunnelQueryContext -from posthog.hogql_queries.insights.funnels.utils import funnel_window_interval_unit_to_sql, get_funnel_actor_class +from posthog.hogql_queries.insights.funnels.utils import ( + funnel_window_interval_unit_to_sql, + get_funnel_actor_class, + use_udf, +) from posthog.hogql_queries.query_runner import QueryRunner from posthog.models import Team from posthog.models.property.util import get_property_string_expr @@ -93,7 +98,7 @@ class FunnelCorrelationQueryRunner(QueryRunner): actors_query: FunnelsActorsQuery correlation_actors_query: Optional[FunnelCorrelationActorsQuery] - _funnel_actors_generator: FunnelActors | FunnelStrictActors | FunnelUnorderedActors + _funnel_actors_generator: FunnelActors | FunnelStrictActors | FunnelUnorderedActors | FunnelUDF def __init__( self, @@ -132,9 +137,11 @@ def __init__( self.context.actorsQuery = self.actors_query # Used for generating the funnel persons cte - funnel_order_actor_class = get_funnel_actor_class(self.context.funnelsFilter)(context=self.context) + funnel_order_actor_class = get_funnel_actor_class( + self.context.funnelsFilter, use_udf(self.context.funnelsFilter, self.team) + )(context=self.context) assert isinstance( - funnel_order_actor_class, FunnelActors | FunnelStrictActors | FunnelUnorderedActors + funnel_order_actor_class, FunnelActors | FunnelStrictActors | FunnelUnorderedActors | FunnelUDF ) # for typings self._funnel_actors_generator = funnel_order_actor_class diff --git a/posthog/hogql_queries/insights/funnels/funnel_udf.py b/posthog/hogql_queries/insights/funnels/funnel_udf.py index 7ec91374dcdee..b2568acc74538 100644 --- a/posthog/hogql_queries/insights/funnels/funnel_udf.py +++ b/posthog/hogql_queries/insights/funnels/funnel_udf.py @@ -1,7 +1,7 @@ -from typing import cast +from typing import cast, Optional from posthog.hogql import ast -from posthog.hogql.parser import parse_select +from posthog.hogql.parser import parse_select, parse_expr from posthog.hogql_queries.insights.funnels.base import FunnelBase from posthog.schema import BreakdownType, BreakdownAttributionType from posthog.utils import DATERANGE_MAP @@ -11,6 +11,16 @@ class FunnelUDF(FunnelBase): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # In base, these fields only get added if you're running an actors query + if "uuid" not in self._extra_event_fields: + self._extra_event_fields.append("uuid") + for property in ("$session_id", "$window_id"): + if property not in self._extra_event_properties: + self._extra_event_properties.append(property) + + # I think I can delete this def get_step_counts_query(self): max_steps = self.context.max_steps return self._get_step_counts_query( @@ -27,7 +37,9 @@ def conversion_window_limit(self) -> int: self.context.funnelWindowInterval * DATERANGE_MAP[self.context.funnelWindowIntervalUnit].total_seconds() ) - def get_query(self) -> ast.SelectQuery: + # This is the function that calls the UDF + # This is used by both the query itself and the actors query + def _inner_aggregation_query(self): if self.context.funnelsFilter.funnelOrderType == "strict": inner_event_query = self._get_inner_event_query_for_udf( entity_name="events", skip_step_filter=True, skip_entity_filter=True @@ -63,30 +75,20 @@ def get_query(self) -> ast.SelectQuery: breakdown_attribution_string = f"{self.context.breakdownAttributionType}{f'_{self.context.funnelsFilter.breakdownAttributionValue}' if self.context.breakdownAttributionType == BreakdownAttributionType.STEP else ''}" - # test - ''' - inner_select = parse_select( - f""" - SELECT - arrayJoin({fn}( - {self.context.max_steps}, - {self.conversion_window_limit()}, - '{breakdown_attribution_string}', - '{self.context.funnelsFilter.funnelOrderType}', - {prop_vals}, - arraySort(t -> t.1, groupArray(tuple(toFloat(timestamp), {prop_selector}, arrayFilter((x) -> x != 0, [{steps}{exclusions}])))) - )) as af_tuple, - af_tuple.1 as af, - af_tuple.2 as breakdown, - af_tuple.3 as timings - FROM {{inner_event_query}} - GROUP BY aggregation_target{breakdown_prop} - HAVING af >= 0 - """, - {"inner_event_query": inner_event_query}, - ) - return inner_select - ''' + def matched_event_arrays_selects(): + # We use matched events to get timestamps for the funnel as well as recordings + if ( + self._include_matched_events() + or self.context.includePrecedingTimestamp + or self.context.includeTimestamp + ): + return """ + af_tuple.4 as matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, $session_id, $window_id)) as user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) as user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) as matched_events_array, + """ + return "" inner_select = parse_select( f""" @@ -97,20 +99,27 @@ def get_query(self) -> ast.SelectQuery: '{breakdown_attribution_string}', '{self.context.funnelsFilter.funnelOrderType}', {prop_vals}, - arraySort(t -> t.1, groupArray(tuple(toFloat(timestamp), {prop_selector}, arrayFilter((x) -> x != 0, [{steps}{exclusions}])))) + arraySort(t -> t.1, groupArray(tuple(toFloat(timestamp), uuid, {prop_selector}, arrayFilter((x) -> x != 0, [{steps}{exclusions}])))) )) as af_tuple, - af_tuple.1 as af, + af_tuple.1 as step_reached, + af_tuple.1 + 1 as steps, -- Backward compatibility af_tuple.2 as breakdown, - af_tuple.3 as timings + af_tuple.3 as timings, + {matched_event_arrays_selects()} + aggregation_target FROM {{inner_event_query}} GROUP BY aggregation_target{breakdown_prop} - HAVING af >= 0 + HAVING step_reached >= 0 """, {"inner_event_query": inner_event_query}, ) + return inner_select + + def get_query(self) -> ast.SelectQuery: + inner_select = self._inner_aggregation_query() step_results = ",".join( - [f"countIf(ifNull(equals(af, {i}), 0)) AS step_{i+1}" for i in range(self.context.max_steps)] + [f"countIf(ifNull(equals(step_reached, {i}), 0)) AS step_{i+1}" for i in range(self.context.max_steps)] ) step_results2 = ",".join([f"sum(step_{i+1}) AS step_{i+1}" for i in range(self.context.max_steps)]) @@ -182,3 +191,112 @@ def get_query(self) -> ast.SelectQuery: ) return cast(ast.SelectQuery, s) + + def _get_funnel_person_step_condition(self) -> ast.Expr: + actorsQuery, breakdownType = ( + self.context.actorsQuery, + self.context.breakdownType, + ) + assert actorsQuery is not None + + funnelStep = actorsQuery.funnelStep + funnelCustomSteps = actorsQuery.funnelCustomSteps + funnelStepBreakdown = actorsQuery.funnelStepBreakdown + + conditions: list[ast.Expr] = [] + + if funnelCustomSteps: + # this is an adjustment for how UDF funnels represent steps + funnelCustomSteps = [x - 1 for x in funnelCustomSteps] + conditions.append(parse_expr(f"step_reached IN {funnelCustomSteps}")) + elif funnelStep is not None: + if funnelStep >= 0: + conditions.append(parse_expr(f"step_reached >= {funnelStep - 1}")) + else: + conditions.append(parse_expr(f"step_reached = {-funnelStep - 2}")) + else: + raise ValueError("Missing both funnelStep and funnelCustomSteps") + + if funnelStepBreakdown is not None: + if isinstance(funnelStepBreakdown, int) and breakdownType != "cohort": + funnelStepBreakdown = str(funnelStepBreakdown) + + conditions.append( + parse_expr( + "arrayFlatten(array(breakdown)) = arrayFlatten(array({funnelStepBreakdown}))", + {"funnelStepBreakdown": ast.Constant(value=funnelStepBreakdown)}, + ) + ) + + return ast.And(exprs=conditions) + + def _get_funnel_person_step_events(self) -> list[ast.Expr]: + if ( + hasattr(self.context, "actorsQuery") + and self.context.actorsQuery is not None + and self.context.actorsQuery.includeRecordings + ): + if self.context.includeFinalMatchingEvents: + # Always returns the user's final step of the funnel, 1 indexed + return [parse_expr("matched_events_array[step_reached + 1] as matching_events")] + + absolute_actors_step = self._absolute_actors_step + if absolute_actors_step is None: + raise ValueError("Missing funnelStep actors query property") + return [parse_expr(f"matched_events_array[{absolute_actors_step + 1}] as matching_events")] + return [] + + def _get_timestamp_outer_select(self) -> list[ast.Expr]: + """ + Returns timestamp selectors for the target step and optionally the preceding step. + In the former case, always returns the timestamp for the first and last step as well. + """ + target_step = self._absolute_actors_step + + if target_step is None: + return [] + + # We pull timestamps from matched_events_array and SQL arrays are 1-indexed + target_step += 1 + + final_step = self.context.max_steps + first_step = 1 + + if self.context.includePrecedingTimestamp: + if target_step == 0: + raise ValueError("Cannot request preceding step timestamp if target funnel step is the first step") + + return [ + parse_expr(f"matched_events_array[{target_step}][1].1 AS max_timestamp"), + parse_expr(f"matched_events_array[{target_step - 1}][1].1 AS min_timestamp"), + ] + elif self.context.includeTimestamp: + return [ + parse_expr(f"matched_events_array[{target_step}][1].1 AS timestamp"), + # Correlation code expects null if user hasn't made it to this step + parse_expr(f"nullIf(matched_events_array[{final_step}][1].1, 0) AS final_timestamp"), + parse_expr(f"matched_events_array[{first_step}][1].1 as first_timestamp"), + ] + else: + return [] + + def actor_query( + self, + extra_fields: Optional[list[str]] = None, + ) -> ast.SelectQuery: + select: list[ast.Expr] = [ + ast.Alias(alias="actor_id", expr=ast.Field(chain=["aggregation_target"])), + *self._get_funnel_person_step_events(), + *self._get_timestamp_outer_select(), + *([ast.Field(chain=[field]) for field in extra_fields or []]), + ] + select_from = ast.JoinExpr(table=self._inner_aggregation_query()) + where = self._get_funnel_person_step_condition() + order_by = [ast.OrderExpr(expr=ast.Field(chain=["aggregation_target"]))] + + return ast.SelectQuery( + select=select, + select_from=select_from, + order_by=order_by, + where=where, + ) diff --git a/posthog/hogql_queries/insights/funnels/funnels_query_runner.py b/posthog/hogql_queries/insights/funnels/funnels_query_runner.py index dcd5056d70bfa..0e8a7919a76ec 100644 --- a/posthog/hogql_queries/insights/funnels/funnels_query_runner.py +++ b/posthog/hogql_queries/insights/funnels/funnels_query_runner.py @@ -19,11 +19,7 @@ from posthog.hogql_queries.insights.funnels.funnel_time_to_convert import FunnelTimeToConvert from posthog.hogql_queries.insights.funnels.funnel_trends import FunnelTrends from posthog.hogql_queries.insights.funnels.funnel_trends_udf import FunnelTrendsUDF -from posthog.hogql_queries.insights.funnels.utils import get_funnel_actor_class, get_funnel_order_class -from posthog.hogql_queries.legacy_compatibility.feature_flag import ( - insight_funnels_use_udf, - insight_funnels_use_udf_trends, -) +from posthog.hogql_queries.insights.funnels.utils import get_funnel_actor_class, get_funnel_order_class, use_udf from posthog.hogql_queries.query_runner import QueryRunner from posthog.hogql_queries.utils.query_date_range import QueryDateRange from posthog.models import Team @@ -115,14 +111,7 @@ def calculate(self): @cached_property def _use_udf(self): - if self.context.funnelsFilter.useUdf: - return True - funnelVizType = self.context.funnelsFilter.funnelVizType - if funnelVizType == FunnelVizType.TRENDS and insight_funnels_use_udf_trends(self.team): - return True - if funnelVizType == FunnelVizType.STEPS and insight_funnels_use_udf(self.team): - return True - return False + return use_udf(self.context.funnelsFilter, self.team) @cached_property def funnel_order_class(self): @@ -145,7 +134,7 @@ def funnel_class(self): @cached_property def funnel_actor_class(self): - return get_funnel_actor_class(self.context.funnelsFilter)(context=self.context) + return get_funnel_actor_class(self.context.funnelsFilter, self._use_udf)(context=self.context) @cached_property def query_date_range(self): diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlations_persons.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_actors.ambr similarity index 96% rename from posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlations_persons.ambr rename to posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_actors.ambr index 7577455686ab6..fc9dcbeb30399 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlations_persons.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_actors.ambr @@ -1,5 +1,5 @@ # serializer version: 1 -# name: TestFunnelCorrelationsActors.test_funnel_correlation_on_event_with_recordings +# name: TestFunnelCorrelationActors.test_funnel_correlation_on_event_with_recordings ''' SELECT persons.id, persons.id AS id, @@ -172,7 +172,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestFunnelCorrelationsActors.test_funnel_correlation_on_event_with_recordings.1 +# name: TestFunnelCorrelationActors.test_funnel_correlation_on_event_with_recordings.1 ''' SELECT DISTINCT session_replay_events.session_id AS session_id FROM session_replay_events @@ -186,7 +186,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestFunnelCorrelationsActors.test_funnel_correlation_on_event_with_recordings.2 +# name: TestFunnelCorrelationActors.test_funnel_correlation_on_event_with_recordings.2 ''' SELECT persons.id, persons.id AS id, @@ -437,7 +437,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestFunnelCorrelationsActors.test_funnel_correlation_on_event_with_recordings.3 +# name: TestFunnelCorrelationActors.test_funnel_correlation_on_event_with_recordings.3 ''' SELECT DISTINCT session_replay_events.session_id AS session_id FROM session_replay_events @@ -451,7 +451,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestFunnelCorrelationsActors.test_funnel_correlation_on_properties_with_recordings +# name: TestFunnelCorrelationActors.test_funnel_correlation_on_properties_with_recordings ''' SELECT persons.id, persons.id AS id, @@ -628,7 +628,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestFunnelCorrelationsActors.test_funnel_correlation_on_properties_with_recordings.1 +# name: TestFunnelCorrelationActors.test_funnel_correlation_on_properties_with_recordings.1 ''' SELECT DISTINCT session_replay_events.session_id AS session_id FROM session_replay_events @@ -642,7 +642,71 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestFunnelCorrelationsActors.test_strict_funnel_correlation_with_recordings +# name: TestFunnelCorrelationActors.test_strict_funnel_correlation_with_recordings + ''' + SELECT countIf(ifNull(equals(steps, 1), 0)) AS step_1, + countIf(ifNull(equals(steps, 2), 0)) AS step_2, + avg(step_1_average_conversion_time_inner) AS step_1_average_conversion_time, + median(step_1_median_conversion_time_inner) AS step_1_median_conversion_time + FROM + (SELECT aggregation_target AS aggregation_target, + steps AS steps, + avg(step_1_conversion_time) AS step_1_average_conversion_time_inner, + median(step_1_conversion_time) AS step_1_median_conversion_time_inner + FROM + (SELECT aggregation_target AS aggregation_target, + steps AS steps, + max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, + step_1_conversion_time AS step_1_conversion_time + FROM + (SELECT aggregation_target AS aggregation_target, + timestamp AS timestamp, + step_0 AS step_0, + latest_0 AS latest_0, + step_1 AS step_1, + latest_1 AS latest_1, + if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1) AS steps, + if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time + FROM + (SELECT aggregation_target AS aggregation_target, + timestamp AS timestamp, + step_0 AS step_0, + latest_0 AS latest_0, + step_1 AS step_1, + min(latest_1) OVER (PARTITION BY aggregation_target + ORDER BY timestamp DESC ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS latest_1 + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + if(equals(e.event, '$pageview'), 1, 0) AS step_0, + if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, + if(equals(e.event, 'insight analyzed'), 1, 0) AS step_1, + if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-08 23:59:59.999999', 6, 'UTC')))))) + WHERE ifNull(equals(step_0, 1), 0))) + GROUP BY aggregation_target, + steps + HAVING ifNull(equals(steps, max(max_steps)), isNull(steps) + and isNull(max(max_steps)))) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + allow_experimental_analyzer=1 + ''' +# --- +# name: TestFunnelCorrelationActors.test_strict_funnel_correlation_with_recordings.1 ''' SELECT persons.id, persons.id AS id, @@ -819,7 +883,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestFunnelCorrelationsActors.test_strict_funnel_correlation_with_recordings.1 +# name: TestFunnelCorrelationActors.test_strict_funnel_correlation_with_recordings.2 ''' SELECT DISTINCT session_replay_events.session_id AS session_id FROM session_replay_events @@ -833,7 +897,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestFunnelCorrelationsActors.test_strict_funnel_correlation_with_recordings.2 +# name: TestFunnelCorrelationActors.test_strict_funnel_correlation_with_recordings.3 ''' SELECT persons.id, persons.id AS id, @@ -1010,7 +1074,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestFunnelCorrelationsActors.test_strict_funnel_correlation_with_recordings.3 +# name: TestFunnelCorrelationActors.test_strict_funnel_correlation_with_recordings.4 ''' SELECT DISTINCT session_replay_events.session_id AS session_id FROM session_replay_events diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_actors_udf.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_actors_udf.ambr new file mode 100644 index 0000000000000..c55def5ee7da4 --- /dev/null +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_actors_udf.ambr @@ -0,0 +1,677 @@ +# serializer version: 1 +# name: TestFunnelCorrelationsActorsUDF.test_funnel_correlation_on_event_with_recordings + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM events AS event + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS event__override ON equals(event.distinct_id, event__override.distinct_id) + JOIN + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, '$pageview'), 1, 0) AS step_0, + if(equals(e.event, 'insight analyzed'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-08 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('$pageview', 'insight analyzed'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(if(not(empty(event__override.distinct_id)), event__override.person_id, event.person_id), funnel_actors.actor_id) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-08 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-08 23:59:59', 6, 'UTC')))), notIn(event.event, ['$pageview', 'insight analyzed']), equals(event.event, 'insight loaded'), ifNull(equals(funnel_actors.steps, 2), 0)) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, any(funnel_actors.matching_events) AS matching_events + FROM events AS event + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS event__override ON equals(event.distinct_id, event__override.distinct_id) + JOIN + (SELECT aggregation_target AS actor_id, matched_events_array[plus(step_reached, 1)] AS matching_events, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps, final_timestamp, first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, '$pageview'), 1, 0) AS step_0, if(equals(e.event, 'insight analyzed'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-08 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('$pageview', 'insight analyzed'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(if(not(empty(event__override.distinct_id)), event__override.person_id, event.person_id), funnel_actors.actor_id) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-08 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-08 23:59:59', 6, 'UTC')))), notIn(event.event, ['$pageview', 'insight analyzed']), equals(event.event, 'insight loaded'), ifNull(equals(funnel_actors.steps, 2), 0)) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestFunnelCorrelationsActorsUDF.test_funnel_correlation_on_event_with_recordings.1 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2021-01-02 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, ['s2'])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestFunnelCorrelationsActorsUDF.test_funnel_correlation_on_event_with_recordings.2 + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM events AS event + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS event__override ON equals(event.distinct_id, event__override.distinct_id) + JOIN + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[3][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(3, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, '$pageview'), 1, 0) AS step_0, + if(equals(e.event, 'insight analyzed'), 1, 0) AS step_1, + if(equals(e.event, 'insight updated'), 1, 0) AS step_2 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-08 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('$pageview', 'insight analyzed', 'insight updated'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(if(not(empty(event__override.distinct_id)), event__override.person_id, event.person_id), funnel_actors.actor_id) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-08 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-08 23:59:59', 6, 'UTC')))), notIn(event.event, ['$pageview', 'insight analyzed', 'insight updated']), equals(event.event, 'insight loaded'), ifNull(notEquals(funnel_actors.steps, 3), 1)) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, any(funnel_actors.matching_events) AS matching_events + FROM events AS event + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS event__override ON equals(event.distinct_id, event__override.distinct_id) + JOIN + (SELECT aggregation_target AS actor_id, matched_events_array[plus(step_reached, 1)] AS matching_events, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[3][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps, final_timestamp, first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(3, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, '$pageview'), 1, 0) AS step_0, if(equals(e.event, 'insight analyzed'), 1, 0) AS step_1, if(equals(e.event, 'insight updated'), 1, 0) AS step_2 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-08 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('$pageview', 'insight analyzed', 'insight updated'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(if(not(empty(event__override.distinct_id)), event__override.person_id, event.person_id), funnel_actors.actor_id) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-08 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2021-01-08 23:59:59', 6, 'UTC')))), notIn(event.event, ['$pageview', 'insight analyzed', 'insight updated']), equals(event.event, 'insight loaded'), ifNull(notEquals(funnel_actors.steps, 3), 1)) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestFunnelCorrelationsActorsUDF.test_funnel_correlation_on_event_with_recordings.3 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2021-01-02 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, ['s2'])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestFunnelCorrelationsActorsUDF.test_funnel_correlation_on_properties_with_recordings + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, '$pageview'), 1, 0) AS step_0, + if(equals(e.event, 'insight analyzed'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'foo'), ''), 'null'), '^"|"$', '') AS properties___foo + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-08 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('$pageview', 'insight analyzed')), ifNull(equals(e__person.properties___foo, 'bar'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, matched_events_array[plus(step_reached, 1)] AS matching_events, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps, final_timestamp, first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, '$pageview'), 1, 0) AS step_0, if(equals(e.event, 'insight analyzed'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'foo'), ''), 'null'), '^"|"$', '') AS properties___foo + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-08 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('$pageview', 'insight analyzed')), ifNull(equals(e__person.properties___foo, 'bar'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestFunnelCorrelationsActorsUDF.test_funnel_correlation_on_properties_with_recordings.1 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2021-01-02 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, ['s2'])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestFunnelCorrelationsActorsUDF.test_strict_funnel_correlation_with_recordings + ''' + SELECT sum(step_1) AS step_1, + sum(step_2) AS step_2, + arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, + groupArray(row_number) AS row_number, + final_prop AS final_prop + FROM + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, + groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, + rowNumberInBlock() AS row_number, + breakdown AS final_prop + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'strict', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, '$pageview'), 1, 0) AS step_0, + if(equals(e.event, 'insight analyzed'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-08 23:59:59.999999', 6, 'UTC'))))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + GROUP BY breakdown + ORDER BY step_2 DESC, step_1 DESC) + GROUP BY final_prop + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + allow_experimental_analyzer=1 + ''' +# --- +# name: TestFunnelCorrelationsActorsUDF.test_strict_funnel_correlation_with_recordings.1 + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'strict', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, '$pageview'), 1, 0) AS step_0, + if(equals(e.event, 'insight analyzed'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'foo'), ''), 'null'), '^"|"$', '') AS properties___foo + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-08 23:59:59.999999', 6, 'UTC'))), ifNull(equals(e__person.properties___foo, 'bar'), 0))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, matched_events_array[plus(step_reached, 1)] AS matching_events, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps, final_timestamp, first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'strict', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, '$pageview'), 1, 0) AS step_0, if(equals(e.event, 'insight analyzed'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'foo'), ''), 'null'), '^"|"$', '') AS properties___foo + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-08 23:59:59.999999', 6, 'UTC'))), ifNull(equals(e__person.properties___foo, 'bar'), 0))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestFunnelCorrelationsActorsUDF.test_strict_funnel_correlation_with_recordings.2 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2021-01-02 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, ['s2'])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestFunnelCorrelationsActorsUDF.test_strict_funnel_correlation_with_recordings.3 + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'strict', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, '$pageview'), 1, 0) AS step_0, + if(equals(e.event, 'insight analyzed'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'foo'), ''), 'null'), '^"|"$', '') AS properties___foo + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-08 23:59:59.999999', 6, 'UTC'))), ifNull(equals(e__person.properties___foo, 'bar'), 0))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, matched_events_array[plus(step_reached, 1)] AS matching_events, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps, final_timestamp, first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'strict', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, '$pageview'), 1, 0) AS step_0, if(equals(e.event, 'insight analyzed'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'foo'), ''), 'null'), '^"|"$', '') AS properties___foo + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-08 23:59:59.999999', 6, 'UTC'))), ifNull(equals(e__person.properties___foo, 'bar'), 0))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestFunnelCorrelationsActorsUDF.test_strict_funnel_correlation_with_recordings.4 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2021-01-02 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, ['s3'])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_udf.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_udf.ambr new file mode 100644 index 0000000000000..e1ddaaabb7ece --- /dev/null +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation_udf.ambr @@ -0,0 +1,5080 @@ +# serializer version: 1 +# name: TestClickhouseFunnelCorrelationUDF.test_action_events_are_excluded_from_correlations + ''' + SELECT event.event AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM events AS event + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS event__override ON equals(event.distinct_id, event__override.distinct_id) + JOIN + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(and(equals(e.event, 'user signed up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0)), 1, 0) AS step_0, + if(and(equals(e.event, 'paid'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0)), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(if(not(empty(event__override.distinct_id)), event__override.person_id, event.person_id), funnel_actors.actor_id) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), notIn(event.event, [])) + GROUP BY name + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(and(equals(e.event, 'user signed up'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0)), 1, 0) AS step_0, + if(and(equals(e.event, 'paid'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, 'key'), ''), 'null'), '^"|"$', ''), 'val'), 0)), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_basic_funnel_correlation_with_properties + ''' + SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, 2), 0)) AS success_count, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, 2), 1)) AS failure_count + FROM + (SELECT funnel_actors.actor_id AS actor_id, + funnel_actors.steps AS steps, + arrayJoin(arrayZip(['$browser'], [JSONExtractString(persons.person_props, '$browser')])) AS prop + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + JOIN + (SELECT persons.id AS id, + persons.properties AS person_props + FROM + (SELECT person.id AS id, + person.properties AS properties + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons) AS persons ON equals(persons.id, funnel_actors.actor_id)) AS aggregation_target_with_props + GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 + HAVING ifNull(notIn((aggregation_target_with_props.prop).1, []), 0) + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_basic_funnel_correlation_with_properties.1 + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, matched_events_array[plus(step_reached, 1)] AS matching_events, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps, final_timestamp, first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_basic_funnel_correlation_with_properties.2 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2019-12-31 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, [''])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_basic_funnel_correlation_with_properties.3 + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, matched_events_array[plus(step_reached, 1)] AS matching_events, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps, final_timestamp, first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_basic_funnel_correlation_with_properties.4 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2019-12-31 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, [''])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_basic_funnel_correlation_with_properties.5 + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, matched_events_array[plus(step_reached, 1)] AS matching_events, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps, final_timestamp, first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_basic_funnel_correlation_with_properties.6 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2019-12-31 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, [''])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_basic_funnel_correlation_with_properties.7 + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, matched_events_array[plus(step_reached, 1)] AS matching_events, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps, final_timestamp, first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_basic_funnel_correlation_with_properties.8 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2019-12-31 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, [''])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_basic_funnel_correlation_with_properties_materialized + ''' + SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, 2), 0)) AS success_count, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, 2), 1)) AS failure_count + FROM + (SELECT funnel_actors.actor_id AS actor_id, + funnel_actors.steps AS steps, + arrayJoin(arrayZip(['$browser'], [JSONExtractString(persons.person_props, '$browser')])) AS prop + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + JOIN + (SELECT persons.id AS id, + persons.properties AS person_props + FROM + (SELECT person.id AS id, + person.properties AS properties + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons) AS persons ON equals(persons.id, funnel_actors.actor_id)) AS aggregation_target_with_props + GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 + HAVING ifNull(notIn((aggregation_target_with_props.prop).1, []), 0) + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_basic_funnel_correlation_with_properties_materialized.1 + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + nullIf(nullIf(person.`pmat_$browser`, ''), 'null') AS `properties___$browser` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, matched_events_array[plus(step_reached, 1)] AS matching_events, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps, final_timestamp, first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, nullIf(nullIf(person.`pmat_$browser`, ''), 'null') AS `properties___$browser` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_basic_funnel_correlation_with_properties_materialized.2 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2019-12-31 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, [''])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_basic_funnel_correlation_with_properties_materialized.3 + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + nullIf(nullIf(person.`pmat_$browser`, ''), 'null') AS `properties___$browser` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, matched_events_array[plus(step_reached, 1)] AS matching_events, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps, final_timestamp, first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, nullIf(nullIf(person.`pmat_$browser`, ''), 'null') AS `properties___$browser` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_basic_funnel_correlation_with_properties_materialized.4 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2019-12-31 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, [''])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_basic_funnel_correlation_with_properties_materialized.5 + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + nullIf(nullIf(person.`pmat_$browser`, ''), 'null') AS `properties___$browser` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, matched_events_array[plus(step_reached, 1)] AS matching_events, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps, final_timestamp, first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, nullIf(nullIf(person.`pmat_$browser`, ''), 'null') AS `properties___$browser` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_basic_funnel_correlation_with_properties_materialized.6 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2019-12-31 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, [''])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_basic_funnel_correlation_with_properties_materialized.7 + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + nullIf(nullIf(person.`pmat_$browser`, ''), 'null') AS `properties___$browser` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, matched_events_array[plus(step_reached, 1)] AS matching_events, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps, final_timestamp, first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, nullIf(nullIf(person.`pmat_$browser`, ''), 'null') AS `properties___$browser` + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_basic_funnel_correlation_with_properties_materialized.8 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2019-12-31 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, [''])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_event_properties_and_groups + ''' + SELECT concat(ifNull(toString(event_name), ''), '::', ifNull(toString((prop).1), ''), '::', ifNull(toString((prop).2), '')) AS name, + countDistinctIf(actor_id, ifNull(equals(steps, 2), 0)) AS success_count, + countDistinctIf(actor_id, ifNull(notEquals(steps, 2), 1)) AS failure_count + FROM + (SELECT funnel_actors.actor_id AS actor_id, + funnel_actors.steps AS steps, + event.event AS event_name, + arrayJoin(JSONExtractKeysAndValues(event.properties, 'String')) AS prop + FROM events AS event + JOIN + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_1` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_1`) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), in(event.event, ['positively_related', 'negatively_related']))) + GROUP BY name + HAVING and(ifNull(greater(plus(success_count, failure_count), 2), 0), ifNull(notIn((prop).1, []), 0)) + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_1` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_event_properties_and_groups_materialized + ''' + SELECT concat(ifNull(toString(event_name), ''), '::', ifNull(toString((prop).1), ''), '::', ifNull(toString((prop).2), '')) AS name, + countDistinctIf(actor_id, ifNull(equals(steps, 2), 0)) AS success_count, + countDistinctIf(actor_id, ifNull(notEquals(steps, 2), 1)) AS failure_count + FROM + (SELECT funnel_actors.actor_id AS actor_id, + funnel_actors.steps AS steps, + event.event AS event_name, + arrayJoin(JSONExtractKeysAndValues(event.properties, 'String')) AS prop + FROM events AS event + JOIN + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_1` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_1`) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), in(event.event, ['positively_related', 'negatively_related']))) + GROUP BY name + HAVING and(ifNull(greater(plus(success_count, failure_count), 2), 0), ifNull(notIn((prop).1, []), 0)) + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_1` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_events_and_groups + ''' + SELECT event.event AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM events AS event + JOIN + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), notIn(event.event, [])) + GROUP BY name + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_events_and_groups.1 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM events AS event + JOIN + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'positively_related'), ifNull(equals(funnel_actors.steps, 2), 0)) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_events_and_groups.2 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM events AS event + JOIN + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'positively_related'), ifNull(notEquals(funnel_actors.steps, 2), 1)) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_events_and_groups.3 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM events AS event + JOIN + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, 2), 0)) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_events_and_groups.4 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM events AS event + JOIN + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, 2), 1)) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_events_and_groups.5 + ''' + SELECT event.event AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM events AS event + JOIN + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'finance'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), notIn(event.event, [])) + GROUP BY name + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'finance'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_events_and_groups.6 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM events AS event + JOIN + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, 2), 0)) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_events_and_groups.7 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM events AS event + JOIN + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, 2), 1)) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_events_and_groups_poe_v2 + ''' + SELECT event.event AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM events AS event + JOIN + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), notIn(event.event, [])) + GROUP BY name + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_events_and_groups_poe_v2.1 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM events AS event + JOIN + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'positively_related'), ifNull(equals(funnel_actors.steps, 2), 0)) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_events_and_groups_poe_v2.2 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM events AS event + JOIN + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'positively_related'), ifNull(notEquals(funnel_actors.steps, 2), 1)) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_events_and_groups_poe_v2.3 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM events AS event + JOIN + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, 2), 0)) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_events_and_groups_poe_v2.4 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM events AS event + JOIN + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, 2), 1)) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_events_and_groups_poe_v2.5 + ''' + SELECT event.event AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM events AS event + JOIN + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'finance'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), notIn(event.event, [])) + GROUP BY name + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'finance'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_events_and_groups_poe_v2.6 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM events AS event + JOIN + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, 2), 0)) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_events_and_groups_poe_v2.7 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM events AS event + JOIN + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 2), greaterOrEquals(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-01 00:00:00', 6, 'UTC'))), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC'))), equals(event.team_id, 2), greater(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toTimeZone(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-14 23:59:59', 6, 'UTC')))), notIn(event.event, ['paid', 'user signed up']), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, 2), 1)) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups + ''' + SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, 2), 0)) AS success_count, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, 2), 1)) AS failure_count + FROM + (SELECT funnel_actors.actor_id AS actor_id, + funnel_actors.steps AS steps, + arrayJoin(arrayZip(['industry'], [JSONExtractString(groups_0.properties, 'industry')])) AS prop + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LEFT JOIN + (SELECT groups.key AS key, + groups.properties AS properties + FROM + (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups + WHERE ifNull(equals(groups.index, 0), 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props + GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 + HAVING ifNull(notIn((aggregation_target_with_props.prop).1, []), 0) + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups.1 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups.2 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups.3 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups.4 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups.5 + ''' + SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, 2), 0)) AS success_count, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, 2), 1)) AS failure_count + FROM + (SELECT funnel_actors.actor_id AS actor_id, + funnel_actors.steps AS steps, + arrayJoin(JSONExtractKeysAndValues(groups_0.properties, 'String')) AS prop + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LEFT JOIN + (SELECT groups.key AS key, + groups.properties AS properties + FROM + (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups + WHERE ifNull(equals(groups.index, 0), 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props + GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 + HAVING ifNull(notIn((aggregation_target_with_props.prop).1, []), 0) + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_materialized + ''' + SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, 2), 0)) AS success_count, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, 2), 1)) AS failure_count + FROM + (SELECT funnel_actors.actor_id AS actor_id, + funnel_actors.steps AS steps, + arrayJoin(arrayZip(['industry'], [JSONExtractString(groups_0.properties, 'industry')])) AS prop + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LEFT JOIN + (SELECT groups.key AS key, + groups.properties AS properties + FROM + (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups + WHERE ifNull(equals(groups.index, 0), 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props + GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 + HAVING ifNull(notIn((aggregation_target_with_props.prop).1, []), 0) + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_materialized.1 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_materialized.2 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_materialized.3 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_materialized.4 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_materialized.5 + ''' + SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, 2), 0)) AS success_count, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, 2), 1)) AS failure_count + FROM + (SELECT funnel_actors.actor_id AS actor_id, + funnel_actors.steps AS steps, + arrayJoin(JSONExtractKeysAndValues(groups_0.properties, 'String')) AS prop + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LEFT JOIN + (SELECT groups.key AS key, + groups.properties AS properties + FROM + (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups + WHERE ifNull(equals(groups.index, 0), 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props + GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 + HAVING ifNull(notIn((aggregation_target_with_props.prop).1, []), 0) + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_person_on_events + ''' + SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, 2), 0)) AS success_count, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, 2), 1)) AS failure_count + FROM + (SELECT funnel_actors.actor_id AS actor_id, + funnel_actors.steps AS steps, + arrayJoin(arrayZip(['industry'], [JSONExtractString(groups_0.properties, 'industry')])) AS prop + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LEFT JOIN + (SELECT groups.key AS key, + groups.properties AS properties + FROM + (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups + WHERE ifNull(equals(groups.index, 0), 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props + GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 + HAVING ifNull(notIn((aggregation_target_with_props.prop).1, []), 0) + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_person_on_events.1 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_person_on_events.2 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_person_on_events.3 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_person_on_events.4 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_person_on_events.5 + ''' + SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, 2), 0)) AS success_count, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, 2), 1)) AS failure_count + FROM + (SELECT funnel_actors.actor_id AS actor_id, + funnel_actors.steps AS steps, + arrayJoin(JSONExtractKeysAndValues(groups_0.properties, 'String')) AS prop + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LEFT JOIN + (SELECT groups.key AS key, + groups.properties AS properties + FROM + (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups + WHERE ifNull(equals(groups.index, 0), 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props + GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 + HAVING ifNull(notIn((aggregation_target_with_props.prop).1, []), 0) + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_person_on_events_materialized + ''' + SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, 2), 0)) AS success_count, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, 2), 1)) AS failure_count + FROM + (SELECT funnel_actors.actor_id AS actor_id, + funnel_actors.steps AS steps, + arrayJoin(arrayZip(['industry'], [JSONExtractString(groups_0.properties, 'industry')])) AS prop + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LEFT JOIN + (SELECT groups.key AS key, + groups.properties AS properties + FROM + (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups + WHERE ifNull(equals(groups.index, 0), 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props + GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 + HAVING ifNull(notIn((aggregation_target_with_props.prop).1, []), 0) + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_person_on_events_materialized.1 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_person_on_events_materialized.2 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_person_on_events_materialized.3 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_person_on_events_materialized.4 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_person_on_events_materialized.5 + ''' + SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, 2), 0)) AS success_count, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, 2), 1)) AS failure_count + FROM + (SELECT funnel_actors.actor_id AS actor_id, + funnel_actors.steps AS steps, + arrayJoin(JSONExtractKeysAndValues(groups_0.properties, 'String')) AS prop + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LEFT JOIN + (SELECT groups.key AS key, + groups.properties AS properties + FROM + (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups + WHERE ifNull(equals(groups.index, 0), 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props + GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 + HAVING ifNull(notIn((aggregation_target_with_props.prop).1, []), 0) + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_person_on_events_poe_v2 + ''' + SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, 2), 0)) AS success_count, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, 2), 1)) AS failure_count + FROM + (SELECT funnel_actors.actor_id AS actor_id, + funnel_actors.steps AS steps, + arrayJoin(arrayZip(['industry'], [JSONExtractString(groups_0.properties, 'industry')])) AS prop + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LEFT JOIN + (SELECT groups.key AS key, + groups.properties AS properties + FROM + (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups + WHERE ifNull(equals(groups.index, 0), 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props + GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 + HAVING ifNull(notIn((aggregation_target_with_props.prop).1, []), 0) + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_person_on_events_poe_v2.1 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_person_on_events_poe_v2.2 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_person_on_events_poe_v2.3 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(equals(funnel_actors.steps, 2), 0) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_person_on_events_poe_v2.4 + ''' + SELECT source.actor_id AS actor_id + FROM + (SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[plus(step_reached, 1)] AS matching_events, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + WHERE ifNull(notEquals(funnel_actors.steps, 2), 1) + GROUP BY funnel_actors.actor_id + ORDER BY funnel_actors.actor_id ASC) AS source + INNER JOIN + (SELECT groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups ON equals(groups.key, source.actor_id) + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelationUDF.test_funnel_correlation_with_properties_and_groups_person_on_events_poe_v2.5 + ''' + SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, 2), 0)) AS success_count, + countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, 2), 1)) AS failure_count + FROM + (SELECT funnel_actors.actor_id AS actor_id, + funnel_actors.steps AS steps, + arrayJoin(JSONExtractKeysAndValues(groups_0.properties, 'String')) AS prop + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LEFT JOIN + (SELECT groups.key AS key, + groups.properties AS properties + FROM + (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE equals(groups.team_id, 2) + GROUP BY groups.group_type_index, + groups.group_key) AS groups + WHERE ifNull(equals(groups.index, 0), 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props + GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 + HAVING ifNull(notIn((aggregation_target_with_props.prop).1, []), 0) + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, 2), 0)) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, 2), 1)) AS failure_count + FROM + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps, + final_timestamp, + first_timestamp + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_persons_udf.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_persons_udf.ambr new file mode 100644 index 0000000000000..90e66603e5963 --- /dev/null +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_persons_udf.ambr @@ -0,0 +1,220 @@ +# serializer version: 1 +# name: TestFunnelPersonsUDF.test_funnel_person_recordings + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[1] AS matching_events + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(3, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'step one'), 1, 0) AS step_0, + if(equals(e.event, 'step two'), 1, 0) AS step_1, + if(equals(e.event, 'step three'), 1, 0) AS step_2 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-08 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + allow_experimental_analyzer=1 + ''' +# --- +# name: TestFunnelPersonsUDF.test_funnel_person_recordings.1 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2021-01-02 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, ['s1'])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestFunnelPersonsUDF.test_funnel_person_recordings.2 + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[2] AS matching_events + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(3, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'step one'), 1, 0) AS step_0, + if(equals(e.event, 'step two'), 1, 0) AS step_1, + if(equals(e.event, 'step three'), 1, 0) AS step_2 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-08 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 1), 0) + ORDER BY aggregation_target ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + allow_experimental_analyzer=1 + ''' +# --- +# name: TestFunnelPersonsUDF.test_funnel_person_recordings.3 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2021-01-02 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, ['s2'])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestFunnelPersonsUDF.test_funnel_person_recordings.4 + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[2] AS matching_events + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(3, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'step one'), 1, 0) AS step_0, + if(equals(e.event, 'step two'), 1, 0) AS step_1, + if(equals(e.event, 'step three'), 1, 0) AS step_2 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-08 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(equals(step_reached, 1), 0) + ORDER BY aggregation_target ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + allow_experimental_analyzer=1 + ''' +# --- +# name: TestFunnelPersonsUDF.test_funnel_person_recordings.5 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2021-01-02 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, ['s2'])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_persons_udf.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_persons_udf.ambr new file mode 100644 index 0000000000000..1dd99c28faa3a --- /dev/null +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_persons_udf.ambr @@ -0,0 +1,220 @@ +# serializer version: 1 +# name: TestFunnelStrictStepsPersons.test_strict_funnel_person_recordings + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[1] AS matching_events + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(3, 1209600, 'first_touch', 'strict', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'step one'), 1, 0) AS step_0, + if(equals(e.event, 'step two'), 1, 0) AS step_1, + if(equals(e.event, 'step three'), 1, 0) AS step_2 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-08 23:59:59.999999', 6, 'UTC'))))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) + ORDER BY aggregation_target ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + allow_experimental_analyzer=1 + ''' +# --- +# name: TestFunnelStrictStepsPersons.test_strict_funnel_person_recordings.1 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2021-01-02 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, ['s1'])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestFunnelStrictStepsPersons.test_strict_funnel_person_recordings.2 + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[2] AS matching_events + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(3, 1209600, 'first_touch', 'strict', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'step one'), 1, 0) AS step_0, + if(equals(e.event, 'step two'), 1, 0) AS step_1, + if(equals(e.event, 'step three'), 1, 0) AS step_2 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-08 23:59:59.999999', 6, 'UTC'))))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 1), 0) + ORDER BY aggregation_target ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + allow_experimental_analyzer=1 + ''' +# --- +# name: TestFunnelStrictStepsPersons.test_strict_funnel_person_recordings.3 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2021-01-02 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, ['s2'])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestFunnelStrictStepsPersons.test_strict_funnel_person_recordings.4 + ''' + SELECT persons.id, + persons.id AS id, + source.matching_events AS matching_events + FROM + (SELECT aggregation_target AS actor_id, + matched_events_array[2] AS matching_events + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(3, 1209600, 'first_touch', 'strict', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'step one'), 1, 0) AS step_0, + if(equals(e.event, 'step two'), 1, 0) AS step_1, + if(equals(e.event, 'step three'), 1, 0) AS step_2 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-01-08 23:59:59.999999', 6, 'UTC'))))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(equals(step_reached, 1), 0) + ORDER BY aggregation_target ASC) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + allow_experimental_analyzer=1 + ''' +# --- +# name: TestFunnelStrictStepsPersons.test_strict_funnel_person_recordings.5 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2021-01-02 00:00:00.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, ['s2'])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_udf.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_udf.ambr index 66b5642a0d5f4..ca74dfd1d449e 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_udf.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_strict_udf.ambr @@ -1337,6 +1337,217 @@ allow_experimental_analyzer=1 ''' # --- +# name: TestFunnelStrictStepsBreakdownUDF.test_funnel_breakdown_correct_breakdown_props_are_chosen + ''' + SELECT sum(step_1) AS step_1, + sum(step_2) AS step_2, + arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, + groupArray(row_number) AS row_number, + final_prop AS final_prop + FROM + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, + groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, + rowNumberInBlock() AS row_number, + if(ifNull(less(row_number, 25), 0), breakdown, ['Other']) AS final_prop + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'strict', groupUniqArray(prop), arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, prop, arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + aggregation_target AS aggregation_target + FROM + (SELECT timestamp AS timestamp, + aggregation_target AS aggregation_target, + uuid AS uuid, + `$session_id` AS `$session_id`, + `$window_id` AS `$window_id`, + step_0 AS step_0, + step_1 AS step_1, + prop_basic AS prop_basic, + prop, + prop_vals AS prop_vals, + if(notEmpty(arrayFilter(x -> notEmpty(x), prop_vals)), prop_vals, ['']) AS prop + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'sign up'), 1, 0) AS step_0, + if(and(equals(e.event, 'buy'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$version'), ''), 'null'), '^"|"$', ''), 'xyz'), 0)), 1, 0) AS step_1, + [ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', '')), '')] AS prop_basic, + prop_basic AS prop, + argMinIf(prop, timestamp, notEmpty(arrayFilter(x -> notEmpty(x), prop))) OVER (PARTITION BY aggregation_target) AS prop_vals + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-08 23:59:59.999999', 6, 'UTC')))))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + GROUP BY breakdown + ORDER BY step_2 DESC, step_1 DESC) + GROUP BY final_prop + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + allow_experimental_analyzer=1 + ''' +# --- +# name: TestFunnelStrictStepsBreakdownUDF.test_funnel_breakdown_correct_breakdown_props_are_chosen_for_step + ''' + SELECT sum(step_1) AS step_1, + sum(step_2) AS step_2, + arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, + groupArray(row_number) AS row_number, + final_prop AS final_prop + FROM + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, + groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, + rowNumberInBlock() AS row_number, + if(ifNull(less(row_number, 25), 0), breakdown, ['Other']) AS final_prop + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'step_1', 'strict', groupUniqArray(prop), arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, prop, arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + aggregation_target AS aggregation_target + FROM + (SELECT timestamp AS timestamp, + aggregation_target AS aggregation_target, + uuid AS uuid, + `$session_id` AS `$session_id`, + `$window_id` AS `$window_id`, + step_0 AS step_0, + step_1 AS step_1, + prop_basic AS prop_basic, + prop_0 AS prop_0, + prop_1 AS prop_1, + prop, + prop_vals AS prop_vals, + prop + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'sign up'), 1, 0) AS step_0, + if(and(equals(e.event, 'buy'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$version'), ''), 'null'), '^"|"$', ''), 'xyz'), 0)), 1, 0) AS step_1, + [ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', '')), '')] AS prop_basic, + if(ifNull(equals(step_0, 1), 0), prop_basic, []) AS prop_0, + if(ifNull(equals(step_1, 1), 0), prop_basic, []) AS prop_1, + prop_1 AS prop, + groupUniqArray(prop) OVER (PARTITION BY aggregation_target) AS prop_vals + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-08 23:59:59.999999', 6, 'UTC'))))) ARRAY + JOIN prop_vals AS prop + WHERE ifNull(notEquals(prop, []), isNotNull(prop) + or isNotNull([]))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + GROUP BY breakdown + ORDER BY step_2 DESC, step_1 DESC) + GROUP BY final_prop + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + allow_experimental_analyzer=1 + ''' +# --- +# name: TestFunnelStrictStepsBreakdownUDF.test_funnel_step_multiple_breakdown_snapshot + ''' + SELECT sum(step_1) AS step_1, + sum(step_2) AS step_2, + arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, + groupArray(row_number) AS row_number, + final_prop AS final_prop + FROM + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, + groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, + rowNumberInBlock() AS row_number, + if(ifNull(less(row_number, 25), 0), breakdown, ['Other']) AS final_prop + FROM + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'strict', groupUniqArray(prop), arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, prop, arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + aggregation_target AS aggregation_target + FROM + (SELECT timestamp AS timestamp, + aggregation_target AS aggregation_target, + uuid AS uuid, + `$session_id` AS `$session_id`, + `$window_id` AS `$window_id`, + step_0 AS step_0, + step_1 AS step_1, + prop_basic AS prop_basic, + prop, + prop_vals AS prop_vals, + if(notEmpty(arrayFilter(x -> notEmpty(x), prop_vals)), prop_vals, ['', '']) AS prop + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'sign up'), 1, 0) AS step_0, + if(equals(e.event, 'buy'), 1, 0) AS step_1, + [ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', '')), ''), ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$version'), ''), 'null'), '^"|"$', '')), '')] AS prop_basic, + prop_basic AS prop, + argMinIf(prop, timestamp, notEmpty(arrayFilter(x -> notEmpty(x), prop))) OVER (PARTITION BY aggregation_target) AS prop_vals + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-08 23:59:59.999999', 6, 'UTC')))))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + GROUP BY breakdown + ORDER BY step_2 DESC, step_1 DESC) + GROUP BY final_prop + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + allow_experimental_analyzer=1 + ''' +# --- # name: TestStrictFunnelGroupBreakdown.test_funnel_aggregate_by_groups_breakdown_group_person_on_events ''' SELECT sum(step_1) AS step_1, @@ -2064,3 +2275,718 @@ max_expanded_ast_elements=1000000 ''' # --- +# name: TestStrictFunnelGroupBreakdownUDF.test_funnel_aggregate_by_groups_breakdown_group_person_on_events + ''' + SELECT sum(step_1) AS step_1, + sum(step_2) AS step_2, + sum(step_3) AS step_3, + arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_2_conversion_times)])[1] AS step_2_average_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_2_conversion_times)])[1] AS step_2_median_conversion_time, + groupArray(row_number) AS row_number, + final_prop AS final_prop + FROM + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, + countIf(ifNull(ifNull(equals(step_reached, 2), 0), 0)) AS step_3, + groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, + groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, + rowNumberInBlock() AS row_number, + if(ifNull(less(row_number, 25), 0), breakdown, 'Other') AS final_prop + FROM + (SELECT arrayJoin(aggregate_funnel_v0(3, 1209600, 'first_touch', 'strict', groupUniqArray(prop), arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, prop, arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + aggregation_target AS aggregation_target + FROM + (SELECT timestamp AS timestamp, + aggregation_target AS aggregation_target, + uuid AS uuid, + `$session_id` AS `$session_id`, + `$window_id` AS `$window_id`, + step_0 AS step_0, + step_1 AS step_1, + step_2 AS step_2, + prop_basic AS prop_basic, + prop, + prop_vals AS prop_vals, + prop_vals AS prop + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'sign up'), 1, 0) AS step_0, + if(equals(e.event, 'play movie'), 1, 0) AS step_1, + if(equals(e.event, 'buy'), 1, 0) AS step_2, + ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, + prop_basic AS prop, + argMinIf(prop, timestamp, isNotNull(prop)) OVER (PARTITION BY aggregation_target) AS prop_vals + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-08 23:59:59.999999', 6, 'UTC')))))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + GROUP BY breakdown + ORDER BY step_3 DESC, step_2 DESC, step_1 DESC) + GROUP BY final_prop + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + allow_experimental_analyzer=1 + ''' +# --- +# name: TestStrictFunnelGroupBreakdownUDF.test_funnel_aggregate_by_groups_breakdown_group_person_on_events_poe_v2 + ''' + SELECT sum(step_1) AS step_1, + sum(step_2) AS step_2, + sum(step_3) AS step_3, + arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_2_conversion_times)])[1] AS step_2_average_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_2_conversion_times)])[1] AS step_2_median_conversion_time, + groupArray(row_number) AS row_number, + final_prop AS final_prop + FROM + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, + countIf(ifNull(ifNull(equals(step_reached, 2), 0), 0)) AS step_3, + groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, + groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, + rowNumberInBlock() AS row_number, + if(ifNull(less(row_number, 25), 0), breakdown, 'Other') AS final_prop + FROM + (SELECT arrayJoin(aggregate_funnel_v0(3, 1209600, 'first_touch', 'strict', groupUniqArray(prop), arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, prop, arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + aggregation_target AS aggregation_target + FROM + (SELECT timestamp AS timestamp, + aggregation_target AS aggregation_target, + uuid AS uuid, + `$session_id` AS `$session_id`, + `$window_id` AS `$window_id`, + step_0 AS step_0, + step_1 AS step_1, + step_2 AS step_2, + prop_basic AS prop_basic, + prop, + prop_vals AS prop_vals, + prop_vals AS prop + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'sign up'), 1, 0) AS step_0, + if(equals(e.event, 'play movie'), 1, 0) AS step_1, + if(equals(e.event, 'buy'), 1, 0) AS step_2, + ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, + prop_basic AS prop, + argMinIf(prop, timestamp, isNotNull(prop)) OVER (PARTITION BY aggregation_target) AS prop_vals + FROM events AS e + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-08 23:59:59.999999', 6, 'UTC')))))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + GROUP BY breakdown + ORDER BY step_3 DESC, step_2 DESC, step_1 DESC) + GROUP BY final_prop + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + allow_experimental_analyzer=1 + ''' +# --- +# name: TestStrictFunnelGroupBreakdownUDF.test_funnel_breakdown_group + ''' + SELECT sum(step_1) AS step_1, + sum(step_2) AS step_2, + sum(step_3) AS step_3, + arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_2_conversion_times)])[1] AS step_2_average_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_2_conversion_times)])[1] AS step_2_median_conversion_time, + groupArray(row_number) AS row_number, + final_prop AS final_prop + FROM + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, + countIf(ifNull(ifNull(equals(step_reached, 2), 0), 0)) AS step_3, + groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, + groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, + rowNumberInBlock() AS row_number, + if(ifNull(less(row_number, 25), 0), breakdown, 'Other') AS final_prop + FROM + (SELECT arrayJoin(aggregate_funnel_v0(3, 1209600, 'first_touch', 'strict', groupUniqArray(prop), arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, prop, arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + aggregation_target AS aggregation_target + FROM + (SELECT timestamp AS timestamp, + aggregation_target AS aggregation_target, + uuid AS uuid, + `$session_id` AS `$session_id`, + `$window_id` AS `$window_id`, + step_0 AS step_0, + step_1 AS step_1, + step_2 AS step_2, + prop_basic AS prop_basic, + prop, + prop_vals AS prop_vals, + prop_vals AS prop + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'sign up'), 1, 0) AS step_0, + if(equals(e.event, 'play movie'), 1, 0) AS step_1, + if(equals(e.event, 'buy'), 1, 0) AS step_2, + ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, + prop_basic AS prop, + argMinIf(prop, timestamp, isNotNull(prop)) OVER (PARTITION BY aggregation_target) AS prop_vals + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 2), and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-08 23:59:59.999999', 6, 'UTC')))))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + GROUP BY breakdown + ORDER BY step_3 DESC, step_2 DESC, step_1 DESC) + GROUP BY final_prop + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + allow_experimental_analyzer=1 + ''' +# --- +# name: TestStrictFunnelGroupBreakdownUDF.test_funnel_breakdown_group.1 + ''' + + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups + WHERE team_id = 2 + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 26 + OFFSET 0 + ''' +# --- +# name: TestStrictFunnelGroupBreakdownUDF.test_funnel_breakdown_group.2 + ''' + + SELECT aggregation_target AS actor_id + FROM + (SELECT aggregation_target, + steps, + avg(step_1_conversion_time) step_1_average_conversion_time_inner, + avg(step_2_conversion_time) step_2_average_conversion_time_inner, + median(step_1_conversion_time) step_1_median_conversion_time_inner, + median(step_2_conversion_time) step_2_median_conversion_time_inner, + prop + FROM + (SELECT aggregation_target, + steps, + max(steps) over (PARTITION BY aggregation_target, + prop) as max_steps, + step_1_conversion_time, + step_2_conversion_time, + prop + FROM + (SELECT *, + if(latest_0 <= latest_1 + AND latest_1 <= latest_0 + INTERVAL 7 DAY + AND latest_1 <= latest_2 + AND latest_2 <= latest_0 + INTERVAL 7 DAY, 3, if(latest_0 <= latest_1 + AND latest_1 <= latest_0 + INTERVAL 7 DAY, 2, 1)) AS steps, + if(isNotNull(latest_1) + AND latest_1 <= latest_0 + INTERVAL 7 DAY, dateDiff('second', toDateTime(latest_0), toDateTime(latest_1)), NULL) step_1_conversion_time, + if(isNotNull(latest_2) + AND latest_2 <= latest_1 + INTERVAL 7 DAY, dateDiff('second', toDateTime(latest_1), toDateTime(latest_2)), NULL) step_2_conversion_time + FROM + (SELECT aggregation_target, timestamp, step_0, + latest_0, + step_1, + min(latest_1) over (PARTITION by aggregation_target, + prop + ORDER BY timestamp DESC ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) latest_1, + step_2, + min(latest_2) over (PARTITION by aggregation_target, + prop + ORDER BY timestamp DESC ROWS BETWEEN 2 PRECEDING AND 2 PRECEDING) latest_2 , + if(has(['technology', 'finance'], prop), prop, 'Other') as prop + FROM + (SELECT *, + prop_vals as prop + FROM + (SELECT e.timestamp as timestamp, + if(notEmpty(pdi.distinct_id), pdi.person_id, e.person_id) as aggregation_target, + if(notEmpty(pdi.distinct_id), pdi.person_id, e.person_id) as person_id, + if(event = 'sign up', 1, 0) as step_0, + if(step_0 = 1, timestamp, null) as latest_0, + if(event = 'play movie', 1, 0) as step_1, + if(step_1 = 1, timestamp, null) as latest_1, + if(event = 'buy', 1, 0) as step_2, + if(step_2 = 1, timestamp, null) as latest_2, + replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS prop_basic, + prop_basic as prop, + argMinIf(prop, timestamp, isNotNull(prop)) over (PARTITION by aggregation_target) as prop_vals + FROM events e + LEFT OUTER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 + WHERE team_id = 2 + AND distinct_id IN + (SELECT distinct_id + FROM events + WHERE team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') ) + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups + WHERE team_id = 2 + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + AND (1=1) ))) + WHERE step_0 = 1 )) + GROUP BY aggregation_target, + steps, + prop + HAVING steps = max_steps) + WHERE steps IN [1, 2, 3] + AND arrayFlatten(array(prop)) = arrayFlatten(array('finance')) + ORDER BY aggregation_target + LIMIT 100 + OFFSET 0 SETTINGS max_ast_elements=1000000, + max_expanded_ast_elements=1000000 + ''' +# --- +# name: TestStrictFunnelGroupBreakdownUDF.test_funnel_breakdown_group.3 + ''' + + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups + WHERE team_id = 2 + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 26 + OFFSET 0 + ''' +# --- +# name: TestStrictFunnelGroupBreakdownUDF.test_funnel_breakdown_group.4 + ''' + + SELECT aggregation_target AS actor_id + FROM + (SELECT aggregation_target, + steps, + avg(step_1_conversion_time) step_1_average_conversion_time_inner, + avg(step_2_conversion_time) step_2_average_conversion_time_inner, + median(step_1_conversion_time) step_1_median_conversion_time_inner, + median(step_2_conversion_time) step_2_median_conversion_time_inner, + prop + FROM + (SELECT aggregation_target, + steps, + max(steps) over (PARTITION BY aggregation_target, + prop) as max_steps, + step_1_conversion_time, + step_2_conversion_time, + prop + FROM + (SELECT *, + if(latest_0 <= latest_1 + AND latest_1 <= latest_0 + INTERVAL 7 DAY + AND latest_1 <= latest_2 + AND latest_2 <= latest_0 + INTERVAL 7 DAY, 3, if(latest_0 <= latest_1 + AND latest_1 <= latest_0 + INTERVAL 7 DAY, 2, 1)) AS steps, + if(isNotNull(latest_1) + AND latest_1 <= latest_0 + INTERVAL 7 DAY, dateDiff('second', toDateTime(latest_0), toDateTime(latest_1)), NULL) step_1_conversion_time, + if(isNotNull(latest_2) + AND latest_2 <= latest_1 + INTERVAL 7 DAY, dateDiff('second', toDateTime(latest_1), toDateTime(latest_2)), NULL) step_2_conversion_time + FROM + (SELECT aggregation_target, timestamp, step_0, + latest_0, + step_1, + min(latest_1) over (PARTITION by aggregation_target, + prop + ORDER BY timestamp DESC ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) latest_1, + step_2, + min(latest_2) over (PARTITION by aggregation_target, + prop + ORDER BY timestamp DESC ROWS BETWEEN 2 PRECEDING AND 2 PRECEDING) latest_2 , + if(has(['technology', 'finance'], prop), prop, 'Other') as prop + FROM + (SELECT *, + prop_vals as prop + FROM + (SELECT e.timestamp as timestamp, + if(notEmpty(pdi.distinct_id), pdi.person_id, e.person_id) as aggregation_target, + if(notEmpty(pdi.distinct_id), pdi.person_id, e.person_id) as person_id, + if(event = 'sign up', 1, 0) as step_0, + if(step_0 = 1, timestamp, null) as latest_0, + if(event = 'play movie', 1, 0) as step_1, + if(step_1 = 1, timestamp, null) as latest_1, + if(event = 'buy', 1, 0) as step_2, + if(step_2 = 1, timestamp, null) as latest_2, + replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS prop_basic, + prop_basic as prop, + argMinIf(prop, timestamp, isNotNull(prop)) over (PARTITION by aggregation_target) as prop_vals + FROM events e + LEFT OUTER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 + WHERE team_id = 2 + AND distinct_id IN + (SELECT distinct_id + FROM events + WHERE team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') ) + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups + WHERE team_id = 2 + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + AND (1=1) ))) + WHERE step_0 = 1 )) + GROUP BY aggregation_target, + steps, + prop + HAVING steps = max_steps) + WHERE steps IN [2, 3] + AND arrayFlatten(array(prop)) = arrayFlatten(array('finance')) + ORDER BY aggregation_target + LIMIT 100 + OFFSET 0 SETTINGS max_ast_elements=1000000, + max_expanded_ast_elements=1000000 + ''' +# --- +# name: TestStrictFunnelGroupBreakdownUDF.test_funnel_breakdown_group.5 + ''' + + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups + WHERE team_id = 2 + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 26 + OFFSET 0 + ''' +# --- +# name: TestStrictFunnelGroupBreakdownUDF.test_funnel_breakdown_group.6 + ''' + + SELECT aggregation_target AS actor_id + FROM + (SELECT aggregation_target, + steps, + avg(step_1_conversion_time) step_1_average_conversion_time_inner, + avg(step_2_conversion_time) step_2_average_conversion_time_inner, + median(step_1_conversion_time) step_1_median_conversion_time_inner, + median(step_2_conversion_time) step_2_median_conversion_time_inner, + prop + FROM + (SELECT aggregation_target, + steps, + max(steps) over (PARTITION BY aggregation_target, + prop) as max_steps, + step_1_conversion_time, + step_2_conversion_time, + prop + FROM + (SELECT *, + if(latest_0 <= latest_1 + AND latest_1 <= latest_0 + INTERVAL 7 DAY + AND latest_1 <= latest_2 + AND latest_2 <= latest_0 + INTERVAL 7 DAY, 3, if(latest_0 <= latest_1 + AND latest_1 <= latest_0 + INTERVAL 7 DAY, 2, 1)) AS steps, + if(isNotNull(latest_1) + AND latest_1 <= latest_0 + INTERVAL 7 DAY, dateDiff('second', toDateTime(latest_0), toDateTime(latest_1)), NULL) step_1_conversion_time, + if(isNotNull(latest_2) + AND latest_2 <= latest_1 + INTERVAL 7 DAY, dateDiff('second', toDateTime(latest_1), toDateTime(latest_2)), NULL) step_2_conversion_time + FROM + (SELECT aggregation_target, timestamp, step_0, + latest_0, + step_1, + min(latest_1) over (PARTITION by aggregation_target, + prop + ORDER BY timestamp DESC ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) latest_1, + step_2, + min(latest_2) over (PARTITION by aggregation_target, + prop + ORDER BY timestamp DESC ROWS BETWEEN 2 PRECEDING AND 2 PRECEDING) latest_2 , + if(has(['technology', 'finance'], prop), prop, 'Other') as prop + FROM + (SELECT *, + prop_vals as prop + FROM + (SELECT e.timestamp as timestamp, + if(notEmpty(pdi.distinct_id), pdi.person_id, e.person_id) as aggregation_target, + if(notEmpty(pdi.distinct_id), pdi.person_id, e.person_id) as person_id, + if(event = 'sign up', 1, 0) as step_0, + if(step_0 = 1, timestamp, null) as latest_0, + if(event = 'play movie', 1, 0) as step_1, + if(step_1 = 1, timestamp, null) as latest_1, + if(event = 'buy', 1, 0) as step_2, + if(step_2 = 1, timestamp, null) as latest_2, + replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS prop_basic, + prop_basic as prop, + argMinIf(prop, timestamp, isNotNull(prop)) over (PARTITION by aggregation_target) as prop_vals + FROM events e + LEFT OUTER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 + WHERE team_id = 2 + AND distinct_id IN + (SELECT distinct_id + FROM events + WHERE team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') ) + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups + WHERE team_id = 2 + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + AND (1=1) ))) + WHERE step_0 = 1 )) + GROUP BY aggregation_target, + steps, + prop + HAVING steps = max_steps) + WHERE steps IN [1, 2, 3] + AND arrayFlatten(array(prop)) = arrayFlatten(array('technology')) + ORDER BY aggregation_target + LIMIT 100 + OFFSET 0 SETTINGS max_ast_elements=1000000, + max_expanded_ast_elements=1000000 + ''' +# --- +# name: TestStrictFunnelGroupBreakdownUDF.test_funnel_breakdown_group.7 + ''' + + SELECT replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS value, + count(*) as count + FROM events e + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups + WHERE team_id = 2 + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND event IN ['buy', 'play movie', 'sign up'] + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + GROUP BY value + ORDER BY count DESC, value DESC + LIMIT 26 + OFFSET 0 + ''' +# --- +# name: TestStrictFunnelGroupBreakdownUDF.test_funnel_breakdown_group.8 + ''' + + SELECT aggregation_target AS actor_id + FROM + (SELECT aggregation_target, + steps, + avg(step_1_conversion_time) step_1_average_conversion_time_inner, + avg(step_2_conversion_time) step_2_average_conversion_time_inner, + median(step_1_conversion_time) step_1_median_conversion_time_inner, + median(step_2_conversion_time) step_2_median_conversion_time_inner, + prop + FROM + (SELECT aggregation_target, + steps, + max(steps) over (PARTITION BY aggregation_target, + prop) as max_steps, + step_1_conversion_time, + step_2_conversion_time, + prop + FROM + (SELECT *, + if(latest_0 <= latest_1 + AND latest_1 <= latest_0 + INTERVAL 7 DAY + AND latest_1 <= latest_2 + AND latest_2 <= latest_0 + INTERVAL 7 DAY, 3, if(latest_0 <= latest_1 + AND latest_1 <= latest_0 + INTERVAL 7 DAY, 2, 1)) AS steps, + if(isNotNull(latest_1) + AND latest_1 <= latest_0 + INTERVAL 7 DAY, dateDiff('second', toDateTime(latest_0), toDateTime(latest_1)), NULL) step_1_conversion_time, + if(isNotNull(latest_2) + AND latest_2 <= latest_1 + INTERVAL 7 DAY, dateDiff('second', toDateTime(latest_1), toDateTime(latest_2)), NULL) step_2_conversion_time + FROM + (SELECT aggregation_target, timestamp, step_0, + latest_0, + step_1, + min(latest_1) over (PARTITION by aggregation_target, + prop + ORDER BY timestamp DESC ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) latest_1, + step_2, + min(latest_2) over (PARTITION by aggregation_target, + prop + ORDER BY timestamp DESC ROWS BETWEEN 2 PRECEDING AND 2 PRECEDING) latest_2 , + if(has(['technology', 'finance'], prop), prop, 'Other') as prop + FROM + (SELECT *, + prop_vals as prop + FROM + (SELECT e.timestamp as timestamp, + if(notEmpty(pdi.distinct_id), pdi.person_id, e.person_id) as aggregation_target, + if(notEmpty(pdi.distinct_id), pdi.person_id, e.person_id) as person_id, + if(event = 'sign up', 1, 0) as step_0, + if(step_0 = 1, timestamp, null) as latest_0, + if(event = 'play movie', 1, 0) as step_1, + if(step_1 = 1, timestamp, null) as latest_1, + if(event = 'buy', 1, 0) as step_2, + if(step_2 = 1, timestamp, null) as latest_2, + replaceRegexpAll(JSONExtractRaw(group_properties_0, 'industry'), '^"|"$', '') AS prop_basic, + prop_basic as prop, + argMinIf(prop, timestamp, isNotNull(prop)) over (PARTITION by aggregation_target) as prop_vals + FROM events e + LEFT OUTER JOIN + (SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id2 + WHERE team_id = 2 + AND distinct_id IN + (SELECT distinct_id + FROM events + WHERE team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') ) + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id + LEFT JOIN + (SELECT group_key, + argMax(group_properties, _timestamp) AS group_properties_0 + FROM groups + WHERE team_id = 2 + AND group_type_index = 0 + GROUP BY group_key) groups_0 ON "$group_0" == groups_0.group_key + WHERE team_id = 2 + AND toTimeZone(timestamp, 'UTC') >= toDateTime('2020-01-01 00:00:00', 'UTC') + AND toTimeZone(timestamp, 'UTC') <= toDateTime('2020-01-08 23:59:59', 'UTC') + AND (1=1) ))) + WHERE step_0 = 1 )) + GROUP BY aggregation_target, + steps, + prop + HAVING steps = max_steps) + WHERE steps IN [2, 3] + AND arrayFlatten(array(prop)) = arrayFlatten(array('technology')) + ORDER BY aggregation_target + LIMIT 100 + OFFSET 0 SETTINGS max_ast_elements=1000000, + max_expanded_ast_elements=1000000 + ''' +# --- diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends.ambr index 16be898651577..a8ce810a87cfd 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends.ambr @@ -1,366 +1,4 @@ # serializer version: 1 -# name: BaseTestFunnelTrends.test_timezones_trends - ''' - SELECT fill.entrance_period_start AS entrance_period_start, - data.reached_from_step_count AS reached_from_step_count, - data.reached_to_step_count AS reached_to_step_count, - if(ifNull(greater(data.reached_from_step_count, 0), 0), round(multiply(divide(data.reached_to_step_count, data.reached_from_step_count), 100), 2), 0) AS conversion_rate - FROM - (SELECT plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull(('2021-04-30 00:00:00'), 6, 'UTC'))), toIntervalDay(period_offsets.number)) AS entrance_period_start - FROM numbers(plus(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull(('2021-04-30 00:00:00'), 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull(('2021-05-07 23:59:59'), 6, 'UTC')))), 1)) AS period_offsets) AS fill - LEFT OUTER JOIN - (SELECT entrance_period_start AS entrance_period_start, - countIf(ifNull(greaterOrEquals(steps_completed, 1), 0)) AS reached_from_step_count, - countIf(ifNull(greaterOrEquals(steps_completed, 3), 0)) AS reached_to_step_count - FROM - (SELECT aggregation_target AS aggregation_target, - toStartOfDay(timestamp) AS entrance_period_start, - max(steps) AS steps_completed - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - latest_2 AS latest_2, - if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps, - if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, - if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - if(ifNull(less(latest_2, latest_1), 0), NULL, latest_2) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - min(latest_1) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - if(equals(e.event, 'step one'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, - if(equals(e.event, 'step two'), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - if(equals(e.event, 'step three'), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2 - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 2) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-04-30 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-05-07 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0))))))) - WHERE ifNull(equals(step_0, 1), 0)) - GROUP BY aggregation_target, - entrance_period_start) - GROUP BY entrance_period_start) AS data ON equals(data.entrance_period_start, fill.entrance_period_start) - ORDER BY fill.entrance_period_start ASC - LIMIT 1000 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - allow_experimental_analyzer=1 - ''' -# --- -# name: BaseTestFunnelTrends.test_timezones_trends.1 - ''' - SELECT fill.entrance_period_start AS entrance_period_start, - data.reached_from_step_count AS reached_from_step_count, - data.reached_to_step_count AS reached_to_step_count, - if(ifNull(greater(data.reached_from_step_count, 0), 0), round(multiply(divide(data.reached_to_step_count, data.reached_from_step_count), 100), 2), 0) AS conversion_rate - FROM - (SELECT plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull(('2021-04-30 00:00:00'), 6, 'US/Pacific'))), toIntervalDay(period_offsets.number)) AS entrance_period_start - FROM numbers(plus(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull(('2021-04-30 00:00:00'), 6, 'US/Pacific'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull(('2021-05-07 23:59:59'), 6, 'US/Pacific')))), 1)) AS period_offsets) AS fill - LEFT OUTER JOIN - (SELECT entrance_period_start AS entrance_period_start, - countIf(ifNull(greaterOrEquals(steps_completed, 1), 0)) AS reached_from_step_count, - countIf(ifNull(greaterOrEquals(steps_completed, 3), 0)) AS reached_to_step_count - FROM - (SELECT aggregation_target AS aggregation_target, - toStartOfDay(timestamp) AS entrance_period_start, - max(steps) AS steps_completed - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - latest_2 AS latest_2, - if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps, - if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, - if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - if(ifNull(less(latest_2, latest_1), 0), NULL, latest_2) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - min(latest_1) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT toTimeZone(e.timestamp, 'US/Pacific') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - if(equals(e.event, 'step one'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, - if(equals(e.event, 'step two'), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - if(equals(e.event, 'step three'), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2 - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 2) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'US/Pacific'), toDateTime64('2021-04-30 00:00:00.000000', 6, 'US/Pacific')), lessOrEquals(toTimeZone(e.timestamp, 'US/Pacific'), toDateTime64('2021-05-07 23:59:59.999999', 6, 'US/Pacific'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0))))))) - WHERE ifNull(equals(step_0, 1), 0)) - GROUP BY aggregation_target, - entrance_period_start) - GROUP BY entrance_period_start) AS data ON equals(data.entrance_period_start, fill.entrance_period_start) - ORDER BY fill.entrance_period_start ASC - LIMIT 1000 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - allow_experimental_analyzer=1 - ''' -# --- -# name: BaseTestFunnelTrends.test_week_interval - ''' - SELECT fill.entrance_period_start AS entrance_period_start, - data.reached_from_step_count AS reached_from_step_count, - data.reached_to_step_count AS reached_to_step_count, - if(ifNull(greater(data.reached_from_step_count, 0), 0), round(multiply(divide(data.reached_to_step_count, data.reached_from_step_count), 100), 2), 0) AS conversion_rate - FROM - (SELECT plus(toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull(('2021-05-01 00:00:00'), 6, 'UTC')), 0), toIntervalWeek(period_offsets.number)) AS entrance_period_start - FROM numbers(plus(dateDiff('week', toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull(('2021-05-01 00:00:00'), 6, 'UTC')), 0), toStartOfWeek(assumeNotNull(parseDateTime64BestEffortOrNull(('2021-05-07 23:59:59'), 6, 'UTC')), 0)), 1)) AS period_offsets) AS fill - LEFT OUTER JOIN - (SELECT entrance_period_start AS entrance_period_start, - countIf(ifNull(greaterOrEquals(steps_completed, 1), 0)) AS reached_from_step_count, - countIf(ifNull(greaterOrEquals(steps_completed, 3), 0)) AS reached_to_step_count - FROM - (SELECT aggregation_target AS aggregation_target, - toStartOfWeek(timestamp, 0) AS entrance_period_start, - max(steps) AS steps_completed - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - latest_2 AS latest_2, - if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps, - if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, - if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - if(ifNull(less(latest_2, latest_1), 0), NULL, latest_2) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - min(latest_1) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - if(equals(e.event, 'step one'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, - if(equals(e.event, 'step two'), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - if(equals(e.event, 'step three'), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2 - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 2) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-05-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-05-07 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0))))))) - WHERE ifNull(equals(step_0, 1), 0)) - GROUP BY aggregation_target, - entrance_period_start) - GROUP BY entrance_period_start) AS data ON equals(data.entrance_period_start, fill.entrance_period_start) - ORDER BY fill.entrance_period_start ASC - LIMIT 1000 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - format_csv_allow_double_quotes=0, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - allow_experimental_analyzer=1 - ''' -# --- -# name: BaseTestFunnelTrends.test_week_interval.1 - ''' - - SELECT aggregation_target AS actor_id - FROM - (SELECT aggregation_target, - toStartOfWeek(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC'), 0) AS entrance_period_start, - max(steps) AS steps_completed - FROM - (SELECT *, - if(latest_0 <= latest_1 - AND latest_1 <= latest_0 + INTERVAL 7 DAY - AND latest_1 <= latest_2 - AND latest_2 <= latest_0 + INTERVAL 7 DAY, 3, if(latest_0 <= latest_1 - AND latest_1 <= latest_0 + INTERVAL 7 DAY, 2, 1)) AS steps , - if(isNotNull(latest_1) - AND latest_1 <= latest_0 + INTERVAL 7 DAY, dateDiff('second', toDateTime(latest_0), toDateTime(latest_1)), NULL) step_1_conversion_time, - if(isNotNull(latest_2) - AND latest_2 <= latest_1 + INTERVAL 7 DAY, dateDiff('second', toDateTime(latest_1), toDateTime(latest_2)), NULL) step_2_conversion_time - FROM - (SELECT aggregation_target, timestamp, step_0, - latest_0, - step_1, - latest_1, - step_2, - min(latest_2) over (PARTITION by aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) latest_2 - FROM - (SELECT aggregation_target, timestamp, step_0, - latest_0, - step_1, - latest_1, - step_2, - if(latest_2 < latest_1, NULL, latest_2) as latest_2 - FROM - (SELECT aggregation_target, timestamp, step_0, - latest_0, - step_1, - min(latest_1) over (PARTITION by aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) latest_1, - step_2, - min(latest_2) over (PARTITION by aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) latest_2 - FROM - (SELECT e.timestamp as timestamp, - pdi.person_id as aggregation_target, - pdi.person_id as person_id, - if(event = 'step one', 1, 0) as step_0, - if(step_0 = 1, timestamp, null) as latest_0, - if(event = 'step two', 1, 0) as step_1, - if(step_1 = 1, timestamp, null) as latest_1, - if(event = 'step three', 1, 0) as step_2, - if(step_2 = 1, timestamp, null) as latest_2 - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - AND distinct_id IN - (SELECT distinct_id - FROM events - WHERE team_id = 2 - AND event IN ['step one', 'step three', 'step two'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2021-05-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2021-05-07 23:59:59', 'UTC') ) - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - WHERE team_id = 2 - AND event IN ['step one', 'step three', 'step two'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2021-05-01 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2021-05-07 23:59:59', 'UTC') - AND (step_0 = 1 - OR step_1 = 1 - OR step_2 = 1) )))) - WHERE step_0 = 1 ) - WHERE toDateTime(entrance_period_start) = '2021-04-25 00:00:00' - GROUP BY aggregation_target, - entrance_period_start) - WHERE steps_completed >= 3 - ORDER BY aggregation_target - LIMIT 100 - OFFSET 0 SETTINGS max_ast_elements=1000000, - max_expanded_ast_elements=1000000 - ''' -# --- # name: TestFunnelTrends.test_timezones_trends ''' SELECT fill.entrance_period_start AS entrance_period_start, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_udf.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_udf.ambr index 8467ca0eff992..bded4b095c3f5 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_udf.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_trends_udf.ambr @@ -15,14 +15,8 @@ (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, if(equals(e.event, 'step one'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, if(equals(e.event, 'step two'), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - if(equals(e.event, 'step three'), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2, - 0 AS exclusion_0, - 0 AS exclusion_1, - 0 AS exclusion_2 + if(equals(e.event, 'step three'), 1, 0) AS step_2 FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, @@ -66,14 +60,8 @@ (SELECT toTimeZone(e.timestamp, 'US/Pacific') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, if(equals(e.event, 'step one'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, if(equals(e.event, 'step two'), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - if(equals(e.event, 'step three'), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2, - 0 AS exclusion_0, - 0 AS exclusion_1, - 0 AS exclusion_2 + if(equals(e.event, 'step three'), 1, 0) AS step_2 FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, @@ -117,14 +105,8 @@ (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, if(equals(e.event, 'step one'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, if(equals(e.event, 'step two'), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - if(equals(e.event, 'step three'), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2, - 0 AS exclusion_0, - 0 AS exclusion_1, - 0 AS exclusion_2 + if(equals(e.event, 'step three'), 1, 0) AS step_2 FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_udf.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_udf.ambr index 82dd4f740c62b..fc27315723526 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_udf.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_udf.ambr @@ -11,30 +11,29 @@ groupArray(row_number) AS row_number, final_prop AS final_prop FROM - (SELECT countIf(ifNull(ifNull(equals(af, 0), 0), 0)) AS step_1, - countIf(ifNull(ifNull(equals(af, 1), 0), 0)) AS step_2, - countIf(ifNull(ifNull(equals(af, 2), 0), 0)) AS step_3, + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, + countIf(ifNull(ifNull(equals(step_reached, 2), 0), 0)) AS step_3, groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, rowNumberInBlock() AS row_number, breakdown AS final_prop FROM - (SELECT arrayJoin(aggregate_funnel_array_v0(3, 15, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, - af_tuple.1 AS af, + (SELECT arrayJoin(aggregate_funnel_array_v0(3, 15, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, - af_tuple.3 AS timings + af_tuple.3 AS timings, + aggregation_target AS aggregation_target FROM (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, if(equals(e.event, 'step one'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, if(equals(e.event, 'step two'), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - if(equals(e.event, 'step three'), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2, - 0 AS exclusion_0, - 0 AS exclusion_1, - 0 AS exclusion_2 + if(equals(e.event, 'step three'), 1, 0) AS step_2 FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, @@ -45,7 +44,7 @@ HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-05-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-05-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))) GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(af, 0), 0)) + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) GROUP BY breakdown ORDER BY step_3 DESC, step_2 DESC, step_1 DESC) GROUP BY final_prop @@ -68,84 +67,33 @@ FROM (SELECT aggregation_target AS actor_id FROM - (SELECT aggregation_target AS aggregation_target, - steps AS steps, - avg(step_1_conversion_time) AS step_1_average_conversion_time_inner, - avg(step_2_conversion_time) AS step_2_average_conversion_time_inner, - median(step_1_conversion_time) AS step_1_median_conversion_time_inner, - median(step_2_conversion_time) AS step_2_median_conversion_time_inner + (SELECT arrayJoin(aggregate_funnel_array_v0(3, 15, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + aggregation_target AS aggregation_target FROM - (SELECT aggregation_target AS aggregation_target, - steps AS steps, - max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, - step_1_conversion_time AS step_1_conversion_time, - step_2_conversion_time AS step_2_conversion_time - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - latest_2 AS latest_2, - if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalSecond(15))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalSecond(15))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalSecond(15))), 0)), 2, 1)) AS steps, - if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalSecond(15))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, - if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalSecond(15))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - if(ifNull(less(latest_2, latest_1), 0), NULL, latest_2) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - min(latest_1) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - if(equals(e.event, 'step one'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, - if(equals(e.event, 'step two'), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - if(equals(e.event, 'step three'), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2 - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 2) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-05-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-05-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0))))))) - WHERE ifNull(equals(step_0, 1), 0))) - GROUP BY aggregation_target, - steps - HAVING ifNull(equals(steps, max(max_steps)), isNull(steps) - and isNull(max(max_steps)))) - WHERE ifNull(in(steps, [2, 3]), 0) + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'step one'), 1, 0) AS step_0, + if(equals(e.event, 'step two'), 1, 0) AS step_1, + if(equals(e.event, 'step three'), 1, 0) AS step_2 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-05-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2021-05-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 1), 0) ORDER BY aggregation_target ASC) AS source INNER JOIN (SELECT argMax(toTimeZone(person.created_at, 'UTC'), person.version) AS created_at, @@ -187,30 +135,29 @@ groupArray(row_number) AS row_number, final_prop AS final_prop FROM - (SELECT countIf(ifNull(ifNull(equals(af, 0), 0), 0)) AS step_1, - countIf(ifNull(ifNull(equals(af, 1), 0), 0)) AS step_2, - countIf(ifNull(ifNull(equals(af, 2), 0), 0)) AS step_3, + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, + countIf(ifNull(ifNull(equals(step_reached, 2), 0), 0)) AS step_3, groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, rowNumberInBlock() AS row_number, breakdown AS final_prop FROM - (SELECT arrayJoin(aggregate_funnel_array_v0(3, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, - af_tuple.1 AS af, + (SELECT arrayJoin(aggregate_funnel_array_v0(3, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, - af_tuple.3 AS timings + af_tuple.3 AS timings, + aggregation_target AS aggregation_target FROM (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, if(and(equals(e.event, '$autocapture'), match(e.elements_chain, '(^|;)button(\\.|$|;|:)'), arrayExists(x -> ifNull(equals(x, 'Pay $10'), 0), e.elements_chain_texts)), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - if(and(equals(e.event, '$autocapture'), match(e.elements_chain, '(^|;)a(\\.|$|;|:)'), equals(e.elements_chain_href, '/movie')), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2, - 0 AS exclusion_0, - 0 AS exclusion_1, - 0 AS exclusion_2 + if(and(equals(e.event, '$autocapture'), match(e.elements_chain, '(^|;)a(\\.|$|;|:)'), equals(e.elements_chain_href, '/movie')), 1, 0) AS step_2 FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, @@ -221,7 +168,7 @@ HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2011-12-25 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2012-01-01 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('$autocapture', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))) GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(af, 0), 0)) + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) GROUP BY breakdown ORDER BY step_3 DESC, step_2 DESC, step_1 DESC) GROUP BY final_prop @@ -264,28 +211,29 @@ groupArray(row_number) AS row_number, final_prop AS final_prop FROM - (SELECT countIf(ifNull(ifNull(equals(af, 0), 0), 0)) AS step_1, - countIf(ifNull(ifNull(equals(af, 1), 0), 0)) AS step_2, + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, rowNumberInBlock() AS row_number, breakdown AS final_prop FROM - (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, - af_tuple.1 AS af, + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, - af_tuple.3 AS timings + af_tuple.3 AS timings, + aggregation_target AS aggregation_target FROM (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, if(and(equals(e.event, 'user signed up'), ifNull(in(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), (SELECT cohortpeople.person_id AS person_id FROM cohortpeople WHERE and(equals(cohortpeople.team_id, 2), equals(cohortpeople.cohort_id, 2), equals(cohortpeople.version, 0)))), 0)), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - 0 AS exclusion_0, - 0 AS exclusion_1 + if(equals(e.event, 'paid'), 1, 0) AS step_1 FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, @@ -296,7 +244,7 @@ HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(af, 0), 0)) + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) GROUP BY breakdown ORDER BY step_2 DESC, step_1 DESC) GROUP BY final_prop @@ -322,30 +270,29 @@ groupArray(row_number) AS row_number, final_prop AS final_prop FROM - (SELECT countIf(ifNull(ifNull(equals(af, 0), 0), 0)) AS step_1, - countIf(ifNull(ifNull(equals(af, 1), 0), 0)) AS step_2, - countIf(ifNull(ifNull(equals(af, 2), 0), 0)) AS step_3, + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, + countIf(ifNull(ifNull(equals(step_reached, 2), 0), 0)) AS step_3, groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, rowNumberInBlock() AS row_number, breakdown AS final_prop FROM - (SELECT arrayJoin(aggregate_funnel_array_v0(3, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, - af_tuple.1 AS af, + (SELECT arrayJoin(aggregate_funnel_array_v0(3, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, - af_tuple.3 AS timings + af_tuple.3 AS timings, + aggregation_target AS aggregation_target FROM (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha.com'), 0)), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha2.com'), 0)), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2, - 0 AS exclusion_0, - 0 AS exclusion_1, - 0 AS exclusion_2 + if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha2.com'), 0)), 1, 0) AS step_2 FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, @@ -367,7 +314,7 @@ HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-07-01 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(e__person.properties___email, '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(e__person.properties___email, '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))) GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(af, 0), 0)) + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) GROUP BY breakdown ORDER BY step_3 DESC, step_2 DESC, step_1 DESC) GROUP BY final_prop @@ -390,95 +337,44 @@ FROM (SELECT aggregation_target AS actor_id FROM - (SELECT aggregation_target AS aggregation_target, - steps AS steps, - avg(step_1_conversion_time) AS step_1_average_conversion_time_inner, - avg(step_2_conversion_time) AS step_2_average_conversion_time_inner, - median(step_1_conversion_time) AS step_1_median_conversion_time_inner, - median(step_2_conversion_time) AS step_2_median_conversion_time_inner + (SELECT arrayJoin(aggregate_funnel_array_v0(3, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + aggregation_target AS aggregation_target FROM - (SELECT aggregation_target AS aggregation_target, - steps AS steps, - max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, - step_1_conversion_time AS step_1_conversion_time, - step_2_conversion_time AS step_2_conversion_time - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - latest_2 AS latest_2, - if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps, - if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, - if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - if(ifNull(less(latest_2, latest_1), 0), NULL, latest_2) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - min(latest_1) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, - if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha.com'), 0)), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha2.com'), 0)), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2 - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 2) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age - FROM person - WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 2) - GROUP BY person.id - HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-07-01 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(e__person.properties___email, '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(e__person.properties___email, '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0))))))) - WHERE ifNull(equals(step_0, 1), 0))) - GROUP BY aggregation_target, - steps - HAVING ifNull(equals(steps, max(max_steps)), isNull(steps) - and isNull(max(max_steps)))) - WHERE ifNull(in(steps, [1, 2, 3]), 0) + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha.com'), 0)), 1, 0) AS step_1, + if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha2.com'), 0)), 1, 0) AS step_2 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-07-01 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(e__person.properties___email, '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(e__person.properties___email, '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 0), 0) ORDER BY aggregation_target ASC) AS source INNER JOIN (SELECT argMax(toTimeZone(person.created_at, 'UTC'), person.version) AS created_at, @@ -508,95 +404,44 @@ FROM (SELECT aggregation_target AS actor_id FROM - (SELECT aggregation_target AS aggregation_target, - steps AS steps, - avg(step_1_conversion_time) AS step_1_average_conversion_time_inner, - avg(step_2_conversion_time) AS step_2_average_conversion_time_inner, - median(step_1_conversion_time) AS step_1_median_conversion_time_inner, - median(step_2_conversion_time) AS step_2_median_conversion_time_inner + (SELECT arrayJoin(aggregate_funnel_array_v0(3, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + aggregation_target AS aggregation_target FROM - (SELECT aggregation_target AS aggregation_target, - steps AS steps, - max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, - step_1_conversion_time AS step_1_conversion_time, - step_2_conversion_time AS step_2_conversion_time - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - latest_2 AS latest_2, - if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps, - if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, - if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - if(ifNull(less(latest_2, latest_1), 0), NULL, latest_2) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - min(latest_1) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, - if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha.com'), 0)), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha2.com'), 0)), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2 - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 2) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age - FROM person - WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 2) - GROUP BY person.id - HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-07-01 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(e__person.properties___email, '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(e__person.properties___email, '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0))))))) - WHERE ifNull(equals(step_0, 1), 0))) - GROUP BY aggregation_target, - steps - HAVING ifNull(equals(steps, max(max_steps)), isNull(steps) - and isNull(max(max_steps)))) - WHERE ifNull(in(steps, [2, 3]), 0) + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha.com'), 0)), 1, 0) AS step_1, + if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha2.com'), 0)), 1, 0) AS step_2 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-07-01 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(e__person.properties___email, '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(e__person.properties___email, '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 1), 0) ORDER BY aggregation_target ASC) AS source INNER JOIN (SELECT argMax(toTimeZone(person.created_at, 'UTC'), person.version) AS created_at, @@ -626,95 +471,44 @@ FROM (SELECT aggregation_target AS actor_id FROM - (SELECT aggregation_target AS aggregation_target, - steps AS steps, - avg(step_1_conversion_time) AS step_1_average_conversion_time_inner, - avg(step_2_conversion_time) AS step_2_average_conversion_time_inner, - median(step_1_conversion_time) AS step_1_median_conversion_time_inner, - median(step_2_conversion_time) AS step_2_median_conversion_time_inner + (SELECT arrayJoin(aggregate_funnel_array_v0(3, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + aggregation_target AS aggregation_target FROM - (SELECT aggregation_target AS aggregation_target, - steps AS steps, - max(steps) OVER (PARTITION BY aggregation_target) AS max_steps, - step_1_conversion_time AS step_1_conversion_time, - step_2_conversion_time AS step_2_conversion_time - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - latest_2 AS latest_2, - if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0), ifNull(lessOrEquals(latest_1, latest_2), 0), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 3, if(and(ifNull(lessOrEquals(latest_0, latest_1), 0), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), 2, 1)) AS steps, - if(and(isNotNull(latest_1), ifNull(lessOrEquals(latest_1, plus(toTimeZone(latest_0, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_0, latest_1), NULL) AS step_1_conversion_time, - if(and(isNotNull(latest_2), ifNull(lessOrEquals(latest_2, plus(toTimeZone(latest_1, 'UTC'), toIntervalDay(14))), 0)), dateDiff('second', latest_1, latest_2), NULL) AS step_2_conversion_time - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - if(ifNull(less(latest_2, latest_1), 0), NULL, latest_2) AS latest_2 - FROM - (SELECT aggregation_target AS aggregation_target, - timestamp AS timestamp, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - min(latest_1) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_1, - step_2 AS step_2, - min(latest_2) OVER (PARTITION BY aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) AS latest_2 - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, - if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha.com'), 0)), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha2.com'), 0)), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2 - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 2) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age - FROM person - WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 2) - GROUP BY person.id - HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-07-01 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(e__person.properties___email, '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(e__person.properties___email, '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0))))))) - WHERE ifNull(equals(step_0, 1), 0))) - GROUP BY aggregation_target, - steps - HAVING ifNull(equals(steps, max(max_steps)), isNull(steps) - and isNull(max(max_steps)))) - WHERE ifNull(in(steps, [3]), 0) + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha.com'), 0)), 1, 0) AS step_1, + if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha2.com'), 0)), 1, 0) AS step_2 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age + FROM person + WHERE and(equals(person.team_id, 2), ifNull(in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 2) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)))), 0)) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-07-01 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(e__person.properties___email, '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(e__person.properties___email, '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE ifNull(greaterOrEquals(step_reached, 2), 0) ORDER BY aggregation_target ASC) AS source INNER JOIN (SELECT argMax(toTimeZone(person.created_at, 'UTC'), person.version) AS created_at, @@ -753,28 +547,29 @@ groupArray(row_number) AS row_number, final_prop AS final_prop FROM - (SELECT countIf(ifNull(ifNull(equals(af, 0), 0), 0)) AS step_1, - countIf(ifNull(ifNull(equals(af, 1), 0), 0)) AS step_2, + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, rowNumberInBlock() AS row_number, breakdown AS final_prop FROM - (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, - af_tuple.1 AS af, + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, - af_tuple.3 AS timings + af_tuple.3 AS timings, + aggregation_target AS aggregation_target FROM (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, if(and(equals(e.event, 'user signed up'), ifNull(in(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), (SELECT person_static_cohort.person_id AS person_id FROM person_static_cohort WHERE and(equals(person_static_cohort.team_id, 2), equals(person_static_cohort.cohort_id, 2)))), 0)), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - 0 AS exclusion_0, - 0 AS exclusion_1 + if(equals(e.event, 'paid'), 1, 0) AS step_1 FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, @@ -785,7 +580,7 @@ HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(af, 0), 0)) + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) GROUP BY breakdown ORDER BY step_2 DESC, step_1 DESC) GROUP BY final_prop @@ -808,25 +603,26 @@ groupArray(row_number) AS row_number, final_prop AS final_prop FROM - (SELECT countIf(ifNull(ifNull(equals(af, 0), 0), 0)) AS step_1, - countIf(ifNull(ifNull(equals(af, 1), 0), 0)) AS step_2, + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, rowNumberInBlock() AS row_number, breakdown AS final_prop FROM - (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, - af_tuple.1 AS af, + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', [[]], arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, [], arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, - af_tuple.3 AS timings + af_tuple.3 AS timings, + aggregation_target AS aggregation_target FROM (SELECT toTimeZone(e.timestamp, 'US/Pacific') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - 0 AS exclusion_0, - 0 AS exclusion_1 + if(equals(e.event, 'paid'), 1, 0) AS step_1 FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, @@ -837,7 +633,7 @@ HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'US/Pacific'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'US/Pacific')), lessOrEquals(toTimeZone(e.timestamp, 'US/Pacific'), toDateTime64('2020-01-14 23:59:59.999999', 6, 'US/Pacific'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(af, 0), 0)) + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) GROUP BY breakdown ORDER BY step_2 DESC, step_1 DESC) GROUP BY final_prop @@ -860,38 +656,38 @@ groupArray(row_number) AS row_number, final_prop AS final_prop FROM - (SELECT countIf(ifNull(ifNull(equals(af, 0), 0), 0)) AS step_1, - countIf(ifNull(ifNull(equals(af, 1), 0), 0)) AS step_2, + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, rowNumberInBlock() AS row_number, if(ifNull(less(row_number, 25), 0), breakdown, ['Other']) AS final_prop FROM - (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', groupUniqArray(prop), arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), prop, arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, - af_tuple.1 AS af, + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', groupUniqArray(prop), arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, prop, arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, - af_tuple.3 AS timings + af_tuple.3 AS timings, + aggregation_target AS aggregation_target FROM (SELECT timestamp AS timestamp, aggregation_target AS aggregation_target, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - exclusion_0 AS exclusion_0, - exclusion_1 AS exclusion_1, - prop_basic AS prop_basic, - prop, - prop_vals AS prop_vals, - if(notEmpty(arrayFilter(x -> notEmpty(x), prop_vals)), prop_vals, ['']) AS prop + uuid AS uuid, + `$session_id` AS `$session_id`, + `$window_id` AS `$window_id`, + step_0 AS step_0, + step_1 AS step_1, + prop_basic AS prop_basic, + prop, + prop_vals AS prop_vals, + if(notEmpty(arrayFilter(x -> notEmpty(x), prop_vals)), prop_vals, ['']) AS prop FROM (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, if(and(equals(e.event, 'buy'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$version'), ''), 'null'), '^"|"$', ''), 'xyz'), 0)), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - 0 AS exclusion_0, - 0 AS exclusion_1, [ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', '')), '')] AS prop_basic, prop_basic AS prop, argMinIf(prop, timestamp, notEmpty(arrayFilter(x -> notEmpty(x), prop))) OVER (PARTITION BY aggregation_target) AS prop_vals @@ -905,7 +701,7 @@ HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-08 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('buy', 'sign up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0))))) GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(af, 0), 0)) + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) GROUP BY breakdown ORDER BY step_2 DESC, step_1 DESC) GROUP BY final_prop @@ -928,40 +724,40 @@ groupArray(row_number) AS row_number, final_prop AS final_prop FROM - (SELECT countIf(ifNull(ifNull(equals(af, 0), 0), 0)) AS step_1, - countIf(ifNull(ifNull(equals(af, 1), 0), 0)) AS step_2, + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, rowNumberInBlock() AS row_number, if(ifNull(less(row_number, 25), 0), breakdown, ['Other']) AS final_prop FROM - (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'step_1', 'ordered', groupUniqArray(prop), arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), prop, arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, - af_tuple.1 AS af, + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'step_1', 'ordered', groupUniqArray(prop), arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, prop, arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, - af_tuple.3 AS timings + af_tuple.3 AS timings, + aggregation_target AS aggregation_target FROM (SELECT timestamp AS timestamp, aggregation_target AS aggregation_target, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - exclusion_0 AS exclusion_0, - exclusion_1 AS exclusion_1, - prop_basic AS prop_basic, - prop_0 AS prop_0, - prop_1 AS prop_1, - prop, - prop_vals AS prop_vals, - prop + uuid AS uuid, + `$session_id` AS `$session_id`, + `$window_id` AS `$window_id`, + step_0 AS step_0, + step_1 AS step_1, + prop_basic AS prop_basic, + prop_0 AS prop_0, + prop_1 AS prop_1, + prop, + prop_vals AS prop_vals, + prop FROM (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, if(and(equals(e.event, 'buy'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$version'), ''), 'null'), '^"|"$', ''), 'xyz'), 0)), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - 0 AS exclusion_0, - 0 AS exclusion_1, [ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', '')), '')] AS prop_basic, if(ifNull(equals(step_0, 1), 0), prop_basic, []) AS prop_0, if(ifNull(equals(step_1, 1), 0), prop_basic, []) AS prop_1, @@ -980,7 +776,7 @@ WHERE ifNull(notEquals(prop, []), isNotNull(prop) or isNotNull([]))) GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(af, 0), 0)) + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) GROUP BY breakdown ORDER BY step_2 DESC, step_1 DESC) GROUP BY final_prop @@ -1003,38 +799,38 @@ groupArray(row_number) AS row_number, final_prop AS final_prop FROM - (SELECT countIf(ifNull(ifNull(equals(af, 0), 0), 0)) AS step_1, - countIf(ifNull(ifNull(equals(af, 1), 0), 0)) AS step_2, + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, rowNumberInBlock() AS row_number, if(ifNull(less(row_number, 25), 0), breakdown, ['Other']) AS final_prop FROM - (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', groupUniqArray(prop), arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), prop, arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, - af_tuple.1 AS af, + (SELECT arrayJoin(aggregate_funnel_array_v0(2, 1209600, 'first_touch', 'ordered', groupUniqArray(prop), arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, prop, arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, - af_tuple.3 AS timings + af_tuple.3 AS timings, + aggregation_target AS aggregation_target FROM (SELECT timestamp AS timestamp, aggregation_target AS aggregation_target, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - exclusion_0 AS exclusion_0, - exclusion_1 AS exclusion_1, - prop_basic AS prop_basic, - prop, - prop_vals AS prop_vals, - if(notEmpty(arrayFilter(x -> notEmpty(x), prop_vals)), prop_vals, ['', '']) AS prop + uuid AS uuid, + `$session_id` AS `$session_id`, + `$window_id` AS `$window_id`, + step_0 AS step_0, + step_1 AS step_1, + prop_basic AS prop_basic, + prop, + prop_vals AS prop_vals, + if(notEmpty(arrayFilter(x -> notEmpty(x), prop_vals)), prop_vals, ['', '']) AS prop FROM (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, if(equals(e.event, 'buy'), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, - 0 AS exclusion_0, - 0 AS exclusion_1, [ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', '')), ''), ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$version'), ''), 'null'), '^"|"$', '')), '')] AS prop_basic, prop_basic AS prop, argMinIf(prop, timestamp, notEmpty(arrayFilter(x -> notEmpty(x), prop))) OVER (PARTITION BY aggregation_target) AS prop_vals @@ -1048,7 +844,7 @@ HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-08 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('buy', 'sign up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0))))) GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(af, 0), 0)) + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) GROUP BY breakdown ORDER BY step_2 DESC, step_1 DESC) GROUP BY final_prop @@ -1074,46 +870,42 @@ groupArray(row_number) AS row_number, final_prop AS final_prop FROM - (SELECT countIf(ifNull(ifNull(equals(af, 0), 0), 0)) AS step_1, - countIf(ifNull(ifNull(equals(af, 1), 0), 0)) AS step_2, - countIf(ifNull(ifNull(equals(af, 2), 0), 0)) AS step_3, + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, + countIf(ifNull(ifNull(equals(step_reached, 2), 0), 0)) AS step_3, groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, rowNumberInBlock() AS row_number, if(ifNull(less(row_number, 25), 0), breakdown, 'Other') AS final_prop FROM - (SELECT arrayJoin(aggregate_funnel_v0(3, 1209600, 'first_touch', 'ordered', groupUniqArray(prop), arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), prop, arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, - af_tuple.1 AS af, + (SELECT arrayJoin(aggregate_funnel_v0(3, 1209600, 'first_touch', 'ordered', groupUniqArray(prop), arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, prop, arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, - af_tuple.3 AS timings + af_tuple.3 AS timings, + aggregation_target AS aggregation_target FROM (SELECT timestamp AS timestamp, aggregation_target AS aggregation_target, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - latest_2 AS latest_2, - exclusion_0 AS exclusion_0, - exclusion_1 AS exclusion_1, - exclusion_2 AS exclusion_2, - prop_basic AS prop_basic, - prop, - prop_vals AS prop_vals, - prop_vals AS prop + uuid AS uuid, + `$session_id` AS `$session_id`, + `$window_id` AS `$window_id`, + step_0 AS step_0, + step_1 AS step_1, + step_2 AS step_2, + prop_basic AS prop_basic, + prop, + prop_vals AS prop_vals, + prop_vals AS prop FROM (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, if(equals(e.event, 'play movie'), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, if(equals(e.event, 'buy'), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2, - 0 AS exclusion_0, - 0 AS exclusion_1, - 0 AS exclusion_2, ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, prop_basic AS prop, argMinIf(prop, timestamp, isNotNull(prop)) OVER (PARTITION BY aggregation_target) AS prop_vals @@ -1128,7 +920,7 @@ groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-08 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('buy', 'play movie', 'sign up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0))))) GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(af, 0), 0)) + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) GROUP BY breakdown ORDER BY step_3 DESC, step_2 DESC, step_1 DESC) GROUP BY final_prop @@ -1154,46 +946,42 @@ groupArray(row_number) AS row_number, final_prop AS final_prop FROM - (SELECT countIf(ifNull(ifNull(equals(af, 0), 0), 0)) AS step_1, - countIf(ifNull(ifNull(equals(af, 1), 0), 0)) AS step_2, - countIf(ifNull(ifNull(equals(af, 2), 0), 0)) AS step_3, + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, + countIf(ifNull(ifNull(equals(step_reached, 2), 0), 0)) AS step_3, groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, rowNumberInBlock() AS row_number, if(ifNull(less(row_number, 25), 0), breakdown, 'Other') AS final_prop FROM - (SELECT arrayJoin(aggregate_funnel_v0(3, 1209600, 'first_touch', 'ordered', groupUniqArray(prop), arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), prop, arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, - af_tuple.1 AS af, + (SELECT arrayJoin(aggregate_funnel_v0(3, 1209600, 'first_touch', 'ordered', groupUniqArray(prop), arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, prop, arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, - af_tuple.3 AS timings + af_tuple.3 AS timings, + aggregation_target AS aggregation_target FROM (SELECT timestamp AS timestamp, aggregation_target AS aggregation_target, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - latest_2 AS latest_2, - exclusion_0 AS exclusion_0, - exclusion_1 AS exclusion_1, - exclusion_2 AS exclusion_2, - prop_basic AS prop_basic, - prop, - prop_vals AS prop_vals, - prop_vals AS prop + uuid AS uuid, + `$session_id` AS `$session_id`, + `$window_id` AS `$window_id`, + step_0 AS step_0, + step_1 AS step_1, + step_2 AS step_2, + prop_basic AS prop_basic, + prop, + prop_vals AS prop_vals, + prop_vals AS prop FROM (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, e.`$group_0` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, if(equals(e.event, 'play movie'), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, if(equals(e.event, 'buy'), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2, - 0 AS exclusion_0, - 0 AS exclusion_1, - 0 AS exclusion_2, ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, prop_basic AS prop, argMinIf(prop, timestamp, isNotNull(prop)) OVER (PARTITION BY aggregation_target) AS prop_vals @@ -1208,7 +996,7 @@ groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-08 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('buy', 'play movie', 'sign up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0))))) GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(af, 0), 0)) + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) GROUP BY breakdown ORDER BY step_3 DESC, step_2 DESC, step_1 DESC) GROUP BY final_prop @@ -1234,46 +1022,42 @@ groupArray(row_number) AS row_number, final_prop AS final_prop FROM - (SELECT countIf(ifNull(ifNull(equals(af, 0), 0), 0)) AS step_1, - countIf(ifNull(ifNull(equals(af, 1), 0), 0)) AS step_2, - countIf(ifNull(ifNull(equals(af, 2), 0), 0)) AS step_3, + (SELECT countIf(ifNull(ifNull(equals(step_reached, 0), 0), 0)) AS step_1, + countIf(ifNull(ifNull(equals(step_reached, 1), 0), 0)) AS step_2, + countIf(ifNull(ifNull(equals(step_reached, 2), 0), 0)) AS step_3, groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, rowNumberInBlock() AS row_number, if(ifNull(less(row_number, 25), 0), breakdown, 'Other') AS final_prop FROM - (SELECT arrayJoin(aggregate_funnel_v0(3, 1209600, 'first_touch', 'ordered', groupUniqArray(prop), arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), prop, arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, - af_tuple.1 AS af, + (SELECT arrayJoin(aggregate_funnel_v0(3, 1209600, 'first_touch', 'ordered', groupUniqArray(prop), arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, prop, arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, - af_tuple.3 AS timings + af_tuple.3 AS timings, + aggregation_target AS aggregation_target FROM (SELECT timestamp AS timestamp, aggregation_target AS aggregation_target, - step_0 AS step_0, - latest_0 AS latest_0, - step_1 AS step_1, - latest_1 AS latest_1, - step_2 AS step_2, - latest_2 AS latest_2, - exclusion_0 AS exclusion_0, - exclusion_1 AS exclusion_1, - exclusion_2 AS exclusion_2, - prop_basic AS prop_basic, - prop, - prop_vals AS prop_vals, - prop_vals AS prop + uuid AS uuid, + `$session_id` AS `$session_id`, + `$window_id` AS `$window_id`, + step_0 AS step_0, + step_1 AS step_1, + step_2 AS step_2, + prop_basic AS prop_basic, + prop, + prop_vals AS prop_vals, + prop_vals AS prop FROM (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(ifNull(equals(step_0, 1), 0), timestamp, NULL) AS latest_0, if(equals(e.event, 'play movie'), 1, 0) AS step_1, - if(ifNull(equals(step_1, 1), 0), timestamp, NULL) AS latest_1, if(equals(e.event, 'buy'), 1, 0) AS step_2, - if(ifNull(equals(step_2, 1), 0), timestamp, NULL) AS latest_2, - 0 AS exclusion_0, - 0 AS exclusion_1, - 0 AS exclusion_2, ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, prop_basic AS prop, argMinIf(prop, timestamp, isNotNull(prop)) OVER (PARTITION BY aggregation_target) AS prop_vals @@ -1295,7 +1079,7 @@ groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) WHERE and(equals(e.team_id, 2), and(and(greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), toDateTime64('2020-01-08 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('buy', 'play movie', 'sign up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0))))) GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(af, 0), 0)) + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) GROUP BY breakdown ORDER BY step_3 DESC, step_2 DESC, step_1 DESC) GROUP BY final_prop diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_correlation.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_correlation.py index da85943dcd894..44b92a5579b4a 100644 --- a/posthog/hogql_queries/insights/funnels/test/test_funnel_correlation.py +++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_correlation.py @@ -1,5 +1,6 @@ from typing import Any, cast import unittest +from unittest import skip from freezegun import freeze_time from rest_framework.exceptions import ValidationError @@ -10,7 +11,7 @@ EventStats, FunnelCorrelationQueryRunner, ) -from posthog.hogql_queries.insights.funnels.test.test_funnel_correlations_persons import get_actors +from posthog.hogql_queries.insights.funnels.test.test_funnel_correlation_actors import get_actors from posthog.hogql_queries.legacy_compatibility.filter_to_query import filter_to_query from posthog.models.action import Action from posthog.models.element import Element @@ -49,7 +50,8 @@ def _create_action(**kwargs): return action -class TestClickhouseFunnelCorrelation(ClickhouseTestMixin, APIBaseTest): +class BaseTestClickhouseFunnelCorrelation(ClickhouseTestMixin, APIBaseTest): + __test__ = False maxDiff = None def _get_events_for_filters( @@ -1206,6 +1208,7 @@ def test_correlation_with_properties_raises_validation_error(self): @also_test_with_materialized_columns( event_properties=[], person_properties=["$browser"], verify_no_jsonextract=False ) + @skip("Works locally and works after you tmate onto github actions and run it, but fails in CI") def test_correlation_with_multiple_properties(self): filters = { "events": [ @@ -1303,6 +1306,11 @@ def test_correlation_with_multiple_properties(self): timestamp="2020-01-04T14:00:00Z", ) + result, _ = self._get_events_for_filters( + filters, + funnelCorrelationType=FunnelCorrelationResultsType.PROPERTIES, + funnelCorrelationNames=["$browser", "$nice"], + ) result, _ = self._get_events_for_filters( filters, funnelCorrelationType=FunnelCorrelationResultsType.PROPERTIES, @@ -2076,6 +2084,10 @@ def test_funnel_correlation_with_event_properties_autocapture(self): # ) +class TestClickhouseFunnelCorrelation(BaseTestClickhouseFunnelCorrelation): + __test__ = True + + class TestCorrelationFunctions(unittest.TestCase): def test_are_results_insignificant(self): # Same setup as above test: test_discarding_insignificant_events diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_correlations_persons.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_correlation_actors.py similarity index 98% rename from posthog/hogql_queries/insights/funnels/test/test_funnel_correlations_persons.py rename to posthog/hogql_queries/insights/funnels/test/test_funnel_correlation_actors.py index a6694deed8330..594d075b426d5 100644 --- a/posthog/hogql_queries/insights/funnels/test/test_funnel_correlations_persons.py +++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_correlation_actors.py @@ -7,6 +7,7 @@ from posthog.constants import INSIGHT_FUNNELS from posthog.hogql_queries.actors_query_runner import ActorsQueryRunner +from posthog.hogql_queries.insights.funnels.funnels_query_runner import FunnelsQueryRunner from posthog.hogql_queries.legacy_compatibility.filter_to_query import filter_to_query from posthog.models.team.team import Team from posthog.schema import ( @@ -73,7 +74,8 @@ def get_actors( return response.results -class TestFunnelCorrelationsActors(ClickhouseTestMixin, APIBaseTest): +class BaseTestFunnelCorrelationActors(ClickhouseTestMixin, APIBaseTest): + __test__ = False maxDiff = None def _setup_basic_test(self): @@ -575,6 +577,8 @@ def test_strict_funnel_correlation_with_recordings(self): {"id": "insight analyzed", "order": 1}, ], } + query = cast(FunnelsQuery, filter_to_query(filters)) + results = FunnelsQueryRunner(query=query, team=self.team).calculate().results results = get_actors( filters, @@ -641,3 +645,7 @@ def test_strict_funnel_correlation_with_recordings(self): } ], ) + + +class TestFunnelCorrelationActors(BaseTestFunnelCorrelationActors): + __test__ = True diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_correlation_actors_udf.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_correlation_actors_udf.py new file mode 100644 index 0000000000000..aa562e0260b73 --- /dev/null +++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_correlation_actors_udf.py @@ -0,0 +1,9 @@ +from unittest.mock import Mock, patch + +from posthog.hogql_queries.insights.funnels.test.test_funnel_correlation_actors import BaseTestFunnelCorrelationActors +from posthog.hogql_queries.insights.funnels.test.test_funnel_udf import use_udf_funnel_flag_side_effect + + +@patch("posthoganalytics.feature_enabled", new=Mock(side_effect=use_udf_funnel_flag_side_effect)) +class TestFunnelCorrelationsActorsUDF(BaseTestFunnelCorrelationActors): + __test__ = True diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_correlation_udf.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_correlation_udf.py new file mode 100644 index 0000000000000..0882bad888059 --- /dev/null +++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_correlation_udf.py @@ -0,0 +1,9 @@ +from unittest.mock import Mock, patch + +from posthog.hogql_queries.insights.funnels.test.test_funnel_correlation import BaseTestClickhouseFunnelCorrelation +from posthog.hogql_queries.insights.funnels.test.test_funnel_udf import use_udf_funnel_flag_side_effect + + +@patch("posthoganalytics.feature_enabled", new=Mock(side_effect=use_udf_funnel_flag_side_effect)) +class TestClickhouseFunnelCorrelationUDF(BaseTestClickhouseFunnelCorrelation): + __test__ = True diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_persons.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_persons.py index e240e54f3f963..c62fefddcc57c 100644 --- a/posthog/hogql_queries/insights/funnels/test/test_funnel_persons.py +++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_persons.py @@ -60,7 +60,9 @@ def get_actors( return response.results -class TestFunnelPersons(ClickhouseTestMixin, APIBaseTest): +class BaseTestFunnelPersons(ClickhouseTestMixin, APIBaseTest): + __test__ = False + def _create_sample_data_multiple_dropoffs(self): for i in range(35): bulk_create_persons([{"distinct_ids": [f"user_{i}"], "team_id": self.team.pk}]) @@ -809,3 +811,7 @@ def test_first_time_math_multiple_ids(self): self.assertEqual(len(results), 2) self.assertCountEqual(set(results[0][1]["distinct_ids"]), {"person1", "anon1"}) self.assertCountEqual(set(results[1][1]["distinct_ids"]), {"person2", "anon2"}) + + +class TestFunnelPersons(BaseTestFunnelPersons): + __test__ = True diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_persons_udf.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_persons_udf.py new file mode 100644 index 0000000000000..14c5144fc7610 --- /dev/null +++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_persons_udf.py @@ -0,0 +1,31 @@ +from unittest.mock import patch, Mock + + +from posthog.constants import INSIGHT_FUNNELS +from posthog.hogql_queries.insights.funnels.test.test_funnel_persons import ( + get_actors, + BaseTestFunnelPersons, +) +from posthog.hogql_queries.insights.funnels.test.test_funnel_udf import use_udf_funnel_flag_side_effect + + +@patch("posthoganalytics.feature_enabled", new=Mock(side_effect=use_udf_funnel_flag_side_effect)) +class TestFunnelPersonsUDF(BaseTestFunnelPersons): + __test__ = True + + @patch("posthog.hogql_queries.insights.funnels.funnel_udf.FunnelUDF.actor_query", return_value=None) + def test_uses_udf(self, obj): + self._create_sample_data_multiple_dropoffs() + filters = { + "insight": INSIGHT_FUNNELS, + "interval": "day", + "date_from": "2021-05-01 00:00:00", + "date_to": "2021-05-07 00:00:00", + "funnel_window_days": 7, + "events": [ + {"id": "step one", "order": 0}, + {"id": "step two", "order": 1}, + {"id": "step three", "order": 2}, + ], + } + self.assertRaises(Exception, lambda: get_actors(filters, self.team, funnel_step=1)) diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_strict.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_strict.py index 7be35d81324d1..dbc215fbb5f6a 100644 --- a/posthog/hogql_queries/insights/funnels/test/test_funnel_strict.py +++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_strict.py @@ -48,6 +48,7 @@ class BaseTestFunnelStrictStepsBreakdown( _create_person, ), ): + __test__ = False maxDiff = None def test_basic_funnel_default_funnel_days_breakdown_event(self): @@ -186,7 +187,7 @@ class BaseTestStrictFunnelGroupBreakdown( ClickhouseFunnelStrictActors, ), ): - pass + __test__ = False class BaseTestFunnelStrictStepsConversionTime( @@ -194,11 +195,12 @@ class BaseTestFunnelStrictStepsConversionTime( funnel_conversion_time_test_factory(FunnelOrderType.ORDERED, ClickhouseFunnelStrictActors), # type: ignore ): maxDiff = None - pass + __test__ = False class BaseTestFunnelStrictSteps(ClickhouseTestMixin, APIBaseTest): maxDiff = None + __test__ = False def _get_actor_ids_at_step(self, filter, funnel_step, breakdown_value=None): filter = Filter(data=filter, team=self.team) @@ -629,19 +631,19 @@ def test_basic_strict_funnel_conversion_times(self): @patch("posthoganalytics.feature_enabled", new=Mock(return_value=False)) class TestFunnelStrictStepsBreakdown(BaseTestFunnelStrictStepsBreakdown): - pass + __test__ = True @patch("posthoganalytics.feature_enabled", new=Mock(return_value=False)) class TestFunnelStrictSteps(BaseTestFunnelStrictSteps): - pass + __test__ = True @patch("posthoganalytics.feature_enabled", new=Mock(return_value=False)) class TestStrictFunnelGroupBreakdown(BaseTestStrictFunnelGroupBreakdown): - pass + __test__ = True @patch("posthoganalytics.feature_enabled", new=Mock(return_value=False)) class TestFunnelStrictStepsConversionTime(BaseTestFunnelStrictStepsConversionTime): - pass + __test__ = True diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_strict_persons.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_strict_persons.py index 33fcdea328426..311fa1c76ccf4 100644 --- a/posthog/hogql_queries/insights/funnels/test/test_funnel_strict_persons.py +++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_strict_persons.py @@ -20,7 +20,9 @@ FORMAT_TIME = "%Y-%m-%d 00:00:00" -class TestFunnelStrictStepsPersons(ClickhouseTestMixin, APIBaseTest): +class BaseTestFunnelStrictStepsPersons(ClickhouseTestMixin, APIBaseTest): + __test__ = False + def _create_sample_data_multiple_dropoffs(self): events_by_person = {} for i in range(5): @@ -265,3 +267,7 @@ def test_strict_funnel_person_recordings(self): } ], ) + + +class TestFunnelStrictStepsPersons(BaseTestFunnelStrictStepsPersons): + __test__ = True diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_strict_persons_udf.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_strict_persons_udf.py new file mode 100644 index 0000000000000..e28999447b524 --- /dev/null +++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_strict_persons_udf.py @@ -0,0 +1,9 @@ +from unittest.mock import patch, Mock + +from posthog.hogql_queries.insights.funnels.test.test_funnel_strict_persons import BaseTestFunnelStrictStepsPersons +from posthog.hogql_queries.insights.funnels.test.test_funnel_udf import use_udf_funnel_flag_side_effect + + +@patch("posthoganalytics.feature_enabled", new=Mock(side_effect=use_udf_funnel_flag_side_effect)) +class TestFunnelStrictStepsPersons(BaseTestFunnelStrictStepsPersons): + __test__ = True diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_strict_udf.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_strict_udf.py index 178e329d3748e..fdde235e5bc7e 100644 --- a/posthog/hogql_queries/insights/funnels/test/test_funnel_strict_udf.py +++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_strict_udf.py @@ -9,20 +9,20 @@ @patch("posthoganalytics.feature_enabled", new=Mock(return_value=True)) -class TestFunnelStrictStepsBreakdown(BaseTestFunnelStrictStepsBreakdown): - pass +class TestFunnelStrictStepsBreakdownUDF(BaseTestFunnelStrictStepsBreakdown): + __test__ = True @patch("posthoganalytics.feature_enabled", new=Mock(return_value=True)) -class TestFunnelStrictSteps(BaseTestFunnelStrictSteps): - pass +class TestFunnelStrictStepsUDF(BaseTestFunnelStrictSteps): + __test__ = True @patch("posthoganalytics.feature_enabled", new=Mock(return_value=True)) -class TestStrictFunnelGroupBreakdown(BaseTestStrictFunnelGroupBreakdown): - pass +class TestStrictFunnelGroupBreakdownUDF(BaseTestStrictFunnelGroupBreakdown): + __test__ = True @patch("posthoganalytics.feature_enabled", new=Mock(return_value=True)) -class TestFunnelStrictStepsConversionTime(BaseTestFunnelStrictStepsConversionTime): - pass +class TestFunnelStrictStepsConversionTimeUDF(BaseTestFunnelStrictStepsConversionTime): + __test__ = True diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py index 446a529113f9f..d7f8d326e2a7d 100644 --- a/posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py +++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_trends.py @@ -1,6 +1,5 @@ from datetime import date, datetime, timedelta from typing import cast -from unittest.mock import patch, Mock from zoneinfo import ZoneInfo from freezegun.api import freeze_time @@ -1615,11 +1614,10 @@ def test_parses_breakdown_correctly(self): self.assertEqual(len(results), 1) -@patch("posthoganalytics.feature_enabled", new=Mock(return_value=False)) class TestFunnelTrends(BaseTestFunnelTrends): __test__ = True - def test_assert_flag_is_working(self): + def test_assert_udf_flag_is_working(self): filters = { "insight": INSIGHT_FUNNELS, "funnel_viz_type": "trends", diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_trends_udf.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_trends_udf.py index 0b28f49fb952c..2829bb93ce7dc 100644 --- a/posthog/hogql_queries/insights/funnels/test/test_funnel_trends_udf.py +++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_trends_udf.py @@ -15,7 +15,7 @@ class TestFunnelTrendsUDF(BaseTestFunnelTrends): __test__ = True - def test_assert_trends_flag_is_on(self): + def test_assert_udf_flag_is_working(self): filters = { "insight": INSIGHT_FUNNELS, "funnel_viz_type": "trends", diff --git a/posthog/hogql_queries/insights/funnels/test/test_funnel_udf.py b/posthog/hogql_queries/insights/funnels/test/test_funnel_udf.py index 849760ed262a3..5fba037657116 100644 --- a/posthog/hogql_queries/insights/funnels/test/test_funnel_udf.py +++ b/posthog/hogql_queries/insights/funnels/test/test_funnel_udf.py @@ -31,10 +31,10 @@ def _create_action(**kwargs): return action -funnel_flag_side_effect = lambda key, *args, **kwargs: key == "insight-funnels-use-udf" +use_udf_funnel_flag_side_effect = lambda key, *args, **kwargs: key == "insight-funnels-use-udf" -@patch("posthoganalytics.feature_enabled", new=Mock(side_effect=funnel_flag_side_effect)) +@patch("posthoganalytics.feature_enabled", new=Mock(side_effect=use_udf_funnel_flag_side_effect)) class TestFunnelBreakdownUDF( ClickhouseTestMixin, funnel_breakdown_test_factory( # type: ignore @@ -48,7 +48,7 @@ class TestFunnelBreakdownUDF( pass -@patch("posthoganalytics.feature_enabled", new=Mock(side_effect=funnel_flag_side_effect)) +@patch("posthoganalytics.feature_enabled", new=Mock(side_effect=use_udf_funnel_flag_side_effect)) class TestFunnelGroupBreakdownUDF( ClickhouseTestMixin, funnel_breakdown_group_test_factory( # type: ignore @@ -59,7 +59,7 @@ class TestFunnelGroupBreakdownUDF( pass -@patch("posthoganalytics.feature_enabled", new=Mock(side_effect=funnel_flag_side_effect)) +@patch("posthoganalytics.feature_enabled", new=Mock(side_effect=use_udf_funnel_flag_side_effect)) class TestFOSSFunnelUDF(funnel_test_factory(Funnel, _create_event, _create_person)): # type: ignore def test_assert_flag_is_on(self): filters = { @@ -102,7 +102,7 @@ def test_assert_trends_flag_is_off(self): maxDiff = None -@patch("posthoganalytics.feature_enabled", new=Mock(side_effect=funnel_flag_side_effect)) +@patch("posthoganalytics.feature_enabled", new=Mock(side_effect=use_udf_funnel_flag_side_effect)) class TestFunnelConversionTimeUDF( ClickhouseTestMixin, funnel_conversion_time_test_factory(FunnelOrderType.ORDERED, ClickhouseFunnelActors), # type: ignore diff --git a/posthog/hogql_queries/insights/funnels/utils.py b/posthog/hogql_queries/insights/funnels/utils.py index ba8425843e5ed..b071bcd7a1d86 100644 --- a/posthog/hogql_queries/insights/funnels/utils.py +++ b/posthog/hogql_queries/insights/funnels/utils.py @@ -1,10 +1,26 @@ from posthog.constants import FUNNEL_WINDOW_INTERVAL_TYPES from posthog.hogql import ast from posthog.hogql.parser import parse_expr +from posthog.hogql_queries.legacy_compatibility.feature_flag import ( + insight_funnels_use_udf_trends, + insight_funnels_use_udf, +) +from posthog.models import Team from posthog.schema import FunnelConversionWindowTimeUnit, FunnelVizType, FunnelsFilter, StepOrderValue from rest_framework.exceptions import ValidationError +def use_udf(funnelsFilter: FunnelsFilter, team: Team): + if funnelsFilter.useUdf: + return True + funnelVizType = funnelsFilter.funnelVizType + if funnelVizType == FunnelVizType.TRENDS and insight_funnels_use_udf_trends(team): + return True + if funnelVizType == FunnelVizType.STEPS and insight_funnels_use_udf(team): + return True + return False + + def get_funnel_order_class(funnelsFilter: FunnelsFilter, use_udf=False): from posthog.hogql_queries.insights.funnels import ( Funnel, @@ -22,8 +38,9 @@ def get_funnel_order_class(funnelsFilter: FunnelsFilter, use_udf=False): return Funnel -def get_funnel_actor_class(funnelsFilter: FunnelsFilter): +def get_funnel_actor_class(funnelsFilter: FunnelsFilter, use_udf=False): from posthog.hogql_queries.insights.funnels import ( + FunnelUDF, FunnelActors, FunnelStrictActors, FunnelUnorderedActors, @@ -34,6 +51,8 @@ def get_funnel_actor_class(funnelsFilter: FunnelsFilter): return FunnelTrendsActors if funnelsFilter.funnelOrderType == StepOrderValue.UNORDERED: return FunnelUnorderedActors + if use_udf: + return FunnelUDF if funnelsFilter.funnelOrderType == StepOrderValue.STRICT: return FunnelStrictActors return FunnelActors diff --git a/posthog/hogql_queries/insights/test/__snapshots__/test_paths_query_runner_ee.ambr b/posthog/hogql_queries/insights/test/__snapshots__/test_paths_query_runner_ee.ambr index e32831815ddea..326acba019703 100644 --- a/posthog/hogql_queries/insights/test/__snapshots__/test_paths_query_runner_ee.ambr +++ b/posthog/hogql_queries/insights/test/__snapshots__/test_paths_query_runner_ee.ambr @@ -1,4 +1,5931 @@ # serializer version: 1 +# name: ClickhousePathsUDF.test_end + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + if(ifNull(greater(target_index, 0), 0), arrayResize(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, -5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arrayResize(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, -5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE ifNull(greater(target_index, 0), 0))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_end.1 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + if(ifNull(greater(target_index, 0), 0), arrayResize(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, -5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arrayResize(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, -5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE ifNull(greater(target_index, 0), 0))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_end_materialized + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + if(ifNull(greater(target_index, 0), 0), arrayResize(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, -5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arrayResize(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, -5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), nullIf(nullIf(events.`mat_$screen_name`, ''), 'null'), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(nullIf(nullIf(events.`mat_$current_url`, ''), 'null'), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE ifNull(greater(target_index, 0), 0))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_end_materialized.1 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + if(ifNull(greater(target_index, 0), 0), arrayResize(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, -5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arrayResize(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, -5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), nullIf(nullIf(events.`mat_$screen_name`, ''), 'null'), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(nullIf(nullIf(events.`mat_$current_url`, ''), 'null'), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE ifNull(greater(target_index, 0), 0))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_event_exclusion_filters_with_wildcard_groups + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + ['/bar/*/foo'] AS groupings, + multiMatchAnyIndex(path_item_ungrouped, ['/bar/.*/foo']) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC')))), and(equals(events.event, '$pageview'), ifNull(notIn(path_item, ['/bar/*/foo']), 0))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_event_exclusion_filters_with_wildcard_groups.1 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + ['/xxx/invalid/*'] AS groupings, + multiMatchAnyIndex(path_item_ungrouped, ['/xxx/invalid/.*']) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC')))), and(equals(events.event, '$pageview'), ifNull(notIn(path_item, ['/bar/*/foo']), 0))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_event_inclusion_exclusion_filters + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_event_inclusion_exclusion_filters.1 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC')))), equals(events.event, '$screen')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_event_inclusion_exclusion_filters.2 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_event_inclusion_exclusion_filters.3 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC')))), and(or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$'))), ifNull(notIn(path_item, ['/custom1', '/1', '/2', '/3']), 0))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_event_ordering + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-03 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_groups_filtering_person_on_events + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS events__group_0 ON equals(events.`$group_0`, events__group_0.key) + WHERE and(equals(events.team_id, 2), ifNull(equals(events__group_0.properties___industry, 'finance'), 0), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-02-01 23:59:59', 6, 'UTC')))), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$')))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_groups_filtering_person_on_events.1 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + events.person_id AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS events__group_0 ON equals(events.`$group_0`, events__group_0.key) + WHERE and(equals(events.team_id, 2), ifNull(equals(events__group_0.properties___industry, 'technology'), 0), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-02-01 23:59:59', 6, 'UTC')))), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$')))) + ORDER BY events.person_id ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_groups_filtering_person_on_events.2 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + events.person_id AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 1)) + GROUP BY groups.group_type_index, + groups.group_key) AS events__group_1 ON equals(events.`$group_1`, events__group_1.key) + WHERE and(equals(events.team_id, 2), ifNull(equals(events__group_1.properties___industry, 'technology'), 0), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-02-01 23:59:59', 6, 'UTC')))), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$')))) + ORDER BY events.person_id ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_person_dropoffs + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '2_step two'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(events.event, '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '2_step two'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_person_dropoffs.1 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '2_step two'), 0) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(events.event, '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '2_step two'), 0) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_person_dropoffs.2 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_step two'), 0), 1) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(events.event, '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_step two'), 0), 1) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_person_dropoffs.3 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '3_step three'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(events.event, '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '3_step three'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_person_dropoffs.4 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '3_step three'), 0) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(events.event, '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '3_step three'), 0) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_person_dropoffs.5 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '3_step three'), 0), 1) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(events.event, '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '3_step three'), 0), 1) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_person_dropoffs.6 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '4_step four'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(events.event, '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '4_step four'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_person_dropoffs.7 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '4_step four'), 0) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(events.event, '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '4_step four'), 0) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_person_dropoffs.8 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '4_step four'), 0), 1) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(events.event, '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '4_step four'), 0), 1) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_person_on_events_v2 + ''' + + SELECT DISTINCT person_id + FROM events + WHERE team_id = 2 + AND distinct_id = 'poev2_p2' + ''' +# --- +# name: ClickhousePathsUDF.test_person_on_events_v2.1 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-02-01 23:59:59', 6, 'UTC')))), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$')))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_recording + ''' + SELECT persons.id AS id, + persons.created_at AS created_at, + source.event_count AS event_count, + source.matching_events AS matching_events + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '2_/2'), 0) + GROUP BY person_id) AS source + INNER JOIN + (SELECT argMax(toTimeZone(person.created_at, 'UTC'), person.version) AS created_at, + person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '2_/2'), 0) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_recording.1 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2012-01-01 03:21:34.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, ['s3', 's1', 's5'])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_recording_for_dropoff + ''' + SELECT persons.id AS id, + persons.created_at AS created_at, + source.event_count AS event_count, + source.matching_events AS matching_events + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '2_/2'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source + INNER JOIN + (SELECT argMax(toTimeZone(person.created_at, 'UTC'), person.version) AS created_at, + person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '2_/2'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_recording_for_dropoff.1 + ''' + SELECT persons.id AS id, + persons.created_at AS created_at, + source.event_count AS event_count, + source.matching_events AS matching_events + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '3_/3'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source + INNER JOIN + (SELECT argMax(toTimeZone(person.created_at, 'UTC'), person.version) AS created_at, + person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '3_/3'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_recording_for_dropoff.2 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2012-01-01 03:21:34.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, ['s1'])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_recording_with_no_window_or_session_id + ''' + SELECT persons.id AS id, + persons.created_at AS created_at, + source.event_count AS event_count, + source.matching_events AS matching_events + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '2_/2'), 0) + GROUP BY person_id) AS source + INNER JOIN + (SELECT argMax(toTimeZone(person.created_at, 'UTC'), person.version) AS created_at, + person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '2_/2'), 0) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_recording_with_no_window_or_session_id.1 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2012-01-01 03:21:34.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, [''])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_recording_with_start_and_end + ''' + SELECT persons.id AS id, + persons.created_at AS created_at, + source.event_count AS event_count, + source.matching_events AS matching_events + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/3') AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + indexOf(compact_path, '/1') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/3') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, + if(ifNull(greater(length(filtered_uuid), 5), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(5, 2)), [filtered_uuid[plus(1, intDiv(5, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_uuid) AS limited_uuid, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, + if(ifNull(greater(length(filtered_timestamp), 5), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(5, 2)), [filtered_timestamp[plus(1, intDiv(5, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timestamp) AS limited_timestamp, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, + if(ifNull(greater(length(filtered_session_id), 5), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(5, 2)), [filtered_session_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_session_id) AS limited_session_id, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, + if(ifNull(greater(length(filtered_window_id), 5), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(5, 2)), [filtered_window_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_window_id) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE ifNull(equals(path_key, '2_/2'), 0) + GROUP BY person_id) AS source + INNER JOIN + (SELECT argMax(toTimeZone(person.created_at, 'UTC'), person.version) AS created_at, + person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, '/3') AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, indexOf(compact_path, '/1') AS start_target_index, if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, indexOf(start_filtered_path, '/3') AS end_target_index, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, if(ifNull(greater(length(filtered_uuid), 5), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(5, 2)), [filtered_uuid[plus(1, intDiv(5, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_uuid) AS limited_uuid, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, if(ifNull(greater(length(filtered_timestamp), 5), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(5, 2)), [filtered_timestamp[plus(1, intDiv(5, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timestamp) AS limited_timestamp, if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, if(ifNull(greater(length(filtered_session_id), 5), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(5, 2)), [filtered_session_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_session_id) AS limited_session_id, if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, if(ifNull(greater(length(filtered_window_id), 5), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(5, 2)), [filtered_window_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_window_id) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE ifNull(equals(path_key, '2_/2'), 0) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_recording_with_start_and_end.1 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2012-01-01 03:21:34.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, ['s1'])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_respect_session_limits + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_start_and_end + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + indexOf(compact_path, '/5') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_start_and_end.1 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + indexOf(compact_path, '/5') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, + if(ifNull(greater(length(filtered_uuid), 5), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(5, 2)), [filtered_uuid[plus(1, intDiv(5, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_uuid) AS limited_uuid, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, + if(ifNull(greater(length(filtered_timestamp), 5), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(5, 2)), [filtered_timestamp[plus(1, intDiv(5, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timestamp) AS limited_timestamp, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, + if(ifNull(greater(length(filtered_session_id), 5), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(5, 2)), [filtered_session_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_session_id) AS limited_session_id, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, + if(ifNull(greater(length(filtered_window_id), 5), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(5, 2)), [filtered_window_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_window_id) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE and(ifNull(equals(last_path_key, '1_/5'), 0), ifNull(equals(path_key, '2_/about'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, '/about') AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, indexOf(compact_path, '/5') AS start_target_index, if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, indexOf(start_filtered_path, '/about') AS end_target_index, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, if(ifNull(greater(length(filtered_uuid), 5), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(5, 2)), [filtered_uuid[plus(1, intDiv(5, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_uuid) AS limited_uuid, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, if(ifNull(greater(length(filtered_timestamp), 5), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(5, 2)), [filtered_timestamp[plus(1, intDiv(5, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timestamp) AS limited_timestamp, if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, if(ifNull(greater(length(filtered_session_id), 5), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(5, 2)), [filtered_session_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_session_id) AS limited_session_id, if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, if(ifNull(greater(length(filtered_window_id), 5), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(5, 2)), [filtered_window_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_window_id) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE and(ifNull(equals(last_path_key, '1_/5'), 0), ifNull(equals(path_key, '2_/about'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_start_and_end.2 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + indexOf(compact_path, '/2') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 4), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(4, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 4), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(4, 2)), [filtered_timings[plus(1, intDiv(4, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timings) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_start_and_end.3 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + indexOf(compact_path, '/2') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 4), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(4, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 4), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(4, 2)), [filtered_timings[plus(1, intDiv(4, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timings) AS limited_timings, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, + if(ifNull(greater(length(filtered_uuid), 4), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(4, 2)), [filtered_uuid[plus(1, intDiv(4, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_uuid) AS limited_uuid, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, + if(ifNull(greater(length(filtered_timestamp), 4), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(4, 2)), [filtered_timestamp[plus(1, intDiv(4, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timestamp) AS limited_timestamp, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, + if(ifNull(greater(length(filtered_session_id), 4), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(4, 2)), [filtered_session_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_session_id) AS limited_session_id, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, + if(ifNull(greater(length(filtered_window_id), 4), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(4, 2)), [filtered_window_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_window_id) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE and(ifNull(equals(last_path_key, '3_...'), 0), ifNull(equals(path_key, '4_/5'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, '/about') AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, indexOf(compact_path, '/2') AS start_target_index, if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, indexOf(start_filtered_path, '/about') AS end_target_index, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, if(ifNull(greater(length(filtered_path), 4), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(4, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_path) AS limited_path, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, if(ifNull(greater(length(filtered_timings), 4), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(4, 2)), [filtered_timings[plus(1, intDiv(4, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timings) AS limited_timings, if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, if(ifNull(greater(length(filtered_uuid), 4), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(4, 2)), [filtered_uuid[plus(1, intDiv(4, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_uuid) AS limited_uuid, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, if(ifNull(greater(length(filtered_timestamp), 4), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(4, 2)), [filtered_timestamp[plus(1, intDiv(4, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timestamp) AS limited_timestamp, if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, if(ifNull(greater(length(filtered_session_id), 4), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(4, 2)), [filtered_session_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_session_id) AS limited_session_id, if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, if(ifNull(greater(length(filtered_window_id), 4), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(4, 2)), [filtered_window_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_window_id) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE and(ifNull(equals(last_path_key, '3_...'), 0), ifNull(equals(path_key, '4_/5'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_start_and_end_materialized + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + indexOf(compact_path, '/5') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(nullIf(nullIf(events.`mat_$current_url`, ''), 'null'), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_start_and_end_materialized.1 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + indexOf(compact_path, '/5') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, + if(ifNull(greater(length(filtered_uuid), 5), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(5, 2)), [filtered_uuid[plus(1, intDiv(5, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_uuid) AS limited_uuid, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, + if(ifNull(greater(length(filtered_timestamp), 5), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(5, 2)), [filtered_timestamp[plus(1, intDiv(5, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timestamp) AS limited_timestamp, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, + if(ifNull(greater(length(filtered_session_id), 5), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(5, 2)), [filtered_session_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_session_id) AS limited_session_id, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, + if(ifNull(greater(length(filtered_window_id), 5), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(5, 2)), [filtered_window_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_window_id) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(nullIf(nullIf(events.`mat_$current_url`, ''), 'null'), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE and(ifNull(equals(last_path_key, '1_/5'), 0), ifNull(equals(path_key, '2_/about'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, '/about') AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, indexOf(compact_path, '/5') AS start_target_index, if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, indexOf(start_filtered_path, '/about') AS end_target_index, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, if(ifNull(greater(length(filtered_uuid), 5), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(5, 2)), [filtered_uuid[plus(1, intDiv(5, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_uuid) AS limited_uuid, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, if(ifNull(greater(length(filtered_timestamp), 5), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(5, 2)), [filtered_timestamp[plus(1, intDiv(5, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timestamp) AS limited_timestamp, if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, if(ifNull(greater(length(filtered_session_id), 5), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(5, 2)), [filtered_session_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_session_id) AS limited_session_id, if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, if(ifNull(greater(length(filtered_window_id), 5), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(5, 2)), [filtered_window_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_window_id) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(nullIf(nullIf(events.`mat_$current_url`, ''), 'null'), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE and(ifNull(equals(last_path_key, '1_/5'), 0), ifNull(equals(path_key, '2_/about'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_start_and_end_materialized.2 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + indexOf(compact_path, '/2') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 4), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(4, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 4), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(4, 2)), [filtered_timings[plus(1, intDiv(4, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timings) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(nullIf(nullIf(events.`mat_$current_url`, ''), 'null'), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_start_and_end_materialized.3 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + indexOf(compact_path, '/2') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 4), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(4, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 4), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(4, 2)), [filtered_timings[plus(1, intDiv(4, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timings) AS limited_timings, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, + if(ifNull(greater(length(filtered_uuid), 4), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(4, 2)), [filtered_uuid[plus(1, intDiv(4, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_uuid) AS limited_uuid, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, + if(ifNull(greater(length(filtered_timestamp), 4), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(4, 2)), [filtered_timestamp[plus(1, intDiv(4, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timestamp) AS limited_timestamp, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, + if(ifNull(greater(length(filtered_session_id), 4), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(4, 2)), [filtered_session_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_session_id) AS limited_session_id, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, + if(ifNull(greater(length(filtered_window_id), 4), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(4, 2)), [filtered_window_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_window_id) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(nullIf(nullIf(events.`mat_$current_url`, ''), 'null'), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE and(ifNull(equals(last_path_key, '3_...'), 0), ifNull(equals(path_key, '4_/5'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, '/about') AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, indexOf(compact_path, '/2') AS start_target_index, if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, indexOf(start_filtered_path, '/about') AS end_target_index, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, if(ifNull(greater(length(filtered_path), 4), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(4, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_path) AS limited_path, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, if(ifNull(greater(length(filtered_timings), 4), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(4, 2)), [filtered_timings[plus(1, intDiv(4, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timings) AS limited_timings, if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, if(ifNull(greater(length(filtered_uuid), 4), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(4, 2)), [filtered_uuid[plus(1, intDiv(4, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_uuid) AS limited_uuid, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, if(ifNull(greater(length(filtered_timestamp), 4), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(4, 2)), [filtered_timestamp[plus(1, intDiv(4, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timestamp) AS limited_timestamp, if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, if(ifNull(greater(length(filtered_session_id), 4), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(4, 2)), [filtered_session_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_session_id) AS limited_session_id, if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, if(ifNull(greater(length(filtered_window_id), 4), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(4, 2)), [filtered_window_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_window_id) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(nullIf(nullIf(events.`mat_$current_url`, ''), 'null'), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE and(ifNull(equals(last_path_key, '3_...'), 0), ifNull(equals(path_key, '4_/5'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_start_dropping_orphaned_edges + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/2') AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE ifNull(greater(target_index, 0), 0))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 6 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_step_conversion_times + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_step_limit + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 2) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 2) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_step_limit.1 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 2) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 2) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 2) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 2) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 2) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 2) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '1_/1'), 0), ifNull(equals(path_key, '2_/2'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 2) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 2) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 2) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 2) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 2) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 2) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '1_/1'), 0), ifNull(equals(path_key, '2_/2'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_step_limit.2 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 2) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 2) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 2) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 2) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 2) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 2) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_/2'), 0), ifNull(equals(path_key, '3_/3'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 2) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 2) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 2) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 2) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 2) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 2) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_/2'), 0), ifNull(equals(path_key, '3_/3'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_step_limit.3 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 3) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 3) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_step_limit.4 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 3) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 3) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 3) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 3) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 3) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 3) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_/2'), 0), ifNull(equals(path_key, '3_/3'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 3) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 3) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 3) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 3) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 3) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 3) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_/2'), 0), ifNull(equals(path_key, '3_/3'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_step_limit.5 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_step_limit.6 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 4) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 4) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 4) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 4) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '1_/1'), 0), ifNull(equals(path_key, '2_/2'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 4) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 4) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 4) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 4) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 4) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 4) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '1_/1'), 0), ifNull(equals(path_key, '2_/2'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_step_limit.7 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 4) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 4) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 4) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 4) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_/2'), 0), ifNull(equals(path_key, '3_/3'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 4) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 4) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 4) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 4) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 4) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 4) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_/2'), 0), ifNull(equals(path_key, '3_/3'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_step_limit.8 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 4) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 4) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 4) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 4) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '3_/3'), 0), ifNull(equals(path_key, '4_/4'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 4) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 4) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 4) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 4) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 4) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 4) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '3_/3'), 0), ifNull(equals(path_key, '4_/4'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: ClickhousePathsUDF.test_wildcard_groups_across_people + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + ['/bar/*/foo'] AS groupings, + multiMatchAnyIndex(path_item_ungrouped, ['/bar/.*/foo']) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: ClickhousePathsUDF.test_wildcard_groups_evil_input + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + ['(a+)+', + '[aaa|aaaa]+', + '1.*', + '.*', + '/3?q=1', + '/3*'] AS groupings, + multiMatchAnyIndex(path_item_ungrouped, ['\\(a\\+\\)\\+', '\\[aaa\\|aaaa\\]\\+', '1\\..*', '\\..*', '/3\\?q=1', '/3.*']) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- # name: TestClickhousePaths.test_end ''' SELECT last_path_key AS source_event, @@ -82,7 +6009,5934 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_end.1 +# name: TestClickhousePaths.test_end.1 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + if(ifNull(greater(target_index, 0), 0), arrayResize(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, -5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arrayResize(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, -5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE ifNull(greater(target_index, 0), 0))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_end_materialized + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + if(ifNull(greater(target_index, 0), 0), arrayResize(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, -5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arrayResize(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, -5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), nullIf(nullIf(events.`mat_$screen_name`, ''), 'null'), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(nullIf(nullIf(events.`mat_$current_url`, ''), 'null'), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE ifNull(greater(target_index, 0), 0))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_end_materialized.1 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + if(ifNull(greater(target_index, 0), 0), arrayResize(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, -5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arrayResize(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, -5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), nullIf(nullIf(events.`mat_$screen_name`, ''), 'null'), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(nullIf(nullIf(events.`mat_$current_url`, ''), 'null'), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE ifNull(greater(target_index, 0), 0))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_event_exclusion_filters_with_wildcard_groups + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + ['/bar/*/foo'] AS groupings, + multiMatchAnyIndex(path_item_ungrouped, ['/bar/.*/foo']) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC')))), and(equals(events.event, '$pageview'), ifNull(notIn(path_item, ['/bar/*/foo']), 0))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_event_exclusion_filters_with_wildcard_groups.1 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + ['/xxx/invalid/*'] AS groupings, + multiMatchAnyIndex(path_item_ungrouped, ['/xxx/invalid/.*']) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC')))), and(equals(events.event, '$pageview'), ifNull(notIn(path_item, ['/bar/*/foo']), 0))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_event_inclusion_exclusion_filters + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_event_inclusion_exclusion_filters.1 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC')))), equals(events.event, '$screen')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_event_inclusion_exclusion_filters.2 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_event_inclusion_exclusion_filters.3 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC')))), and(or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$'))), ifNull(notIn(path_item, ['/custom1', '/1', '/2', '/3']), 0))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_event_ordering + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-03 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_groups_filtering_person_on_events + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS events__group_0 ON equals(events.`$group_0`, events__group_0.key) + WHERE and(equals(events.team_id, 2), ifNull(equals(events__group_0.properties___industry, 'finance'), 0), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-02-01 23:59:59', 6, 'UTC')))), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$')))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_groups_filtering_person_on_events.1 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + events.person_id AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS events__group_0 ON equals(events.`$group_0`, events__group_0.key) + WHERE and(equals(events.team_id, 2), ifNull(equals(events__group_0.properties___industry, 'technology'), 0), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-02-01 23:59:59', 6, 'UTC')))), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$')))) + ORDER BY events.person_id ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_groups_filtering_person_on_events.2 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + events.person_id AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT JOIN + (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 2), equals(index, 1)) + GROUP BY groups.group_type_index, + groups.group_key) AS events__group_1 ON equals(events.`$group_1`, events__group_1.key) + WHERE and(equals(events.team_id, 2), ifNull(equals(events__group_1.properties___industry, 'technology'), 0), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-02-01 23:59:59', 6, 'UTC')))), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$')))) + ORDER BY events.person_id ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_person_dropoffs + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '2_step two'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(events.event, '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '2_step two'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_person_dropoffs.1 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '2_step two'), 0) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(events.event, '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '2_step two'), 0) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_person_dropoffs.2 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_step two'), 0), 1) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(events.event, '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_step two'), 0), 1) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_person_dropoffs.3 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '3_step three'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(events.event, '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '3_step three'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_person_dropoffs.4 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '3_step three'), 0) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(events.event, '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '3_step three'), 0) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_person_dropoffs.5 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '3_step three'), 0), 1) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(events.event, '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '3_step three'), 0), 1) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_person_dropoffs.6 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '4_step four'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(events.event, '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '4_step four'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_person_dropoffs.7 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '4_step four'), 0) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(events.event, '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '4_step four'), 0) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_person_dropoffs.8 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '4_step four'), 0), 1) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(events.event, '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '4_step four'), 0), 1) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_person_on_events_v2 + ''' + + SELECT DISTINCT person_id + FROM events + WHERE team_id = 2 + AND distinct_id = 'poev2_p2' + ''' +# --- +# name: TestClickhousePaths.test_person_on_events_v2.1 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-02-01 23:59:59', 6, 'UTC')))), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$')))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_recording + ''' + SELECT persons.id AS id, + persons.created_at AS created_at, + source.event_count AS event_count, + source.matching_events AS matching_events + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '2_/2'), 0) + GROUP BY person_id) AS source + INNER JOIN + (SELECT argMax(toTimeZone(person.created_at, 'UTC'), person.version) AS created_at, + person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '2_/2'), 0) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_recording.1 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2012-01-01 03:21:34.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, ['s3', 's1', 's5'])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_recording_for_dropoff + ''' + SELECT persons.id AS id, + persons.created_at AS created_at, + source.event_count AS event_count, + source.matching_events AS matching_events + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '2_/2'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source + INNER JOIN + (SELECT argMax(toTimeZone(person.created_at, 'UTC'), person.version) AS created_at, + person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '2_/2'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_recording_for_dropoff.1 + ''' + SELECT persons.id AS id, + persons.created_at AS created_at, + source.event_count AS event_count, + source.matching_events AS matching_events + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '3_/3'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source + INNER JOIN + (SELECT argMax(toTimeZone(person.created_at, 'UTC'), person.version) AS created_at, + person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(path_dropoff_key, '3_/3'), 0), ifNull(equals(path_dropoff_key, path_key), isNull(path_dropoff_key) + and isNull(path_key))) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_recording_for_dropoff.2 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2012-01-01 03:21:34.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, ['s1'])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_recording_with_no_window_or_session_id + ''' + SELECT persons.id AS id, + persons.created_at AS created_at, + source.event_count AS event_count, + source.matching_events AS matching_events + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 5) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 5) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 5) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '2_/2'), 0) + GROUP BY person_id) AS source + INNER JOIN + (SELECT argMax(toTimeZone(person.created_at, 'UTC'), person.version) AS created_at, + person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 5) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 5) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 5) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 5) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 5) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 5) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE ifNull(equals(path_key, '2_/2'), 0) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_recording_with_no_window_or_session_id.1 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2012-01-01 03:21:34.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, [''])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_recording_with_start_and_end + ''' + SELECT persons.id AS id, + persons.created_at AS created_at, + source.event_count AS event_count, + source.matching_events AS matching_events + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/3') AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + indexOf(compact_path, '/1') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/3') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, + if(ifNull(greater(length(filtered_uuid), 5), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(5, 2)), [filtered_uuid[plus(1, intDiv(5, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_uuid) AS limited_uuid, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, + if(ifNull(greater(length(filtered_timestamp), 5), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(5, 2)), [filtered_timestamp[plus(1, intDiv(5, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timestamp) AS limited_timestamp, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, + if(ifNull(greater(length(filtered_session_id), 5), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(5, 2)), [filtered_session_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_session_id) AS limited_session_id, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, + if(ifNull(greater(length(filtered_window_id), 5), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(5, 2)), [filtered_window_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_window_id) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE ifNull(equals(path_key, '2_/2'), 0) + GROUP BY person_id) AS source + INNER JOIN + (SELECT argMax(toTimeZone(person.created_at, 'UTC'), person.version) AS created_at, + person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, '/3') AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, indexOf(compact_path, '/1') AS start_target_index, if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, indexOf(start_filtered_path, '/3') AS end_target_index, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, if(ifNull(greater(length(filtered_uuid), 5), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(5, 2)), [filtered_uuid[plus(1, intDiv(5, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_uuid) AS limited_uuid, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, if(ifNull(greater(length(filtered_timestamp), 5), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(5, 2)), [filtered_timestamp[plus(1, intDiv(5, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timestamp) AS limited_timestamp, if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, if(ifNull(greater(length(filtered_session_id), 5), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(5, 2)), [filtered_session_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_session_id) AS limited_session_id, if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, if(ifNull(greater(length(filtered_window_id), 5), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(5, 2)), [filtered_window_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_window_id) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-02 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE ifNull(equals(path_key, '2_/2'), 0) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_recording_with_start_and_end.1 + ''' + SELECT DISTINCT session_replay_events.session_id AS session_id + FROM session_replay_events + WHERE and(equals(session_replay_events.team_id, 2), ifNull(greaterOrEquals(toTimeZone(session_replay_events.min_first_timestamp, 'UTC'), minus(toDateTime64('2012-01-01 03:21:34.000000', 6, 'UTC'), toIntervalDay(21))), 0), in(session_replay_events.session_id, ['s1'])) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_respect_session_limits + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_start_and_end + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + indexOf(compact_path, '/5') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_start_and_end.1 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + indexOf(compact_path, '/5') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, + if(ifNull(greater(length(filtered_uuid), 5), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(5, 2)), [filtered_uuid[plus(1, intDiv(5, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_uuid) AS limited_uuid, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, + if(ifNull(greater(length(filtered_timestamp), 5), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(5, 2)), [filtered_timestamp[plus(1, intDiv(5, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timestamp) AS limited_timestamp, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, + if(ifNull(greater(length(filtered_session_id), 5), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(5, 2)), [filtered_session_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_session_id) AS limited_session_id, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, + if(ifNull(greater(length(filtered_window_id), 5), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(5, 2)), [filtered_window_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_window_id) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE and(ifNull(equals(last_path_key, '1_/5'), 0), ifNull(equals(path_key, '2_/about'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, '/about') AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, indexOf(compact_path, '/5') AS start_target_index, if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, indexOf(start_filtered_path, '/about') AS end_target_index, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, if(ifNull(greater(length(filtered_uuid), 5), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(5, 2)), [filtered_uuid[plus(1, intDiv(5, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_uuid) AS limited_uuid, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, if(ifNull(greater(length(filtered_timestamp), 5), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(5, 2)), [filtered_timestamp[plus(1, intDiv(5, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timestamp) AS limited_timestamp, if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, if(ifNull(greater(length(filtered_session_id), 5), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(5, 2)), [filtered_session_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_session_id) AS limited_session_id, if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, if(ifNull(greater(length(filtered_window_id), 5), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(5, 2)), [filtered_window_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_window_id) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE and(ifNull(equals(last_path_key, '1_/5'), 0), ifNull(equals(path_key, '2_/about'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_start_and_end.2 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + indexOf(compact_path, '/2') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 4), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(4, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 4), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(4, 2)), [filtered_timings[plus(1, intDiv(4, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timings) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_start_and_end.3 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + indexOf(compact_path, '/2') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 4), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(4, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 4), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(4, 2)), [filtered_timings[plus(1, intDiv(4, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timings) AS limited_timings, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, + if(ifNull(greater(length(filtered_uuid), 4), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(4, 2)), [filtered_uuid[plus(1, intDiv(4, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_uuid) AS limited_uuid, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, + if(ifNull(greater(length(filtered_timestamp), 4), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(4, 2)), [filtered_timestamp[plus(1, intDiv(4, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timestamp) AS limited_timestamp, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, + if(ifNull(greater(length(filtered_session_id), 4), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(4, 2)), [filtered_session_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_session_id) AS limited_session_id, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, + if(ifNull(greater(length(filtered_window_id), 4), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(4, 2)), [filtered_window_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_window_id) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE and(ifNull(equals(last_path_key, '3_...'), 0), ifNull(equals(path_key, '4_/5'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, '/about') AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, indexOf(compact_path, '/2') AS start_target_index, if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, indexOf(start_filtered_path, '/about') AS end_target_index, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, if(ifNull(greater(length(filtered_path), 4), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(4, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_path) AS limited_path, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, if(ifNull(greater(length(filtered_timings), 4), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(4, 2)), [filtered_timings[plus(1, intDiv(4, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timings) AS limited_timings, if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, if(ifNull(greater(length(filtered_uuid), 4), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(4, 2)), [filtered_uuid[plus(1, intDiv(4, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_uuid) AS limited_uuid, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, if(ifNull(greater(length(filtered_timestamp), 4), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(4, 2)), [filtered_timestamp[plus(1, intDiv(4, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timestamp) AS limited_timestamp, if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, if(ifNull(greater(length(filtered_session_id), 4), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(4, 2)), [filtered_session_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_session_id) AS limited_session_id, if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, if(ifNull(greater(length(filtered_window_id), 4), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(4, 2)), [filtered_window_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_window_id) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE and(ifNull(equals(last_path_key, '3_...'), 0), ifNull(equals(path_key, '4_/5'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_start_and_end_materialized + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + indexOf(compact_path, '/5') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(nullIf(nullIf(events.`mat_$current_url`, ''), 'null'), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_start_and_end_materialized.1 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + indexOf(compact_path, '/5') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, + if(ifNull(greater(length(filtered_uuid), 5), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(5, 2)), [filtered_uuid[plus(1, intDiv(5, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_uuid) AS limited_uuid, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, + if(ifNull(greater(length(filtered_timestamp), 5), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(5, 2)), [filtered_timestamp[plus(1, intDiv(5, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timestamp) AS limited_timestamp, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, + if(ifNull(greater(length(filtered_session_id), 5), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(5, 2)), [filtered_session_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_session_id) AS limited_session_id, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, + if(ifNull(greater(length(filtered_window_id), 5), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(5, 2)), [filtered_window_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_window_id) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(nullIf(nullIf(events.`mat_$current_url`, ''), 'null'), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE and(ifNull(equals(last_path_key, '1_/5'), 0), ifNull(equals(path_key, '2_/about'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, '/about') AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, indexOf(compact_path, '/5') AS start_target_index, if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, indexOf(start_filtered_path, '/about') AS end_target_index, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, if(ifNull(greater(length(filtered_uuid), 5), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(5, 2)), [filtered_uuid[plus(1, intDiv(5, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_uuid) AS limited_uuid, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, if(ifNull(greater(length(filtered_timestamp), 5), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(5, 2)), [filtered_timestamp[plus(1, intDiv(5, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timestamp) AS limited_timestamp, if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, if(ifNull(greater(length(filtered_session_id), 5), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(5, 2)), [filtered_session_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_session_id) AS limited_session_id, if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, if(ifNull(greater(length(filtered_window_id), 5), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(5, 2)), [filtered_window_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_window_id) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(nullIf(nullIf(events.`mat_$current_url`, ''), 'null'), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE and(ifNull(equals(last_path_key, '1_/5'), 0), ifNull(equals(path_key, '2_/about'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_start_and_end_materialized.2 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + indexOf(compact_path, '/2') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 4), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(4, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 4), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(4, 2)), [filtered_timings[plus(1, intDiv(4, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timings) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(nullIf(nullIf(events.`mat_$current_url`, ''), 'null'), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_start_and_end_materialized.3 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + indexOf(compact_path, '/2') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 4), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(4, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 4), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(4, 2)), [filtered_timings[plus(1, intDiv(4, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timings) AS limited_timings, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, + if(ifNull(greater(length(filtered_uuid), 4), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(4, 2)), [filtered_uuid[plus(1, intDiv(4, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_uuid) AS limited_uuid, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, + if(ifNull(greater(length(filtered_timestamp), 4), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(4, 2)), [filtered_timestamp[plus(1, intDiv(4, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timestamp) AS limited_timestamp, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, + if(ifNull(greater(length(filtered_session_id), 4), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(4, 2)), [filtered_session_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_session_id) AS limited_session_id, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, + if(ifNull(greater(length(filtered_window_id), 4), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(4, 2)), [filtered_window_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_window_id) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(nullIf(nullIf(events.`mat_$current_url`, ''), 'null'), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE and(ifNull(equals(last_path_key, '3_...'), 0), ifNull(equals(path_key, '4_/5'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, '/about') AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, indexOf(compact_path, '/2') AS start_target_index, if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, indexOf(start_filtered_path, '/about') AS end_target_index, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, if(ifNull(greater(length(filtered_path), 4), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(4, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_path) AS limited_path, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, if(ifNull(greater(length(filtered_timings), 4), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(4, 2)), [filtered_timings[plus(1, intDiv(4, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timings) AS limited_timings, if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, if(ifNull(greater(length(filtered_uuid), 4), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(4, 2)), [filtered_uuid[plus(1, intDiv(4, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_uuid) AS limited_uuid, if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, if(ifNull(greater(length(filtered_timestamp), 4), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(4, 2)), [filtered_timestamp[plus(1, intDiv(4, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timestamp) AS limited_timestamp, if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, if(ifNull(greater(length(filtered_session_id), 4), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(4, 2)), [filtered_session_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_session_id) AS limited_session_id, if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, if(ifNull(greater(length(filtered_window_id), 4), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(4, 2)), [filtered_window_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_window_id) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(nullIf(nullIf(events.`mat_$current_url`, ''), 'null'), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE and(ifNull(equals(last_path_key, '3_...'), 0), ifNull(equals(path_key, '4_/5'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_start_dropping_orphaned_edges + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/2') AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE ifNull(greater(target_index, 0), 0))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 6 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_step_conversion_times + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_step_limit + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 2) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 2) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_step_limit.1 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 2) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 2) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 2) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 2) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 2) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 2) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '1_/1'), 0), ifNull(equals(path_key, '2_/2'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 2) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 2) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 2) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 2) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 2) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 2) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '1_/1'), 0), ifNull(equals(path_key, '2_/2'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_step_limit.2 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 2) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 2) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 2) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 2) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 2) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 2) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_/2'), 0), ifNull(equals(path_key, '3_/3'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 2) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 2) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 2) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 2) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 2) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 2) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_/2'), 0), ifNull(equals(path_key, '3_/3'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_step_limit.3 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 3) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 3) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_step_limit.4 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 3) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 3) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 3) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 3) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 3) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 3) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_/2'), 0), ifNull(equals(path_key, '3_/3'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 3) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 3) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 3) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 3) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 3) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 3) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_/2'), 0), ifNull(equals(path_key, '3_/3'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_step_limit.5 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_step_limit.6 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 4) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 4) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 4) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 4) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '1_/1'), 0), ifNull(equals(path_key, '2_/2'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 4) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 4) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 4) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 4) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 4) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 4) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '1_/1'), 0), ifNull(equals(path_key, '2_/2'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_step_limit.7 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 4) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 4) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 4) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 4) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_/2'), 0), ifNull(equals(path_key, '3_/3'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 4) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 4) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 4) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 4) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 4) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 4) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_/2'), 0), ifNull(equals(path_key, '3_/3'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_step_limit.8 + ''' + SELECT persons.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 4) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 4) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 4) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 4) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '3_/3'), 0), ifNull(equals(path_key, '4_/4'), 0)) + GROUP BY person_id) AS source + INNER JOIN + (SELECT person.id AS id + FROM person + WHERE and(equals(person.team_id, 2), in(id, + (SELECT source.actor_id AS actor_id + FROM + (SELECT person_id AS actor_id, groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, count(*) AS event_count + FROM + (SELECT person_id AS person_id, path AS path, conversion_time AS conversion_time, event_in_session_index AS event_in_session_index, concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, path_dropoff_key AS path_dropoff_key, final_uuid AS uuid, final_timestamp AS timestamp, final_session_id AS session_id, final_window_id AS window_id + FROM + (SELECT person_id AS person_id, joined_path_tuple.1 AS path, joined_path_tuple.2 AS conversion_time, joined_path_tuple.3 AS prev_path, event_in_session_index, session_index AS session_index, arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, arrayFilter((x, y) -> y, time, mapping) AS timings, arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, indexOf(compact_path, NULL) AS target_index, joined_path_tuple.4 AS final_uuid, joined_path_tuple.5 AS final_timestamp, joined_path_tuple.6 AS final_session_id, joined_path_tuple.7 AS final_window_id, arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, arraySlice(filtered_path, 1, 4) AS limited_path, if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, arraySlice(filtered_timings, 1, 4) AS limited_timings, if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, arraySlice(filtered_uuid, 1, 4) AS limited_uuid, if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, arraySlice(filtered_timestamp, 1, 4) AS limited_timestamp, if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, arraySlice(filtered_session_id, 1, 4) AS limited_session_id, if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, arraySlice(filtered_window_id, 1, 4) AS limited_window_id, arrayDifference(limited_timings) AS timings_diff, concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, path_time_tuple.1 AS path_basic, path_time_tuple.2 AS time, session_index, arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, path_time_tuple.4 AS uuid_items, path_time_tuple.5 AS timestamp_items, path_time_tuple.6 AS session_id_items, path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, groupArray(timestamp) AS timing_list, groupArray(path_item) AS path_list, groupArray(uuid) AS uuid_list, groupArray(timestamp) AS timestamp_list, groupArray(session_id) AS session_id_list, groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, events.uuid AS uuid, toTimeZone(events.timestamp, 'UTC') AS timestamp, ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, NULL AS groupings, multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2011-12-31 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '3_/3'), 0), ifNull(equals(path_key, '4_/4'), 0)) + GROUP BY person_id) AS source))) + GROUP BY person.id + HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0)) SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, source.actor_id) + ORDER BY persons.id ASC + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 + ''' +# --- +# name: TestClickhousePaths.test_wildcard_groups_across_people + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + ['/bar/*/foo'] AS groupings, + multiMatchAnyIndex(path_item_ungrouped, ['/bar/.*/foo']) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePaths.test_wildcard_groups_evil_input + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + ['(a+)+', + '[aaa|aaaa]+', + '1.*', + '.*', + '/3?q=1', + '/3*'] AS groupings, + multiMatchAnyIndex(path_item_ungrouped, ['\\(a\\+\\)\\+', '\\[aaa\\|aaaa\\]\\+', '1\\..*', '\\..*', '/3\\?q=1', '/3.*']) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2012-01-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2023-05-23 23:59:59', 6, 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePathsUDF.test_end + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + if(ifNull(greater(target_index, 0), 0), arrayResize(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, -5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arrayResize(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, -5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 2), and(greaterOrEquals(toTimeZone(events.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-01 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(events.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2021-05-07 23:59:59', 6, 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE ifNull(greater(target_index, 0), 0))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128 + ''' +# --- +# name: TestClickhousePathsUDF.test_end.1 ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -165,7 +12019,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_end_materialized +# name: TestClickhousePathsUDF.test_end_materialized ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -248,7 +12102,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_end_materialized.1 +# name: TestClickhousePathsUDF.test_end_materialized.1 ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -331,7 +12185,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_event_exclusion_filters_with_wildcard_groups +# name: TestClickhousePathsUDF.test_event_exclusion_filters_with_wildcard_groups ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -413,7 +12267,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_event_exclusion_filters_with_wildcard_groups.1 +# name: TestClickhousePathsUDF.test_event_exclusion_filters_with_wildcard_groups.1 ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -495,7 +12349,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_event_inclusion_exclusion_filters +# name: TestClickhousePathsUDF.test_event_inclusion_exclusion_filters ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -577,7 +12431,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_event_inclusion_exclusion_filters.1 +# name: TestClickhousePathsUDF.test_event_inclusion_exclusion_filters.1 ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -659,7 +12513,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_event_inclusion_exclusion_filters.2 +# name: TestClickhousePathsUDF.test_event_inclusion_exclusion_filters.2 ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -741,7 +12595,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_event_inclusion_exclusion_filters.3 +# name: TestClickhousePathsUDF.test_event_inclusion_exclusion_filters.3 ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -823,7 +12677,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_event_ordering +# name: TestClickhousePathsUDF.test_event_ordering ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -905,7 +12759,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_groups_filtering_person_on_events +# name: TestClickhousePathsUDF.test_groups_filtering_person_on_events ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -995,7 +12849,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_groups_filtering_person_on_events.1 +# name: TestClickhousePathsUDF.test_groups_filtering_person_on_events.1 ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -1078,7 +12932,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_groups_filtering_person_on_events.2 +# name: TestClickhousePathsUDF.test_groups_filtering_person_on_events.2 ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -1161,7 +13015,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_person_dropoffs +# name: TestClickhousePathsUDF.test_person_dropoffs ''' SELECT persons.id AS id FROM @@ -1310,7 +13164,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_person_dropoffs.1 +# name: TestClickhousePathsUDF.test_person_dropoffs.1 ''' SELECT persons.id AS id FROM @@ -1457,7 +13311,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_person_dropoffs.2 +# name: TestClickhousePathsUDF.test_person_dropoffs.2 ''' SELECT persons.id AS id FROM @@ -1604,7 +13458,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_person_dropoffs.3 +# name: TestClickhousePathsUDF.test_person_dropoffs.3 ''' SELECT persons.id AS id FROM @@ -1753,7 +13607,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_person_dropoffs.4 +# name: TestClickhousePathsUDF.test_person_dropoffs.4 ''' SELECT persons.id AS id FROM @@ -1900,7 +13754,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_person_dropoffs.5 +# name: TestClickhousePathsUDF.test_person_dropoffs.5 ''' SELECT persons.id AS id FROM @@ -2047,7 +13901,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_person_dropoffs.6 +# name: TestClickhousePathsUDF.test_person_dropoffs.6 ''' SELECT persons.id AS id FROM @@ -2196,7 +14050,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_person_dropoffs.7 +# name: TestClickhousePathsUDF.test_person_dropoffs.7 ''' SELECT persons.id AS id FROM @@ -2343,7 +14197,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_person_dropoffs.8 +# name: TestClickhousePathsUDF.test_person_dropoffs.8 ''' SELECT persons.id AS id FROM @@ -2490,7 +14344,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_person_on_events_v2 +# name: TestClickhousePathsUDF.test_person_on_events_v2 ''' SELECT DISTINCT person_id @@ -2499,7 +14353,7 @@ AND distinct_id = 'poev2_p2' ''' # --- -# name: TestClickhousePaths.test_person_on_events_v2.1 +# name: TestClickhousePathsUDF.test_person_on_events_v2.1 ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -2581,7 +14435,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_recording +# name: TestClickhousePathsUDF.test_recording ''' SELECT persons.id AS id, persons.created_at AS created_at, @@ -2732,7 +14586,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_recording.1 +# name: TestClickhousePathsUDF.test_recording.1 ''' SELECT DISTINCT session_replay_events.session_id AS session_id FROM session_replay_events @@ -2746,7 +14600,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_recording_for_dropoff +# name: TestClickhousePathsUDF.test_recording_for_dropoff ''' SELECT persons.id AS id, persons.created_at AS created_at, @@ -2899,7 +14753,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_recording_for_dropoff.1 +# name: TestClickhousePathsUDF.test_recording_for_dropoff.1 ''' SELECT persons.id AS id, persons.created_at AS created_at, @@ -3052,7 +14906,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_recording_for_dropoff.2 +# name: TestClickhousePathsUDF.test_recording_for_dropoff.2 ''' SELECT DISTINCT session_replay_events.session_id AS session_id FROM session_replay_events @@ -3066,7 +14920,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_recording_with_no_window_or_session_id +# name: TestClickhousePathsUDF.test_recording_with_no_window_or_session_id ''' SELECT persons.id AS id, persons.created_at AS created_at, @@ -3217,7 +15071,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_recording_with_no_window_or_session_id.1 +# name: TestClickhousePathsUDF.test_recording_with_no_window_or_session_id.1 ''' SELECT DISTINCT session_replay_events.session_id AS session_id FROM session_replay_events @@ -3231,7 +15085,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_recording_with_start_and_end +# name: TestClickhousePathsUDF.test_recording_with_start_and_end ''' SELECT persons.id AS id, persons.created_at AS created_at, @@ -3392,7 +15246,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_recording_with_start_and_end.1 +# name: TestClickhousePathsUDF.test_recording_with_start_and_end.1 ''' SELECT DISTINCT session_replay_events.session_id AS session_id FROM session_replay_events @@ -3406,7 +15260,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_respect_session_limits +# name: TestClickhousePathsUDF.test_respect_session_limits ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -3488,7 +15342,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_start_and_end +# name: TestClickhousePathsUDF.test_start_and_end ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -3575,7 +15429,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_start_and_end.1 +# name: TestClickhousePathsUDF.test_start_and_end.1 ''' SELECT persons.id AS id FROM @@ -3732,7 +15586,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_start_and_end.2 +# name: TestClickhousePathsUDF.test_start_and_end.2 ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -3819,7 +15673,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_start_and_end.3 +# name: TestClickhousePathsUDF.test_start_and_end.3 ''' SELECT persons.id AS id FROM @@ -3976,7 +15830,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_start_and_end_materialized +# name: TestClickhousePathsUDF.test_start_and_end_materialized ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -4063,7 +15917,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_start_and_end_materialized.1 +# name: TestClickhousePathsUDF.test_start_and_end_materialized.1 ''' SELECT persons.id AS id FROM @@ -4220,7 +16074,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_start_and_end_materialized.2 +# name: TestClickhousePathsUDF.test_start_and_end_materialized.2 ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -4307,7 +16161,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_start_and_end_materialized.3 +# name: TestClickhousePathsUDF.test_start_and_end_materialized.3 ''' SELECT persons.id AS id FROM @@ -4464,7 +16318,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_start_dropping_orphaned_edges +# name: TestClickhousePathsUDF.test_start_dropping_orphaned_edges ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -4547,7 +16401,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_step_conversion_times +# name: TestClickhousePathsUDF.test_step_conversion_times ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -4629,7 +16483,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_step_limit +# name: TestClickhousePathsUDF.test_step_limit ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -4711,7 +16565,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_step_limit.1 +# name: TestClickhousePathsUDF.test_step_limit.1 ''' SELECT persons.id AS id FROM @@ -4858,7 +16712,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_step_limit.2 +# name: TestClickhousePathsUDF.test_step_limit.2 ''' SELECT persons.id AS id FROM @@ -5005,7 +16859,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_step_limit.3 +# name: TestClickhousePathsUDF.test_step_limit.3 ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -5087,7 +16941,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_step_limit.4 +# name: TestClickhousePathsUDF.test_step_limit.4 ''' SELECT persons.id AS id FROM @@ -5234,7 +17088,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_step_limit.5 +# name: TestClickhousePathsUDF.test_step_limit.5 ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -5316,7 +17170,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_step_limit.6 +# name: TestClickhousePathsUDF.test_step_limit.6 ''' SELECT persons.id AS id FROM @@ -5463,7 +17317,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_step_limit.7 +# name: TestClickhousePathsUDF.test_step_limit.7 ''' SELECT persons.id AS id FROM @@ -5610,7 +17464,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_step_limit.8 +# name: TestClickhousePathsUDF.test_step_limit.8 ''' SELECT persons.id AS id FROM @@ -5757,7 +17611,7 @@ max_bytes_before_external_group_by=0 ''' # --- -# name: TestClickhousePaths.test_wildcard_groups_across_people +# name: TestClickhousePathsUDF.test_wildcard_groups_across_people ''' SELECT last_path_key AS source_event, path_key AS target_event, @@ -5839,7 +17693,7 @@ max_bytes_before_external_group_by=23622320128 ''' # --- -# name: TestClickhousePaths.test_wildcard_groups_evil_input +# name: TestClickhousePathsUDF.test_wildcard_groups_evil_input ''' SELECT last_path_key AS source_event, path_key AS target_event, diff --git a/posthog/hogql_queries/insights/test/test_paths_query_runner_ee.py b/posthog/hogql_queries/insights/test/test_paths_query_runner_ee.py index 4b593c75e8e6d..a00ae50de973e 100644 --- a/posthog/hogql_queries/insights/test/test_paths_query_runner_ee.py +++ b/posthog/hogql_queries/insights/test/test_paths_query_runner_ee.py @@ -1,7 +1,7 @@ from datetime import timedelta from typing import Any from unittest import skip -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch, Mock from uuid import UUID from django.test import TestCase @@ -40,7 +40,8 @@ ONE_MINUTE = 60_000 # 1 minute in milliseconds -class TestClickhousePaths(ClickhouseTestMixin, APIBaseTest): +class BaseTestClickhousePaths(ClickhouseTestMixin, APIBaseTest): + __test__ = False maxDiff = None def _create_groups(self): @@ -4721,6 +4722,18 @@ def test_wildcard_groups_with_sampling(self): ) +insight_funnels_use_udf_funnel_flag_side_effect = lambda key, *args, **kwargs: key == "insight-funnels-use-udf" + + +class ClickhousePathsUDF(BaseTestClickhousePaths): + __test__ = True + + +@patch("posthoganalytics.feature_enabled", new=Mock(side_effect=insight_funnels_use_udf_funnel_flag_side_effect)) +class TestClickhousePathsUDF(BaseTestClickhousePaths): + __test__ = True + + class TestClickhousePathsEdgeValidation(TestCase): BASIC_PATH = [("1_a", "2_b"), ("2_b", "3_c"), ("3_c", "4_d")] # a->b->c->d BASIC_PATH_2 = [("1_x", "2_y"), ("2_y", "3_z")] # x->y->z diff --git a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr index 35a6ce39c4b15..ed0ddee0e0f34 100644 --- a/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr +++ b/posthog/hogql_queries/insights/trends/test/__snapshots__/test_trends.ambr @@ -851,14 +851,49 @@ # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.1 ''' - /* celery:posthog.tasks.tasks.sync_insight_caching_state */ - SELECT team_id, - date_diff('second', max(timestamp), now()) AS age - FROM events - WHERE timestamp > date_sub(DAY, 3, now()) - AND timestamp < now() - GROUP BY team_id - ORDER BY age; + SELECT groupArray(1)(date)[1] AS date, + arrayFold((acc, x) -> arrayMap(i -> plus(acc[i], x[i]), range(1, plus(length(date), 1))), groupArray(total), arrayWithConstant(length(date), reinterpretAsFloat64(0))) AS total, + if(ifNull(ifNull(greaterOrEquals(row_number, 25), 0), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + breakdown_value AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(DISTINCT if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id)) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1.0 + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC) + WHERE isNotNull(breakdown_value) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 ''' # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.10 @@ -1075,38 +1110,143 @@ # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.2 ''' - /* celery:posthog.tasks.tasks.sync_insight_caching_state */ - SELECT team_id, - date_diff('second', max(timestamp), now()) AS age - FROM events - WHERE timestamp > date_sub(DAY, 3, now()) - AND timestamp < now() - GROUP BY team_id - ORDER BY age; + SELECT groupArray(1)(date)[1] AS date, + arrayFold((acc, x) -> arrayMap(i -> plus(acc[i], x[i]), range(1, plus(length(date), 1))), groupArray(total), arrayWithConstant(length(date), reinterpretAsFloat64(0))) AS total, + if(ifNull(ifNull(greaterOrEquals(row_number, 25), 0), 0), '$$_posthog_breakdown_other_$$', breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + breakdown_value AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + breakdown_value AS breakdown_value + FROM + (SELECT count(DISTINCT if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id)) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value + FROM events AS e SAMPLE 1.0 + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')) + GROUP BY day_start, + breakdown_value) + GROUP BY day_start, + breakdown_value + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC) + WHERE isNotNull(breakdown_value) + GROUP BY breakdown_value + ORDER BY if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_other_$$'), 0), 2, if(ifNull(equals(breakdown_value, '$$_posthog_breakdown_null_$$'), 0), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 ''' # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.3 ''' - /* celery:posthog.tasks.tasks.sync_insight_caching_state */ - SELECT team_id, - date_diff('second', max(timestamp), now()) AS age - FROM events - WHERE timestamp > date_sub(DAY, 3, now()) - AND timestamp < now() - GROUP BY team_id - ORDER BY age; + SELECT groupArray(1)(date)[1] AS date, + arrayFold((acc, x) -> arrayMap(i -> plus(acc[i], x[i]), range(1, plus(length(date), 1))), groupArray(total), arrayWithConstant(length(date), reinterpretAsFloat64(0))) AS total, + arrayMap(i -> if(ifNull(ifNull(greaterOrEquals(row_number, 25), 0), 0), '$$_posthog_breakdown_other_$$', i), breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + breakdown_value AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + [ifNull(toString(breakdown_value_1), '$$_posthog_breakdown_null_$$')] AS breakdown_value + FROM + (SELECT count(DISTINCT if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id)) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value_1 + FROM events AS e SAMPLE 1.0 + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')) + GROUP BY day_start, + breakdown_value_1) + GROUP BY day_start, + breakdown_value_1 + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(has(breakdown_value, '$$_posthog_breakdown_other_$$'), 2, if(has(breakdown_value, '$$_posthog_breakdown_null_$$'), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC) + WHERE arrayExists(x -> isNotNull(x), breakdown_value) + GROUP BY breakdown_value + ORDER BY if(has(breakdown_value, '$$_posthog_breakdown_other_$$'), 2, if(has(breakdown_value, '$$_posthog_breakdown_null_$$'), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 ''' # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.4 ''' - /* celery:posthog.tasks.tasks.sync_insight_caching_state */ - SELECT team_id, - date_diff('second', max(timestamp), now()) AS age - FROM events - WHERE timestamp > date_sub(DAY, 3, now()) - AND timestamp < now() - GROUP BY team_id - ORDER BY age; + SELECT groupArray(1)(date)[1] AS date, + arrayFold((acc, x) -> arrayMap(i -> plus(acc[i], x[i]), range(1, plus(length(date), 1))), groupArray(total), arrayWithConstant(length(date), reinterpretAsFloat64(0))) AS total, + arrayMap(i -> if(ifNull(ifNull(greaterOrEquals(row_number, 25), 0), 0), '$$_posthog_breakdown_other_$$', i), breakdown_value) AS breakdown_value + FROM + (SELECT arrayMap(number -> plus(toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toIntervalDay(number)), range(0, plus(coalesce(dateDiff('day', toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC'))), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))))), 1))) AS date, + arrayMap(_match_date -> arraySum(arraySlice(groupArray(count), indexOf(groupArray(day_start) AS _days_for_count, _match_date) AS _index, plus(minus(arrayLastIndex(x -> ifNull(equals(x, _match_date), isNull(x) + and isNull(_match_date)), _days_for_count), _index), 1))), date) AS total, + breakdown_value AS breakdown_value, + rowNumberInAllBlocks() AS row_number + FROM + (SELECT sum(total) AS count, + day_start AS day_start, + [ifNull(toString(breakdown_value_1), '$$_posthog_breakdown_null_$$')] AS breakdown_value + FROM + (SELECT count(DISTINCT if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id)) AS total, + toStartOfDay(toTimeZone(e.timestamp, 'UTC')) AS day_start, + ifNull(nullIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$some_property'), ''), 'null'), '^"|"$', '')), ''), '$$_posthog_breakdown_null_$$') AS breakdown_value_1 + FROM events AS e SAMPLE 1.0 + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 2) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING ifNull(equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + WHERE and(equals(e.team_id, 2), greaterOrEquals(toTimeZone(e.timestamp, 'UTC'), toStartOfDay(assumeNotNull(parseDateTime64BestEffortOrNull('2019-12-28 00:00:00', 6, 'UTC')))), lessOrEquals(toTimeZone(e.timestamp, 'UTC'), assumeNotNull(parseDateTime64BestEffortOrNull('2020-01-04 23:59:59', 6, 'UTC'))), equals(e.event, 'sign up')) + GROUP BY day_start, + breakdown_value_1) + GROUP BY day_start, + breakdown_value_1 + ORDER BY day_start ASC, breakdown_value ASC) + GROUP BY breakdown_value + ORDER BY if(has(breakdown_value, '$$_posthog_breakdown_other_$$'), 2, if(has(breakdown_value, '$$_posthog_breakdown_null_$$'), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC) + WHERE arrayExists(x -> isNotNull(x), breakdown_value) + GROUP BY breakdown_value + ORDER BY if(has(breakdown_value, '$$_posthog_breakdown_other_$$'), 2, if(has(breakdown_value, '$$_posthog_breakdown_null_$$'), 1, 0)) ASC, arraySum(total) DESC, breakdown_value ASC + LIMIT 50000 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + format_csv_allow_double_quotes=0, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0 ''' # --- # name: TestTrends.test_dau_with_breakdown_filtering_with_sampling.5 diff --git a/posthog/queries/funnels/test/__snapshots__/test_funnel_time_to_convert.ambr b/posthog/queries/funnels/test/__snapshots__/test_funnel_time_to_convert.ambr index 6ccb7f1a035d0..3ccc12d354191 100644 --- a/posthog/queries/funnels/test/__snapshots__/test_funnel_time_to_convert.ambr +++ b/posthog/queries/funnels/test/__snapshots__/test_funnel_time_to_convert.ambr @@ -126,260 +126,6 @@ max_expanded_ast_elements=1000000 ''' # --- -# name: TestFunnelTimeToConvert.test_auto_bin_count_total - ''' - WITH step_runs AS - (SELECT aggregation_target, - steps, - avg(step_1_conversion_time) step_1_average_conversion_time_inner, - avg(step_2_conversion_time) step_2_average_conversion_time_inner, - median(step_1_conversion_time) step_1_median_conversion_time_inner, - median(step_2_conversion_time) step_2_median_conversion_time_inner - FROM - (SELECT aggregation_target, - steps, - max(steps) over (PARTITION BY aggregation_target) as max_steps, - step_1_conversion_time, - step_2_conversion_time - FROM - (SELECT *, - if(latest_0 <= latest_1 - AND latest_1 <= latest_0 + INTERVAL 7 DAY - AND latest_1 <= latest_2 - AND latest_2 <= latest_0 + INTERVAL 7 DAY, 3, if(latest_0 <= latest_1 - AND latest_1 <= latest_0 + INTERVAL 7 DAY, 2, 1)) AS steps , - if(isNotNull(latest_1) - AND latest_1 <= latest_0 + INTERVAL 7 DAY, dateDiff('second', toDateTime(latest_0), toDateTime(latest_1)), NULL) step_1_conversion_time, - if(isNotNull(latest_2) - AND latest_2 <= latest_1 + INTERVAL 7 DAY, dateDiff('second', toDateTime(latest_1), toDateTime(latest_2)), NULL) step_2_conversion_time - FROM - (SELECT aggregation_target, timestamp, step_0, - latest_0, - step_1, - latest_1, - step_2, - min(latest_2) over (PARTITION by aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) latest_2 - FROM - (SELECT aggregation_target, timestamp, step_0, - latest_0, - step_1, - latest_1, - step_2, - if(latest_2 < latest_1, NULL, latest_2) as latest_2 - FROM - (SELECT aggregation_target, timestamp, step_0, - latest_0, - step_1, - min(latest_1) over (PARTITION by aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) latest_1, - step_2, - min(latest_2) over (PARTITION by aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) latest_2 - FROM - (SELECT e.timestamp as timestamp, - pdi.person_id as aggregation_target, - pdi.person_id as person_id, - if(event = 'step one', 1, 0) as step_0, - if(step_0 = 1, timestamp, null) as latest_0, - if(event = 'step two', 1, 0) as step_1, - if(step_1 = 1, timestamp, null) as latest_1, - if(event = 'step three', 1, 0) as step_2, - if(step_2 = 1, timestamp, null) as latest_2 - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - AND distinct_id IN - (SELECT distinct_id - FROM events - WHERE team_id = 2 - AND event IN ['step one', 'step three', 'step two'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2021-06-07 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2021-06-13 23:59:59', 'UTC') ) - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - WHERE team_id = 2 - AND event IN ['step one', 'step three', 'step two'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2021-06-07 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2021-06-13 23:59:59', 'UTC') - AND (step_0 = 1 - OR step_1 = 1 - OR step_2 = 1) )))) - WHERE step_0 = 1 )) - GROUP BY aggregation_target, - steps - HAVING steps = max_steps), - histogram_params AS - (SELECT ifNull(floor(min(step_1_average_conversion_time_inner + step_2_average_conversion_time_inner)), 0) AS from_seconds, - ifNull(ceil(max(step_1_average_conversion_time_inner + step_2_average_conversion_time_inner)), 1) AS to_seconds, - round(avg(step_1_average_conversion_time_inner + step_2_average_conversion_time_inner), 2) AS average_conversion_time, - count() AS sample_count, - least(60, greatest(1, ceil(cbrt(ifNull(sample_count, 0))))) AS bin_count, - ceil((to_seconds - from_seconds) / bin_count) AS bin_width_seconds_raw, - if(bin_width_seconds_raw > 0, bin_width_seconds_raw, 60) AS bin_width_seconds - FROM step_runs - WHERE step_2_average_conversion_time_inner IS NOT NULL ), - - (SELECT bin_width_seconds - FROM histogram_params) AS bin_width_seconds, - - (SELECT bin_count - FROM histogram_params) AS bin_count, - - (SELECT from_seconds - FROM histogram_params) AS histogram_from_seconds, - - (SELECT to_seconds - FROM histogram_params) AS histogram_to_seconds, - - (SELECT average_conversion_time - FROM histogram_params) AS histogram_average_conversion_time - SELECT bin_from_seconds, - person_count, - histogram_average_conversion_time AS average_conversion_time - FROM - (SELECT histogram_from_seconds + floor((step_1_average_conversion_time_inner + step_2_average_conversion_time_inner - histogram_from_seconds) / bin_width_seconds) * bin_width_seconds AS bin_from_seconds, - count() AS person_count - FROM step_runs - GROUP BY bin_from_seconds) results - RIGHT OUTER JOIN - (SELECT histogram_from_seconds + number * bin_width_seconds AS bin_from_seconds - FROM system.numbers - LIMIT ifNull(bin_count, 0) + 1) fill USING (bin_from_seconds) - ORDER BY bin_from_seconds SETTINGS max_ast_elements=1000000, - max_expanded_ast_elements=1000000 - ''' -# --- -# name: TestFunnelTimeToConvert.test_auto_bin_count_total.1 - ''' - WITH step_runs AS - (SELECT aggregation_target, - steps, - avg(step_1_conversion_time) step_1_average_conversion_time_inner, - avg(step_2_conversion_time) step_2_average_conversion_time_inner, - median(step_1_conversion_time) step_1_median_conversion_time_inner, - median(step_2_conversion_time) step_2_median_conversion_time_inner - FROM - (SELECT aggregation_target, - steps, - max(steps) over (PARTITION BY aggregation_target) as max_steps, - step_1_conversion_time, - step_2_conversion_time - FROM - (SELECT *, - if(latest_0 <= latest_1 - AND latest_1 <= latest_0 + INTERVAL 7 DAY - AND latest_1 <= latest_2 - AND latest_2 <= latest_0 + INTERVAL 7 DAY, 3, if(latest_0 <= latest_1 - AND latest_1 <= latest_0 + INTERVAL 7 DAY, 2, 1)) AS steps , - if(isNotNull(latest_1) - AND latest_1 <= latest_0 + INTERVAL 7 DAY, dateDiff('second', toDateTime(latest_0), toDateTime(latest_1)), NULL) step_1_conversion_time, - if(isNotNull(latest_2) - AND latest_2 <= latest_1 + INTERVAL 7 DAY, dateDiff('second', toDateTime(latest_1), toDateTime(latest_2)), NULL) step_2_conversion_time - FROM - (SELECT aggregation_target, timestamp, step_0, - latest_0, - step_1, - latest_1, - step_2, - min(latest_2) over (PARTITION by aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) latest_2 - FROM - (SELECT aggregation_target, timestamp, step_0, - latest_0, - step_1, - latest_1, - step_2, - if(latest_2 < latest_1, NULL, latest_2) as latest_2 - FROM - (SELECT aggregation_target, timestamp, step_0, - latest_0, - step_1, - min(latest_1) over (PARTITION by aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) latest_1, - step_2, - min(latest_2) over (PARTITION by aggregation_target - ORDER BY timestamp DESC ROWS BETWEEN UNBOUNDED PRECEDING AND 0 PRECEDING) latest_2 - FROM - (SELECT e.timestamp as timestamp, - pdi.person_id as aggregation_target, - pdi.person_id as person_id, - if(event = 'step one', 1, 0) as step_0, - if(step_0 = 1, timestamp, null) as latest_0, - if(event = 'step two', 1, 0) as step_1, - if(step_1 = 1, timestamp, null) as latest_1, - if(event = 'step three', 1, 0) as step_2, - if(step_2 = 1, timestamp, null) as latest_2 - FROM events e - INNER JOIN - (SELECT distinct_id, - argMax(person_id, version) as person_id - FROM person_distinct_id2 - WHERE team_id = 2 - AND distinct_id IN - (SELECT distinct_id - FROM events - WHERE team_id = 2 - AND event IN ['step one', 'step three', 'step two'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2021-06-07 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2021-06-13 23:59:59', 'UTC') ) - GROUP BY distinct_id - HAVING argMax(is_deleted, version) = 0) AS pdi ON e.distinct_id = pdi.distinct_id - WHERE team_id = 2 - AND event IN ['step one', 'step three', 'step two'] - AND toTimeZone(timestamp, 'UTC') >= toDateTime('2021-06-07 00:00:00', 'UTC') - AND toTimeZone(timestamp, 'UTC') <= toDateTime('2021-06-13 23:59:59', 'UTC') - AND (step_0 = 1 - OR step_1 = 1 - OR step_2 = 1) )))) - WHERE step_0 = 1 )) - GROUP BY aggregation_target, - steps - HAVING steps = max_steps), - histogram_params AS - (SELECT ifNull(floor(min(step_1_average_conversion_time_inner + step_2_average_conversion_time_inner)), 0) AS from_seconds, - ifNull(ceil(max(step_1_average_conversion_time_inner + step_2_average_conversion_time_inner)), 1) AS to_seconds, - round(avg(step_1_average_conversion_time_inner + step_2_average_conversion_time_inner), 2) AS average_conversion_time, - count() AS sample_count, - least(60, greatest(1, ceil(cbrt(ifNull(sample_count, 0))))) AS bin_count, - ceil((to_seconds - from_seconds) / bin_count) AS bin_width_seconds_raw, - if(bin_width_seconds_raw > 0, bin_width_seconds_raw, 60) AS bin_width_seconds - FROM step_runs - WHERE step_2_average_conversion_time_inner IS NOT NULL ), - - (SELECT bin_width_seconds - FROM histogram_params) AS bin_width_seconds, - - (SELECT bin_count - FROM histogram_params) AS bin_count, - - (SELECT from_seconds - FROM histogram_params) AS histogram_from_seconds, - - (SELECT to_seconds - FROM histogram_params) AS histogram_to_seconds, - - (SELECT average_conversion_time - FROM histogram_params) AS histogram_average_conversion_time - SELECT bin_from_seconds, - person_count, - histogram_average_conversion_time AS average_conversion_time - FROM - (SELECT histogram_from_seconds + floor((step_1_average_conversion_time_inner + step_2_average_conversion_time_inner - histogram_from_seconds) / bin_width_seconds) * bin_width_seconds AS bin_from_seconds, - count() AS person_count - FROM step_runs - GROUP BY bin_from_seconds) results - RIGHT OUTER JOIN - (SELECT histogram_from_seconds + number * bin_width_seconds AS bin_from_seconds - FROM system.numbers - LIMIT ifNull(bin_count, 0) + 1) fill USING (bin_from_seconds) - ORDER BY bin_from_seconds SETTINGS max_ast_elements=1000000, - max_expanded_ast_elements=1000000 - ''' -# --- # name: TestFunnelTimeToConvert.test_basic_strict ''' WITH step_runs AS diff --git a/posthog/test/user_scripts/test_aggregate_funnel.py b/posthog/test/user_scripts/test_aggregate_funnel.py deleted file mode 100644 index 2b20929002f83..0000000000000 --- a/posthog/test/user_scripts/test_aggregate_funnel.py +++ /dev/null @@ -1,4206 +0,0 @@ -from typing import Any - -from posthog.user_scripts.aggregate_funnel import calculate_funnel_from_user_events - - -def test(): - y = [ - [(1577973600.0, "", [1]), (1577980800.0, "", [2]), (1577984400.0, "", [3])], - [(1577880000.0, "", [1]), (1577883600.0, "", [2]), (1577890800.0, "", [3])], - [(1577973600.0, "", [1]), (1577980800.0, "", [2])], - ] - - for x in y: - calculate_funnel_from_user_events(3, 1209600, "first_touch", "strict", [""], x) - - -def test2(): - a: list[Any] = [ - [(1720051532.484019, [], [1, 2, 3, 4, 5, 6])], - [(1720105713.331995, [], [1, 2, 3, 4, 5, 6])], - [(1720329565.847159, [], [1, 2, 3, 4, 5, 6])], - [ - (1720186008.567886, [], [1, 2, 3, 4, 5, 6]), - (1720326697.522923, [], [1, 2, 3, 4, 5, 6]), - (1720482974.426314, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720327526.250804, [], [1, 2, 3, 4, 5, 6]), (1720497558.23414, [], [1, 2, 3, 4, 5, 6])], - [ - (1719979738.339271, [], [1, 2, 3, 4, 5, 6]), - (1720025384.961105, [], [1, 2, 3, 4, 5, 6]), - (1720504618.55439, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720136408.619257, [], [1, 2, 3, 4, 5, 6]), - (1720136458.666712, [], [1, 2, 3, 4, 5, 6]), - (1720136460.776795, [], [1, 2, 3, 4, 5, 6]), - (1720136463.761667, [], [1, 2, 3, 4, 5, 6]), - (1720136465.813823, [], [1, 2, 3, 4, 5, 6]), - (1720153490.167176, [], [1, 2, 3, 4, 5, 6]), - (1720153611.687424, [], [1, 2, 3, 4, 5, 6]), - (1720153613.813758, [], [1, 2, 3, 4, 5, 6]), - (1720221238.819741, [], [1, 2, 3, 4, 5, 6]), - (1720221389.412602, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720234125.717526, [], [1, 2, 3, 4, 5, 6])], - [ - (1720245095.229565, [], [1, 2, 3, 4, 5, 6]), - (1720396821.910578, [], [1, 2, 3, 4, 5, 6]), - (1720502554.801179, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720237286.585886, [], [1, 2, 3, 4, 5, 6]), (1720492842.0014, [], [1, 2, 3, 4, 5, 6])], - [(1720145259.463577, [], [1, 2, 3, 4, 5, 6])], - [(1720173037.951133, [], [1, 2, 3, 4, 5, 6]), (1720398629.834351, [], [1, 2, 3, 4, 5, 6])], - [(1720576515.470242, [], [1, 2, 3, 4, 5, 6])], - [(1720488634.248776, [], [1, 2, 3, 4, 5, 6])], - [ - (1719966672.507604, [], [1, 2, 3, 4, 5, 6]), - (1720379305.230415, [], [1, 2, 3, 4, 5, 6]), - (1720485725.30467, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720056848.984567, [], [1, 2, 3, 4, 5, 6]), - (1720234634.97164, [], [1, 2, 3, 4, 5, 6]), - (1720326372.083307, [], [1, 2, 3, 4, 5, 6]), - (1720487169.130815, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719979630.05111, [], [1, 2, 3, 4, 5, 6])], - [(1720067082.599895, [], [1, 2, 3, 4, 5, 6])], - [(1720065455.678956, [], [1, 2, 3, 4, 5, 6])], - [(1720141594.235645, [], [1, 2, 3, 4, 5, 6]), (1720479638.868071, [], [1, 2, 3, 4, 5, 6])], - [(1720172558.775714, [], [1, 2, 3, 4, 5, 6]), (1720589944.987293, [], [1, 2, 3, 4, 5, 6])], - [(1720240665.403432, [], [1, 2, 3, 4, 5, 6]), (1720403456.771406, [], [1, 2, 3, 4, 5, 6])], - [ - (1720151433.593775, [], [1, 2, 3, 4, 5, 6]), - (1720397705.729741, [], [1, 2, 3, 4, 5, 6]), - (1720407937.654196, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720063019.413544, [], [1, 2, 3, 4, 5, 6]), - (1720230670.007217, [], [1, 2, 3, 4, 5, 6]), - (1720572529.432945, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720062676.566511, [], [1, 2, 3, 4, 5, 6]), - (1720062768.411832, [], [1, 2, 3, 4, 5, 6]), - (1720062770.476807, [], [1, 2, 3, 4, 5, 6]), - (1720062771.394614, [], [1, 2, 3, 4, 5, 6]), - (1720156065.434007, [], [1, 2, 3, 4, 5, 6]), - (1720156180.339675, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720053274.311851, [], [1, 2, 3, 4, 5, 6]), (1720574916.370766, [], [1, 2, 3, 4, 5, 6])], - [(1720403600.103166, [], [1, 2, 3, 4, 5, 6])], - [(1720070524.509752, [], [1, 2, 3, 4, 5, 6]), (1720330735.128105, [], [1, 2, 3, 4, 5, 6])], - [ - (1719980823.099161, [], [1, 2, 3, 4, 5, 6]), - (1720109783.667678, [], [1, 2, 3, 4, 5, 6]), - (1720488536.75761, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720120539.020908, [], [1, 2, 3, 4, 5, 6]), - (1720235556.263511, [], [1, 2, 3, 4, 5, 6]), - (1720404531.8727, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720461710.602542, [], [1, 2, 3, 4, 5, 6])], - [(1720142147.27027, [], [1, 2, 3, 4, 5, 6]), (1720463509.177443, [], [1, 2, 3, 4, 5, 6])], - [(1720609249.094945, [], [1, 2, 3, 4, 5, 6])], - [ - (1720061653.09558, [], [1, 2, 3, 4, 5, 6]), - (1720331923.364924, [], [1, 2, 3, 4, 5, 6]), - (1720493879.336969, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719981455.944035, [], [1, 2, 3, 4, 5, 6]), - (1719981517.705732, [], [1, 2, 3, 4, 5, 6]), - (1719994503.81365, [], [1, 2, 3, 4, 5, 6]), - (1719994621.6397, [], [1, 2, 3, 4, 5, 6]), - (1719994623.698368, [], [1, 2, 3, 4, 5, 6]), - (1719994627.578717, [], [1, 2, 3, 4, 5, 6]), - (1719994629.663136, [], [1, 2, 3, 4, 5, 6]), - (1719994631.068061, [], [1, 2, 3, 4, 5, 6]), - (1719994633.142381, [], [1, 2, 3, 4, 5, 6]), - (1720027463.767433, [], [1, 2, 3, 4, 5, 6]), - (1720027502.563106, [], [1, 2, 3, 4, 5, 6]), - (1720027504.670674, [], [1, 2, 3, 4, 5, 6]), - (1720057341.723675, [], [1, 2, 3, 4, 5, 6]), - (1720057343.781939, [], [1, 2, 3, 4, 5, 6]), - (1720145087.601179, [], [1, 2, 3, 4, 5, 6]), - (1720145089.680587, [], [1, 2, 3, 4, 5, 6]), - (1720243008.749524, [], [1, 2, 3, 4, 5, 6]), - (1720243068.439551, [], [1, 2, 3, 4, 5, 6]), - (1720318425.097956, [], [1, 2, 3, 4, 5, 6]), - (1720318427.16319, [], [1, 2, 3, 4, 5, 6]), - (1720318432.221956, [], [1, 2, 3, 4, 5, 6]), - (1720318434.329525, [], [1, 2, 3, 4, 5, 6]), - (1720418148.778433, [], [1, 2, 3, 4, 5, 6]), - (1720418150.861104, [], [1, 2, 3, 4, 5, 6]), - (1720488202.399436, [], [1, 2, 3, 4, 5, 6]), - (1720488212.260625, [], [1, 2, 3, 4, 5, 6]), - (1720488214.365566, [], [1, 2, 3, 4, 5, 6]), - (1720572393.815712, [], [1, 2, 3, 4, 5, 6]), - (1720613041.916708, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720057097.342555, [], [1, 2, 3, 4, 5, 6]), - (1720317039.904735, [], [1, 2, 3, 4, 5, 6]), - (1720483178.967836, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720181661.187285, [], [1, 2, 3, 4, 5, 6])], - [(1720199552.174104, [], [1, 2, 3, 4, 5, 6]), (1720568803.062761, [], [1, 2, 3, 4, 5, 6])], - [ - (1720247391.136136, [], [1, 2, 3, 4, 5, 6]), - (1720410696.088339, [], [1, 2, 3, 4, 5, 6]), - (1720599399.171422, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720312357.61565, [], [1, 2, 3, 4, 5, 6])], - [(1720052008.103911, [], [1, 2, 3, 4, 5, 6]), (1720400141.042944, [], [1, 2, 3, 4, 5, 6])], - [(1720210751.331903, [], [1, 2, 3, 4, 5, 6]), (1720503558.839248, [], [1, 2, 3, 4, 5, 6])], - [ - (1720241352.747626, [], [1, 2, 3, 4, 5, 6]), - (1720321677.766712, [], [1, 2, 3, 4, 5, 6]), - (1720409706.122052, [], [1, 2, 3, 4, 5, 6]), - (1720519728.980875, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719977467.931514, [], [1, 2, 3, 4, 5, 6]), (1720568695.132969, [], [1, 2, 3, 4, 5, 6])], - [(1720071302.148667, [], [1, 2, 3, 4, 5, 6]), (1720238096.092618, [], [1, 2, 3, 4, 5, 6])], - [(1720057437.769059, [], [1, 2, 3, 4, 5, 6])], - [ - (1720221473.506037, [], [1, 2, 3, 4, 5, 6]), - (1720348129.55283, [], [1, 2, 3, 4, 5, 6]), - (1720482938.000889, [], [1, 2, 3, 4, 5, 6]), - (1720576755.035308, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720235902.362301, [], [1, 2, 3, 4, 5, 6])], - [(1720024782.723245, [], [1, 2, 3, 4, 5, 6])], - [(1720070158.75827, [], [1, 2, 3, 4, 5, 6])], - [ - (1720000651.858702, [], [1, 2, 3, 4, 5, 6]), - (1720244645.395695, [], [1, 2, 3, 4, 5, 6]), - (1720411107.259775, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720191076.938109, [], [1, 2, 3, 4, 5, 6]), - (1720322967.081356, [], [1, 2, 3, 4, 5, 6]), - (1720323158.146239, [], [1, 2, 3, 4, 5, 6]), - (1720323172.234517, [], [1, 2, 3, 4, 5, 6]), - (1720323206.302768, [], [1, 2, 3, 4, 5, 6]), - (1720323313.146535, [], [1, 2, 3, 4, 5, 6]), - (1720323364.511129, [], [1, 2, 3, 4, 5, 6]), - (1720323458.282407, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720065260.493236, [], [1, 2, 3, 4, 5, 6]), - (1720065334.608797, [], [1, 2, 3, 4, 5, 6]), - (1720141650.234015, [], [1, 2, 3, 4, 5, 6]), - (1720141749.547675, [], [1, 2, 3, 4, 5, 6]), - (1720141751.641012, [], [1, 2, 3, 4, 5, 6]), - (1720154278.705276, [], [1, 2, 3, 4, 5, 6]), - (1720154280.760532, [], [1, 2, 3, 4, 5, 6]), - (1720229499.077048, [], [1, 2, 3, 4, 5, 6]), - (1720229572.436301, [], [1, 2, 3, 4, 5, 6]), - (1720259010.216367, [], [1, 2, 3, 4, 5, 6]), - (1720259234.335094, [], [1, 2, 3, 4, 5, 6]), - (1720259236.42606, [], [1, 2, 3, 4, 5, 6]), - (1720318686.64822, [], [1, 2, 3, 4, 5, 6]), - (1720318843.45613, [], [1, 2, 3, 4, 5, 6]), - (1720318845.509738, [], [1, 2, 3, 4, 5, 6]), - (1720363113.918907, [], [1, 2, 3, 4, 5, 6]), - (1720363184.856665, [], [1, 2, 3, 4, 5, 6]), - (1720400947.604003, [], [1, 2, 3, 4, 5, 6]), - (1720400949.633637, [], [1, 2, 3, 4, 5, 6]), - (1720498232.720406, [], [1, 2, 3, 4, 5, 6]), - (1720498253.802808, [], [1, 2, 3, 4, 5, 6]), - (1720498255.908508, [], [1, 2, 3, 4, 5, 6]), - (1720586991.26782, [], [1, 2, 3, 4, 5, 6]), - (1720587059.251675, [], [1, 2, 3, 4, 5, 6]), - (1720587061.383312, [], [1, 2, 3, 4, 5, 6]), - (1720638042.876812, [], [1, 2, 3, 4, 5, 6]), - (1720638133.182092, [], [1, 2, 3, 4, 5, 6]), - (1720638135.286491, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719978486.488845, [], [1, 2, 3, 4, 5, 6]), (1720406010.994165, [], [1, 2, 3, 4, 5, 6])], - [(1720062931.526777, [], [1, 2, 3, 4, 5, 6])], - [ - (1720142330.725196, [], [1, 2, 3, 4, 5, 6]), - (1720238332.287607, [], [1, 2, 3, 4, 5, 6]), - (1720404745.279674, [], [1, 2, 3, 4, 5, 6]), - (1720577388.350881, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719973721.653682, [], [1, 2, 3, 4, 5, 6]), - (1720045556.714061, [], [1, 2, 3, 4, 5, 6]), - (1720286335.062706, [], [1, 2, 3, 4, 5, 6]), - (1720408637.593505, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719980831.540691, [], [1, 2, 3, 4, 5, 6]), - (1719980890.3872, [], [1, 2, 3, 4, 5, 6]), - (1719980892.464391, [], [1, 2, 3, 4, 5, 6]), - (1720027957.165729, [], [1, 2, 3, 4, 5, 6]), - (1720027959.212697, [], [1, 2, 3, 4, 5, 6]), - (1720055928.682589, [], [1, 2, 3, 4, 5, 6]), - (1720055930.747743, [], [1, 2, 3, 4, 5, 6]), - (1720138782.333308, [], [1, 2, 3, 4, 5, 6]), - (1720138842.547168, [], [1, 2, 3, 4, 5, 6]), - (1720138844.667335, [], [1, 2, 3, 4, 5, 6]), - (1720138846.225705, [], [1, 2, 3, 4, 5, 6]), - (1720153595.409537, [], [1, 2, 3, 4, 5, 6]), - (1720153694.792152, [], [1, 2, 3, 4, 5, 6]), - (1720222583.234486, [], [1, 2, 3, 4, 5, 6]), - (1720222651.732326, [], [1, 2, 3, 4, 5, 6]), - (1720222653.840022, [], [1, 2, 3, 4, 5, 6]), - (1720231129.338916, [], [1, 2, 3, 4, 5, 6]), - (1720231262.508465, [], [1, 2, 3, 4, 5, 6]), - (1720315761.130281, [], [1, 2, 3, 4, 5, 6]), - (1720315844.746953, [], [1, 2, 3, 4, 5, 6]), - (1720315846.831435, [], [1, 2, 3, 4, 5, 6]), - (1720406873.849957, [], [1, 2, 3, 4, 5, 6]), - (1720406879.412626, [], [1, 2, 3, 4, 5, 6]), - (1720485467.197531, [], [1, 2, 3, 4, 5, 6]), - (1720485486.733099, [], [1, 2, 3, 4, 5, 6]), - (1720485488.847143, [], [1, 2, 3, 4, 5, 6]), - (1720485492.354688, [], [1, 2, 3, 4, 5, 6]), - (1720485494.434006, [], [1, 2, 3, 4, 5, 6]), - (1720581292.87898, [], [1, 2, 3, 4, 5, 6]), - (1720581372.990683, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719965396.997192, [], [1, 2, 3, 4, 5, 6]), - (1720109840.37035, [], [1, 2, 3, 4, 5, 6]), - (1720224849.338664, [], [1, 2, 3, 4, 5, 6]), - (1720311680.960628, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720593841.069028, [], [1, 2, 3, 4, 5, 6])], - [(1720193711.631887, [], [1, 2, 3, 4, 5, 6]), (1720355386.424798, [], [1, 2, 3, 4, 5, 6])], - [(1720137394.637585, [], [1, 2, 3, 4, 5, 6]), (1720227526.549035, [], [1, 2, 3, 4, 5, 6])], - [(1720601724.604091, [], [1, 2, 3, 4, 5, 6])], - [(1720242114.286726, [], [1, 2, 3, 4, 5, 6]), (1720495287.866943, [], [1, 2, 3, 4, 5, 6])], - [ - (1719984060.976083, [], [1, 2, 3, 4, 5, 6]), - (1720233353.478142, [], [1, 2, 3, 4, 5, 6]), - (1720331822.027661, [], [1, 2, 3, 4, 5, 6]), - (1720499420.953642, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720327908.649598, [], [1, 2, 3, 4, 5, 6]), - (1720327957.004146, [], [1, 2, 3, 4, 5, 6]), - (1720328002.921775, [], [1, 2, 3, 4, 5, 6]), - (1720328054.34555, [], [1, 2, 3, 4, 5, 6]), - (1720394578.210396, [], [1, 2, 3, 4, 5, 6]), - (1720394668.213374, [], [1, 2, 3, 4, 5, 6]), - (1720394670.323029, [], [1, 2, 3, 4, 5, 6]), - (1720410358.68385, [], [1, 2, 3, 4, 5, 6]), - (1720410430.047079, [], [1, 2, 3, 4, 5, 6]), - (1720410432.093006, [], [1, 2, 3, 4, 5, 6]), - (1720485479.982584, [], [1, 2, 3, 4, 5, 6]), - (1720485552.035405, [], [1, 2, 3, 4, 5, 6]), - (1720485554.099771, [], [1, 2, 3, 4, 5, 6]), - (1720576265.461408, [], [1, 2, 3, 4, 5, 6]), - (1720576267.553332, [], [1, 2, 3, 4, 5, 6]), - (1720580196.882833, [], [1, 2, 3, 4, 5, 6]), - (1720580198.938581, [], [1, 2, 3, 4, 5, 6]), - (1720580201.66793, [], [1, 2, 3, 4, 5, 6]), - (1720580203.765767, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720512918.56325, [], [1, 2, 3, 4, 5, 6])], - [(1720587573.354151, [], [1, 2, 3, 4, 5, 6])], - [ - (1720059581.380866, [], [1, 2, 3, 4, 5, 6]), - (1720226059.821101, [], [1, 2, 3, 4, 5, 6]), - (1720569936.860231, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720483812.243251, [], [1, 2, 3, 4, 5, 6])], - [ - (1720047890.599986, [], [1, 2, 3, 4, 5, 6]), - (1720151933.610926, [], [1, 2, 3, 4, 5, 6]), - (1720426395.237753, [], [1, 2, 3, 4, 5, 6]), - (1720589584.479646, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720050613.849179, [], [1, 2, 3, 4, 5, 6]), - (1720183728.480776, [], [1, 2, 3, 4, 5, 6]), - (1720245305.222942, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719974717.393306, [], [1, 2, 3, 4, 5, 6]), - (1720238913.058213, [], [1, 2, 3, 4, 5, 6]), - (1720403863.202175, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720066616.245506, [], [1, 2, 3, 4, 5, 6]), (1720329802.077257, [], [1, 2, 3, 4, 5, 6])], - [(1720058023.220462, [], [1, 2, 3, 4, 5, 6]), (1720273250.296181, [], [1, 2, 3, 4, 5, 6])], - [(1720400521.741834, [], [1, 2, 3, 4, 5, 6])], - [(1720590006.799829, [], [1, 2, 3, 4, 5, 6])], - [ - (1719977522.311193, [], [1, 2, 3, 4, 5, 6]), - (1720394307.490994, [], [1, 2, 3, 4, 5, 6]), - (1720541599.758133, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720059720.64277, [], [1, 2, 3, 4, 5, 6]), - (1720225557.277258, [], [1, 2, 3, 4, 5, 6]), - (1720318879.528985, [], [1, 2, 3, 4, 5, 6]), - (1720448939.738279, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720159019.782951, [], [1, 2, 3, 4, 5, 6]), - (1720232688.231366, [], [1, 2, 3, 4, 5, 6]), - (1720312031.934367, [], [1, 2, 3, 4, 5, 6]), - (1720420263.292336, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720051467.327131, [], [1, 2, 3, 4, 5, 6]), - (1720226107.259649, [], [1, 2, 3, 4, 5, 6]), - (1720410027.350582, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719966308.30787, [], [1, 2, 3, 4, 5, 6]), - (1720136775.382126, [], [1, 2, 3, 4, 5, 6]), - (1720453167.302523, [], [1, 2, 3, 4, 5, 6]), - (1720578911.142536, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719965705.478301, [], [1, 2, 3, 4, 5, 6]), - (1720089492.274268, [], [1, 2, 3, 4, 5, 6]), - (1720458943.365803, [], [1, 2, 3, 4, 5, 6]), - (1720588170.374851, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720137202.259506, [], [1, 2, 3, 4, 5, 6]), (1720309242.730837, [], [1, 2, 3, 4, 5, 6])], - [(1720397209.557207, [], [1, 2, 3, 4, 5, 6]), (1720628958.303298, [], [1, 2, 3, 4, 5, 6])], - [(1719967303.936898, [], [1, 2, 3, 4, 5, 6]), (1720069496.922345, [], [1, 2, 3, 4, 5, 6])], - [ - (1719970996.586184, [], [1, 2, 3, 4, 5, 6]), - (1720168525.715398, [], [1, 2, 3, 4, 5, 6]), - (1720504098.515479, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720601826.09111, [], [1, 2, 3, 4, 5, 6])], - [ - (1719971199.057468, [], [1, 2, 3, 4, 5, 6]), - (1720068258.0151, [], [1, 2, 3, 4, 5, 6]), - (1720137337.044491, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719971421.175757, [], [1, 2, 3, 4, 5, 6]), (1720134959.956933, [], [1, 2, 3, 4, 5, 6])], - [ - (1719976305.13246, [], [1, 2, 3, 4, 5, 6]), - (1720059646.658845, [], [1, 2, 3, 4, 5, 6]), - (1720145964.773181, [], [1, 2, 3, 4, 5, 6]), - (1720233116.664838, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720436748.603567, [], [1, 2, 3, 4, 5, 6])], - [(1719969550.144929, [], [1, 2, 3, 4, 5, 6]), (1720315269.690666, [], [1, 2, 3, 4, 5, 6])], - [ - (1720274096.315691, [], [1, 2, 3, 4, 5, 6]), - (1720274154.981534, [], [1, 2, 3, 4, 5, 6]), - (1720274184.028094, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720268306.026574, [], [1, 2, 3, 4, 5, 6]), (1720323182.163554, [], [1, 2, 3, 4, 5, 6])], - [ - (1720032173.053995, [], [1, 2, 3, 4, 5, 6]), - (1720157155.365383, [], [1, 2, 3, 4, 5, 6]), - (1720314424.94755, [], [1, 2, 3, 4, 5, 6]), - (1720481047.114281, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720010572.095008, [], [1, 2, 3, 4, 5, 6])], - [ - (1719968638.302149, [], [1, 2, 3, 4, 5, 6]), - (1719968762.98274, [], [1, 2, 3, 4, 5, 6]), - (1719968765.072701, [], [1, 2, 3, 4, 5, 6]), - (1720005012.137582, [], [1, 2, 3, 4, 5, 6]), - (1720005160.858454, [], [1, 2, 3, 4, 5, 6]), - (1720005162.913788, [], [1, 2, 3, 4, 5, 6]), - (1720175681.69546, [], [1, 2, 3, 4, 5, 6]), - (1720175774.033356, [], [1, 2, 3, 4, 5, 6]), - (1720175776.151125, [], [1, 2, 3, 4, 5, 6]), - (1720220252.732147, [], [1, 2, 3, 4, 5, 6]), - (1720220252.777516, [], [1, 2, 3, 4, 5, 6]), - (1720220256.747294, [], [1, 2, 3, 4, 5, 6]), - (1720492139.162569, [], [1, 2, 3, 4, 5, 6]), - (1720492141.256483, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719970987.569565, [], [1, 2, 3, 4, 5, 6])], - [(1720054508.915859, [], [1, 2, 3, 4, 5, 6]), (1720315224.08896, [], [1, 2, 3, 4, 5, 6])], - [(1720325854.156535, [], [1, 2, 3, 4, 5, 6]), (1720483854.080251, [], [1, 2, 3, 4, 5, 6])], - [(1720406510.418443, [], [1, 2, 3, 4, 5, 6])], - [ - (1720030600.544521, [], [1, 2, 3, 4, 5, 6]), - (1720069524.719771, [], [1, 2, 3, 4, 5, 6]), - (1720484112.369653, [], [1, 2, 3, 4, 5, 6]), - (1720568851.121099, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720304785.385271, [], [1, 2, 3, 4, 5, 6])], - [ - (1720313035.494802, [], [1, 2, 3, 4, 5, 6]), - (1720313090.718424, [], [1, 2, 3, 4, 5, 6]), - (1720313239.783577, [], [1, 2, 3, 4, 5, 6]), - (1720313343.042083, [], [1, 2, 3, 4, 5, 6]), - (1720404935.491204, [], [1, 2, 3, 4, 5, 6]), - (1720404979.552845, [], [1, 2, 3, 4, 5, 6]), - (1720404981.647182, [], [1, 2, 3, 4, 5, 6]), - (1720501716.284759, [], [1, 2, 3, 4, 5, 6]), - (1720501761.154088, [], [1, 2, 3, 4, 5, 6]), - (1720501763.234903, [], [1, 2, 3, 4, 5, 6]), - (1720584650.853158, [], [1, 2, 3, 4, 5, 6]), - (1720584688.899508, [], [1, 2, 3, 4, 5, 6]), - (1720584691.020015, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720229088.205557, [], [1, 2, 3, 4, 5, 6]), (1720487228.612214, [], [1, 2, 3, 4, 5, 6])], - [(1720241088.456982, [], [1, 2, 3, 4, 5, 6])], - [(1720042764.160666, [], [1, 2, 3, 4, 5, 6])], - [ - (1719984325.782697, [], [1, 2, 3, 4, 5, 6]), - (1720047953.098659, [], [1, 2, 3, 4, 5, 6]), - (1720268267.887048, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720054622.362154, [], [1, 2, 3, 4, 5, 6]), - (1720134575.953204, [], [1, 2, 3, 4, 5, 6]), - (1720416355.096939, [], [1, 2, 3, 4, 5, 6]), - (1720500581.691615, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719972690.486054, [], [1, 2, 3, 4, 5, 6]), - (1719972877.103609, [], [1, 2, 3, 4, 5, 6]), - (1719972879.22778, [], [1, 2, 3, 4, 5, 6]), - (1719972881.797102, [], [1, 2, 3, 4, 5, 6]), - (1719972883.859612, [], [1, 2, 3, 4, 5, 6]), - (1720052338.317127, [], [1, 2, 3, 4, 5, 6]), - (1720052340.409592, [], [1, 2, 3, 4, 5, 6]), - (1720069426.554888, [], [1, 2, 3, 4, 5, 6]), - (1720069428.615973, [], [1, 2, 3, 4, 5, 6]), - (1720149027.365317, [], [1, 2, 3, 4, 5, 6]), - (1720149089.951754, [], [1, 2, 3, 4, 5, 6]), - (1720149092.012724, [], [1, 2, 3, 4, 5, 6]), - (1720234565.610403, [], [1, 2, 3, 4, 5, 6]), - (1720239125.82035, [], [1, 2, 3, 4, 5, 6]), - (1720239147.87241, [], [1, 2, 3, 4, 5, 6]), - (1720318243.573983, [], [1, 2, 3, 4, 5, 6]), - (1720318245.63705, [], [1, 2, 3, 4, 5, 6]), - (1720418009.877203, [], [1, 2, 3, 4, 5, 6]), - (1720418011.983148, [], [1, 2, 3, 4, 5, 6]), - (1720418014.464823, [], [1, 2, 3, 4, 5, 6]), - (1720485277.913378, [], [1, 2, 3, 4, 5, 6]), - (1720485280.026695, [], [1, 2, 3, 4, 5, 6]), - (1720574328.529507, [], [1, 2, 3, 4, 5, 6]), - (1720574330.633898, [], [1, 2, 3, 4, 5, 6]), - (1720581736.051228, [], [1, 2, 3, 4, 5, 6]), - (1720581766.135021, [], [1, 2, 3, 4, 5, 6]), - (1720581768.228326, [], [1, 2, 3, 4, 5, 6]), - (1720652888.715284, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720328167.133332, [], [1, 2, 3, 4, 5, 6]), (1720394291.998224, [], [1, 2, 3, 4, 5, 6])], - [(1720138278.025347, [], [1, 2, 3, 4, 5, 6])], - [(1720411684.615562, [], [1, 2, 3, 4, 5, 6])], - [ - (1719979551.790599, [], [1, 2, 3, 4, 5, 6]), - (1720062548.79153, [], [1, 2, 3, 4, 5, 6]), - (1720152645.092565, [], [1, 2, 3, 4, 5, 6]), - (1720273648.542968, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720150050.836492, [], [1, 2, 3, 4, 5, 6])], - [(1720235665.517364, [], [1, 2, 3, 4, 5, 6]), (1720504709.666893, [], [1, 2, 3, 4, 5, 6])], - [(1720502409.011067, [], [1, 2, 3, 4, 5, 6]), (1720652305.691241, [], [1, 2, 3, 4, 5, 6])], - [ - (1719983664.396995, [], [1, 2, 3, 4, 5, 6]), - (1720057964.860551, [], [1, 2, 3, 4, 5, 6]), - (1720058069.016671, [], [1, 2, 3, 4, 5, 6]), - (1720119055.986377, [], [1, 2, 3, 4, 5, 6]), - (1720119177.882926, [], [1, 2, 3, 4, 5, 6]), - (1720146988.862958, [], [1, 2, 3, 4, 5, 6]), - (1720146990.940118, [], [1, 2, 3, 4, 5, 6]), - (1720207091.824328, [], [1, 2, 3, 4, 5, 6]), - (1720207147.984162, [], [1, 2, 3, 4, 5, 6]), - (1720207150.045311, [], [1, 2, 3, 4, 5, 6]), - (1720221686.916464, [], [1, 2, 3, 4, 5, 6]), - (1720221731.792885, [], [1, 2, 3, 4, 5, 6]), - (1720221733.892091, [], [1, 2, 3, 4, 5, 6]), - (1720221736.114027, [], [1, 2, 3, 4, 5, 6]), - (1720221738.1731, [], [1, 2, 3, 4, 5, 6]), - (1720221740.137735, [], [1, 2, 3, 4, 5, 6]), - (1720221742.219472, [], [1, 2, 3, 4, 5, 6]), - (1720319188.083254, [], [1, 2, 3, 4, 5, 6]), - (1720319190.195166, [], [1, 2, 3, 4, 5, 6]), - (1720333160.336537, [], [1, 2, 3, 4, 5, 6]), - (1720333162.39224, [], [1, 2, 3, 4, 5, 6]), - (1720350382.882768, [], [1, 2, 3, 4, 5, 6]), - (1720350384.998862, [], [1, 2, 3, 4, 5, 6]), - (1720350385.400746, [], [1, 2, 3, 4, 5, 6]), - (1720350387.504804, [], [1, 2, 3, 4, 5, 6]), - (1720350390.868713, [], [1, 2, 3, 4, 5, 6]), - (1720350392.929211, [], [1, 2, 3, 4, 5, 6]), - (1720482928.905461, [], [1, 2, 3, 4, 5, 6]), - (1720482987.630373, [], [1, 2, 3, 4, 5, 6]), - (1720549510.694147, [], [1, 2, 3, 4, 5, 6]), - (1720549582.87966, [], [1, 2, 3, 4, 5, 6]), - (1720549584.95341, [], [1, 2, 3, 4, 5, 6]), - (1720581476.586746, [], [1, 2, 3, 4, 5, 6]), - (1720581478.656771, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719977177.729628, [], [1, 2, 3, 4, 5, 6]), (1720393638.078415, [], [1, 2, 3, 4, 5, 6])], - [ - (1719980378.113974, [], [1, 2, 3, 4, 5, 6]), - (1720443102.049493, [], [1, 2, 3, 4, 5, 6]), - (1720590770.939412, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719979883.308045, [], [1, 2, 3, 4, 5, 6]), - (1720230654.923495, [], [1, 2, 3, 4, 5, 6]), - (1720310908.910099, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719964829.898192, [], [1, 2, 3, 4, 5, 6]), - (1719964832.003811, [], [1, 2, 3, 4, 5, 6]), - (1719985240.876339, [], [1, 2, 3, 4, 5, 6]), - (1719985242.911416, [], [1, 2, 3, 4, 5, 6]), - (1719985243.633507, [], [1, 2, 3, 4, 5, 6]), - (1719985245.665729, [], [1, 2, 3, 4, 5, 6]), - (1720060650.829248, [], [1, 2, 3, 4, 5, 6]), - (1720060759.718692, [], [1, 2, 3, 4, 5, 6]), - (1720060761.830242, [], [1, 2, 3, 4, 5, 6]), - (1720070260.443094, [], [1, 2, 3, 4, 5, 6]), - (1720070280.911994, [], [1, 2, 3, 4, 5, 6]), - (1720070282.979246, [], [1, 2, 3, 4, 5, 6]), - (1720134645.425223, [], [1, 2, 3, 4, 5, 6]), - (1720134793.819981, [], [1, 2, 3, 4, 5, 6]), - (1720134795.932398, [], [1, 2, 3, 4, 5, 6]), - (1720155938.192604, [], [1, 2, 3, 4, 5, 6]), - (1720155940.320279, [], [1, 2, 3, 4, 5, 6]), - (1720155945.041101, [], [1, 2, 3, 4, 5, 6]), - (1720155947.088061, [], [1, 2, 3, 4, 5, 6]), - (1720236895.111761, [], [1, 2, 3, 4, 5, 6]), - (1720236912.473535, [], [1, 2, 3, 4, 5, 6]), - (1720236914.593968, [], [1, 2, 3, 4, 5, 6]), - (1720236917.655587, [], [1, 2, 3, 4, 5, 6]), - (1720318871.824625, [], [1, 2, 3, 4, 5, 6]), - (1720318935.358285, [], [1, 2, 3, 4, 5, 6]), - (1720318937.446561, [], [1, 2, 3, 4, 5, 6]), - (1720318940.05207, [], [1, 2, 3, 4, 5, 6]), - (1720318942.106239, [], [1, 2, 3, 4, 5, 6]), - (1720405217.370251, [], [1, 2, 3, 4, 5, 6]), - (1720405312.528519, [], [1, 2, 3, 4, 5, 6]), - (1720405314.627163, [], [1, 2, 3, 4, 5, 6]), - (1720413100.204244, [], [1, 2, 3, 4, 5, 6]), - (1720413102.291035, [], [1, 2, 3, 4, 5, 6]), - (1720496542.308228, [], [1, 2, 3, 4, 5, 6]), - (1720574260.260325, [], [1, 2, 3, 4, 5, 6]), - (1720574343.117651, [], [1, 2, 3, 4, 5, 6]), - (1720574345.20748, [], [1, 2, 3, 4, 5, 6]), - (1720578705.104516, [], [1, 2, 3, 4, 5, 6]), - (1720578717.159504, [], [1, 2, 3, 4, 5, 6]), - (1720578719.26077, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720071370.538676, [], [1, 2, 3, 4, 5, 6])], - [(1719972220.86175, [], [1, 2, 3, 4, 5, 6]), (1720227223.558904, [], [1, 2, 3, 4, 5, 6])], - [ - (1720047692.206123, [], [1, 2, 3, 4, 5, 6]), - (1720074479.043983, [], [1, 2, 3, 4, 5, 6]), - (1720221755.131247, [], [1, 2, 3, 4, 5, 6]), - (1720343377.429715, [], [1, 2, 3, 4, 5, 6]), - (1720581159.65796, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720054906.379171, [], [1, 2, 3, 4, 5, 6]), - (1720326827.193456, [], [1, 2, 3, 4, 5, 6]), - (1720395837.565662, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719972411.855532, [], [1, 2, 3, 4, 5, 6]), - (1719972538.863121, [], [1, 2, 3, 4, 5, 6]), - (1719972540.935712, [], [1, 2, 3, 4, 5, 6]), - (1720063717.900878, [], [1, 2, 3, 4, 5, 6]), - (1720063719.954111, [], [1, 2, 3, 4, 5, 6]), - (1720070114.807467, [], [1, 2, 3, 4, 5, 6]), - (1720070235.024434, [], [1, 2, 3, 4, 5, 6]), - (1720070237.14674, [], [1, 2, 3, 4, 5, 6]), - (1720097819.236115, [], [1, 2, 3, 4, 5, 6]), - (1720097978.260021, [], [1, 2, 3, 4, 5, 6]), - (1720097980.382821, [], [1, 2, 3, 4, 5, 6]), - (1720151026.716063, [], [1, 2, 3, 4, 5, 6]), - (1720151173.670938, [], [1, 2, 3, 4, 5, 6]), - (1720151175.717239, [], [1, 2, 3, 4, 5, 6]), - (1720166439.941955, [], [1, 2, 3, 4, 5, 6]), - (1720166583.693905, [], [1, 2, 3, 4, 5, 6]), - (1720166585.791065, [], [1, 2, 3, 4, 5, 6]), - (1720181553.630642, [], [1, 2, 3, 4, 5, 6]), - (1720181555.746202, [], [1, 2, 3, 4, 5, 6]), - (1720242210.300006, [], [1, 2, 3, 4, 5, 6]), - (1720242331.451228, [], [1, 2, 3, 4, 5, 6]), - (1720316730.127117, [], [1, 2, 3, 4, 5, 6]), - (1720316751.481651, [], [1, 2, 3, 4, 5, 6]), - (1720350332.517593, [], [1, 2, 3, 4, 5, 6]), - (1720350427.724851, [], [1, 2, 3, 4, 5, 6]), - (1720350429.836812, [], [1, 2, 3, 4, 5, 6]), - (1720396153.382808, [], [1, 2, 3, 4, 5, 6]), - (1720396199.106453, [], [1, 2, 3, 4, 5, 6]), - (1720396201.15929, [], [1, 2, 3, 4, 5, 6]), - (1720424092.525755, [], [1, 2, 3, 4, 5, 6]), - (1720424190.959176, [], [1, 2, 3, 4, 5, 6]), - (1720424193.037739, [], [1, 2, 3, 4, 5, 6]), - (1720492456.877253, [], [1, 2, 3, 4, 5, 6]), - (1720492529.103048, [], [1, 2, 3, 4, 5, 6]), - (1720492531.198928, [], [1, 2, 3, 4, 5, 6]), - (1720583806.008143, [], [1, 2, 3, 4, 5, 6]), - (1720583868.43082, [], [1, 2, 3, 4, 5, 6]), - (1720648763.855471, [], [1, 2, 3, 4, 5, 6]), - (1720648878.799852, [], [1, 2, 3, 4, 5, 6]), - (1720648880.882297, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720071793.774403, [], [1, 2, 3, 4, 5, 6]), - (1720309002.505766, [], [1, 2, 3, 4, 5, 6]), - (1720367384.379119, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719978345.677095, [], [1, 2, 3, 4, 5, 6]), (1720134660.416426, [], [1, 2, 3, 4, 5, 6])], - [ - (1720056400.339178, [], [1, 2, 3, 4, 5, 6]), - (1720137451.906538, [], [1, 2, 3, 4, 5, 6]), - (1720581731.115191, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719974999.631834, [], [1, 2, 3, 4, 5, 6]), (1720503857.499785, [], [1, 2, 3, 4, 5, 6])], - [(1720325249.830373, [], [1, 2, 3, 4, 5, 6])], - [ - (1719970175.486046, [], [1, 2, 3, 4, 5, 6]), - (1720061532.244847, [], [1, 2, 3, 4, 5, 6]), - (1720387059.054565, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720157797.242967, [], [1, 2, 3, 4, 5, 6])], - [(1720490173.84352, [], [1, 2, 3, 4, 5, 6])], - [(1720568387.145132, [], [1, 2, 3, 4, 5, 6])], - [(1720027447.264569, [], [1, 2, 3, 4, 5, 6])], - [(1719979106.899872, [], [1, 2, 3, 4, 5, 6]), (1720417473.653713, [], [1, 2, 3, 4, 5, 6])], - [(1720153359.982848, [], [1, 2, 3, 4, 5, 6]), (1720468837.459019, [], [1, 2, 3, 4, 5, 6])], - [(1720047669.218866, [], [1, 2, 3, 4, 5, 6])], - [(1720230050.113895, [], [1, 2, 3, 4, 5, 6])], - [ - (1719975411.228945, [], [1, 2, 3, 4, 5, 6]), - (1720235199.496284, [], [1, 2, 3, 4, 5, 6]), - (1720403154.17646, [], [1, 2, 3, 4, 5, 6]), - (1720626578.282517, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719978136.275137, [], [1, 2, 3, 4, 5, 6]), (1720331670.572264, [], [1, 2, 3, 4, 5, 6])], - [ - (1719975669.597909, [], [1, 2, 3, 4, 5, 6]), - (1719975763.25366, [], [1, 2, 3, 4, 5, 6]), - (1720055306.937976, [], [1, 2, 3, 4, 5, 6]), - (1720055449.351479, [], [1, 2, 3, 4, 5, 6]), - (1720067433.572041, [], [1, 2, 3, 4, 5, 6]), - (1720067452.84519, [], [1, 2, 3, 4, 5, 6]), - (1720067454.935816, [], [1, 2, 3, 4, 5, 6]), - (1720233716.974937, [], [1, 2, 3, 4, 5, 6]), - (1720233884.056907, [], [1, 2, 3, 4, 5, 6]), - (1720233886.107033, [], [1, 2, 3, 4, 5, 6]), - (1720238869.144339, [], [1, 2, 3, 4, 5, 6]), - (1720239054.729577, [], [1, 2, 3, 4, 5, 6]), - (1720239056.811577, [], [1, 2, 3, 4, 5, 6]), - (1720248048.594017, [], [1, 2, 3, 4, 5, 6]), - (1720248147.506317, [], [1, 2, 3, 4, 5, 6]), - (1720248149.540209, [], [1, 2, 3, 4, 5, 6]), - (1720323761.342714, [], [1, 2, 3, 4, 5, 6]), - (1720323763.381547, [], [1, 2, 3, 4, 5, 6]), - (1720400825.240853, [], [1, 2, 3, 4, 5, 6]), - (1720400844.815642, [], [1, 2, 3, 4, 5, 6]), - (1720410954.1329, [], [1, 2, 3, 4, 5, 6]), - (1720410956.230411, [], [1, 2, 3, 4, 5, 6]), - (1720410956.661655, [], [1, 2, 3, 4, 5, 6]), - (1720410958.800282, [], [1, 2, 3, 4, 5, 6]), - (1720414005.128157, [], [1, 2, 3, 4, 5, 6]), - (1720414035.742095, [], [1, 2, 3, 4, 5, 6]), - (1720414037.861356, [], [1, 2, 3, 4, 5, 6]), - (1720414039.522054, [], [1, 2, 3, 4, 5, 6]), - (1720414041.622559, [], [1, 2, 3, 4, 5, 6]), - (1720449836.553695, [], [1, 2, 3, 4, 5, 6]), - (1720449909.88067, [], [1, 2, 3, 4, 5, 6]), - (1720449912.006572, [], [1, 2, 3, 4, 5, 6]), - (1720504478.640048, [], [1, 2, 3, 4, 5, 6]), - (1720504584.183246, [], [1, 2, 3, 4, 5, 6]), - (1720504586.273448, [], [1, 2, 3, 4, 5, 6]), - (1720589586.941948, [], [1, 2, 3, 4, 5, 6]), - (1720589732.653657, [], [1, 2, 3, 4, 5, 6]), - (1720589734.757411, [], [1, 2, 3, 4, 5, 6]), - (1720589735.718174, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719987925.192586, [], [1, 2, 3, 4, 5, 6]), (1720319498.157106, [], [1, 2, 3, 4, 5, 6])], - [(1720140316.935341, [], [1, 2, 3, 4, 5, 6]), (1720581286.138288, [], [1, 2, 3, 4, 5, 6])], - [ - (1719984942.453601, [], [1, 2, 3, 4, 5, 6]), - (1720334036.972544, [], [1, 2, 3, 4, 5, 6]), - (1720568302.136228, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720231990.896895, [], [1, 2, 3, 4, 5, 6]), (1720320392.727402, [], [1, 2, 3, 4, 5, 6])], - [(1720151072.246138, [], [1, 2, 3, 4, 5, 6]), (1720309428.675922, [], [1, 2, 3, 4, 5, 6])], - [(1720652752.302257, [], [1, 2, 3, 4, 5, 6])], - [ - (1719977332.758786, [], [1, 2, 3, 4, 5, 6]), - (1720135118.942837, [], [1, 2, 3, 4, 5, 6]), - (1720498977.766189, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720054300.10225, [], [1, 2, 3, 4, 5, 6]), (1720419250.119038, [], [1, 2, 3, 4, 5, 6])], - [ - (1720170242.586928, [], [1, 2, 3, 4, 5, 6]), - (1720322954.401713, [], [1, 2, 3, 4, 5, 6]), - (1720500416.057333, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719976447.157218, [], [1, 2, 3, 4, 5, 6])], - [(1720134454.623643, [], [1, 2, 3, 4, 5, 6]), (1720482790.529945, [], [1, 2, 3, 4, 5, 6])], - [(1720575291.374898, [], [1, 2, 3, 4, 5, 6])], - [(1720575147.912954, [], [1, 2, 3, 4, 5, 6])], - [ - (1719997197.65312, [], [1, 2, 3, 4, 5, 6]), - (1720137704.47896, [], [1, 2, 3, 4, 5, 6]), - (1720226085.527498, [], [1, 2, 3, 4, 5, 6]), - (1720306837.86921, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720314781.298338, [], [1, 2, 3, 4, 5, 6]), (1720443503.319112, [], [1, 2, 3, 4, 5, 6])], - [ - (1719976515.23989, [], [1, 2, 3, 4, 5, 6]), - (1720225806.720086, [], [1, 2, 3, 4, 5, 6]), - (1720388901.256231, [], [1, 2, 3, 4, 5, 6]), - (1720490185.842396, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720568142.650151, [], [1, 2, 3, 4, 5, 6])], - [(1720587602.828532, [], [1, 2, 3, 4, 5, 6])], - [(1720228569.783763, [], [1, 2, 3, 4, 5, 6]), (1720577136.698764, [], [1, 2, 3, 4, 5, 6])], - [ - (1720058398.793045, [], [1, 2, 3, 4, 5, 6]), - (1720317616.711315, [], [1, 2, 3, 4, 5, 6]), - (1720498994.241943, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719988512.009335, [], [1, 2, 3, 4, 5, 6]), - (1720411879.880695, [], [1, 2, 3, 4, 5, 6]), - (1720575546.218164, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720245099.046699, [], [1, 2, 3, 4, 5, 6]), (1720652539.847041, [], [1, 2, 3, 4, 5, 6])], - [ - (1719965518.303227, [], [1, 2, 3, 4, 5, 6]), - (1720241249.736668, [], [1, 2, 3, 4, 5, 6]), - (1720410560.906617, [], [1, 2, 3, 4, 5, 6]), - (1720566388.427971, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720033619.669265, [], [1, 2, 3, 4, 5, 6]), - (1720309514.690673, [], [1, 2, 3, 4, 5, 6]), - (1720584737.484501, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719981137.891986, [], [1, 2, 3, 4, 5, 6]), - (1719981255.525287, [], [1, 2, 3, 4, 5, 6]), - (1719981257.57542, [], [1, 2, 3, 4, 5, 6]), - (1720156254.934266, [], [1, 2, 3, 4, 5, 6]), - (1720156432.088183, [], [1, 2, 3, 4, 5, 6]), - (1720221245.352908, [], [1, 2, 3, 4, 5, 6]), - (1720221247.415618, [], [1, 2, 3, 4, 5, 6]), - (1720306695.430622, [], [1, 2, 3, 4, 5, 6]), - (1720306697.509606, [], [1, 2, 3, 4, 5, 6]), - (1720399726.625066, [], [1, 2, 3, 4, 5, 6]), - (1720399728.675873, [], [1, 2, 3, 4, 5, 6]), - (1720486842.405361, [], [1, 2, 3, 4, 5, 6]), - (1720486974.649877, [], [1, 2, 3, 4, 5, 6]), - (1720494605.919949, [], [1, 2, 3, 4, 5, 6]), - (1720494724.480053, [], [1, 2, 3, 4, 5, 6]), - (1720494726.541559, [], [1, 2, 3, 4, 5, 6]), - (1720572824.284783, [], [1, 2, 3, 4, 5, 6]), - (1720572826.355789, [], [1, 2, 3, 4, 5, 6]), - (1720652512.753893, [], [1, 2, 3, 4, 5, 6]), - (1720652514.833743, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720044224.653908, [], [1, 2, 3, 4, 5, 6]), (1720583176.852571, [], [1, 2, 3, 4, 5, 6])], - [(1720567262.122889, [], [1, 2, 3, 4, 5, 6])], - [(1720518049.925836, [], [1, 2, 3, 4, 5, 6])], - [(1720148280.678113, [], [1, 2, 3, 4, 5, 6]), (1720391739.484219, [], [1, 2, 3, 4, 5, 6])], - [ - (1719983654.268169, [], [1, 2, 3, 4, 5, 6]), - (1719983761.844014, [], [1, 2, 3, 4, 5, 6]), - (1719983763.902973, [], [1, 2, 3, 4, 5, 6]), - (1720014262.846562, [], [1, 2, 3, 4, 5, 6]), - (1720014264.966534, [], [1, 2, 3, 4, 5, 6]), - (1720014268.064236, [], [1, 2, 3, 4, 5, 6]), - (1720014270.176366, [], [1, 2, 3, 4, 5, 6]), - (1720062164.847608, [], [1, 2, 3, 4, 5, 6]), - (1720062166.93557, [], [1, 2, 3, 4, 5, 6]), - (1720070760.286042, [], [1, 2, 3, 4, 5, 6]), - (1720070793.879274, [], [1, 2, 3, 4, 5, 6]), - (1720070795.974998, [], [1, 2, 3, 4, 5, 6]), - (1720136175.682667, [], [1, 2, 3, 4, 5, 6]), - (1720136177.782735, [], [1, 2, 3, 4, 5, 6]), - (1720150756.421019, [], [1, 2, 3, 4, 5, 6]), - (1720150758.537073, [], [1, 2, 3, 4, 5, 6]), - (1720226712.358545, [], [1, 2, 3, 4, 5, 6]), - (1720320316.371588, [], [1, 2, 3, 4, 5, 6]), - (1720396676.623722, [], [1, 2, 3, 4, 5, 6]), - (1720396759.330429, [], [1, 2, 3, 4, 5, 6]), - (1720482810.511366, [], [1, 2, 3, 4, 5, 6]), - (1720482891.609285, [], [1, 2, 3, 4, 5, 6]), - (1720482893.739553, [], [1, 2, 3, 4, 5, 6]), - (1720502988.652815, [], [1, 2, 3, 4, 5, 6]), - (1720503034.447086, [], [1, 2, 3, 4, 5, 6]), - (1720503036.52898, [], [1, 2, 3, 4, 5, 6]), - (1720503036.606516, [], [1, 2, 3, 4, 5, 6]), - (1720503038.712119, [], [1, 2, 3, 4, 5, 6]), - (1720566567.148583, [], [1, 2, 3, 4, 5, 6]), - (1720566710.618717, [], [1, 2, 3, 4, 5, 6]), - (1720624425.022175, [], [1, 2, 3, 4, 5, 6]), - (1720624567.571474, [], [1, 2, 3, 4, 5, 6]), - (1720624569.66289, [], [1, 2, 3, 4, 5, 6]), - (1720652508.525789, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719978793.297818, [], [1, 2, 3, 4, 5, 6]), (1720312690.624643, [], [1, 2, 3, 4, 5, 6])], - [(1720408103.17786, [], [1, 2, 3, 4, 5, 6]), (1720496665.901316, [], [1, 2, 3, 4, 5, 6])], - [(1720397084.267673, [], [1, 2, 3, 4, 5, 6]), (1720499411.209847, [], [1, 2, 3, 4, 5, 6])], - [ - (1720111853.777887, [], [1, 2, 3, 4, 5, 6]), - (1720111923.412934, [], [1, 2, 3, 4, 5, 6]), - (1720139482.167685, [], [1, 2, 3, 4, 5, 6]), - (1720139533.842338, [], [1, 2, 3, 4, 5, 6]), - (1720139535.907287, [], [1, 2, 3, 4, 5, 6]), - (1720139540.267313, [], [1, 2, 3, 4, 5, 6]), - (1720139542.34773, [], [1, 2, 3, 4, 5, 6]), - (1720139547.051966, [], [1, 2, 3, 4, 5, 6]), - (1720139549.136732, [], [1, 2, 3, 4, 5, 6]), - (1720464344.794745, [], [1, 2, 3, 4, 5, 6]), - (1720464401.900918, [], [1, 2, 3, 4, 5, 6]), - (1720464404.029255, [], [1, 2, 3, 4, 5, 6]), - (1720498850.875209, [], [1, 2, 3, 4, 5, 6]), - (1720499103.608103, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719973523.458065, [], [1, 2, 3, 4, 5, 6]), (1720233566.787523, [], [1, 2, 3, 4, 5, 6])], - [(1720407928.090004, [], [1, 2, 3, 4, 5, 6])], - [(1720032729.148346, [], [1, 2, 3, 4, 5, 6]), (1720062532.225999, [], [1, 2, 3, 4, 5, 6])], - [(1720034095.862663, [], [1, 2, 3, 4, 5, 6]), (1720491285.681862, [], [1, 2, 3, 4, 5, 6])], - [(1720096472.997597, [], [1, 2, 3, 4, 5, 6]), (1720568056.766425, [], [1, 2, 3, 4, 5, 6])], - [(1720138718.911672, [], [1, 2, 3, 4, 5, 6])], - [ - (1720336998.597537, [], [1, 2, 3, 4, 5, 6]), - (1720489473.142035, [], [1, 2, 3, 4, 5, 6]), - (1720574315.596422, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720577328.151149, [], [1, 2, 3, 4, 5, 6])], - [(1720072713.69671, [], [1, 2, 3, 4, 5, 6]), (1720231002.690818, [], [1, 2, 3, 4, 5, 6])], - [(1719966317.997493, [], [1, 2, 3, 4, 5, 6]), (1720238108.647106, [], [1, 2, 3, 4, 5, 6])], - [(1720146847.656681, [], [1, 2, 3, 4, 5, 6])], - [(1720239981.42926, [], [1, 2, 3, 4, 5, 6])], - [(1720081339.444776, [], [1, 2, 3, 4, 5, 6]), (1720234051.371763, [], [1, 2, 3, 4, 5, 6])], - [(1720236937.844197, [], [1, 2, 3, 4, 5, 6]), (1720501314.981075, [], [1, 2, 3, 4, 5, 6])], - [(1720495611.198831, [], [1, 2, 3, 4, 5, 6])], - [(1720071452.84595, [], [1, 2, 3, 4, 5, 6])], - [(1720320971.754361, [], [1, 2, 3, 4, 5, 6])], - [ - (1719982790.224924, [], [1, 2, 3, 4, 5, 6]), - (1719982861.078823, [], [1, 2, 3, 4, 5, 6]), - (1719982863.122702, [], [1, 2, 3, 4, 5, 6]), - (1720052453.241504, [], [1, 2, 3, 4, 5, 6]), - (1720052505.408448, [], [1, 2, 3, 4, 5, 6]), - (1720052507.485592, [], [1, 2, 3, 4, 5, 6]), - (1720078456.868981, [], [1, 2, 3, 4, 5, 6]), - (1720137047.584706, [], [1, 2, 3, 4, 5, 6]), - (1720137124.096958, [], [1, 2, 3, 4, 5, 6]), - (1720137126.192241, [], [1, 2, 3, 4, 5, 6]), - (1720155528.420602, [], [1, 2, 3, 4, 5, 6]), - (1720155596.835697, [], [1, 2, 3, 4, 5, 6]), - (1720155598.919376, [], [1, 2, 3, 4, 5, 6]), - (1720204090.330488, [], [1, 2, 3, 4, 5, 6]), - (1720204222.690243, [], [1, 2, 3, 4, 5, 6]), - (1720204224.805824, [], [1, 2, 3, 4, 5, 6]), - (1720232760.467367, [], [1, 2, 3, 4, 5, 6]), - (1720232859.977733, [], [1, 2, 3, 4, 5, 6]), - (1720242903.930897, [], [1, 2, 3, 4, 5, 6]), - (1720242906.021355, [], [1, 2, 3, 4, 5, 6]), - (1720309697.411345, [], [1, 2, 3, 4, 5, 6]), - (1720309699.483954, [], [1, 2, 3, 4, 5, 6]), - (1720406346.354509, [], [1, 2, 3, 4, 5, 6]), - (1720406523.466919, [], [1, 2, 3, 4, 5, 6]), - (1720406525.535072, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720065979.154591, [], [1, 2, 3, 4, 5, 6]), (1720325699.423285, [], [1, 2, 3, 4, 5, 6])], - [(1720244750.093352, [], [1, 2, 3, 4, 5, 6]), (1720394343.192185, [], [1, 2, 3, 4, 5, 6])], - [(1720193298.590097, [], [1, 2, 3, 4, 5, 6]), (1720315677.193089, [], [1, 2, 3, 4, 5, 6])], - [(1720501281.07252, [], [1, 2, 3, 4, 5, 6])], - [(1720055721.622214, [], [1, 2, 3, 4, 5, 6])], - [ - (1720137839.895581, [], [1, 2, 3, 4, 5, 6]), - (1720231251.548774, [], [1, 2, 3, 4, 5, 6]), - (1720350224.693877, [], [1, 2, 3, 4, 5, 6]), - (1720519304.741337, [], [1, 2, 3, 4, 5, 6]), - (1720586487.784295, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719969181.053709, [], [1, 2, 3, 4, 5, 6]), (1720434586.266895, [], [1, 2, 3, 4, 5, 6])], - [(1720070912.148493, [], [1, 2, 3, 4, 5, 6])], - [(1720244703.673132, [], [1, 2, 3, 4, 5, 6]), (1720494833.034907, [], [1, 2, 3, 4, 5, 6])], - [ - (1720053732.993216, [], [1, 2, 3, 4, 5, 6]), - (1720053735.086697, [], [1, 2, 3, 4, 5, 6]), - (1720081490.114819, [], [1, 2, 3, 4, 5, 6]), - (1720081492.188923, [], [1, 2, 3, 4, 5, 6]), - (1720108329.744396, [], [1, 2, 3, 4, 5, 6]), - (1720108420.858541, [], [1, 2, 3, 4, 5, 6]), - (1720108422.93886, [], [1, 2, 3, 4, 5, 6]), - (1720142971.786605, [], [1, 2, 3, 4, 5, 6]), - (1720143021.896153, [], [1, 2, 3, 4, 5, 6]), - (1720149975.921352, [], [1, 2, 3, 4, 5, 6]), - (1720150041.125351, [], [1, 2, 3, 4, 5, 6]), - (1720150043.15518, [], [1, 2, 3, 4, 5, 6]), - (1720200733.408027, [], [1, 2, 3, 4, 5, 6]), - (1720200744.366236, [], [1, 2, 3, 4, 5, 6]), - (1720200746.48024, [], [1, 2, 3, 4, 5, 6]), - (1720226248.428928, [], [1, 2, 3, 4, 5, 6]), - (1720226387.261389, [], [1, 2, 3, 4, 5, 6]), - (1720325189.273212, [], [1, 2, 3, 4, 5, 6]), - (1720367266.448359, [], [1, 2, 3, 4, 5, 6]), - (1720367268.528501, [], [1, 2, 3, 4, 5, 6]), - (1720397514.119584, [], [1, 2, 3, 4, 5, 6]), - (1720397583.541623, [], [1, 2, 3, 4, 5, 6]), - (1720397585.62972, [], [1, 2, 3, 4, 5, 6]), - (1720407649.068004, [], [1, 2, 3, 4, 5, 6]), - (1720407859.450723, [], [1, 2, 3, 4, 5, 6]), - (1720407861.5267, [], [1, 2, 3, 4, 5, 6]), - (1720418226.184583, [], [1, 2, 3, 4, 5, 6]), - (1720418312.907521, [], [1, 2, 3, 4, 5, 6]), - (1720418312.959891, [], [1, 2, 3, 4, 5, 6]), - (1720418314.508588, [], [1, 2, 3, 4, 5, 6]), - (1720429033.410454, [], [1, 2, 3, 4, 5, 6]), - (1720429217.5183, [], [1, 2, 3, 4, 5, 6]), - (1720429219.58254, [], [1, 2, 3, 4, 5, 6]), - (1720476196.299215, [], [1, 2, 3, 4, 5, 6]), - (1720476290.414317, [], [1, 2, 3, 4, 5, 6]), - (1720476292.497993, [], [1, 2, 3, 4, 5, 6]), - (1720496668.635514, [], [1, 2, 3, 4, 5, 6]), - (1720496670.762669, [], [1, 2, 3, 4, 5, 6]), - (1720566807.578929, [], [1, 2, 3, 4, 5, 6]), - (1720566881.524889, [], [1, 2, 3, 4, 5, 6]), - (1720566883.613068, [], [1, 2, 3, 4, 5, 6]), - (1720575742.398153, [], [1, 2, 3, 4, 5, 6]), - (1720575760.407369, [], [1, 2, 3, 4, 5, 6]), - (1720575762.530879, [], [1, 2, 3, 4, 5, 6]), - (1720623060.799492, [], [1, 2, 3, 4, 5, 6]), - (1720623163.775703, [], [1, 2, 3, 4, 5, 6]), - (1720623165.819144, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720490829.088908, [], [1, 2, 3, 4, 5, 6])], - [ - (1720222040.449568, [], [1, 2, 3, 4, 5, 6]), - (1720328183.580613, [], [1, 2, 3, 4, 5, 6]), - (1720581997.108309, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720631726.024509, [], [1, 2, 3, 4, 5, 6])], - [ - (1719969186.42388, [], [1, 2, 3, 4, 5, 6]), - (1720236467.453142, [], [1, 2, 3, 4, 5, 6]), - (1720460294.599805, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720328624.599959, [], [1, 2, 3, 4, 5, 6]), - (1720411036.02508, [], [1, 2, 3, 4, 5, 6]), - (1720470233.314202, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719966062.758672, [], [1, 2, 3, 4, 5, 6]), - (1720057471.705526, [], [1, 2, 3, 4, 5, 6]), - (1720325034.717518, [], [1, 2, 3, 4, 5, 6]), - (1720407309.902625, [], [1, 2, 3, 4, 5, 6]), - (1720573477.911506, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720063887.70449, [], [1, 2, 3, 4, 5, 6])], - [(1720343326.152899, [], [1, 2, 3, 4, 5, 6])], - [(1720411362.644921, [], [1, 2, 3, 4, 5, 6])], - [(1720395606.751317, [], [1, 2, 3, 4, 5, 6])], - [(1720155980.858558, [], [1, 2, 3, 4, 5, 6])], - [(1720450339.669296, [], [1, 2, 3, 4, 5, 6])], - [(1719981567.460091, [], [1, 2, 3, 4, 5, 6]), (1720587320.169523, [], [1, 2, 3, 4, 5, 6])], - [(1720245122.915738, [], [1, 2, 3, 4, 5, 6])], - [(1719970229.063219, [], [1, 2, 3, 4, 5, 6]), (1720488361.805483, [], [1, 2, 3, 4, 5, 6])], - [(1720320009.047059, [], [1, 2, 3, 4, 5, 6])], - [(1720139484.708505, [], [1, 2, 3, 4, 5, 6]), (1720396780.73649, [], [1, 2, 3, 4, 5, 6])], - [(1720238094.386701, [], [1, 2, 3, 4, 5, 6])], - [(1720627574.598265, [], [1, 2, 3, 4, 5, 6])], - [(1720136834.089355, [], [1, 2, 3, 4, 5, 6]), (1720396824.609765, [], [1, 2, 3, 4, 5, 6])], - [(1720225652.369657, [], [1, 2, 3, 4, 5, 6])], - [ - (1719982133.012616, [], [1, 2, 3, 4, 5, 6]), - (1719982177.631804, [], [1, 2, 3, 4, 5, 6]), - (1719982179.720602, [], [1, 2, 3, 4, 5, 6]), - (1720006591.274361, [], [1, 2, 3, 4, 5, 6]), - (1720006644.930183, [], [1, 2, 3, 4, 5, 6]), - (1720006647.03435, [], [1, 2, 3, 4, 5, 6]), - (1720460548.964008, [], [1, 2, 3, 4, 5, 6]), - (1720460614.237345, [], [1, 2, 3, 4, 5, 6]), - (1720460616.332418, [], [1, 2, 3, 4, 5, 6]), - (1720585282.645498, [], [1, 2, 3, 4, 5, 6]), - (1720585293.462072, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720370572.550118, [], [1, 2, 3, 4, 5, 6])], - [ - (1719973160.879923, [], [1, 2, 3, 4, 5, 6]), - (1720329101.982409, [], [1, 2, 3, 4, 5, 6]), - (1720581501.430356, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720063998.039445, [], [1, 2, 3, 4, 5, 6]), - (1720232764.384684, [], [1, 2, 3, 4, 5, 6]), - (1720502473.633051, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720153647.635598, [], [1, 2, 3, 4, 5, 6]), - (1720225923.85076, [], [1, 2, 3, 4, 5, 6]), - (1720413430.570698, [], [1, 2, 3, 4, 5, 6]), - (1720584131.042756, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719966759.043742, [], [1, 2, 3, 4, 5, 6])], - [(1720405936.570297, [], [1, 2, 3, 4, 5, 6])], - [(1720144919.519677, [], [1, 2, 3, 4, 5, 6]), (1720402676.1685, [], [1, 2, 3, 4, 5, 6])], - [(1720402994.034134, [], [1, 2, 3, 4, 5, 6]), (1720497341.728864, [], [1, 2, 3, 4, 5, 6])], - [ - (1719973785.927392, [], [1, 2, 3, 4, 5, 6]), - (1720142506.754009, [], [1, 2, 3, 4, 5, 6]), - (1720312482.395361, [], [1, 2, 3, 4, 5, 6]), - (1720578049.42885, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720143203.796648, [], [1, 2, 3, 4, 5, 6]), (1720504600.034248, [], [1, 2, 3, 4, 5, 6])], - [(1720138317.024564, [], [1, 2, 3, 4, 5, 6]), (1720307922.860078, [], [1, 2, 3, 4, 5, 6])], - [(1720576710.045341, [], [1, 2, 3, 4, 5, 6])], - [ - (1720237948.24219, [], [1, 2, 3, 4, 5, 6]), - (1720322691.233406, [], [1, 2, 3, 4, 5, 6]), - (1720412663.957815, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720342019.617667, [], [1, 2, 3, 4, 5, 6]), - (1720342090.227667, [], [1, 2, 3, 4, 5, 6]), - (1720342263.731169, [], [1, 2, 3, 4, 5, 6]), - (1720342307.569989, [], [1, 2, 3, 4, 5, 6]), - (1720342413.538738, [], [1, 2, 3, 4, 5, 6]), - (1720342570.868506, [], [1, 2, 3, 4, 5, 6]), - (1720342664.423143, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720053725.982337, [], [1, 2, 3, 4, 5, 6]), - (1720137089.95596, [], [1, 2, 3, 4, 5, 6]), - (1720250340.159455, [], [1, 2, 3, 4, 5, 6]), - (1720408080.82431, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720502215.54816, [], [1, 2, 3, 4, 5, 6])], - [(1720051018.757074, [], [1, 2, 3, 4, 5, 6]), (1720221304.68857, [], [1, 2, 3, 4, 5, 6])], - [ - (1720055639.220711, [], [1, 2, 3, 4, 5, 6]), - (1720242136.136068, [], [1, 2, 3, 4, 5, 6]), - (1720501308.452889, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720210019.832882, [], [1, 2, 3, 4, 5, 6])], - [(1720222496.41532, [], [1, 2, 3, 4, 5, 6])], - [(1720221892.596089, [], [1, 2, 3, 4, 5, 6]), (1720488555.303827, [], [1, 2, 3, 4, 5, 6])], - [ - (1720055240.779901, [], [1, 2, 3, 4, 5, 6]), - (1720485059.84637, [], [1, 2, 3, 4, 5, 6]), - (1720520102.630634, [], [1, 2, 3, 4, 5, 6]), - (1720591031.4354, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720570592.888394, [], [1, 2, 3, 4, 5, 6])], - [ - (1720059956.606064, [], [1, 2, 3, 4, 5, 6]), - (1720232781.82764, [], [1, 2, 3, 4, 5, 6]), - (1720489307.963369, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720466563.789269, [], [1, 2, 3, 4, 5, 6])], - [(1720120332.505828, [], [1, 2, 3, 4, 5, 6]), (1720501386.247192, [], [1, 2, 3, 4, 5, 6])], - [ - (1720045443.968104, [], [1, 2, 3, 4, 5, 6]), - (1720337612.000658, [], [1, 2, 3, 4, 5, 6]), - (1720484793.823359, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720240516.409323, [], [1, 2, 3, 4, 5, 6]), (1720508486.303913, [], [1, 2, 3, 4, 5, 6])], - [ - (1720056682.445295, [], [1, 2, 3, 4, 5, 6]), - (1720239570.480365, [], [1, 2, 3, 4, 5, 6]), - (1720399243.691516, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720065346.577694, [], [1, 2, 3, 4, 5, 6]), - (1720111179.563476, [], [1, 2, 3, 4, 5, 6]), - (1720152182.18393, [], [1, 2, 3, 4, 5, 6]), - (1720456368.150945, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720037842.027886, [], [1, 2, 3, 4, 5, 6])], - [(1720051512.155726, [], [1, 2, 3, 4, 5, 6]), (1720316085.436368, [], [1, 2, 3, 4, 5, 6])], - [ - (1720153922.872643, [], [1, 2, 3, 4, 5, 6]), - (1720316484.292604, [], [1, 2, 3, 4, 5, 6]), - (1720481626.562697, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720304528.044157, [], [1, 2, 3, 4, 5, 6]), (1720587171.914424, [], [1, 2, 3, 4, 5, 6])], - [ - (1719969690.052003, [], [1, 2, 3, 4, 5, 6]), - (1720098093.259497, [], [1, 2, 3, 4, 5, 6]), - (1720589467.401983, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720182994.851728, [], [1, 2, 3, 4, 5, 6]), (1720490206.204252, [], [1, 2, 3, 4, 5, 6])], - [(1720305269.133214, [], [1, 2, 3, 4, 5, 6]), (1720580679.401674, [], [1, 2, 3, 4, 5, 6])], - [(1720582113.001824, [], [1, 2, 3, 4, 5, 6])], - [ - (1719971867.373391, [], [1, 2, 3, 4, 5, 6]), - (1719971869.452767, [], [1, 2, 3, 4, 5, 6]), - (1719983561.418747, [], [1, 2, 3, 4, 5, 6]), - (1719983615.306689, [], [1, 2, 3, 4, 5, 6]), - (1719983617.371374, [], [1, 2, 3, 4, 5, 6]), - (1719983622.154397, [], [1, 2, 3, 4, 5, 6]), - (1719983624.239597, [], [1, 2, 3, 4, 5, 6]), - (1720057585.854293, [], [1, 2, 3, 4, 5, 6]), - (1720127843.991043, [], [1, 2, 3, 4, 5, 6]), - (1720127952.545227, [], [1, 2, 3, 4, 5, 6]), - (1720150451.197164, [], [1, 2, 3, 4, 5, 6]), - (1720150472.889245, [], [1, 2, 3, 4, 5, 6]), - (1720229579.372015, [], [1, 2, 3, 4, 5, 6]), - (1720229585.29839, [], [1, 2, 3, 4, 5, 6]), - (1720229587.33746, [], [1, 2, 3, 4, 5, 6]), - (1720272362.151724, [], [1, 2, 3, 4, 5, 6]), - (1720272395.494166, [], [1, 2, 3, 4, 5, 6]), - (1720272397.584197, [], [1, 2, 3, 4, 5, 6]), - (1720325287.360716, [], [1, 2, 3, 4, 5, 6]), - (1720325289.430457, [], [1, 2, 3, 4, 5, 6]), - (1720392144.674955, [], [1, 2, 3, 4, 5, 6]), - (1720392146.786158, [], [1, 2, 3, 4, 5, 6]), - (1720406690.885685, [], [1, 2, 3, 4, 5, 6]), - (1720406692.950513, [], [1, 2, 3, 4, 5, 6]), - (1720486441.134231, [], [1, 2, 3, 4, 5, 6]), - (1720486443.192435, [], [1, 2, 3, 4, 5, 6]), - (1720648828.296221, [], [1, 2, 3, 4, 5, 6]), - (1720648830.340132, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719977053.236432, [], [1, 2, 3, 4, 5, 6])], - [(1720146886.388756, [], [1, 2, 3, 4, 5, 6])], - [(1720147721.983335, [], [1, 2, 3, 4, 5, 6]), (1720577319.095652, [], [1, 2, 3, 4, 5, 6])], - [(1720187232.833461, [], [1, 2, 3, 4, 5, 6])], - [(1720309745.334443, [], [1, 2, 3, 4, 5, 6]), (1720525020.981442, [], [1, 2, 3, 4, 5, 6])], - [(1719985270.896874, [], [1, 2, 3, 4, 5, 6]), (1720147203.361104, [], [1, 2, 3, 4, 5, 6])], - [(1719975189.590595, [], [1, 2, 3, 4, 5, 6])], - [ - (1720153681.561666, [], [1, 2, 3, 4, 5, 6]), - (1720315141.854012, [], [1, 2, 3, 4, 5, 6]), - (1720483759.06017, [], [1, 2, 3, 4, 5, 6]), - (1720632532.362134, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719969377.021113, [], [1, 2, 3, 4, 5, 6]), - (1719969547.865829, [], [1, 2, 3, 4, 5, 6]), - (1720050670.589639, [], [1, 2, 3, 4, 5, 6]), - (1720050713.412665, [], [1, 2, 3, 4, 5, 6]), - (1720139076.150907, [], [1, 2, 3, 4, 5, 6]), - (1720139087.933212, [], [1, 2, 3, 4, 5, 6]), - (1720139090.022598, [], [1, 2, 3, 4, 5, 6]), - (1720139092.090332, [], [1, 2, 3, 4, 5, 6]), - (1720148904.698605, [], [1, 2, 3, 4, 5, 6]), - (1720197454.202625, [], [1, 2, 3, 4, 5, 6]), - (1720197456.301898, [], [1, 2, 3, 4, 5, 6]), - (1720221957.937687, [], [1, 2, 3, 4, 5, 6]), - (1720222151.210074, [], [1, 2, 3, 4, 5, 6]), - (1720222153.281944, [], [1, 2, 3, 4, 5, 6]), - (1720231319.785278, [], [1, 2, 3, 4, 5, 6]), - (1720314287.823226, [], [1, 2, 3, 4, 5, 6]), - (1720314375.707773, [], [1, 2, 3, 4, 5, 6]), - (1720314377.787834, [], [1, 2, 3, 4, 5, 6]), - (1720331369.745063, [], [1, 2, 3, 4, 5, 6]), - (1720331582.949466, [], [1, 2, 3, 4, 5, 6]), - (1720331585.058912, [], [1, 2, 3, 4, 5, 6]), - (1720399235.526545, [], [1, 2, 3, 4, 5, 6]), - (1720399237.6268, [], [1, 2, 3, 4, 5, 6]), - (1720410762.341061, [], [1, 2, 3, 4, 5, 6]), - (1720410808.990309, [], [1, 2, 3, 4, 5, 6]), - (1720410811.040448, [], [1, 2, 3, 4, 5, 6]), - (1720493330.828194, [], [1, 2, 3, 4, 5, 6]), - (1720493516.887173, [], [1, 2, 3, 4, 5, 6]), - (1720501442.580123, [], [1, 2, 3, 4, 5, 6]), - (1720501548.316894, [], [1, 2, 3, 4, 5, 6]), - (1720501550.379738, [], [1, 2, 3, 4, 5, 6]), - (1720573012.279738, [], [1, 2, 3, 4, 5, 6]), - (1720573204.24471, [], [1, 2, 3, 4, 5, 6]), - (1720573206.359087, [], [1, 2, 3, 4, 5, 6]), - (1720573210.996145, [], [1, 2, 3, 4, 5, 6]), - (1720573213.096745, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719984464.146305, [], [1, 2, 3, 4, 5, 6])], - [(1719969484.575186, [], [1, 2, 3, 4, 5, 6])], - [(1719967098.321792, [], [1, 2, 3, 4, 5, 6]), (1720140304.171738, [], [1, 2, 3, 4, 5, 6])], - [ - (1720067679.407113, [], [1, 2, 3, 4, 5, 6]), - (1720240007.297001, [], [1, 2, 3, 4, 5, 6]), - (1720499615.946055, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719984745.989291, [], [1, 2, 3, 4, 5, 6]), - (1720088802.060799, [], [1, 2, 3, 4, 5, 6]), - (1720226330.102201, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720057325.702019, [], [1, 2, 3, 4, 5, 6]), (1720499465.567145, [], [1, 2, 3, 4, 5, 6])], - [ - (1720055478.668518, [], [1, 2, 3, 4, 5, 6]), - (1720055619.422527, [], [1, 2, 3, 4, 5, 6]), - (1720138025.392906, [], [1, 2, 3, 4, 5, 6]), - (1720138074.32289, [], [1, 2, 3, 4, 5, 6]), - (1720232045.90559, [], [1, 2, 3, 4, 5, 6]), - (1720232073.337701, [], [1, 2, 3, 4, 5, 6]), - (1720278094.793407, [], [1, 2, 3, 4, 5, 6]), - (1720278096.912409, [], [1, 2, 3, 4, 5, 6]), - (1720278099.569789, [], [1, 2, 3, 4, 5, 6]), - (1720278101.660519, [], [1, 2, 3, 4, 5, 6]), - (1720324663.973123, [], [1, 2, 3, 4, 5, 6]), - (1720324666.034118, [], [1, 2, 3, 4, 5, 6]), - (1720412864.0991, [], [1, 2, 3, 4, 5, 6]), - (1720412932.896312, [], [1, 2, 3, 4, 5, 6]), - (1720412934.95735, [], [1, 2, 3, 4, 5, 6]), - (1720493768.204791, [], [1, 2, 3, 4, 5, 6]), - (1720493848.668367, [], [1, 2, 3, 4, 5, 6]), - (1720493850.800293, [], [1, 2, 3, 4, 5, 6]), - (1720493853.855696, [], [1, 2, 3, 4, 5, 6]), - (1720578407.565863, [], [1, 2, 3, 4, 5, 6]), - (1720578455.012928, [], [1, 2, 3, 4, 5, 6]), - (1720578457.12311, [], [1, 2, 3, 4, 5, 6]), - (1720592507.954368, [], [1, 2, 3, 4, 5, 6]), - (1720592695.674207, [], [1, 2, 3, 4, 5, 6]), - (1720592697.763035, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720448820.538088, [], [1, 2, 3, 4, 5, 6]), - (1720448848.354821, [], [1, 2, 3, 4, 5, 6]), - (1720448968.980221, [], [1, 2, 3, 4, 5, 6]), - (1720449109.647373, [], [1, 2, 3, 4, 5, 6]), - (1720449132.605916, [], [1, 2, 3, 4, 5, 6]), - (1720449141.226924, [], [1, 2, 3, 4, 5, 6]), - (1720449174.132961, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720143295.563285, [], [1, 2, 3, 4, 5, 6])], - [ - (1720009875.184202, [], [1, 2, 3, 4, 5, 6]), - (1720064301.403426, [], [1, 2, 3, 4, 5, 6]), - (1720221459.433168, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720316122.630709, [], [1, 2, 3, 4, 5, 6]), (1720625396.811387, [], [1, 2, 3, 4, 5, 6])], - [(1720064525.079458, [], [1, 2, 3, 4, 5, 6])], - [(1720600790.059805, [], [1, 2, 3, 4, 5, 6])], - [(1720053513.239524, [], [1, 2, 3, 4, 5, 6]), (1720533559.490134, [], [1, 2, 3, 4, 5, 6])], - [(1720222657.803241, [], [1, 2, 3, 4, 5, 6])], - [ - (1719971419.792625, [], [1, 2, 3, 4, 5, 6]), - (1720239049.653382, [], [1, 2, 3, 4, 5, 6]), - (1720497253.487835, [], [1, 2, 3, 4, 5, 6]), - (1720571009.60795, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719978213.57048, [], [1, 2, 3, 4, 5, 6]), - (1719978402.543586, [], [1, 2, 3, 4, 5, 6]), - (1720067921.564313, [], [1, 2, 3, 4, 5, 6]), - (1720068031.2973, [], [1, 2, 3, 4, 5, 6]), - (1720068033.364045, [], [1, 2, 3, 4, 5, 6]), - (1720076693.193638, [], [1, 2, 3, 4, 5, 6]), - (1720076695.234922, [], [1, 2, 3, 4, 5, 6]), - (1720088372.082518, [], [1, 2, 3, 4, 5, 6]), - (1720088448.747115, [], [1, 2, 3, 4, 5, 6]), - (1720222636.476764, [], [1, 2, 3, 4, 5, 6]), - (1720222701.214913, [], [1, 2, 3, 4, 5, 6]), - (1720311136.481341, [], [1, 2, 3, 4, 5, 6]), - (1720311279.356667, [], [1, 2, 3, 4, 5, 6]), - (1720311281.435353, [], [1, 2, 3, 4, 5, 6]), - (1720321937.516249, [], [1, 2, 3, 4, 5, 6]), - (1720321977.750869, [], [1, 2, 3, 4, 5, 6]), - (1720321979.826956, [], [1, 2, 3, 4, 5, 6]), - (1720321983.309368, [], [1, 2, 3, 4, 5, 6]), - (1720417820.177018, [], [1, 2, 3, 4, 5, 6]), - (1720417888.907443, [], [1, 2, 3, 4, 5, 6]), - (1720482544.485269, [], [1, 2, 3, 4, 5, 6]), - (1720482650.874077, [], [1, 2, 3, 4, 5, 6]), - (1720571012.586842, [], [1, 2, 3, 4, 5, 6]), - (1720571014.653099, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720067135.000485, [], [1, 2, 3, 4, 5, 6]), - (1720226886.323383, [], [1, 2, 3, 4, 5, 6]), - (1720626810.190995, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720021468.494681, [], [1, 2, 3, 4, 5, 6]), (1720244311.296556, [], [1, 2, 3, 4, 5, 6])], - [ - (1720054497.052088, [], [1, 2, 3, 4, 5, 6]), - (1720315797.04068, [], [1, 2, 3, 4, 5, 6]), - (1720396623.976121, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719970439.050635, [], [1, 2, 3, 4, 5, 6]), (1720411294.606462, [], [1, 2, 3, 4, 5, 6])], - [ - (1720047660.240807, [], [1, 2, 3, 4, 5, 6]), - (1720209425.126479, [], [1, 2, 3, 4, 5, 6]), - (1720417042.301423, [], [1, 2, 3, 4, 5, 6]), - (1720579466.836909, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720068666.058135, [], [1, 2, 3, 4, 5, 6]), - (1720224717.712974, [], [1, 2, 3, 4, 5, 6]), - (1720313644.184984, [], [1, 2, 3, 4, 5, 6]), - (1720417247.572309, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720217112.012918, [], [1, 2, 3, 4, 5, 6])], - [(1720228893.793094, [], [1, 2, 3, 4, 5, 6])], - [ - (1719965114.583168, [], [1, 2, 3, 4, 5, 6]), - (1720221700.128257, [], [1, 2, 3, 4, 5, 6]), - (1720359492.65181, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720149938.452021, [], [1, 2, 3, 4, 5, 6]), - (1720150083.518978, [], [1, 2, 3, 4, 5, 6]), - (1720150100.711862, [], [1, 2, 3, 4, 5, 6]), - (1720403516.136956, [], [1, 2, 3, 4, 5, 6]), - (1720403602.399166, [], [1, 2, 3, 4, 5, 6]), - (1720403688.061721, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720016151.530651, [], [1, 2, 3, 4, 5, 6]), - (1720126052.51206, [], [1, 2, 3, 4, 5, 6]), - (1720243360.967974, [], [1, 2, 3, 4, 5, 6]), - (1720567481.805169, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720259903.388453, [], [1, 2, 3, 4, 5, 6]), (1720495071.607118, [], [1, 2, 3, 4, 5, 6])], - [ - (1719978731.351246, [], [1, 2, 3, 4, 5, 6]), - (1720142275.008236, [], [1, 2, 3, 4, 5, 6]), - (1720225627.748133, [], [1, 2, 3, 4, 5, 6]), - (1720599835.060544, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720308817.017884, [], [1, 2, 3, 4, 5, 6]), (1720500376.721695, [], [1, 2, 3, 4, 5, 6])], - [ - (1720062080.162523, [], [1, 2, 3, 4, 5, 6]), - (1720424051.051867, [], [1, 2, 3, 4, 5, 6]), - (1720577193.657241, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720226358.301934, [], [1, 2, 3, 4, 5, 6]), (1720611516.599998, [], [1, 2, 3, 4, 5, 6])], - [(1720142831.087971, [], [1, 2, 3, 4, 5, 6]), (1720568727.59182, [], [1, 2, 3, 4, 5, 6])], - [(1720045127.801767, [], [1, 2, 3, 4, 5, 6])], - [(1720598254.557545, [], [1, 2, 3, 4, 5, 6])], - [(1720230498.737196, [], [1, 2, 3, 4, 5, 6]), (1720502519.921733, [], [1, 2, 3, 4, 5, 6])], - [(1720149819.132452, [], [1, 2, 3, 4, 5, 6]), (1720317818.669453, [], [1, 2, 3, 4, 5, 6])], - [(1719965630.184525, [], [1, 2, 3, 4, 5, 6]), (1720566194.006106, [], [1, 2, 3, 4, 5, 6])], - [(1719996710.23806, [], [1, 2, 3, 4, 5, 6])], - [(1720053587.04154, [], [1, 2, 3, 4, 5, 6]), (1720476400.319672, [], [1, 2, 3, 4, 5, 6])], - [(1720238998.499612, [], [1, 2, 3, 4, 5, 6])], - [(1720049964.339669, [], [1, 2, 3, 4, 5, 6]), (1720503256.459045, [], [1, 2, 3, 4, 5, 6])], - [(1720629914.75266, [], [1, 2, 3, 4, 5, 6])], - [ - (1720067406.552276, [], [1, 2, 3, 4, 5, 6]), - (1720192823.078475, [], [1, 2, 3, 4, 5, 6]), - (1720615636.068682, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720050726.320669, [], [1, 2, 3, 4, 5, 6]), - (1720159164.117987, [], [1, 2, 3, 4, 5, 6]), - (1720583837.972687, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720470505.483407, [], [1, 2, 3, 4, 5, 6]), (1720589293.330858, [], [1, 2, 3, 4, 5, 6])], - [ - (1719990309.924021, [], [1, 2, 3, 4, 5, 6]), - (1720242385.881249, [], [1, 2, 3, 4, 5, 6]), - (1720648573.041044, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720059240.88106, [], [1, 2, 3, 4, 5, 6]), (1720486550.385795, [], [1, 2, 3, 4, 5, 6])], - [(1720232277.114726, [], [1, 2, 3, 4, 5, 6])], - [(1720156360.413945, [], [1, 2, 3, 4, 5, 6]), (1720415380.907597, [], [1, 2, 3, 4, 5, 6])], - [(1720143721.130937, [], [1, 2, 3, 4, 5, 6])], - [(1720093040.94431, [], [1, 2, 3, 4, 5, 6]), (1720230385.831757, [], [1, 2, 3, 4, 5, 6])], - [(1720313919.101562, [], [1, 2, 3, 4, 5, 6]), (1720600894.542752, [], [1, 2, 3, 4, 5, 6])], - [(1720008883.059792, [], [1, 2, 3, 4, 5, 6]), (1720151981.800615, [], [1, 2, 3, 4, 5, 6])], - [(1720583883.771582, [], [1, 2, 3, 4, 5, 6])], - [(1720054595.476172, [], [1, 2, 3, 4, 5, 6]), (1720494101.96425, [], [1, 2, 3, 4, 5, 6])], - [ - (1719975987.869421, [], [1, 2, 3, 4, 5, 6]), - (1720072012.445937, [], [1, 2, 3, 4, 5, 6]), - (1720141541.892965, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719974700.775073, [], [1, 2, 3, 4, 5, 6]), - (1719974810.784479, [], [1, 2, 3, 4, 5, 6]), - (1720022010.687673, [], [1, 2, 3, 4, 5, 6]), - (1720022125.15125, [], [1, 2, 3, 4, 5, 6]), - (1720022127.183082, [], [1, 2, 3, 4, 5, 6]), - (1720022127.850327, [], [1, 2, 3, 4, 5, 6]), - (1720022129.972437, [], [1, 2, 3, 4, 5, 6]), - (1720149346.519881, [], [1, 2, 3, 4, 5, 6]), - (1720149423.39517, [], [1, 2, 3, 4, 5, 6]), - (1720232515.945385, [], [1, 2, 3, 4, 5, 6]), - (1720232642.031851, [], [1, 2, 3, 4, 5, 6]), - (1720232644.151326, [], [1, 2, 3, 4, 5, 6]), - (1720232649.102724, [], [1, 2, 3, 4, 5, 6]), - (1720232651.213687, [], [1, 2, 3, 4, 5, 6]), - (1720291284.378849, [], [1, 2, 3, 4, 5, 6]), - (1720291307.651917, [], [1, 2, 3, 4, 5, 6]), - (1720317724.65539, [], [1, 2, 3, 4, 5, 6]), - (1720317962.176994, [], [1, 2, 3, 4, 5, 6]), - (1720317962.221761, [], [1, 2, 3, 4, 5, 6]), - (1720317967.870483, [], [1, 2, 3, 4, 5, 6]), - (1720416284.403485, [], [1, 2, 3, 4, 5, 6]), - (1720416286.45094, [], [1, 2, 3, 4, 5, 6]), - (1720446964.44037, [], [1, 2, 3, 4, 5, 6]), - (1720447111.491786, [], [1, 2, 3, 4, 5, 6]), - (1720447113.551591, [], [1, 2, 3, 4, 5, 6]), - (1720500857.609857, [], [1, 2, 3, 4, 5, 6]), - (1720500933.241251, [], [1, 2, 3, 4, 5, 6]), - (1720500935.342693, [], [1, 2, 3, 4, 5, 6]), - (1720550391.631024, [], [1, 2, 3, 4, 5, 6]), - (1720550393.677097, [], [1, 2, 3, 4, 5, 6]), - (1720571962.115275, [], [1, 2, 3, 4, 5, 6]), - (1720571964.156322, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720300973.659698, [], [1, 2, 3, 4, 5, 6]), (1720502088.420309, [], [1, 2, 3, 4, 5, 6])], - [ - (1720226060.114355, [], [1, 2, 3, 4, 5, 6]), - (1720367668.242413, [], [1, 2, 3, 4, 5, 6]), - (1720580879.469873, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720148122.993839, [], [1, 2, 3, 4, 5, 6]), - (1720283848.988921, [], [1, 2, 3, 4, 5, 6]), - (1720392902.670008, [], [1, 2, 3, 4, 5, 6]), - (1720547569.939146, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720164561.277691, [], [1, 2, 3, 4, 5, 6]), (1720322322.293618, [], [1, 2, 3, 4, 5, 6])], - [(1720394391.029382, [], [1, 2, 3, 4, 5, 6])], - [(1720578227.91725, [], [1, 2, 3, 4, 5, 6])], - [(1720427348.104988, [], [1, 2, 3, 4, 5, 6]), (1720586312.438776, [], [1, 2, 3, 4, 5, 6])], - [ - (1719967279.972433, [], [1, 2, 3, 4, 5, 6]), - (1719967282.055508, [], [1, 2, 3, 4, 5, 6]), - (1719986090.097845, [], [1, 2, 3, 4, 5, 6]), - (1719986173.00659, [], [1, 2, 3, 4, 5, 6]), - (1719986175.082864, [], [1, 2, 3, 4, 5, 6]), - (1720034526.875582, [], [1, 2, 3, 4, 5, 6]), - (1720061201.725715, [], [1, 2, 3, 4, 5, 6]), - (1720061294.240057, [], [1, 2, 3, 4, 5, 6]), - (1720061296.35589, [], [1, 2, 3, 4, 5, 6]), - (1720155141.396653, [], [1, 2, 3, 4, 5, 6]), - (1720155143.510508, [], [1, 2, 3, 4, 5, 6]), - (1720155145.301155, [], [1, 2, 3, 4, 5, 6]), - (1720155147.393972, [], [1, 2, 3, 4, 5, 6]), - (1720231098.024705, [], [1, 2, 3, 4, 5, 6]), - (1720231317.54759, [], [1, 2, 3, 4, 5, 6]), - (1720231319.611985, [], [1, 2, 3, 4, 5, 6]), - (1720271983.621164, [], [1, 2, 3, 4, 5, 6]), - (1720271985.710974, [], [1, 2, 3, 4, 5, 6]), - (1720316981.40392, [], [1, 2, 3, 4, 5, 6]), - (1720317019.941522, [], [1, 2, 3, 4, 5, 6]), - (1720317022.040965, [], [1, 2, 3, 4, 5, 6]), - (1720411936.226228, [], [1, 2, 3, 4, 5, 6]), - (1720411963.208146, [], [1, 2, 3, 4, 5, 6]), - (1720479757.589657, [], [1, 2, 3, 4, 5, 6]), - (1720479839.302922, [], [1, 2, 3, 4, 5, 6]), - (1720582109.835415, [], [1, 2, 3, 4, 5, 6]), - (1720582111.914294, [], [1, 2, 3, 4, 5, 6]), - (1720652093.707438, [], [1, 2, 3, 4, 5, 6]), - (1720652211.598303, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720579951.356488, [], [1, 2, 3, 4, 5, 6])], - [(1720593973.655643, [], [1, 2, 3, 4, 5, 6])], - [ - (1720061475.003195, [], [1, 2, 3, 4, 5, 6]), - (1720270392.101123, [], [1, 2, 3, 4, 5, 6]), - (1720415797.057544, [], [1, 2, 3, 4, 5, 6]), - (1720574029.592383, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719987865.032004, [], [1, 2, 3, 4, 5, 6]), (1720212776.214811, [], [1, 2, 3, 4, 5, 6])], - [(1720315089.869542, [], [1, 2, 3, 4, 5, 6]), (1720578088.622431, [], [1, 2, 3, 4, 5, 6])], - [(1720575422.335555, [], [1, 2, 3, 4, 5, 6])], - [(1720329438.482756, [], [1, 2, 3, 4, 5, 6]), (1720443842.432414, [], [1, 2, 3, 4, 5, 6])], - [ - (1720135846.308239, [], [1, 2, 3, 4, 5, 6]), - (1720221161.535587, [], [1, 2, 3, 4, 5, 6]), - (1720326226.738859, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719977789.721113, [], [1, 2, 3, 4, 5, 6]), - (1719977899.031956, [], [1, 2, 3, 4, 5, 6]), - (1719977901.119465, [], [1, 2, 3, 4, 5, 6]), - (1719982587.985388, [], [1, 2, 3, 4, 5, 6]), - (1719982666.211377, [], [1, 2, 3, 4, 5, 6]), - (1719982668.29279, [], [1, 2, 3, 4, 5, 6]), - (1719982672.56956, [], [1, 2, 3, 4, 5, 6]), - (1720063592.708606, [], [1, 2, 3, 4, 5, 6]), - (1720063594.776009, [], [1, 2, 3, 4, 5, 6]), - (1720145103.906614, [], [1, 2, 3, 4, 5, 6]), - (1720145165.665926, [], [1, 2, 3, 4, 5, 6]), - (1720157026.459569, [], [1, 2, 3, 4, 5, 6]), - (1720223512.011646, [], [1, 2, 3, 4, 5, 6]), - (1720223586.453989, [], [1, 2, 3, 4, 5, 6]), - (1720223588.535794, [], [1, 2, 3, 4, 5, 6]), - (1720329405.565358, [], [1, 2, 3, 4, 5, 6]), - (1720398313.307695, [], [1, 2, 3, 4, 5, 6]), - (1720398429.724071, [], [1, 2, 3, 4, 5, 6]), - (1720414381.775047, [], [1, 2, 3, 4, 5, 6]), - (1720446240.471098, [], [1, 2, 3, 4, 5, 6]), - (1720481889.793923, [], [1, 2, 3, 4, 5, 6]), - (1720481891.93036, [], [1, 2, 3, 4, 5, 6]), - (1720489136.015971, [], [1, 2, 3, 4, 5, 6]), - (1720489247.728734, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720318482.752639, [], [1, 2, 3, 4, 5, 6])], - [ - (1720242162.48487, [], [1, 2, 3, 4, 5, 6]), - (1720503535.294123, [], [1, 2, 3, 4, 5, 6]), - (1720590538.582039, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720148407.104121, [], [1, 2, 3, 4, 5, 6])], - [(1720154168.367205, [], [1, 2, 3, 4, 5, 6]), (1720568213.544423, [], [1, 2, 3, 4, 5, 6])], - [ - (1720069001.717509, [], [1, 2, 3, 4, 5, 6]), - (1720346135.538471, [], [1, 2, 3, 4, 5, 6]), - (1720489854.284499, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719967260.706099, [], [1, 2, 3, 4, 5, 6]), - (1720082538.484733, [], [1, 2, 3, 4, 5, 6]), - (1720240732.567635, [], [1, 2, 3, 4, 5, 6]), - (1720395713.187024, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719968640.279026, [], [1, 2, 3, 4, 5, 6]), - (1720058387.048155, [], [1, 2, 3, 4, 5, 6]), - (1720240163.514327, [], [1, 2, 3, 4, 5, 6]), - (1720391336.792179, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720310285.653948, [], [1, 2, 3, 4, 5, 6]), (1720589147.207321, [], [1, 2, 3, 4, 5, 6])], - [(1719973140.021275, [], [1, 2, 3, 4, 5, 6]), (1720504055.006021, [], [1, 2, 3, 4, 5, 6])], - [(1720115792.85023, [], [1, 2, 3, 4, 5, 6])], - [ - (1720140689.444004, [], [1, 2, 3, 4, 5, 6]), - (1720312169.980048, [], [1, 2, 3, 4, 5, 6]), - (1720399894.527727, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720316227.642169, [], [1, 2, 3, 4, 5, 6]), (1720484121.740556, [], [1, 2, 3, 4, 5, 6])], - [ - (1720150629.632571, [], [1, 2, 3, 4, 5, 6]), - (1720312593.72112, [], [1, 2, 3, 4, 5, 6]), - (1720584121.246833, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719966806.196729, [], [1, 2, 3, 4, 5, 6]), (1720492831.262792, [], [1, 2, 3, 4, 5, 6])], - [ - (1720069584.25825, [], [1, 2, 3, 4, 5, 6]), - (1720233172.76065, [], [1, 2, 3, 4, 5, 6]), - (1720317363.164219, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720227600.733956, [], [1, 2, 3, 4, 5, 6]), - (1720227600.784387, [], [1, 2, 3, 4, 5, 6]), - (1720227605.27419, [], [1, 2, 3, 4, 5, 6]), - (1720269710.791405, [], [1, 2, 3, 4, 5, 6]), - (1720269759.332462, [], [1, 2, 3, 4, 5, 6]), - (1720326344.424672, [], [1, 2, 3, 4, 5, 6]), - (1720369614.287387, [], [1, 2, 3, 4, 5, 6]), - (1720369719.27491, [], [1, 2, 3, 4, 5, 6]), - (1720369719.331218, [], [1, 2, 3, 4, 5, 6]), - (1720369721.899004, [], [1, 2, 3, 4, 5, 6]), - (1720385493.685201, [], [1, 2, 3, 4, 5, 6]), - (1720385551.219825, [], [1, 2, 3, 4, 5, 6]), - (1720385553.316418, [], [1, 2, 3, 4, 5, 6]), - (1720450115.39061, [], [1, 2, 3, 4, 5, 6]), - (1720450117.502598, [], [1, 2, 3, 4, 5, 6]), - (1720450118.78177, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719969800.978378, [], [1, 2, 3, 4, 5, 6]), - (1720222415.35262, [], [1, 2, 3, 4, 5, 6]), - (1720434706.74629, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720237451.24597, [], [1, 2, 3, 4, 5, 6]), (1720585495.150654, [], [1, 2, 3, 4, 5, 6])], - [(1719970937.04025, [], [1, 2, 3, 4, 5, 6])], - [ - (1719983075.420902, [], [1, 2, 3, 4, 5, 6]), - (1720313367.078665, [], [1, 2, 3, 4, 5, 6]), - (1720413122.113225, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720155749.238687, [], [1, 2, 3, 4, 5, 6]), (1720491353.243799, [], [1, 2, 3, 4, 5, 6])], - [(1720060021.000595, [], [1, 2, 3, 4, 5, 6])], - [(1719988378.536367, [], [1, 2, 3, 4, 5, 6]), (1720228662.183092, [], [1, 2, 3, 4, 5, 6])], - [(1719981886.782157, [], [1, 2, 3, 4, 5, 6]), (1720156878.496962, [], [1, 2, 3, 4, 5, 6])], - [(1720582313.689559, [], [1, 2, 3, 4, 5, 6])], - [(1720232302.477057, [], [1, 2, 3, 4, 5, 6]), (1720493756.958556, [], [1, 2, 3, 4, 5, 6])], - [(1720295778.241704, [], [1, 2, 3, 4, 5, 6])], - [(1720021503.203052, [], [1, 2, 3, 4, 5, 6]), (1720325452.491454, [], [1, 2, 3, 4, 5, 6])], - [(1720527219.478404, [], [1, 2, 3, 4, 5, 6]), (1720567646.306507, [], [1, 2, 3, 4, 5, 6])], - [(1720223792.29193, [], [1, 2, 3, 4, 5, 6])], - [ - (1720006636.772706, [], [1, 2, 3, 4, 5, 6]), - (1720006795.60427, [], [1, 2, 3, 4, 5, 6]), - (1720006845.799981, [], [1, 2, 3, 4, 5, 6]), - (1720007022.741945, [], [1, 2, 3, 4, 5, 6]), - (1720007095.581047, [], [1, 2, 3, 4, 5, 6]), - (1720007134.850115, [], [1, 2, 3, 4, 5, 6]), - (1720025117.762503, [], [1, 2, 3, 4, 5, 6]), - (1720025308.512649, [], [1, 2, 3, 4, 5, 6]), - (1720025310.568037, [], [1, 2, 3, 4, 5, 6]), - (1720052547.163003, [], [1, 2, 3, 4, 5, 6]), - (1720052600.03312, [], [1, 2, 3, 4, 5, 6]), - (1720068046.902248, [], [1, 2, 3, 4, 5, 6]), - (1720068213.189912, [], [1, 2, 3, 4, 5, 6]), - (1720144711.311281, [], [1, 2, 3, 4, 5, 6]), - (1720144713.407177, [], [1, 2, 3, 4, 5, 6]), - (1720222638.332245, [], [1, 2, 3, 4, 5, 6]), - (1720222640.418838, [], [1, 2, 3, 4, 5, 6]), - (1720242141.813366, [], [1, 2, 3, 4, 5, 6]), - (1720242245.921587, [], [1, 2, 3, 4, 5, 6]), - (1720242248.011768, [], [1, 2, 3, 4, 5, 6]), - (1720333146.03005, [], [1, 2, 3, 4, 5, 6]), - (1720333287.562561, [], [1, 2, 3, 4, 5, 6]), - (1720333289.592652, [], [1, 2, 3, 4, 5, 6]), - (1720333292.319879, [], [1, 2, 3, 4, 5, 6]), - (1720333294.386109, [], [1, 2, 3, 4, 5, 6]), - (1720396984.211837, [], [1, 2, 3, 4, 5, 6]), - (1720397094.401782, [], [1, 2, 3, 4, 5, 6]), - (1720486134.144443, [], [1, 2, 3, 4, 5, 6]), - (1720486136.211044, [], [1, 2, 3, 4, 5, 6]), - (1720486140.873481, [], [1, 2, 3, 4, 5, 6]), - (1720486142.970428, [], [1, 2, 3, 4, 5, 6]), - (1720497754.706526, [], [1, 2, 3, 4, 5, 6]), - (1720497979.155047, [], [1, 2, 3, 4, 5, 6]), - (1720531991.462042, [], [1, 2, 3, 4, 5, 6]), - (1720532199.030662, [], [1, 2, 3, 4, 5, 6]), - (1720588796.771517, [], [1, 2, 3, 4, 5, 6]), - (1720588842.077879, [], [1, 2, 3, 4, 5, 6]), - (1720588844.116306, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720239926.764196, [], [1, 2, 3, 4, 5, 6])], - [(1720395045.1902, [], [1, 2, 3, 4, 5, 6])], - [(1720431147.297621, [], [1, 2, 3, 4, 5, 6])], - [(1720240748.713179, [], [1, 2, 3, 4, 5, 6])], - [(1719972432.742571, [], [1, 2, 3, 4, 5, 6])], - [(1720410198.607466, [], [1, 2, 3, 4, 5, 6]), (1720566548.549011, [], [1, 2, 3, 4, 5, 6])], - [(1720455428.865155, [], [1, 2, 3, 4, 5, 6])], - [(1720498325.755933, [], [1, 2, 3, 4, 5, 6])], - [ - (1719983684.033908, [], [1, 2, 3, 4, 5, 6]), - (1720319741.991515, [], [1, 2, 3, 4, 5, 6]), - (1720414800.645761, [], [1, 2, 3, 4, 5, 6]), - (1720484979.12583, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720134283.600008, [], [1, 2, 3, 4, 5, 6])], - [(1720409485.01654, [], [1, 2, 3, 4, 5, 6])], - [ - (1719981074.661088, [], [1, 2, 3, 4, 5, 6]), - (1720143880.41593, [], [1, 2, 3, 4, 5, 6]), - (1720229983.175788, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720060903.203334, [], [1, 2, 3, 4, 5, 6])], - [ - (1719968419.743912, [], [1, 2, 3, 4, 5, 6]), - (1719968490.438903, [], [1, 2, 3, 4, 5, 6]), - (1719968620.857174, [], [1, 2, 3, 4, 5, 6]), - (1720498112.351156, [], [1, 2, 3, 4, 5, 6]), - (1720498358.36836, [], [1, 2, 3, 4, 5, 6]), - (1720498468.250047, [], [1, 2, 3, 4, 5, 6]), - (1720574778.111823, [], [1, 2, 3, 4, 5, 6]), - (1720574806.5479, [], [1, 2, 3, 4, 5, 6]), - (1720574917.425735, [], [1, 2, 3, 4, 5, 6]), - (1720574933.603291, [], [1, 2, 3, 4, 5, 6]), - (1720575020.164914, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720406407.483318, [], [1, 2, 3, 4, 5, 6])], - [(1720412099.352018, [], [1, 2, 3, 4, 5, 6]), (1720498223.084881, [], [1, 2, 3, 4, 5, 6])], - [ - (1719979024.598321, [], [1, 2, 3, 4, 5, 6]), - (1720114791.984992, [], [1, 2, 3, 4, 5, 6]), - (1720241390.157269, [], [1, 2, 3, 4, 5, 6]), - (1720500283.345509, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720569522.382331, [], [1, 2, 3, 4, 5, 6])], - [(1720146840.111051, [], [1, 2, 3, 4, 5, 6]), (1720287268.372643, [], [1, 2, 3, 4, 5, 6])], - [(1720583508.926048, [], [1, 2, 3, 4, 5, 6])], - [ - (1720229146.528014, [], [1, 2, 3, 4, 5, 6]), - (1720229340.131801, [], [1, 2, 3, 4, 5, 6]), - (1720229424.480475, [], [1, 2, 3, 4, 5, 6]), - (1720229565.859999, [], [1, 2, 3, 4, 5, 6]), - (1720229567.783491, [], [1, 2, 3, 4, 5, 6]), - (1720229693.297904, [], [1, 2, 3, 4, 5, 6]), - (1720229755.453165, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720098291.146186, [], [1, 2, 3, 4, 5, 6])], - [(1719983104.788269, [], [1, 2, 3, 4, 5, 6]), (1720070626.816099, [], [1, 2, 3, 4, 5, 6])], - [ - (1720226820.995006, [], [1, 2, 3, 4, 5, 6]), - (1720316639.892049, [], [1, 2, 3, 4, 5, 6]), - (1720589368.875624, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720070877.576019, [], [1, 2, 3, 4, 5, 6]), (1720234703.959519, [], [1, 2, 3, 4, 5, 6])], - [ - (1719978870.060134, [], [1, 2, 3, 4, 5, 6]), - (1720137971.413991, [], [1, 2, 3, 4, 5, 6]), - (1720491059.303159, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719995043.481796, [], [1, 2, 3, 4, 5, 6]), (1720148819.805573, [], [1, 2, 3, 4, 5, 6])], - [ - (1720052005.359573, [], [1, 2, 3, 4, 5, 6]), - (1720052127.565063, [], [1, 2, 3, 4, 5, 6]), - (1720052129.679258, [], [1, 2, 3, 4, 5, 6]), - (1720064915.795875, [], [1, 2, 3, 4, 5, 6]), - (1720064917.840723, [], [1, 2, 3, 4, 5, 6]), - (1720064919.250429, [], [1, 2, 3, 4, 5, 6]), - (1720140563.359858, [], [1, 2, 3, 4, 5, 6]), - (1720140686.221967, [], [1, 2, 3, 4, 5, 6]), - (1720147133.126896, [], [1, 2, 3, 4, 5, 6]), - (1720154606.237768, [], [1, 2, 3, 4, 5, 6]), - (1720208312.107821, [], [1, 2, 3, 4, 5, 6]), - (1720208397.77235, [], [1, 2, 3, 4, 5, 6]), - (1720208399.88578, [], [1, 2, 3, 4, 5, 6]), - (1720226692.740751, [], [1, 2, 3, 4, 5, 6]), - (1720226809.874422, [], [1, 2, 3, 4, 5, 6]), - (1720226811.929607, [], [1, 2, 3, 4, 5, 6]), - (1720320735.680282, [], [1, 2, 3, 4, 5, 6]), - (1720320737.781583, [], [1, 2, 3, 4, 5, 6]), - (1720394544.101953, [], [1, 2, 3, 4, 5, 6]), - (1720394546.228449, [], [1, 2, 3, 4, 5, 6]), - (1720411628.159882, [], [1, 2, 3, 4, 5, 6]), - (1720411765.678009, [], [1, 2, 3, 4, 5, 6]), - (1720411765.737071, [], [1, 2, 3, 4, 5, 6]), - (1720411771.063593, [], [1, 2, 3, 4, 5, 6]), - (1720493021.815332, [], [1, 2, 3, 4, 5, 6]), - (1720493023.89141, [], [1, 2, 3, 4, 5, 6]), - (1720547092.818141, [], [1, 2, 3, 4, 5, 6]), - (1720547133.337079, [], [1, 2, 3, 4, 5, 6]), - (1720566405.934125, [], [1, 2, 3, 4, 5, 6]), - (1720566407.979963, [], [1, 2, 3, 4, 5, 6]), - (1720592934.864349, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720407281.391395, [], [1, 2, 3, 4, 5, 6]), (1720578489.911262, [], [1, 2, 3, 4, 5, 6])], - [(1720393905.799101, [], [1, 2, 3, 4, 5, 6])], - [ - (1719965431.440319, [], [1, 2, 3, 4, 5, 6]), - (1719965547.770505, [], [1, 2, 3, 4, 5, 6]), - (1719965549.880668, [], [1, 2, 3, 4, 5, 6]), - (1720010279.644796, [], [1, 2, 3, 4, 5, 6]), - (1720010481.117481, [], [1, 2, 3, 4, 5, 6]), - (1720060297.061777, [], [1, 2, 3, 4, 5, 6]), - (1720060299.106745, [], [1, 2, 3, 4, 5, 6]), - (1720072795.790373, [], [1, 2, 3, 4, 5, 6]), - (1720072933.175213, [], [1, 2, 3, 4, 5, 6]), - (1720138923.382269, [], [1, 2, 3, 4, 5, 6]), - (1720138952.892452, [], [1, 2, 3, 4, 5, 6]), - (1720138954.952138, [], [1, 2, 3, 4, 5, 6]), - (1720243737.055635, [], [1, 2, 3, 4, 5, 6]), - (1720243742.725476, [], [1, 2, 3, 4, 5, 6]), - (1720243744.812736, [], [1, 2, 3, 4, 5, 6]), - (1720278868.092914, [], [1, 2, 3, 4, 5, 6]), - (1720278981.120539, [], [1, 2, 3, 4, 5, 6]), - (1720278983.221413, [], [1, 2, 3, 4, 5, 6]), - (1720312851.319112, [], [1, 2, 3, 4, 5, 6]), - (1720312961.59678, [], [1, 2, 3, 4, 5, 6]), - (1720312963.701002, [], [1, 2, 3, 4, 5, 6]), - (1720401167.589016, [], [1, 2, 3, 4, 5, 6]), - (1720401192.232905, [], [1, 2, 3, 4, 5, 6]), - (1720488671.153932, [], [1, 2, 3, 4, 5, 6]), - (1720488673.262556, [], [1, 2, 3, 4, 5, 6]), - (1720498148.914747, [], [1, 2, 3, 4, 5, 6]), - (1720498151.01221, [], [1, 2, 3, 4, 5, 6]), - (1720585902.00157, [], [1, 2, 3, 4, 5, 6]), - (1720585904.068243, [], [1, 2, 3, 4, 5, 6]), - (1720627230.183177, [], [1, 2, 3, 4, 5, 6]), - (1720627251.343451, [], [1, 2, 3, 4, 5, 6]), - (1720627253.395817, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720049050.269443, [], [1, 2, 3, 4, 5, 6])], - [(1720406587.77676, [], [1, 2, 3, 4, 5, 6])], - [ - (1720201120.604139, [], [1, 2, 3, 4, 5, 6]), - (1720237348.670203, [], [1, 2, 3, 4, 5, 6]), - (1720503188.882528, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720137961.069782, [], [1, 2, 3, 4, 5, 6]), (1720233994.333193, [], [1, 2, 3, 4, 5, 6])], - [(1720148673.115174, [], [1, 2, 3, 4, 5, 6])], - [ - (1720017472.013793, [], [1, 2, 3, 4, 5, 6]), - (1720238395.438066, [], [1, 2, 3, 4, 5, 6]), - (1720481118.520931, [], [1, 2, 3, 4, 5, 6]), - (1720624077.141735, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720460985.277631, [], [1, 2, 3, 4, 5, 6])], - [(1720065960.10824, [], [1, 2, 3, 4, 5, 6]), (1720568092.250721, [], [1, 2, 3, 4, 5, 6])], - [(1719964803.220143, [], [1, 2, 3, 4, 5, 6]), (1720072690.78503, [], [1, 2, 3, 4, 5, 6])], - [(1719973118.028284, [], [1, 2, 3, 4, 5, 6])], - [(1720148963.270876, [], [1, 2, 3, 4, 5, 6])], - [ - (1720055800.056897, [], [1, 2, 3, 4, 5, 6]), - (1720147428.656208, [], [1, 2, 3, 4, 5, 6]), - (1720237025.123131, [], [1, 2, 3, 4, 5, 6]), - (1720326902.440989, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719967443.988967, [], [1, 2, 3, 4, 5, 6]), - (1719967507.96168, [], [1, 2, 3, 4, 5, 6]), - (1719967510.01003, [], [1, 2, 3, 4, 5, 6]), - (1719967510.547419, [], [1, 2, 3, 4, 5, 6]), - (1719967512.630086, [], [1, 2, 3, 4, 5, 6]), - (1720138300.130825, [], [1, 2, 3, 4, 5, 6]), - (1720138300.228601, [], [1, 2, 3, 4, 5, 6]), - (1720138302.137449, [], [1, 2, 3, 4, 5, 6]), - (1720266243.760636, [], [1, 2, 3, 4, 5, 6]), - (1720266262.29505, [], [1, 2, 3, 4, 5, 6]), - (1720266262.382243, [], [1, 2, 3, 4, 5, 6]), - (1720266267.714044, [], [1, 2, 3, 4, 5, 6]), - (1720376066.44502, [], [1, 2, 3, 4, 5, 6]), - (1720376075.005446, [], [1, 2, 3, 4, 5, 6]), - (1720376075.055395, [], [1, 2, 3, 4, 5, 6]), - (1720376078.271297, [], [1, 2, 3, 4, 5, 6]), - (1720495615.317205, [], [1, 2, 3, 4, 5, 6]), - (1720495625.121167, [], [1, 2, 3, 4, 5, 6]), - (1720495627.190587, [], [1, 2, 3, 4, 5, 6]), - (1720495631.668389, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720094115.933922, [], [1, 2, 3, 4, 5, 6])], - [(1720327035.126978, [], [1, 2, 3, 4, 5, 6]), (1720652576.382878, [], [1, 2, 3, 4, 5, 6])], - [(1720494001.577927, [], [1, 2, 3, 4, 5, 6])], - [ - (1720102356.301353, [], [1, 2, 3, 4, 5, 6]), - (1720244955.2084, [], [1, 2, 3, 4, 5, 6]), - (1720393949.41044, [], [1, 2, 3, 4, 5, 6]), - (1720576986.579566, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720033048.609785, [], [1, 2, 3, 4, 5, 6])], - [(1720402968.773862, [], [1, 2, 3, 4, 5, 6])], - [(1720230211.716966, [], [1, 2, 3, 4, 5, 6])], - [ - (1720059787.289306, [], [1, 2, 3, 4, 5, 6]), - (1720397378.208597, [], [1, 2, 3, 4, 5, 6]), - (1720481196.422422, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720003149.057462, [], [1, 2, 3, 4, 5, 6]), - (1720003372.883061, [], [1, 2, 3, 4, 5, 6]), - (1720003374.955179, [], [1, 2, 3, 4, 5, 6]), - (1720039541.861276, [], [1, 2, 3, 4, 5, 6]), - (1720039688.427965, [], [1, 2, 3, 4, 5, 6]), - (1720039690.485555, [], [1, 2, 3, 4, 5, 6]), - (1720048625.116329, [], [1, 2, 3, 4, 5, 6]), - (1720048725.117697, [], [1, 2, 3, 4, 5, 6]), - (1720141659.610639, [], [1, 2, 3, 4, 5, 6]), - (1720141661.665952, [], [1, 2, 3, 4, 5, 6]), - (1720196426.042225, [], [1, 2, 3, 4, 5, 6]), - (1720196487.09087, [], [1, 2, 3, 4, 5, 6]), - (1720196489.183893, [], [1, 2, 3, 4, 5, 6]), - (1720207066.952798, [], [1, 2, 3, 4, 5, 6]), - (1720207237.857105, [], [1, 2, 3, 4, 5, 6]), - (1720207239.919375, [], [1, 2, 3, 4, 5, 6]), - (1720271033.503072, [], [1, 2, 3, 4, 5, 6]), - (1720271035.587795, [], [1, 2, 3, 4, 5, 6]), - (1720583123.471438, [], [1, 2, 3, 4, 5, 6]), - (1720583125.576798, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720154856.626343, [], [1, 2, 3, 4, 5, 6]), - (1720226072.346309, [], [1, 2, 3, 4, 5, 6]), - (1720310601.449016, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720242303.35601, [], [1, 2, 3, 4, 5, 6])], - [(1720078404.748142, [], [1, 2, 3, 4, 5, 6]), (1720147584.809447, [], [1, 2, 3, 4, 5, 6])], - [(1720178488.289574, [], [1, 2, 3, 4, 5, 6]), (1720306985.894457, [], [1, 2, 3, 4, 5, 6])], - [(1720146748.830901, [], [1, 2, 3, 4, 5, 6]), (1720406666.368212, [], [1, 2, 3, 4, 5, 6])], - [ - (1720023013.684634, [], [1, 2, 3, 4, 5, 6]), - (1720091577.184398, [], [1, 2, 3, 4, 5, 6]), - (1720415121.299085, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720056649.932043, [], [1, 2, 3, 4, 5, 6])], - [ - (1720323285.584787, [], [1, 2, 3, 4, 5, 6]), - (1720415154.592994, [], [1, 2, 3, 4, 5, 6]), - (1720437978.9498, [], [1, 2, 3, 4, 5, 6]), - (1720473849.744602, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719984649.0919, [], [1, 2, 3, 4, 5, 6]), - (1719984690.14033, [], [1, 2, 3, 4, 5, 6]), - (1720049900.611104, [], [1, 2, 3, 4, 5, 6]), - (1720049902.719326, [], [1, 2, 3, 4, 5, 6]), - (1720070202.827545, [], [1, 2, 3, 4, 5, 6]), - (1720070219.8164, [], [1, 2, 3, 4, 5, 6]), - (1720070221.892056, [], [1, 2, 3, 4, 5, 6]), - (1720137406.074377, [], [1, 2, 3, 4, 5, 6]), - (1720137495.69452, [], [1, 2, 3, 4, 5, 6]), - (1720137497.797134, [], [1, 2, 3, 4, 5, 6]), - (1720328661.139393, [], [1, 2, 3, 4, 5, 6]), - (1720328739.939669, [], [1, 2, 3, 4, 5, 6]), - (1720498901.295947, [], [1, 2, 3, 4, 5, 6]), - (1720570987.624349, [], [1, 2, 3, 4, 5, 6]), - (1720602646.235039, [], [1, 2, 3, 4, 5, 6]), - (1720602812.376711, [], [1, 2, 3, 4, 5, 6]), - (1720602814.446349, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719974528.289353, [], [1, 2, 3, 4, 5, 6]), - (1720395723.591687, [], [1, 2, 3, 4, 5, 6]), - (1720617442.004095, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719975592.337919, [], [1, 2, 3, 4, 5, 6])], - [(1720057591.780745, [], [1, 2, 3, 4, 5, 6]), (1720488152.255523, [], [1, 2, 3, 4, 5, 6])], - [ - (1720148926.955422, [], [1, 2, 3, 4, 5, 6]), - (1720232410.538746, [], [1, 2, 3, 4, 5, 6]), - (1720408447.752538, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720242892.942203, [], [1, 2, 3, 4, 5, 6]), (1720322823.10199, [], [1, 2, 3, 4, 5, 6])], - [(1720400088.852099, [], [1, 2, 3, 4, 5, 6]), (1720652752.741385, [], [1, 2, 3, 4, 5, 6])], - [(1720412428.936544, [], [1, 2, 3, 4, 5, 6]), (1720574790.414977, [], [1, 2, 3, 4, 5, 6])], - [(1720069130.346249, [], [1, 2, 3, 4, 5, 6])], - [ - (1720157382.402748, [], [1, 2, 3, 4, 5, 6]), - (1720157476.064866, [], [1, 2, 3, 4, 5, 6]), - (1720157570.728611, [], [1, 2, 3, 4, 5, 6]), - (1720157662.381552, [], [1, 2, 3, 4, 5, 6]), - (1720168580.805279, [], [1, 2, 3, 4, 5, 6]), - (1720168760.793692, [], [1, 2, 3, 4, 5, 6]), - (1720230288.646191, [], [1, 2, 3, 4, 5, 6]), - (1720230430.103977, [], [1, 2, 3, 4, 5, 6]), - (1720243393.667459, [], [1, 2, 3, 4, 5, 6]), - (1720311604.919662, [], [1, 2, 3, 4, 5, 6]), - (1720311771.258364, [], [1, 2, 3, 4, 5, 6]), - (1720311773.310317, [], [1, 2, 3, 4, 5, 6]), - (1720324823.664232, [], [1, 2, 3, 4, 5, 6]), - (1720324825.721978, [], [1, 2, 3, 4, 5, 6]), - (1720393374.344255, [], [1, 2, 3, 4, 5, 6]), - (1720410680.226051, [], [1, 2, 3, 4, 5, 6]), - (1720410682.252163, [], [1, 2, 3, 4, 5, 6]), - (1720498394.961086, [], [1, 2, 3, 4, 5, 6]), - (1720498440.73496, [], [1, 2, 3, 4, 5, 6]), - (1720569716.948564, [], [1, 2, 3, 4, 5, 6]), - (1720569812.275586, [], [1, 2, 3, 4, 5, 6]), - (1720569814.333894, [], [1, 2, 3, 4, 5, 6]), - (1720569814.53133, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719978988.407662, [], [1, 2, 3, 4, 5, 6]), - (1719978990.511958, [], [1, 2, 3, 4, 5, 6]), - (1719984066.593696, [], [1, 2, 3, 4, 5, 6]), - (1719984323.803693, [], [1, 2, 3, 4, 5, 6]), - (1719984325.863773, [], [1, 2, 3, 4, 5, 6]), - (1720152078.595081, [], [1, 2, 3, 4, 5, 6]), - (1720225565.930941, [], [1, 2, 3, 4, 5, 6]), - (1720280378.036955, [], [1, 2, 3, 4, 5, 6]), - (1720326955.218979, [], [1, 2, 3, 4, 5, 6]), - (1720326957.292481, [], [1, 2, 3, 4, 5, 6]), - (1720331017.031137, [], [1, 2, 3, 4, 5, 6]), - (1720331019.121504, [], [1, 2, 3, 4, 5, 6]), - (1720410756.685202, [], [1, 2, 3, 4, 5, 6]), - (1720410758.796614, [], [1, 2, 3, 4, 5, 6]), - (1720417802.859919, [], [1, 2, 3, 4, 5, 6]), - (1720417804.989442, [], [1, 2, 3, 4, 5, 6]), - (1720487313.059882, [], [1, 2, 3, 4, 5, 6]), - (1720487498.533155, [], [1, 2, 3, 4, 5, 6]), - (1720487500.652063, [], [1, 2, 3, 4, 5, 6]), - (1720487503.708405, [], [1, 2, 3, 4, 5, 6]), - (1720487505.805861, [], [1, 2, 3, 4, 5, 6]), - (1720501546.266299, [], [1, 2, 3, 4, 5, 6]), - (1720501655.51812, [], [1, 2, 3, 4, 5, 6]), - (1720575741.153236, [], [1, 2, 3, 4, 5, 6]), - (1720575891.79104, [], [1, 2, 3, 4, 5, 6]), - (1720589520.786652, [], [1, 2, 3, 4, 5, 6]), - (1720589642.390304, [], [1, 2, 3, 4, 5, 6]), - (1720589644.452771, [], [1, 2, 3, 4, 5, 6]), - (1720615233.591986, [], [1, 2, 3, 4, 5, 6]), - (1720615337.890481, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719971542.018584, [], [1, 2, 3, 4, 5, 6]), - (1719971788.12398, [], [1, 2, 3, 4, 5, 6]), - (1720117268.913487, [], [1, 2, 3, 4, 5, 6]), - (1720117270.991206, [], [1, 2, 3, 4, 5, 6]), - (1720143198.612101, [], [1, 2, 3, 4, 5, 6]), - (1720143429.084839, [], [1, 2, 3, 4, 5, 6]), - (1720238625.308642, [], [1, 2, 3, 4, 5, 6]), - (1720238757.81922, [], [1, 2, 3, 4, 5, 6]), - (1720238759.894117, [], [1, 2, 3, 4, 5, 6]), - (1720330550.917977, [], [1, 2, 3, 4, 5, 6]), - (1720330626.459228, [], [1, 2, 3, 4, 5, 6]), - (1720378937.313156, [], [1, 2, 3, 4, 5, 6]), - (1720379038.375789, [], [1, 2, 3, 4, 5, 6]), - (1720386358.224787, [], [1, 2, 3, 4, 5, 6]), - (1720386360.275601, [], [1, 2, 3, 4, 5, 6]), - (1720416486.117358, [], [1, 2, 3, 4, 5, 6]), - (1720416608.109114, [], [1, 2, 3, 4, 5, 6]), - (1720493716.833205, [], [1, 2, 3, 4, 5, 6]), - (1720493844.641363, [], [1, 2, 3, 4, 5, 6]), - (1720493846.67691, [], [1, 2, 3, 4, 5, 6]), - (1720568118.486107, [], [1, 2, 3, 4, 5, 6]), - (1720568219.230995, [], [1, 2, 3, 4, 5, 6]), - (1720568221.334344, [], [1, 2, 3, 4, 5, 6]), - (1720574746.351324, [], [1, 2, 3, 4, 5, 6]), - (1720574815.297689, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720233195.120615, [], [1, 2, 3, 4, 5, 6]), - (1720393731.964556, [], [1, 2, 3, 4, 5, 6]), - (1720570257.699261, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720149655.238203, [], [1, 2, 3, 4, 5, 6]), - (1720352361.227124, [], [1, 2, 3, 4, 5, 6]), - (1720578697.147852, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720411277.985506, [], [1, 2, 3, 4, 5, 6]), (1720572981.673421, [], [1, 2, 3, 4, 5, 6])], - [(1720569584.93865, [], [1, 2, 3, 4, 5, 6])], - [(1720311303.894177, [], [1, 2, 3, 4, 5, 6])], - [(1720576463.87807, [], [1, 2, 3, 4, 5, 6])], - [ - (1719982989.782732, [], [1, 2, 3, 4, 5, 6]), - (1720080708.007665, [], [1, 2, 3, 4, 5, 6]), - (1720234553.333259, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719983711.203136, [], [1, 2, 3, 4, 5, 6]), (1720395076.590109, [], [1, 2, 3, 4, 5, 6])], - [ - (1719968905.802345, [], [1, 2, 3, 4, 5, 6]), - (1720054751.228152, [], [1, 2, 3, 4, 5, 6]), - (1720393228.571573, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720151965.57438, [], [1, 2, 3, 4, 5, 6]), - (1720265917.280767, [], [1, 2, 3, 4, 5, 6]), - (1720414597.498797, [], [1, 2, 3, 4, 5, 6]), - (1720569352.211054, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720139623.448558, [], [1, 2, 3, 4, 5, 6]), (1720413909.371495, [], [1, 2, 3, 4, 5, 6])], - [ - (1720332156.972433, [], [1, 2, 3, 4, 5, 6]), - (1720486770.808084, [], [1, 2, 3, 4, 5, 6]), - (1720570506.129092, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720046377.309817, [], [1, 2, 3, 4, 5, 6]), (1720144405.103977, [], [1, 2, 3, 4, 5, 6])], - [(1720311749.460887, [], [1, 2, 3, 4, 5, 6]), (1720480404.801297, [], [1, 2, 3, 4, 5, 6])], - [(1719965504.779832, [], [1, 2, 3, 4, 5, 6])], - [(1720489219.425884, [], [1, 2, 3, 4, 5, 6])], - [(1720480774.306063, [], [1, 2, 3, 4, 5, 6])], - [(1719968486.759348, [], [1, 2, 3, 4, 5, 6]), (1720229505.650825, [], [1, 2, 3, 4, 5, 6])], - [(1720122526.844622, [], [1, 2, 3, 4, 5, 6]), (1720142320.524851, [], [1, 2, 3, 4, 5, 6])], - [(1720320995.267452, [], [1, 2, 3, 4, 5, 6])], - [ - (1719984296.990609, [], [1, 2, 3, 4, 5, 6]), - (1719984350.22369, [], [1, 2, 3, 4, 5, 6]), - (1719984352.257572, [], [1, 2, 3, 4, 5, 6]), - (1720052997.807277, [], [1, 2, 3, 4, 5, 6]), - (1720053049.304507, [], [1, 2, 3, 4, 5, 6]), - (1720053051.378602, [], [1, 2, 3, 4, 5, 6]), - (1720150389.546489, [], [1, 2, 3, 4, 5, 6]), - (1720150391.660121, [], [1, 2, 3, 4, 5, 6]), - (1720168392.833864, [], [1, 2, 3, 4, 5, 6]), - (1720168394.931787, [], [1, 2, 3, 4, 5, 6]), - (1720218901.13549, [], [1, 2, 3, 4, 5, 6]), - (1720218984.364651, [], [1, 2, 3, 4, 5, 6]), - (1720218986.444335, [], [1, 2, 3, 4, 5, 6]), - (1720236871.413173, [], [1, 2, 3, 4, 5, 6]), - (1720236873.52, [], [1, 2, 3, 4, 5, 6]), - (1720318899.245869, [], [1, 2, 3, 4, 5, 6]), - (1720319077.080816, [], [1, 2, 3, 4, 5, 6]), - (1720319079.181328, [], [1, 2, 3, 4, 5, 6]), - (1720421585.966107, [], [1, 2, 3, 4, 5, 6]), - (1720421692.177002, [], [1, 2, 3, 4, 5, 6]), - (1720421694.269891, [], [1, 2, 3, 4, 5, 6]), - (1720484559.101295, [], [1, 2, 3, 4, 5, 6]), - (1720484634.686657, [], [1, 2, 3, 4, 5, 6]), - (1720484636.791229, [], [1, 2, 3, 4, 5, 6]), - (1720484640.876498, [], [1, 2, 3, 4, 5, 6]), - (1720484642.914839, [], [1, 2, 3, 4, 5, 6]), - (1720568978.955929, [], [1, 2, 3, 4, 5, 6]), - (1720569083.551067, [], [1, 2, 3, 4, 5, 6]), - (1720603472.634189, [], [1, 2, 3, 4, 5, 6]), - (1720603622.57534, [], [1, 2, 3, 4, 5, 6]), - (1720603624.69381, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720204831.715905, [], [1, 2, 3, 4, 5, 6])], - [(1719985836.773241, [], [1, 2, 3, 4, 5, 6]), (1720227063.151216, [], [1, 2, 3, 4, 5, 6])], - [(1720226998.434162, [], [1, 2, 3, 4, 5, 6])], - [(1720413326.470543, [], [1, 2, 3, 4, 5, 6])], - [(1720484747.542658, [], [1, 2, 3, 4, 5, 6])], - [ - (1719975290.47648, [], [1, 2, 3, 4, 5, 6]), - (1719975421.539502, [], [1, 2, 3, 4, 5, 6]), - (1719975423.609947, [], [1, 2, 3, 4, 5, 6]), - (1719975424.130571, [], [1, 2, 3, 4, 5, 6]), - (1719975426.224942, [], [1, 2, 3, 4, 5, 6]), - (1720024823.782769, [], [1, 2, 3, 4, 5, 6]), - (1720024943.32157, [], [1, 2, 3, 4, 5, 6]), - (1720024945.350078, [], [1, 2, 3, 4, 5, 6]), - (1720024945.905366, [], [1, 2, 3, 4, 5, 6]), - (1720056619.663053, [], [1, 2, 3, 4, 5, 6]), - (1720056822.318399, [], [1, 2, 3, 4, 5, 6]), - (1720056824.36183, [], [1, 2, 3, 4, 5, 6]), - (1720072281.168332, [], [1, 2, 3, 4, 5, 6]), - (1720072283.222603, [], [1, 2, 3, 4, 5, 6]), - (1720134154.949613, [], [1, 2, 3, 4, 5, 6]), - (1720134376.323715, [], [1, 2, 3, 4, 5, 6]), - (1720134378.416906, [], [1, 2, 3, 4, 5, 6]), - (1720140112.874786, [], [1, 2, 3, 4, 5, 6]), - (1720140131.322854, [], [1, 2, 3, 4, 5, 6]), - (1720140133.38169, [], [1, 2, 3, 4, 5, 6]), - (1720238635.597737, [], [1, 2, 3, 4, 5, 6]), - (1720238637.672121, [], [1, 2, 3, 4, 5, 6]), - (1720418306.625113, [], [1, 2, 3, 4, 5, 6]), - (1720418333.5673, [], [1, 2, 3, 4, 5, 6]), - (1720487528.439985, [], [1, 2, 3, 4, 5, 6]), - (1720487546.337876, [], [1, 2, 3, 4, 5, 6]), - (1720487548.449392, [], [1, 2, 3, 4, 5, 6]), - (1720502509.125496, [], [1, 2, 3, 4, 5, 6]), - (1720502624.411704, [], [1, 2, 3, 4, 5, 6]), - (1720585053.028856, [], [1, 2, 3, 4, 5, 6]), - (1720585055.08891, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719982052.592392, [], [1, 2, 3, 4, 5, 6]), - (1719982258.203523, [], [1, 2, 3, 4, 5, 6]), - (1719985183.079021, [], [1, 2, 3, 4, 5, 6]), - (1719985305.780432, [], [1, 2, 3, 4, 5, 6]), - (1720055777.47168, [], [1, 2, 3, 4, 5, 6]), - (1720071284.174477, [], [1, 2, 3, 4, 5, 6]), - (1720152558.44266, [], [1, 2, 3, 4, 5, 6]), - (1720152671.333552, [], [1, 2, 3, 4, 5, 6]), - (1720152673.415669, [], [1, 2, 3, 4, 5, 6]), - (1720182134.94743, [], [1, 2, 3, 4, 5, 6]), - (1720182137.028427, [], [1, 2, 3, 4, 5, 6]), - (1720182138.842693, [], [1, 2, 3, 4, 5, 6]), - (1720182140.958073, [], [1, 2, 3, 4, 5, 6]), - (1720227575.333539, [], [1, 2, 3, 4, 5, 6]), - (1720227736.260264, [], [1, 2, 3, 4, 5, 6]), - (1720227738.300477, [], [1, 2, 3, 4, 5, 6]), - (1720236068.538351, [], [1, 2, 3, 4, 5, 6]), - (1720236070.60483, [], [1, 2, 3, 4, 5, 6]), - (1720317757.071545, [], [1, 2, 3, 4, 5, 6]), - (1720317866.130343, [], [1, 2, 3, 4, 5, 6]), - (1720317868.22788, [], [1, 2, 3, 4, 5, 6]), - (1720330462.980036, [], [1, 2, 3, 4, 5, 6]), - (1720330707.655663, [], [1, 2, 3, 4, 5, 6]), - (1720330709.750072, [], [1, 2, 3, 4, 5, 6]), - (1720373940.747523, [], [1, 2, 3, 4, 5, 6]), - (1720374027.697475, [], [1, 2, 3, 4, 5, 6]), - (1720374029.799405, [], [1, 2, 3, 4, 5, 6]), - (1720406670.90306, [], [1, 2, 3, 4, 5, 6]), - (1720406844.478936, [], [1, 2, 3, 4, 5, 6]), - (1720406846.592556, [], [1, 2, 3, 4, 5, 6]), - (1720406851.281701, [], [1, 2, 3, 4, 5, 6]), - (1720412576.66958, [], [1, 2, 3, 4, 5, 6]), - (1720412652.412012, [], [1, 2, 3, 4, 5, 6]), - (1720412654.533239, [], [1, 2, 3, 4, 5, 6]), - (1720486184.910769, [], [1, 2, 3, 4, 5, 6]), - (1720571066.49819, [], [1, 2, 3, 4, 5, 6]), - (1720626129.788949, [], [1, 2, 3, 4, 5, 6]), - (1720626261.332132, [], [1, 2, 3, 4, 5, 6]), - (1720626263.446226, [], [1, 2, 3, 4, 5, 6]), - (1720626267.27108, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719972691.157799, [], [1, 2, 3, 4, 5, 6]), - (1719972822.458675, [], [1, 2, 3, 4, 5, 6]), - (1719972824.589752, [], [1, 2, 3, 4, 5, 6]), - (1719993329.829737, [], [1, 2, 3, 4, 5, 6]), - (1720053155.834123, [], [1, 2, 3, 4, 5, 6]), - (1720053293.330528, [], [1, 2, 3, 4, 5, 6]), - (1720084358.281667, [], [1, 2, 3, 4, 5, 6]), - (1720084522.554824, [], [1, 2, 3, 4, 5, 6]), - (1720084524.658959, [], [1, 2, 3, 4, 5, 6]), - (1720153146.842639, [], [1, 2, 3, 4, 5, 6]), - (1720153159.620793, [], [1, 2, 3, 4, 5, 6]), - (1720223041.359927, [], [1, 2, 3, 4, 5, 6]), - (1720223107.178767, [], [1, 2, 3, 4, 5, 6]), - (1720223109.301943, [], [1, 2, 3, 4, 5, 6]), - (1720243608.145196, [], [1, 2, 3, 4, 5, 6]), - (1720243715.813915, [], [1, 2, 3, 4, 5, 6]), - (1720310055.295457, [], [1, 2, 3, 4, 5, 6]), - (1720310225.190394, [], [1, 2, 3, 4, 5, 6]), - (1720310227.250668, [], [1, 2, 3, 4, 5, 6]), - (1720374255.295948, [], [1, 2, 3, 4, 5, 6]), - (1720374257.390432, [], [1, 2, 3, 4, 5, 6]), - (1720397033.790744, [], [1, 2, 3, 4, 5, 6]), - (1720397192.93351, [], [1, 2, 3, 4, 5, 6]), - (1720489562.012912, [], [1, 2, 3, 4, 5, 6]), - (1720489620.124167, [], [1, 2, 3, 4, 5, 6]), - (1720489622.2461, [], [1, 2, 3, 4, 5, 6]), - (1720577615.944083, [], [1, 2, 3, 4, 5, 6]), - (1720595908.263871, [], [1, 2, 3, 4, 5, 6]), - (1720596022.795818, [], [1, 2, 3, 4, 5, 6]), - (1720596024.909409, [], [1, 2, 3, 4, 5, 6]), - (1720596025.112291, [], [1, 2, 3, 4, 5, 6]), - (1720596027.181848, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720410184.878056, [], [1, 2, 3, 4, 5, 6])], - [(1720446097.457008, [], [1, 2, 3, 4, 5, 6]), (1720582142.651169, [], [1, 2, 3, 4, 5, 6])], - [(1720496385.651227, [], [1, 2, 3, 4, 5, 6]), (1720588018.159792, [], [1, 2, 3, 4, 5, 6])], - [(1719975737.968693, [], [1, 2, 3, 4, 5, 6]), (1720069758.403541, [], [1, 2, 3, 4, 5, 6])], - [(1720573220.196789, [], [1, 2, 3, 4, 5, 6])], - [(1720435170.4772, [], [1, 2, 3, 4, 5, 6])], - [(1720156838.062846, [], [1, 2, 3, 4, 5, 6]), (1720588244.606338, [], [1, 2, 3, 4, 5, 6])], - [(1720072706.921977, [], [1, 2, 3, 4, 5, 6]), (1720588899.722119, [], [1, 2, 3, 4, 5, 6])], - [(1720058343.607628, [], [1, 2, 3, 4, 5, 6])], - [(1720314365.315505, [], [1, 2, 3, 4, 5, 6])], - [(1720502417.751936, [], [1, 2, 3, 4, 5, 6])], - [(1720302708.367359, [], [1, 2, 3, 4, 5, 6]), (1720585704.559633, [], [1, 2, 3, 4, 5, 6])], - [ - (1719984540.656777, [], [1, 2, 3, 4, 5, 6]), - (1720146973.8651, [], [1, 2, 3, 4, 5, 6]), - (1720341798.159409, [], [1, 2, 3, 4, 5, 6]), - (1720482149.273983, [], [1, 2, 3, 4, 5, 6]), - (1720570969.604085, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720311167.154928, [], [1, 2, 3, 4, 5, 6]), (1720404884.472902, [], [1, 2, 3, 4, 5, 6])], - [(1720481370.176729, [], [1, 2, 3, 4, 5, 6])], - [(1720098969.778426, [], [1, 2, 3, 4, 5, 6]), (1720542708.023885, [], [1, 2, 3, 4, 5, 6])], - [ - (1720149583.597081, [], [1, 2, 3, 4, 5, 6]), - (1720314574.406545, [], [1, 2, 3, 4, 5, 6]), - (1720416038.659142, [], [1, 2, 3, 4, 5, 6]), - (1720572347.697131, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720033450.660601, [], [1, 2, 3, 4, 5, 6]), - (1720033450.752555, [], [1, 2, 3, 4, 5, 6]), - (1720033455.770905, [], [1, 2, 3, 4, 5, 6]), - (1720400227.89913, [], [1, 2, 3, 4, 5, 6]), - (1720400250.299243, [], [1, 2, 3, 4, 5, 6]), - (1720400252.394995, [], [1, 2, 3, 4, 5, 6]), - (1720417432.186774, [], [1, 2, 3, 4, 5, 6]), - (1720417504.33498, [], [1, 2, 3, 4, 5, 6]), - (1720417506.39095, [], [1, 2, 3, 4, 5, 6]), - (1720417509.515927, [], [1, 2, 3, 4, 5, 6]), - (1720417511.647347, [], [1, 2, 3, 4, 5, 6]), - (1720417512.311827, [], [1, 2, 3, 4, 5, 6]), - (1720512787.544525, [], [1, 2, 3, 4, 5, 6]), - (1720512787.637452, [], [1, 2, 3, 4, 5, 6]), - (1720512790.509437, [], [1, 2, 3, 4, 5, 6]), - (1720546384.085434, [], [1, 2, 3, 4, 5, 6]), - (1720546402.635026, [], [1, 2, 3, 4, 5, 6]), - (1720546404.721606, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720091612.414745, [], [1, 2, 3, 4, 5, 6]), (1720394180.692823, [], [1, 2, 3, 4, 5, 6])], - [(1720499565.692616, [], [1, 2, 3, 4, 5, 6])], - [ - (1720050050.500015, [], [1, 2, 3, 4, 5, 6]), - (1720050118.092842, [], [1, 2, 3, 4, 5, 6]), - (1720050225.108458, [], [1, 2, 3, 4, 5, 6]), - (1720065727.65261, [], [1, 2, 3, 4, 5, 6]), - (1720065830.735126, [], [1, 2, 3, 4, 5, 6]), - (1720065888.617825, [], [1, 2, 3, 4, 5, 6]), - (1720065986.346168, [], [1, 2, 3, 4, 5, 6]), - (1720066099.675624, [], [1, 2, 3, 4, 5, 6]), - (1720066274.481661, [], [1, 2, 3, 4, 5, 6]), - (1720066349.195281, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720151340.048882, [], [1, 2, 3, 4, 5, 6]), - (1720314849.375028, [], [1, 2, 3, 4, 5, 6]), - (1720413869.9313, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720054419.118301, [], [1, 2, 3, 4, 5, 6]), (1720399142.008777, [], [1, 2, 3, 4, 5, 6])], - [ - (1720070403.925696, [], [1, 2, 3, 4, 5, 6]), - (1720244978.0606, [], [1, 2, 3, 4, 5, 6]), - (1720588083.280232, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720142944.669634, [], [1, 2, 3, 4, 5, 6]), - (1720317829.682224, [], [1, 2, 3, 4, 5, 6]), - (1720402172.873968, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720135813.27587, [], [1, 2, 3, 4, 5, 6]), - (1720344219.879026, [], [1, 2, 3, 4, 5, 6]), - (1720587780.127476, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720307062.497702, [], [1, 2, 3, 4, 5, 6])], - [(1720145491.250495, [], [1, 2, 3, 4, 5, 6]), (1720586242.178403, [], [1, 2, 3, 4, 5, 6])], - [(1720586566.015615, [], [1, 2, 3, 4, 5, 6])], - [(1720409077.369118, [], [1, 2, 3, 4, 5, 6]), (1720569722.833936, [], [1, 2, 3, 4, 5, 6])], - [(1720098442.029928, [], [1, 2, 3, 4, 5, 6]), (1720391796.053942, [], [1, 2, 3, 4, 5, 6])], - [(1720243240.863801, [], [1, 2, 3, 4, 5, 6]), (1720492060.909226, [], [1, 2, 3, 4, 5, 6])], - [ - (1719967064.859664, [], [1, 2, 3, 4, 5, 6]), - (1720055971.205432, [], [1, 2, 3, 4, 5, 6]), - (1720056017.075129, [], [1, 2, 3, 4, 5, 6]), - (1720069354.406111, [], [1, 2, 3, 4, 5, 6]), - (1720069356.466395, [], [1, 2, 3, 4, 5, 6]), - (1720153990.829373, [], [1, 2, 3, 4, 5, 6]), - (1720154034.130574, [], [1, 2, 3, 4, 5, 6]), - (1720232518.365492, [], [1, 2, 3, 4, 5, 6]), - (1720232668.084829, [], [1, 2, 3, 4, 5, 6]), - (1720232670.147567, [], [1, 2, 3, 4, 5, 6]), - (1720247796.598987, [], [1, 2, 3, 4, 5, 6]), - (1720310742.132713, [], [1, 2, 3, 4, 5, 6]), - (1720310784.012123, [], [1, 2, 3, 4, 5, 6]), - (1720320091.334971, [], [1, 2, 3, 4, 5, 6]), - (1720357505.367765, [], [1, 2, 3, 4, 5, 6]), - (1720357507.406388, [], [1, 2, 3, 4, 5, 6]), - (1720404625.988586, [], [1, 2, 3, 4, 5, 6]), - (1720404726.50447, [], [1, 2, 3, 4, 5, 6]), - (1720404728.609789, [], [1, 2, 3, 4, 5, 6]), - (1720417440.696768, [], [1, 2, 3, 4, 5, 6]), - (1720460381.831877, [], [1, 2, 3, 4, 5, 6]), - (1720460489.831088, [], [1, 2, 3, 4, 5, 6]), - (1720492881.459734, [], [1, 2, 3, 4, 5, 6]), - (1720492883.570789, [], [1, 2, 3, 4, 5, 6]), - (1720580680.591028, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719975556.382414, [], [1, 2, 3, 4, 5, 6]), (1720231475.932226, [], [1, 2, 3, 4, 5, 6])], - [(1720569569.754826, [], [1, 2, 3, 4, 5, 6])], - [(1720043952.413223, [], [1, 2, 3, 4, 5, 6]), (1720225500.222696, [], [1, 2, 3, 4, 5, 6])], - [ - (1719967819.052883, [], [1, 2, 3, 4, 5, 6]), - (1720234292.697748, [], [1, 2, 3, 4, 5, 6]), - (1720397113.348799, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720087477.672876, [], [1, 2, 3, 4, 5, 6]), (1720438489.760021, [], [1, 2, 3, 4, 5, 6])], - [(1720577383.739689, [], [1, 2, 3, 4, 5, 6])], - [ - (1720047896.111507, [], [1, 2, 3, 4, 5, 6]), - (1720200244.93862, [], [1, 2, 3, 4, 5, 6]), - (1720464543.942733, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719970307.394382, [], [1, 2, 3, 4, 5, 6])], - [(1719982131.954439, [], [1, 2, 3, 4, 5, 6]), (1720238111.874625, [], [1, 2, 3, 4, 5, 6])], - [(1719966189.201726, [], [1, 2, 3, 4, 5, 6]), (1720150700.452652, [], [1, 2, 3, 4, 5, 6])], - [(1720142072.057736, [], [1, 2, 3, 4, 5, 6])], - [(1720063956.632264, [], [1, 2, 3, 4, 5, 6])], - [(1720057020.243682, [], [1, 2, 3, 4, 5, 6])], - [ - (1719973656.343743, [], [1, 2, 3, 4, 5, 6]), - (1719973767.615562, [], [1, 2, 3, 4, 5, 6]), - (1719973769.702368, [], [1, 2, 3, 4, 5, 6]), - (1720050422.073716, [], [1, 2, 3, 4, 5, 6]), - (1720060932.515015, [], [1, 2, 3, 4, 5, 6]), - (1720061076.268193, [], [1, 2, 3, 4, 5, 6]), - (1720061078.375626, [], [1, 2, 3, 4, 5, 6]), - (1720061082.695369, [], [1, 2, 3, 4, 5, 6]), - (1720061084.803345, [], [1, 2, 3, 4, 5, 6]), - (1720061088.695247, [], [1, 2, 3, 4, 5, 6]), - (1720061090.795592, [], [1, 2, 3, 4, 5, 6]), - (1720135057.814031, [], [1, 2, 3, 4, 5, 6]), - (1720135295.655111, [], [1, 2, 3, 4, 5, 6]), - (1720135297.734383, [], [1, 2, 3, 4, 5, 6]), - (1720156608.706907, [], [1, 2, 3, 4, 5, 6]), - (1720156610.771323, [], [1, 2, 3, 4, 5, 6]), - (1720208160.885537, [], [1, 2, 3, 4, 5, 6]), - (1720208274.821579, [], [1, 2, 3, 4, 5, 6]), - (1720208276.929569, [], [1, 2, 3, 4, 5, 6]), - (1720223270.224257, [], [1, 2, 3, 4, 5, 6]), - (1720223272.316827, [], [1, 2, 3, 4, 5, 6]), - (1720223275.15326, [], [1, 2, 3, 4, 5, 6]), - (1720261350.082829, [], [1, 2, 3, 4, 5, 6]), - (1720261423.329391, [], [1, 2, 3, 4, 5, 6]), - (1720261425.427693, [], [1, 2, 3, 4, 5, 6]), - (1720319448.712298, [], [1, 2, 3, 4, 5, 6]), - (1720319512.283877, [], [1, 2, 3, 4, 5, 6]), - (1720319514.384024, [], [1, 2, 3, 4, 5, 6]), - (1720397163.860459, [], [1, 2, 3, 4, 5, 6]), - (1720397201.824506, [], [1, 2, 3, 4, 5, 6]), - (1720397203.898302, [], [1, 2, 3, 4, 5, 6]), - (1720487465.098454, [], [1, 2, 3, 4, 5, 6]), - (1720487616.241062, [], [1, 2, 3, 4, 5, 6]), - (1720576783.799559, [], [1, 2, 3, 4, 5, 6]), - (1720576837.625767, [], [1, 2, 3, 4, 5, 6]), - (1720576839.747181, [], [1, 2, 3, 4, 5, 6]), - (1720621584.709553, [], [1, 2, 3, 4, 5, 6]), - (1720621686.14789, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720210871.32264, [], [1, 2, 3, 4, 5, 6]), (1720324564.785043, [], [1, 2, 3, 4, 5, 6])], - [ - (1719976074.774068, [], [1, 2, 3, 4, 5, 6]), - (1720432422.057214, [], [1, 2, 3, 4, 5, 6]), - (1720498059.505822, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720537307.322626, [], [1, 2, 3, 4, 5, 6])], - [(1720051279.943281, [], [1, 2, 3, 4, 5, 6]), (1720326635.291429, [], [1, 2, 3, 4, 5, 6])], - [ - (1720317156.788498, [], [1, 2, 3, 4, 5, 6]), - (1720409687.362687, [], [1, 2, 3, 4, 5, 6]), - (1720499324.086042, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720058380.956986, [], [1, 2, 3, 4, 5, 6]), (1720150021.258812, [], [1, 2, 3, 4, 5, 6])], - [ - (1720053489.906165, [], [1, 2, 3, 4, 5, 6]), - (1720139673.741326, [], [1, 2, 3, 4, 5, 6]), - (1720257769.454421, [], [1, 2, 3, 4, 5, 6]), - (1720393246.483443, [], [1, 2, 3, 4, 5, 6]), - (1720653126.121555, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719980029.84453, [], [1, 2, 3, 4, 5, 6])], - [(1720579706.660021, [], [1, 2, 3, 4, 5, 6])], - [(1720044200.360238, [], [1, 2, 3, 4, 5, 6])], - [(1720081853.12837, [], [1, 2, 3, 4, 5, 6])], - [(1720052881.805602, [], [1, 2, 3, 4, 5, 6])], - [ - (1720072654.6007, [], [1, 2, 3, 4, 5, 6]), - (1720238922.414211, [], [1, 2, 3, 4, 5, 6]), - (1720410048.118631, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720307869.769519, [], [1, 2, 3, 4, 5, 6])], - [ - (1720016591.216479, [], [1, 2, 3, 4, 5, 6]), - (1720157097.134758, [], [1, 2, 3, 4, 5, 6]), - (1720238731.063819, [], [1, 2, 3, 4, 5, 6]), - (1720575486.387284, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719965711.424761, [], [1, 2, 3, 4, 5, 6]), - (1719965713.531779, [], [1, 2, 3, 4, 5, 6]), - (1720049361.440421, [], [1, 2, 3, 4, 5, 6]), - (1720049363.495467, [], [1, 2, 3, 4, 5, 6]), - (1720070584.34326, [], [1, 2, 3, 4, 5, 6]), - (1720070586.439897, [], [1, 2, 3, 4, 5, 6]), - (1720070588.168108, [], [1, 2, 3, 4, 5, 6]), - (1720154076.359366, [], [1, 2, 3, 4, 5, 6]), - (1720154271.555582, [], [1, 2, 3, 4, 5, 6]), - (1720221139.621509, [], [1, 2, 3, 4, 5, 6]), - (1720221264.378154, [], [1, 2, 3, 4, 5, 6]), - (1720221266.479342, [], [1, 2, 3, 4, 5, 6]), - (1720239415.452192, [], [1, 2, 3, 4, 5, 6]), - (1720239475.014596, [], [1, 2, 3, 4, 5, 6]), - (1720239477.07742, [], [1, 2, 3, 4, 5, 6]), - (1720313035.217622, [], [1, 2, 3, 4, 5, 6]), - (1720313041.373706, [], [1, 2, 3, 4, 5, 6]), - (1720313043.420222, [], [1, 2, 3, 4, 5, 6]), - (1720406631.8599, [], [1, 2, 3, 4, 5, 6]), - (1720406659.023715, [], [1, 2, 3, 4, 5, 6]), - (1720484615.165994, [], [1, 2, 3, 4, 5, 6]), - (1720484638.913162, [], [1, 2, 3, 4, 5, 6]), - (1720497880.450011, [], [1, 2, 3, 4, 5, 6]), - (1720497934.842426, [], [1, 2, 3, 4, 5, 6]), - (1720497936.912581, [], [1, 2, 3, 4, 5, 6]), - (1720540604.563371, [], [1, 2, 3, 4, 5, 6]), - (1720540779.42356, [], [1, 2, 3, 4, 5, 6]), - (1720540781.553641, [], [1, 2, 3, 4, 5, 6]), - (1720570083.468668, [], [1, 2, 3, 4, 5, 6]), - (1720570174.103962, [], [1, 2, 3, 4, 5, 6]), - (1720570176.16906, [], [1, 2, 3, 4, 5, 6]), - (1720583667.401678, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719984388.470208, [], [1, 2, 3, 4, 5, 6]), - (1720057777.644161, [], [1, 2, 3, 4, 5, 6]), - (1720057953.012529, [], [1, 2, 3, 4, 5, 6]), - (1720057955.119335, [], [1, 2, 3, 4, 5, 6]), - (1720148795.685708, [], [1, 2, 3, 4, 5, 6]), - (1720148805.802813, [], [1, 2, 3, 4, 5, 6]), - (1720148807.834184, [], [1, 2, 3, 4, 5, 6]), - (1720234649.035149, [], [1, 2, 3, 4, 5, 6]), - (1720234705.911474, [], [1, 2, 3, 4, 5, 6]), - (1720321956.32096, [], [1, 2, 3, 4, 5, 6]), - (1720321958.41442, [], [1, 2, 3, 4, 5, 6]), - (1720409662.220157, [], [1, 2, 3, 4, 5, 6]), - (1720409664.333692, [], [1, 2, 3, 4, 5, 6]), - (1720447795.315077, [], [1, 2, 3, 4, 5, 6]), - (1720447797.391813, [], [1, 2, 3, 4, 5, 6]), - (1720483016.135213, [], [1, 2, 3, 4, 5, 6]), - (1720483018.21033, [], [1, 2, 3, 4, 5, 6]), - (1720483019.948558, [], [1, 2, 3, 4, 5, 6]), - (1720573042.040836, [], [1, 2, 3, 4, 5, 6]), - (1720573166.473551, [], [1, 2, 3, 4, 5, 6]), - (1720624631.359534, [], [1, 2, 3, 4, 5, 6]), - (1720624673.478312, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720072684.014623, [], [1, 2, 3, 4, 5, 6]), - (1720308176.896274, [], [1, 2, 3, 4, 5, 6]), - (1720404626.250723, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719970599.437881, [], [1, 2, 3, 4, 5, 6]), - (1720066959.831691, [], [1, 2, 3, 4, 5, 6]), - (1720067155.392921, [], [1, 2, 3, 4, 5, 6]), - (1720067157.488109, [], [1, 2, 3, 4, 5, 6]), - (1720155932.952678, [], [1, 2, 3, 4, 5, 6]), - (1720156052.447154, [], [1, 2, 3, 4, 5, 6]), - (1720156054.565087, [], [1, 2, 3, 4, 5, 6]), - (1720176991.157569, [], [1, 2, 3, 4, 5, 6]), - (1720177007.156433, [], [1, 2, 3, 4, 5, 6]), - (1720197632.204363, [], [1, 2, 3, 4, 5, 6]), - (1720197634.246364, [], [1, 2, 3, 4, 5, 6]), - (1720245139.96838, [], [1, 2, 3, 4, 5, 6]), - (1720245142.060531, [], [1, 2, 3, 4, 5, 6]), - (1720313085.938317, [], [1, 2, 3, 4, 5, 6]), - (1720313087.991269, [], [1, 2, 3, 4, 5, 6]), - (1720382887.239454, [], [1, 2, 3, 4, 5, 6]), - (1720383102.784337, [], [1, 2, 3, 4, 5, 6]), - (1720383104.858248, [], [1, 2, 3, 4, 5, 6]), - (1720401645.882852, [], [1, 2, 3, 4, 5, 6]), - (1720401875.751914, [], [1, 2, 3, 4, 5, 6]), - (1720401877.871081, [], [1, 2, 3, 4, 5, 6]), - (1720401879.349072, [], [1, 2, 3, 4, 5, 6]), - (1720401881.439362, [], [1, 2, 3, 4, 5, 6]), - (1720414595.608826, [], [1, 2, 3, 4, 5, 6]), - (1720414633.289105, [], [1, 2, 3, 4, 5, 6]), - (1720414635.359202, [], [1, 2, 3, 4, 5, 6]), - (1720417003.580682, [], [1, 2, 3, 4, 5, 6]), - (1720417074.130853, [], [1, 2, 3, 4, 5, 6]), - (1720417076.197409, [], [1, 2, 3, 4, 5, 6]), - (1720480715.553431, [], [1, 2, 3, 4, 5, 6]), - (1720480828.705337, [], [1, 2, 3, 4, 5, 6]), - (1720480830.783164, [], [1, 2, 3, 4, 5, 6]), - (1720579482.271054, [], [1, 2, 3, 4, 5, 6]), - (1720579503.249382, [], [1, 2, 3, 4, 5, 6]), - (1720579505.309044, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720320578.9281, [], [1, 2, 3, 4, 5, 6]), - (1720320637.501704, [], [1, 2, 3, 4, 5, 6]), - (1720320671.3461, [], [1, 2, 3, 4, 5, 6]), - (1720320700.900689, [], [1, 2, 3, 4, 5, 6]), - (1720326925.640353, [], [1, 2, 3, 4, 5, 6]), - (1720326927.740401, [], [1, 2, 3, 4, 5, 6]), - (1720397880.433215, [], [1, 2, 3, 4, 5, 6]), - (1720397991.949085, [], [1, 2, 3, 4, 5, 6]), - (1720407395.883774, [], [1, 2, 3, 4, 5, 6]), - (1720478571.564518, [], [1, 2, 3, 4, 5, 6]), - (1720478573.689929, [], [1, 2, 3, 4, 5, 6]), - (1720496055.710657, [], [1, 2, 3, 4, 5, 6]), - (1720496250.423433, [], [1, 2, 3, 4, 5, 6]), - (1720496252.533919, [], [1, 2, 3, 4, 5, 6]), - (1720567595.861171, [], [1, 2, 3, 4, 5, 6]), - (1720567634.8402, [], [1, 2, 3, 4, 5, 6]), - (1720567636.90682, [], [1, 2, 3, 4, 5, 6]), - (1720652785.237133, [], [1, 2, 3, 4, 5, 6]), - (1720652836.758383, [], [1, 2, 3, 4, 5, 6]), - (1720652838.851539, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720076323.446198, [], [1, 2, 3, 4, 5, 6]), - (1720076387.202961, [], [1, 2, 3, 4, 5, 6]), - (1720076463.503404, [], [1, 2, 3, 4, 5, 6]), - (1720235511.111341, [], [1, 2, 3, 4, 5, 6]), - (1720235587.152102, [], [1, 2, 3, 4, 5, 6]), - (1720235604.104726, [], [1, 2, 3, 4, 5, 6]), - (1720235788.441489, [], [1, 2, 3, 4, 5, 6]), - (1720235897.829327, [], [1, 2, 3, 4, 5, 6]), - (1720235943.361057, [], [1, 2, 3, 4, 5, 6]), - (1720236088.122922, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719969865.146509, [], [1, 2, 3, 4, 5, 6])], - [ - (1720088372.900286, [], [1, 2, 3, 4, 5, 6]), - (1720220494.799398, [], [1, 2, 3, 4, 5, 6]), - (1720488909.409034, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720063420.61895, [], [1, 2, 3, 4, 5, 6]), - (1720320803.577679, [], [1, 2, 3, 4, 5, 6]), - (1720401999.385093, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720051291.94966, [], [1, 2, 3, 4, 5, 6]), (1720313692.587964, [], [1, 2, 3, 4, 5, 6])], - [(1720509709.170957, [], [1, 2, 3, 4, 5, 6]), (1720652928.475583, [], [1, 2, 3, 4, 5, 6])], - [(1719976500.586248, [], [1, 2, 3, 4, 5, 6])], - [(1720063184.061031, [], [1, 2, 3, 4, 5, 6])], - [(1720147998.634564, [], [1, 2, 3, 4, 5, 6]), (1720575037.093899, [], [1, 2, 3, 4, 5, 6])], - [(1720594897.858543, [], [1, 2, 3, 4, 5, 6])], - [ - (1720238660.290085, [], [1, 2, 3, 4, 5, 6]), - (1720306835.46462, [], [1, 2, 3, 4, 5, 6]), - (1720401110.356341, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719970976.422012, [], [1, 2, 3, 4, 5, 6]), - (1720051629.137902, [], [1, 2, 3, 4, 5, 6]), - (1720301759.327348, [], [1, 2, 3, 4, 5, 6]), - (1720646663.705407, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720077214.628033, [], [1, 2, 3, 4, 5, 6]), (1720579842.451312, [], [1, 2, 3, 4, 5, 6])], - [(1720403179.578891, [], [1, 2, 3, 4, 5, 6]), (1720573175.772465, [], [1, 2, 3, 4, 5, 6])], - [ - (1720418161.36421, [], [1, 2, 3, 4, 5, 6]), - (1720418260.71249, [], [1, 2, 3, 4, 5, 6]), - (1720418315.726571, [], [1, 2, 3, 4, 5, 6]), - (1720418380.031953, [], [1, 2, 3, 4, 5, 6]), - (1720491482.634884, [], [1, 2, 3, 4, 5, 6]), - (1720491528.332034, [], [1, 2, 3, 4, 5, 6]), - (1720491530.434212, [], [1, 2, 3, 4, 5, 6]), - (1720573999.084897, [], [1, 2, 3, 4, 5, 6]), - (1720574047.543743, [], [1, 2, 3, 4, 5, 6]), - (1720574049.630747, [], [1, 2, 3, 4, 5, 6]), - (1720616534.181229, [], [1, 2, 3, 4, 5, 6]), - (1720616536.227681, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719971505.975792, [], [1, 2, 3, 4, 5, 6]), - (1720309386.016213, [], [1, 2, 3, 4, 5, 6]), - (1720570539.167487, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720026981.201845, [], [1, 2, 3, 4, 5, 6]), - (1720063393.228975, [], [1, 2, 3, 4, 5, 6]), - (1720569870.489752, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720151334.562097, [], [1, 2, 3, 4, 5, 6]), - (1720311454.894847, [], [1, 2, 3, 4, 5, 6]), - (1720483363.072169, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720051176.858561, [], [1, 2, 3, 4, 5, 6]), (1720567049.0934, [], [1, 2, 3, 4, 5, 6])], - [ - (1720300278.98565, [], [1, 2, 3, 4, 5, 6]), - (1720397622.151994, [], [1, 2, 3, 4, 5, 6]), - (1720487075.583534, [], [1, 2, 3, 4, 5, 6]), - (1720572927.092976, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720588311.663694, [], [1, 2, 3, 4, 5, 6])], - [(1720323338.447054, [], [1, 2, 3, 4, 5, 6])], - [(1720499740.21156, [], [1, 2, 3, 4, 5, 6])], - [(1720239215.924016, [], [1, 2, 3, 4, 5, 6]), (1720407168.197067, [], [1, 2, 3, 4, 5, 6])], - [(1720496803.3956, [], [1, 2, 3, 4, 5, 6])], - [(1720578053.220463, [], [1, 2, 3, 4, 5, 6])], - [(1720400566.962842, [], [1, 2, 3, 4, 5, 6]), (1720652817.676145, [], [1, 2, 3, 4, 5, 6])], - [ - (1720234566.739671, [], [1, 2, 3, 4, 5, 6]), - (1720335553.524142, [], [1, 2, 3, 4, 5, 6]), - (1720576366.993741, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720058095.694475, [], [1, 2, 3, 4, 5, 6])], - [(1720499846.305083, [], [1, 2, 3, 4, 5, 6])], - [(1720069736.856188, [], [1, 2, 3, 4, 5, 6]), (1720327054.018462, [], [1, 2, 3, 4, 5, 6])], - [(1720446989.50202, [], [1, 2, 3, 4, 5, 6]), (1720579246.321269, [], [1, 2, 3, 4, 5, 6])], - [ - (1720065515.046196, [], [1, 2, 3, 4, 5, 6]), - (1720237193.252454, [], [1, 2, 3, 4, 5, 6]), - (1720402549.014306, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719965737.195869, [], [1, 2, 3, 4, 5, 6]), - (1720057334.427369, [], [1, 2, 3, 4, 5, 6]), - (1720221205.840325, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720366343.985963, [], [1, 2, 3, 4, 5, 6])], - [(1720195598.557753, [], [1, 2, 3, 4, 5, 6])], - [(1719980678.939359, [], [1, 2, 3, 4, 5, 6]), (1720144995.169791, [], [1, 2, 3, 4, 5, 6])], - [(1720583721.214132, [], [1, 2, 3, 4, 5, 6])], - [ - (1720054537.756175, [], [1, 2, 3, 4, 5, 6]), - (1720182546.976397, [], [1, 2, 3, 4, 5, 6]), - (1720415420.418491, [], [1, 2, 3, 4, 5, 6]), - (1720491014.558376, [], [1, 2, 3, 4, 5, 6]), - (1720568712.840731, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720061881.320656, [], [1, 2, 3, 4, 5, 6]), - (1720352102.705661, [], [1, 2, 3, 4, 5, 6]), - (1720405007.368123, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720578542.112522, [], [1, 2, 3, 4, 5, 6]), (1720652582.644175, [], [1, 2, 3, 4, 5, 6])], - [(1720243826.422296, [], [1, 2, 3, 4, 5, 6])], - [ - (1719980177.888359, [], [1, 2, 3, 4, 5, 6]), - (1720151191.015847, [], [1, 2, 3, 4, 5, 6]), - (1720578744.147878, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720401711.559261, [], [1, 2, 3, 4, 5, 6])], - [ - (1720166076.219132, [], [1, 2, 3, 4, 5, 6]), - (1720603429.438791, [], [1, 2, 3, 4, 5, 6]), - (1720653029.910009, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720060428.267185, [], [1, 2, 3, 4, 5, 6]), (1720330543.204866, [], [1, 2, 3, 4, 5, 6])], - [(1720023596.681592, [], [1, 2, 3, 4, 5, 6]), (1720492712.578171, [], [1, 2, 3, 4, 5, 6])], - [ - (1719969468.250895, [], [1, 2, 3, 4, 5, 6]), - (1719969673.12603, [], [1, 2, 3, 4, 5, 6]), - (1719984075.563213, [], [1, 2, 3, 4, 5, 6]), - (1719984158.542506, [], [1, 2, 3, 4, 5, 6]), - (1719984160.639094, [], [1, 2, 3, 4, 5, 6]), - (1719984161.840978, [], [1, 2, 3, 4, 5, 6]), - (1720063496.706949, [], [1, 2, 3, 4, 5, 6]), - (1720063498.73801, [], [1, 2, 3, 4, 5, 6]), - (1720071811.706281, [], [1, 2, 3, 4, 5, 6]), - (1720071892.339384, [], [1, 2, 3, 4, 5, 6]), - (1720071894.401443, [], [1, 2, 3, 4, 5, 6]), - (1720141092.670014, [], [1, 2, 3, 4, 5, 6]), - (1720141281.129314, [], [1, 2, 3, 4, 5, 6]), - (1720141283.221705, [], [1, 2, 3, 4, 5, 6]), - (1720242813.561025, [], [1, 2, 3, 4, 5, 6]), - (1720242887.120065, [], [1, 2, 3, 4, 5, 6]), - (1720242889.197122, [], [1, 2, 3, 4, 5, 6]), - (1720314744.527265, [], [1, 2, 3, 4, 5, 6]), - (1720314759.642908, [], [1, 2, 3, 4, 5, 6]), - (1720314761.711826, [], [1, 2, 3, 4, 5, 6]), - (1720371097.307425, [], [1, 2, 3, 4, 5, 6]), - (1720371255.548011, [], [1, 2, 3, 4, 5, 6]), - (1720387058.372995, [], [1, 2, 3, 4, 5, 6]), - (1720387060.502073, [], [1, 2, 3, 4, 5, 6]), - (1720404975.528018, [], [1, 2, 3, 4, 5, 6]), - (1720405017.654969, [], [1, 2, 3, 4, 5, 6]), - (1720405019.759385, [], [1, 2, 3, 4, 5, 6]), - (1720415081.995346, [], [1, 2, 3, 4, 5, 6]), - (1720415260.662438, [], [1, 2, 3, 4, 5, 6]), - (1720415262.742795, [], [1, 2, 3, 4, 5, 6]), - (1720485117.023333, [], [1, 2, 3, 4, 5, 6]), - (1720485119.095263, [], [1, 2, 3, 4, 5, 6]), - (1720499098.798714, [], [1, 2, 3, 4, 5, 6]), - (1720499235.930954, [], [1, 2, 3, 4, 5, 6]), - (1720499238.042676, [], [1, 2, 3, 4, 5, 6]), - (1720569739.038396, [], [1, 2, 3, 4, 5, 6]), - (1720569853.204944, [], [1, 2, 3, 4, 5, 6]), - (1720569855.2363, [], [1, 2, 3, 4, 5, 6]), - (1720602936.713875, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720242969.422529, [], [1, 2, 3, 4, 5, 6])], - [ - (1719964838.834772, [], [1, 2, 3, 4, 5, 6]), - (1720091821.44426, [], [1, 2, 3, 4, 5, 6]), - (1720233192.310563, [], [1, 2, 3, 4, 5, 6]), - (1720328141.79034, [], [1, 2, 3, 4, 5, 6]), - (1720496224.014897, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720152285.903631, [], [1, 2, 3, 4, 5, 6]), - (1720243334.372125, [], [1, 2, 3, 4, 5, 6]), - (1720379463.678962, [], [1, 2, 3, 4, 5, 6]), - (1720542353.379097, [], [1, 2, 3, 4, 5, 6]), - (1720649368.688756, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720226523.433881, [], [1, 2, 3, 4, 5, 6]), - (1720226623.918185, [], [1, 2, 3, 4, 5, 6]), - (1720226651.194519, [], [1, 2, 3, 4, 5, 6]), - (1720226742.683406, [], [1, 2, 3, 4, 5, 6]), - (1720330187.550014, [], [1, 2, 3, 4, 5, 6]), - (1720330210.231169, [], [1, 2, 3, 4, 5, 6]), - (1720410272.539479, [], [1, 2, 3, 4, 5, 6]), - (1720410274.655647, [], [1, 2, 3, 4, 5, 6]), - (1720480303.2625, [], [1, 2, 3, 4, 5, 6]), - (1720480415.671856, [], [1, 2, 3, 4, 5, 6]), - (1720480417.738288, [], [1, 2, 3, 4, 5, 6]), - (1720504693.498524, [], [1, 2, 3, 4, 5, 6]), - (1720504764.21831, [], [1, 2, 3, 4, 5, 6]), - (1720504766.268173, [], [1, 2, 3, 4, 5, 6]), - (1720568377.567722, [], [1, 2, 3, 4, 5, 6]), - (1720568452.706691, [], [1, 2, 3, 4, 5, 6]), - (1720568454.778127, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720150751.139246, [], [1, 2, 3, 4, 5, 6])], - [ - (1720064945.077586, [], [1, 2, 3, 4, 5, 6]), - (1720176851.77124, [], [1, 2, 3, 4, 5, 6]), - (1720413751.53369, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720067667.982332, [], [1, 2, 3, 4, 5, 6]), - (1720498323.491767, [], [1, 2, 3, 4, 5, 6]), - (1720640332.912224, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720586416.962046, [], [1, 2, 3, 4, 5, 6])], - [(1720403065.106609, [], [1, 2, 3, 4, 5, 6]), (1720468529.097883, [], [1, 2, 3, 4, 5, 6])], - [(1719976409.626599, [], [1, 2, 3, 4, 5, 6]), (1720617974.74258, [], [1, 2, 3, 4, 5, 6])], - [(1720155789.338418, [], [1, 2, 3, 4, 5, 6])], - [(1719965523.519862, [], [1, 2, 3, 4, 5, 6])], - [(1720492317.02938, [], [1, 2, 3, 4, 5, 6])], - [ - (1719971602.527103, [], [1, 2, 3, 4, 5, 6]), - (1720069918.036547, [], [1, 2, 3, 4, 5, 6]), - (1720149900.77775, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720143447.493022, [], [1, 2, 3, 4, 5, 6]), (1720292005.708479, [], [1, 2, 3, 4, 5, 6])], - [(1720316731.010367, [], [1, 2, 3, 4, 5, 6])], - [(1720061643.180585, [], [1, 2, 3, 4, 5, 6])], - [ - (1719998587.453659, [], [1, 2, 3, 4, 5, 6]), - (1720141642.077196, [], [1, 2, 3, 4, 5, 6]), - (1720308402.56405, [], [1, 2, 3, 4, 5, 6]), - (1720416584.28358, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720587211.681421, [], [1, 2, 3, 4, 5, 6])], - [(1720155992.271335, [], [1, 2, 3, 4, 5, 6]), (1720572458.818891, [], [1, 2, 3, 4, 5, 6])], - [(1720052898.053761, [], [1, 2, 3, 4, 5, 6])], - [(1720221610.587492, [], [1, 2, 3, 4, 5, 6]), (1720312064.403238, [], [1, 2, 3, 4, 5, 6])], - [ - (1720147178.948005, [], [1, 2, 3, 4, 5, 6]), - (1720315892.264762, [], [1, 2, 3, 4, 5, 6]), - (1720484335.142158, [], [1, 2, 3, 4, 5, 6]), - (1720625426.867126, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720225964.225383, [], [1, 2, 3, 4, 5, 6]), (1720486617.901554, [], [1, 2, 3, 4, 5, 6])], - [(1720064206.744859, [], [1, 2, 3, 4, 5, 6]), (1720148855.512919, [], [1, 2, 3, 4, 5, 6])], - [ - (1719978022.859036, [], [1, 2, 3, 4, 5, 6]), - (1720225065.814898, [], [1, 2, 3, 4, 5, 6]), - (1720379679.901663, [], [1, 2, 3, 4, 5, 6]), - (1720486481.106043, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720072107.565425, [], [1, 2, 3, 4, 5, 6]), (1720244247.598994, [], [1, 2, 3, 4, 5, 6])], - [(1720418305.8177, [], [1, 2, 3, 4, 5, 6])], - [(1719982059.871483, [], [1, 2, 3, 4, 5, 6])], - [(1719978817.113542, [], [1, 2, 3, 4, 5, 6])], - [(1720138229.977179, [], [1, 2, 3, 4, 5, 6])], - [(1720069967.627991, [], [1, 2, 3, 4, 5, 6]), (1720327850.533176, [], [1, 2, 3, 4, 5, 6])], - [(1720047008.96098, [], [1, 2, 3, 4, 5, 6]), (1720399885.70332, [], [1, 2, 3, 4, 5, 6])], - [(1720241326.183043, [], [1, 2, 3, 4, 5, 6])], - [ - (1719974240.573502, [], [1, 2, 3, 4, 5, 6]), - (1719974319.570753, [], [1, 2, 3, 4, 5, 6]), - (1719974321.645618, [], [1, 2, 3, 4, 5, 6]), - (1720023281.111928, [], [1, 2, 3, 4, 5, 6]), - (1720023484.341612, [], [1, 2, 3, 4, 5, 6]), - (1720023486.442918, [], [1, 2, 3, 4, 5, 6]), - (1720056803.840688, [], [1, 2, 3, 4, 5, 6]), - (1720056805.906524, [], [1, 2, 3, 4, 5, 6]), - (1720056810.106719, [], [1, 2, 3, 4, 5, 6]), - (1720056812.20004, [], [1, 2, 3, 4, 5, 6]), - (1720070833.346034, [], [1, 2, 3, 4, 5, 6]), - (1720070835.405627, [], [1, 2, 3, 4, 5, 6]), - (1720070839.751918, [], [1, 2, 3, 4, 5, 6]), - (1720070841.8631, [], [1, 2, 3, 4, 5, 6]), - (1720143274.991396, [], [1, 2, 3, 4, 5, 6]), - (1720143497.358536, [], [1, 2, 3, 4, 5, 6]), - (1720205396.067954, [], [1, 2, 3, 4, 5, 6]), - (1720205567.621928, [], [1, 2, 3, 4, 5, 6]), - (1720235968.291387, [], [1, 2, 3, 4, 5, 6]), - (1720236089.012578, [], [1, 2, 3, 4, 5, 6]), - (1720236091.096483, [], [1, 2, 3, 4, 5, 6]), - (1720299309.484376, [], [1, 2, 3, 4, 5, 6]), - (1720299348.029909, [], [1, 2, 3, 4, 5, 6]), - (1720299350.111093, [], [1, 2, 3, 4, 5, 6]), - (1720320587.866423, [], [1, 2, 3, 4, 5, 6]), - (1720320589.944508, [], [1, 2, 3, 4, 5, 6]), - (1720400218.389368, [], [1, 2, 3, 4, 5, 6]), - (1720400220.487059, [], [1, 2, 3, 4, 5, 6]), - (1720492976.763916, [], [1, 2, 3, 4, 5, 6]), - (1720493019.503907, [], [1, 2, 3, 4, 5, 6]), - (1720493021.579652, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720122662.758187, [], [1, 2, 3, 4, 5, 6])], - [ - (1720005466.226878, [], [1, 2, 3, 4, 5, 6]), - (1720152200.991189, [], [1, 2, 3, 4, 5, 6]), - (1720481835.844194, [], [1, 2, 3, 4, 5, 6]), - (1720580183.828864, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720070662.89957, [], [1, 2, 3, 4, 5, 6])], - [(1720142183.650314, [], [1, 2, 3, 4, 5, 6]), (1720411252.551579, [], [1, 2, 3, 4, 5, 6])], - [(1720072507.675137, [], [1, 2, 3, 4, 5, 6]), (1720238913.204492, [], [1, 2, 3, 4, 5, 6])], - [(1720218011.114445, [], [1, 2, 3, 4, 5, 6])], - [(1720302385.101079, [], [1, 2, 3, 4, 5, 6])], - [(1720570186.762002, [], [1, 2, 3, 4, 5, 6])], - [(1720157661.668233, [], [1, 2, 3, 4, 5, 6])], - [ - (1720002096.475822, [], [1, 2, 3, 4, 5, 6]), - (1720002098.548489, [], [1, 2, 3, 4, 5, 6]), - (1720066947.715378, [], [1, 2, 3, 4, 5, 6]), - (1720066949.844651, [], [1, 2, 3, 4, 5, 6]), - (1720105199.7729, [], [1, 2, 3, 4, 5, 6]), - (1720105370.978068, [], [1, 2, 3, 4, 5, 6]), - (1720105373.072451, [], [1, 2, 3, 4, 5, 6]), - (1720134686.761223, [], [1, 2, 3, 4, 5, 6]), - (1720134688.87969, [], [1, 2, 3, 4, 5, 6]), - (1720154938.58341, [], [1, 2, 3, 4, 5, 6]), - (1720219977.007902, [], [1, 2, 3, 4, 5, 6]), - (1720219979.064108, [], [1, 2, 3, 4, 5, 6]), - (1720250673.20197, [], [1, 2, 3, 4, 5, 6]), - (1720250796.014257, [], [1, 2, 3, 4, 5, 6]), - (1720313894.440119, [], [1, 2, 3, 4, 5, 6]), - (1720313896.495204, [], [1, 2, 3, 4, 5, 6]), - (1720313896.774105, [], [1, 2, 3, 4, 5, 6]), - (1720396853.1643, [], [1, 2, 3, 4, 5, 6]), - (1720396855.257174, [], [1, 2, 3, 4, 5, 6]), - (1720489852.456446, [], [1, 2, 3, 4, 5, 6]), - (1720489986.86769, [], [1, 2, 3, 4, 5, 6]), - (1720569682.852233, [], [1, 2, 3, 4, 5, 6]), - (1720569767.225411, [], [1, 2, 3, 4, 5, 6]), - (1720569769.334261, [], [1, 2, 3, 4, 5, 6]), - (1720581192.763754, [], [1, 2, 3, 4, 5, 6]), - (1720581262.544992, [], [1, 2, 3, 4, 5, 6]), - (1720581264.629216, [], [1, 2, 3, 4, 5, 6]), - (1720581266.203535, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720226441.129883, [], [1, 2, 3, 4, 5, 6]), - (1720226519.948161, [], [1, 2, 3, 4, 5, 6]), - (1720226639.444269, [], [1, 2, 3, 4, 5, 6]), - (1720226731.198095, [], [1, 2, 3, 4, 5, 6]), - (1720226779.385516, [], [1, 2, 3, 4, 5, 6]), - (1720226788.674966, [], [1, 2, 3, 4, 5, 6]), - (1720226923.560385, [], [1, 2, 3, 4, 5, 6]), - (1720487951.436457, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720038002.665527, [], [1, 2, 3, 4, 5, 6]), - (1720488750.698306, [], [1, 2, 3, 4, 5, 6]), - (1720589885.270178, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720580260.501907, [], [1, 2, 3, 4, 5, 6])], - [(1720237662.32438, [], [1, 2, 3, 4, 5, 6]), (1720309542.106408, [], [1, 2, 3, 4, 5, 6])], - [(1720573441.412558, [], [1, 2, 3, 4, 5, 6])], - [(1720598006.382998, [], [1, 2, 3, 4, 5, 6])], - [(1720249262.676111, [], [1, 2, 3, 4, 5, 6])], - [ - (1719968867.096755, [], [1, 2, 3, 4, 5, 6]), - (1720406515.635103, [], [1, 2, 3, 4, 5, 6]), - (1720568473.896114, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720063183.618875, [], [1, 2, 3, 4, 5, 6])], - [(1720313109.473272, [], [1, 2, 3, 4, 5, 6]), (1720418083.174351, [], [1, 2, 3, 4, 5, 6])], - [(1720106024.609, [], [1, 2, 3, 4, 5, 6])], - [(1720058454.321955, [], [1, 2, 3, 4, 5, 6])], - [ - (1720062639.625591, [], [1, 2, 3, 4, 5, 6]), - (1720221670.858026, [], [1, 2, 3, 4, 5, 6]), - (1720496857.495022, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719974856.395033, [], [1, 2, 3, 4, 5, 6]), - (1720053608.017225, [], [1, 2, 3, 4, 5, 6]), - (1720053669.477447, [], [1, 2, 3, 4, 5, 6]), - (1720053671.560518, [], [1, 2, 3, 4, 5, 6]), - (1720062649.578553, [], [1, 2, 3, 4, 5, 6]), - (1720062651.66265, [], [1, 2, 3, 4, 5, 6]), - (1720062653.377042, [], [1, 2, 3, 4, 5, 6]), - (1720062655.44862, [], [1, 2, 3, 4, 5, 6]), - (1720082692.43711, [], [1, 2, 3, 4, 5, 6]), - (1720082771.847834, [], [1, 2, 3, 4, 5, 6]), - (1720153671.808662, [], [1, 2, 3, 4, 5, 6]), - (1720153713.489374, [], [1, 2, 3, 4, 5, 6]), - (1720153715.599402, [], [1, 2, 3, 4, 5, 6]), - (1720153716.387598, [], [1, 2, 3, 4, 5, 6]), - (1720153718.446591, [], [1, 2, 3, 4, 5, 6]), - (1720239246.152588, [], [1, 2, 3, 4, 5, 6]), - (1720239248.241024, [], [1, 2, 3, 4, 5, 6]), - (1720239250.77294, [], [1, 2, 3, 4, 5, 6]), - (1720309956.683905, [], [1, 2, 3, 4, 5, 6]), - (1720310071.602061, [], [1, 2, 3, 4, 5, 6]), - (1720325462.049867, [], [1, 2, 3, 4, 5, 6]), - (1720325503.88631, [], [1, 2, 3, 4, 5, 6]), - (1720325506.004479, [], [1, 2, 3, 4, 5, 6]), - (1720412942.492135, [], [1, 2, 3, 4, 5, 6]), - (1720458829.36376, [], [1, 2, 3, 4, 5, 6]), - (1720458949.800013, [], [1, 2, 3, 4, 5, 6]), - (1720458951.887504, [], [1, 2, 3, 4, 5, 6]), - (1720492407.820081, [], [1, 2, 3, 4, 5, 6]), - (1720492444.404348, [], [1, 2, 3, 4, 5, 6]), - (1720492446.466946, [], [1, 2, 3, 4, 5, 6]), - (1720575932.543872, [], [1, 2, 3, 4, 5, 6]), - (1720576033.410802, [], [1, 2, 3, 4, 5, 6]), - (1720576035.469127, [], [1, 2, 3, 4, 5, 6]), - (1720576036.44253, [], [1, 2, 3, 4, 5, 6]), - (1720647287.059052, [], [1, 2, 3, 4, 5, 6]), - (1720647289.160943, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720330899.088291, [], [1, 2, 3, 4, 5, 6])], - [ - (1720081793.462679, [], [1, 2, 3, 4, 5, 6]), - (1720081976.131384, [], [1, 2, 3, 4, 5, 6]), - (1720081976.187009, [], [1, 2, 3, 4, 5, 6]), - (1720081980.325716, [], [1, 2, 3, 4, 5, 6]), - (1720091170.691618, [], [1, 2, 3, 4, 5, 6]), - (1720091356.840132, [], [1, 2, 3, 4, 5, 6]), - (1720091358.928927, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720238399.047753, [], [1, 2, 3, 4, 5, 6]), (1720567368.683366, [], [1, 2, 3, 4, 5, 6])], - [(1720583646.429286, [], [1, 2, 3, 4, 5, 6])], - [(1720330049.16818, [], [1, 2, 3, 4, 5, 6])], - [(1720578076.222297, [], [1, 2, 3, 4, 5, 6])], - [(1719972331.112716, [], [1, 2, 3, 4, 5, 6]), (1720051745.533132, [], [1, 2, 3, 4, 5, 6])], - [(1720146463.601388, [], [1, 2, 3, 4, 5, 6]), (1720474541.840768, [], [1, 2, 3, 4, 5, 6])], - [ - (1720051898.891617, [], [1, 2, 3, 4, 5, 6]), - (1720173013.609275, [], [1, 2, 3, 4, 5, 6]), - (1720320493.657042, [], [1, 2, 3, 4, 5, 6]), - (1720345690.851927, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720408818.140609, [], [1, 2, 3, 4, 5, 6]), (1720486964.900766, [], [1, 2, 3, 4, 5, 6])], - [ - (1720043728.363322, [], [1, 2, 3, 4, 5, 6]), - (1720311975.271982, [], [1, 2, 3, 4, 5, 6]), - (1720571578.431424, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719972681.535491, [], [1, 2, 3, 4, 5, 6]), (1720481018.937328, [], [1, 2, 3, 4, 5, 6])], - [ - (1720026382.553301, [], [1, 2, 3, 4, 5, 6]), - (1720149100.706808, [], [1, 2, 3, 4, 5, 6]), - (1720307942.507634, [], [1, 2, 3, 4, 5, 6]), - (1720570495.6023, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720488173.923373, [], [1, 2, 3, 4, 5, 6]), (1720568447.027354, [], [1, 2, 3, 4, 5, 6])], - [(1720567270.126647, [], [1, 2, 3, 4, 5, 6])], - [(1720499324.726509, [], [1, 2, 3, 4, 5, 6])], - [(1719967975.358552, [], [1, 2, 3, 4, 5, 6]), (1720326137.056104, [], [1, 2, 3, 4, 5, 6])], - [(1720328587.433829, [], [1, 2, 3, 4, 5, 6])], - [ - (1720055336.001624, [], [1, 2, 3, 4, 5, 6]), - (1720138863.165013, [], [1, 2, 3, 4, 5, 6]), - (1720395924.519387, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720069032.516709, [], [1, 2, 3, 4, 5, 6])], - [(1720055010.518163, [], [1, 2, 3, 4, 5, 6]), (1720099339.150894, [], [1, 2, 3, 4, 5, 6])], - [(1720585549.317132, [], [1, 2, 3, 4, 5, 6])], - [ - (1720409221.504822, [], [1, 2, 3, 4, 5, 6]), - (1720519728.003909, [], [1, 2, 3, 4, 5, 6]), - (1720567616.396835, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720244626.143033, [], [1, 2, 3, 4, 5, 6]), - (1720408635.777109, [], [1, 2, 3, 4, 5, 6]), - (1720575490.310245, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720650559.825556, [], [1, 2, 3, 4, 5, 6])], - [(1720582328.399682, [], [1, 2, 3, 4, 5, 6])], - [(1720394085.218172, [], [1, 2, 3, 4, 5, 6])], - [ - (1720066065.444143, [], [1, 2, 3, 4, 5, 6]), - (1720191993.66672, [], [1, 2, 3, 4, 5, 6]), - (1720491329.586545, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720071639.073353, [], [1, 2, 3, 4, 5, 6]), - (1720147406.207381, [], [1, 2, 3, 4, 5, 6]), - (1720311129.234658, [], [1, 2, 3, 4, 5, 6]), - (1720573354.037576, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720240131.733437, [], [1, 2, 3, 4, 5, 6])], - [ - (1719984704.452873, [], [1, 2, 3, 4, 5, 6]), - (1720154844.587696, [], [1, 2, 3, 4, 5, 6]), - (1720322613.231449, [], [1, 2, 3, 4, 5, 6]), - (1720569714.825725, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720486946.256934, [], [1, 2, 3, 4, 5, 6]), (1720647691.877446, [], [1, 2, 3, 4, 5, 6])], - [ - (1719993805.960143, [], [1, 2, 3, 4, 5, 6]), - (1720143738.25635, [], [1, 2, 3, 4, 5, 6]), - (1720309437.19401, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720398437.265534, [], [1, 2, 3, 4, 5, 6]), - (1720509793.976335, [], [1, 2, 3, 4, 5, 6]), - (1720629661.586274, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720062670.596985, [], [1, 2, 3, 4, 5, 6]), (1720438472.029491, [], [1, 2, 3, 4, 5, 6])], - [(1720329032.038648, [], [1, 2, 3, 4, 5, 6])], - [(1719984110.370743, [], [1, 2, 3, 4, 5, 6])], - [(1719986035.664892, [], [1, 2, 3, 4, 5, 6]), (1720241225.374801, [], [1, 2, 3, 4, 5, 6])], - [ - (1720105075.810739, [], [1, 2, 3, 4, 5, 6]), - (1720150414.362845, [], [1, 2, 3, 4, 5, 6]), - (1720308174.216706, [], [1, 2, 3, 4, 5, 6]), - (1720412581.759663, [], [1, 2, 3, 4, 5, 6]), - (1720567554.209717, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720537016.009206, [], [1, 2, 3, 4, 5, 6]), (1720589963.468469, [], [1, 2, 3, 4, 5, 6])], - [(1720487391.820061, [], [1, 2, 3, 4, 5, 6])], - [ - (1719975756.8334, [], [1, 2, 3, 4, 5, 6]), - (1720155078.652264, [], [1, 2, 3, 4, 5, 6]), - (1720400698.199527, [], [1, 2, 3, 4, 5, 6]), - (1720496832.791723, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720018681.490249, [], [1, 2, 3, 4, 5, 6]), - (1720136925.765051, [], [1, 2, 3, 4, 5, 6]), - (1720299976.82238, [], [1, 2, 3, 4, 5, 6]), - (1720569027.317754, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720154725.361924, [], [1, 2, 3, 4, 5, 6]), - (1720243905.535885, [], [1, 2, 3, 4, 5, 6]), - (1720315512.512864, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720588095.521256, [], [1, 2, 3, 4, 5, 6])], - [(1719983442.727069, [], [1, 2, 3, 4, 5, 6]), (1720152453.391462, [], [1, 2, 3, 4, 5, 6])], - [(1720405182.158931, [], [1, 2, 3, 4, 5, 6]), (1720575807.583852, [], [1, 2, 3, 4, 5, 6])], - [(1720069878.030512, [], [1, 2, 3, 4, 5, 6]), (1720168504.534482, [], [1, 2, 3, 4, 5, 6])], - [(1720239119.238416, [], [1, 2, 3, 4, 5, 6])], - [(1720576430.211377, [], [1, 2, 3, 4, 5, 6])], - [ - (1719972965.48613, [], [1, 2, 3, 4, 5, 6]), - (1720240976.867243, [], [1, 2, 3, 4, 5, 6]), - (1720404977.394327, [], [1, 2, 3, 4, 5, 6]), - (1720537004.359466, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720487019.528315, [], [1, 2, 3, 4, 5, 6]), (1720581959.239135, [], [1, 2, 3, 4, 5, 6])], - [(1720652926.990055, [], [1, 2, 3, 4, 5, 6])], - [ - (1720052099.960777, [], [1, 2, 3, 4, 5, 6]), - (1720138855.98453, [], [1, 2, 3, 4, 5, 6]), - (1720138921.586511, [], [1, 2, 3, 4, 5, 6]), - (1720139014.744606, [], [1, 2, 3, 4, 5, 6]), - (1720139182.595009, [], [1, 2, 3, 4, 5, 6]), - (1720139192.3206, [], [1, 2, 3, 4, 5, 6]), - (1720581909.908771, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720065955.899244, [], [1, 2, 3, 4, 5, 6]), - (1720236828.579322, [], [1, 2, 3, 4, 5, 6]), - (1720308640.597753, [], [1, 2, 3, 4, 5, 6]), - (1720579634.738256, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719980159.276761, [], [1, 2, 3, 4, 5, 6]), - (1720308552.424302, [], [1, 2, 3, 4, 5, 6]), - (1720653256.063729, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719985271.443965, [], [1, 2, 3, 4, 5, 6]), - (1720220543.115385, [], [1, 2, 3, 4, 5, 6]), - (1720315297.143816, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720330015.747888, [], [1, 2, 3, 4, 5, 6]), (1720481610.656596, [], [1, 2, 3, 4, 5, 6])], - [(1720224041.283015, [], [1, 2, 3, 4, 5, 6]), (1720567371.834809, [], [1, 2, 3, 4, 5, 6])], - [ - (1720244217.827624, [], [1, 2, 3, 4, 5, 6]), - (1720402557.505715, [], [1, 2, 3, 4, 5, 6]), - (1720502124.284452, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720415547.576014, [], [1, 2, 3, 4, 5, 6]), (1720531682.711315, [], [1, 2, 3, 4, 5, 6])], - [(1720407411.272993, [], [1, 2, 3, 4, 5, 6]), (1720574508.629738, [], [1, 2, 3, 4, 5, 6])], - [(1720257290.163002, [], [1, 2, 3, 4, 5, 6]), (1720492975.717018, [], [1, 2, 3, 4, 5, 6])], - [(1720144145.711511, [], [1, 2, 3, 4, 5, 6]), (1720401163.125022, [], [1, 2, 3, 4, 5, 6])], - [(1720355601.346282, [], [1, 2, 3, 4, 5, 6]), (1720572069.286814, [], [1, 2, 3, 4, 5, 6])], - [ - (1719968339.260056, [], [1, 2, 3, 4, 5, 6]), - (1719968341.365428, [], [1, 2, 3, 4, 5, 6]), - (1719968343.993978, [], [1, 2, 3, 4, 5, 6]), - (1719968346.031381, [], [1, 2, 3, 4, 5, 6]), - (1719968349.431552, [], [1, 2, 3, 4, 5, 6]), - (1719968351.559689, [], [1, 2, 3, 4, 5, 6]), - (1719976134.941126, [], [1, 2, 3, 4, 5, 6]), - (1719976278.477066, [], [1, 2, 3, 4, 5, 6]), - (1719976280.56988, [], [1, 2, 3, 4, 5, 6]), - (1720052757.855887, [], [1, 2, 3, 4, 5, 6]), - (1720052759.915085, [], [1, 2, 3, 4, 5, 6]), - (1720110094.313929, [], [1, 2, 3, 4, 5, 6]), - (1720142517.707832, [], [1, 2, 3, 4, 5, 6]), - (1720142570.10559, [], [1, 2, 3, 4, 5, 6]), - (1720142572.151412, [], [1, 2, 3, 4, 5, 6]), - (1720142576.179553, [], [1, 2, 3, 4, 5, 6]), - (1720237055.807105, [], [1, 2, 3, 4, 5, 6]), - (1720237203.321556, [], [1, 2, 3, 4, 5, 6]), - (1720237205.419793, [], [1, 2, 3, 4, 5, 6]), - (1720316912.566247, [], [1, 2, 3, 4, 5, 6]), - (1720317055.804333, [], [1, 2, 3, 4, 5, 6]), - (1720317057.925258, [], [1, 2, 3, 4, 5, 6]), - (1720317058.399149, [], [1, 2, 3, 4, 5, 6]), - (1720317060.481448, [], [1, 2, 3, 4, 5, 6]), - (1720393849.766518, [], [1, 2, 3, 4, 5, 6]), - (1720393921.300236, [], [1, 2, 3, 4, 5, 6]), - (1720406796.853939, [], [1, 2, 3, 4, 5, 6]), - (1720406798.933918, [], [1, 2, 3, 4, 5, 6]), - (1720482599.505433, [], [1, 2, 3, 4, 5, 6]), - (1720482663.255581, [], [1, 2, 3, 4, 5, 6]), - (1720482665.27704, [], [1, 2, 3, 4, 5, 6]), - (1720492023.699542, [], [1, 2, 3, 4, 5, 6]), - (1720492025.737059, [], [1, 2, 3, 4, 5, 6]), - (1720500142.609638, [], [1, 2, 3, 4, 5, 6]), - (1720500250.895423, [], [1, 2, 3, 4, 5, 6]), - (1720525828.241699, [], [1, 2, 3, 4, 5, 6]), - (1720525830.335737, [], [1, 2, 3, 4, 5, 6]), - (1720543473.185403, [], [1, 2, 3, 4, 5, 6]), - (1720543629.193018, [], [1, 2, 3, 4, 5, 6]), - (1720543631.258205, [], [1, 2, 3, 4, 5, 6]), - (1720566115.315069, [], [1, 2, 3, 4, 5, 6]), - (1720566235.252146, [], [1, 2, 3, 4, 5, 6]), - (1720566237.371673, [], [1, 2, 3, 4, 5, 6]), - (1720566239.622085, [], [1, 2, 3, 4, 5, 6]), - (1720566241.74061, [], [1, 2, 3, 4, 5, 6]), - (1720652422.12376, [], [1, 2, 3, 4, 5, 6]), - (1720652589.161105, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720332940.235911, [], [1, 2, 3, 4, 5, 6])], - [(1720065527.859172, [], [1, 2, 3, 4, 5, 6])], - [(1720568368.543876, [], [1, 2, 3, 4, 5, 6]), (1720635472.219669, [], [1, 2, 3, 4, 5, 6])], - [ - (1719968828.538353, [], [1, 2, 3, 4, 5, 6]), - (1720051948.377763, [], [1, 2, 3, 4, 5, 6]), - (1720299205.556357, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720235830.179481, [], [1, 2, 3, 4, 5, 6]), - (1720235953.147018, [], [1, 2, 3, 4, 5, 6]), - (1720236018.20472, [], [1, 2, 3, 4, 5, 6]), - (1720236089.138704, [], [1, 2, 3, 4, 5, 6]), - (1720236119.593712, [], [1, 2, 3, 4, 5, 6]), - (1720236141.83499, [], [1, 2, 3, 4, 5, 6]), - (1720311050.201652, [], [1, 2, 3, 4, 5, 6]), - (1720311052.280309, [], [1, 2, 3, 4, 5, 6]), - (1720395484.534496, [], [1, 2, 3, 4, 5, 6]), - (1720491406.080018, [], [1, 2, 3, 4, 5, 6]), - (1720491430.598198, [], [1, 2, 3, 4, 5, 6]), - (1720491432.661821, [], [1, 2, 3, 4, 5, 6]), - (1720572678.481313, [], [1, 2, 3, 4, 5, 6]), - (1720572808.45491, [], [1, 2, 3, 4, 5, 6]), - (1720572810.563889, [], [1, 2, 3, 4, 5, 6]), - (1720603175.70942, [], [1, 2, 3, 4, 5, 6]), - (1720603202.06502, [], [1, 2, 3, 4, 5, 6]), - (1720603204.156746, [], [1, 2, 3, 4, 5, 6]), - (1720652491.405509, [], [1, 2, 3, 4, 5, 6]), - (1720652598.039059, [], [1, 2, 3, 4, 5, 6]), - (1720652600.082367, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720153161.725292, [], [1, 2, 3, 4, 5, 6])], - [(1720494662.408865, [], [1, 2, 3, 4, 5, 6]), (1720568597.855422, [], [1, 2, 3, 4, 5, 6])], - [(1720416466.753208, [], [1, 2, 3, 4, 5, 6])], - [(1719978991.390168, [], [1, 2, 3, 4, 5, 6])], - [(1720072031.976782, [], [1, 2, 3, 4, 5, 6]), (1720584690.251602, [], [1, 2, 3, 4, 5, 6])], - [(1720575076.950008, [], [1, 2, 3, 4, 5, 6])], - [(1720231712.798613, [], [1, 2, 3, 4, 5, 6]), (1720398045.987903, [], [1, 2, 3, 4, 5, 6])], - [(1720056840.047309, [], [1, 2, 3, 4, 5, 6]), (1720625851.477544, [], [1, 2, 3, 4, 5, 6])], - [ - (1720053429.359158, [], [1, 2, 3, 4, 5, 6]), - (1720053508.29626, [], [1, 2, 3, 4, 5, 6]), - (1720053570.605172, [], [1, 2, 3, 4, 5, 6]), - (1720053639.380777, [], [1, 2, 3, 4, 5, 6]), - (1720066181.00997, [], [1, 2, 3, 4, 5, 6]), - (1720066215.053405, [], [1, 2, 3, 4, 5, 6]), - (1720066217.116517, [], [1, 2, 3, 4, 5, 6]), - (1720143779.66573, [], [1, 2, 3, 4, 5, 6]), - (1720143781.713526, [], [1, 2, 3, 4, 5, 6]), - (1720222105.35254, [], [1, 2, 3, 4, 5, 6]), - (1720317654.056711, [], [1, 2, 3, 4, 5, 6]), - (1720317708.563828, [], [1, 2, 3, 4, 5, 6]), - (1720480329.549535, [], [1, 2, 3, 4, 5, 6]), - (1720480520.417693, [], [1, 2, 3, 4, 5, 6]), - (1720480522.54519, [], [1, 2, 3, 4, 5, 6]), - (1720480523.499363, [], [1, 2, 3, 4, 5, 6]), - (1720577037.242221, [], [1, 2, 3, 4, 5, 6]), - (1720577039.306434, [], [1, 2, 3, 4, 5, 6]), - (1720639329.717862, [], [1, 2, 3, 4, 5, 6]), - (1720639469.331454, [], [1, 2, 3, 4, 5, 6]), - (1720639471.36127, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720024453.629276, [], [1, 2, 3, 4, 5, 6]), (1720316176.77774, [], [1, 2, 3, 4, 5, 6])], - [ - (1720068883.919311, [], [1, 2, 3, 4, 5, 6]), - (1720319773.101818, [], [1, 2, 3, 4, 5, 6]), - (1720586957.747953, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720056927.404781, [], [1, 2, 3, 4, 5, 6])], - [ - (1720054270.638059, [], [1, 2, 3, 4, 5, 6]), - (1720153565.870327, [], [1, 2, 3, 4, 5, 6]), - (1720334693.538652, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720152641.309698, [], [1, 2, 3, 4, 5, 6]), (1720576318.23299, [], [1, 2, 3, 4, 5, 6])], - [ - (1720404934.034946, [], [1, 2, 3, 4, 5, 6]), - (1720476937.980269, [], [1, 2, 3, 4, 5, 6]), - (1720652925.317718, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720583121.659416, [], [1, 2, 3, 4, 5, 6])], - [(1719974918.036807, [], [1, 2, 3, 4, 5, 6])], - [(1720052839.789235, [], [1, 2, 3, 4, 5, 6]), (1720309756.887443, [], [1, 2, 3, 4, 5, 6])], - [(1720214816.601474, [], [1, 2, 3, 4, 5, 6]), (1720628643.350003, [], [1, 2, 3, 4, 5, 6])], - [(1719990813.089809, [], [1, 2, 3, 4, 5, 6]), (1720312746.860016, [], [1, 2, 3, 4, 5, 6])], - [(1720072040.294779, [], [1, 2, 3, 4, 5, 6]), (1720573666.820699, [], [1, 2, 3, 4, 5, 6])], - [(1720221192.01312, [], [1, 2, 3, 4, 5, 6])], - [(1720143165.437476, [], [1, 2, 3, 4, 5, 6])], - [ - (1719965593.95106, [], [1, 2, 3, 4, 5, 6]), - (1720062861.422969, [], [1, 2, 3, 4, 5, 6]), - (1720503564.270709, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720310060.645551, [], [1, 2, 3, 4, 5, 6]), - (1720482309.217878, [], [1, 2, 3, 4, 5, 6]), - (1720591036.349001, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720157768.51272, [], [1, 2, 3, 4, 5, 6])], - [(1720149445.473387, [], [1, 2, 3, 4, 5, 6]), (1720438026.869011, [], [1, 2, 3, 4, 5, 6])], - [ - (1720060489.013219, [], [1, 2, 3, 4, 5, 6]), - (1720415886.629529, [], [1, 2, 3, 4, 5, 6]), - (1720580867.871164, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720414273.276015, [], [1, 2, 3, 4, 5, 6]), (1720487097.033528, [], [1, 2, 3, 4, 5, 6])], - [(1719966569.062726, [], [1, 2, 3, 4, 5, 6])], - [ - (1719966533.368239, [], [1, 2, 3, 4, 5, 6]), - (1720318867.369239, [], [1, 2, 3, 4, 5, 6]), - (1720424434.84027, [], [1, 2, 3, 4, 5, 6]), - (1720566607.607309, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720608001.068576, [], [1, 2, 3, 4, 5, 6])], - [ - (1720062804.379267, [], [1, 2, 3, 4, 5, 6]), - (1720226534.49236, [], [1, 2, 3, 4, 5, 6]), - (1720321084.499585, [], [1, 2, 3, 4, 5, 6]), - (1720450358.303395, [], [1, 2, 3, 4, 5, 6]), - (1720577114.020932, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720138095.209939, [], [1, 2, 3, 4, 5, 6])], - [ - (1720109654.682718, [], [1, 2, 3, 4, 5, 6]), - (1720109832.815741, [], [1, 2, 3, 4, 5, 6]), - (1720109959.88633, [], [1, 2, 3, 4, 5, 6]), - (1720110033.900336, [], [1, 2, 3, 4, 5, 6]), - (1720110090.159457, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719973270.54798, [], [1, 2, 3, 4, 5, 6])], - [(1720404747.93651, [], [1, 2, 3, 4, 5, 6])], - [(1719975896.00888, [], [1, 2, 3, 4, 5, 6]), (1720412877.994145, [], [1, 2, 3, 4, 5, 6])], - [(1720488415.324703, [], [1, 2, 3, 4, 5, 6])], - [ - (1719979329.168652, [], [1, 2, 3, 4, 5, 6]), - (1719979433.834943, [], [1, 2, 3, 4, 5, 6]), - (1719979537.706541, [], [1, 2, 3, 4, 5, 6]), - (1720054525.517489, [], [1, 2, 3, 4, 5, 6]), - (1720054576.808031, [], [1, 2, 3, 4, 5, 6]), - (1720054637.310552, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720008269.86362, [], [1, 2, 3, 4, 5, 6]), (1720228186.690288, [], [1, 2, 3, 4, 5, 6])], - [(1719980906.390651, [], [1, 2, 3, 4, 5, 6])], - [(1720049804.519108, [], [1, 2, 3, 4, 5, 6])], - [(1719982824.989151, [], [1, 2, 3, 4, 5, 6])], - [ - (1720052793.231176, [], [1, 2, 3, 4, 5, 6]), - (1720416383.522419, [], [1, 2, 3, 4, 5, 6]), - (1720576203.462386, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720141956.096351, [], [1, 2, 3, 4, 5, 6]), (1720501761.653384, [], [1, 2, 3, 4, 5, 6])], - [(1719968214.670337, [], [1, 2, 3, 4, 5, 6])], - [ - (1720061582.741936, [], [1, 2, 3, 4, 5, 6]), - (1720148352.805998, [], [1, 2, 3, 4, 5, 6]), - (1720320650.836088, [], [1, 2, 3, 4, 5, 6]), - (1720480540.757287, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720066746.296404, [], [1, 2, 3, 4, 5, 6]), (1720470635.779369, [], [1, 2, 3, 4, 5, 6])], - [(1720036143.99876, [], [1, 2, 3, 4, 5, 6]), (1720235115.25383, [], [1, 2, 3, 4, 5, 6])], - [ - (1720325739.301864, [], [1, 2, 3, 4, 5, 6]), - (1720366758.48691, [], [1, 2, 3, 4, 5, 6]), - (1720579671.285769, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720406674.612278, [], [1, 2, 3, 4, 5, 6])], - [(1720569843.609239, [], [1, 2, 3, 4, 5, 6])], - [ - (1720060843.491976, [], [1, 2, 3, 4, 5, 6]), - (1720147665.305258, [], [1, 2, 3, 4, 5, 6]), - (1720501529.904655, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720046373.265399, [], [1, 2, 3, 4, 5, 6]), - (1720046375.332994, [], [1, 2, 3, 4, 5, 6]), - (1720068144.411279, [], [1, 2, 3, 4, 5, 6]), - (1720068261.195225, [], [1, 2, 3, 4, 5, 6]), - (1720131629.331825, [], [1, 2, 3, 4, 5, 6]), - (1720131717.731289, [], [1, 2, 3, 4, 5, 6]), - (1720143208.108341, [], [1, 2, 3, 4, 5, 6]), - (1720224375.336718, [], [1, 2, 3, 4, 5, 6]), - (1720224489.89131, [], [1, 2, 3, 4, 5, 6]), - (1720239983.991454, [], [1, 2, 3, 4, 5, 6]), - (1720240023.957522, [], [1, 2, 3, 4, 5, 6]), - (1720240026.023994, [], [1, 2, 3, 4, 5, 6]), - (1720288870.449116, [], [1, 2, 3, 4, 5, 6]), - (1720319014.330473, [], [1, 2, 3, 4, 5, 6]), - (1720319153.071162, [], [1, 2, 3, 4, 5, 6]), - (1720319155.11854, [], [1, 2, 3, 4, 5, 6]), - (1720356008.754634, [], [1, 2, 3, 4, 5, 6]), - (1720356170.017209, [], [1, 2, 3, 4, 5, 6]), - (1720414281.753569, [], [1, 2, 3, 4, 5, 6]), - (1720466888.126284, [], [1, 2, 3, 4, 5, 6]), - (1720466890.175399, [], [1, 2, 3, 4, 5, 6]), - (1720496852.884055, [], [1, 2, 3, 4, 5, 6]), - (1720496900.157534, [], [1, 2, 3, 4, 5, 6]), - (1720496902.257177, [], [1, 2, 3, 4, 5, 6]), - (1720517711.484252, [], [1, 2, 3, 4, 5, 6]), - (1720517785.617389, [], [1, 2, 3, 4, 5, 6]), - (1720517787.722386, [], [1, 2, 3, 4, 5, 6]), - (1720574636.301281, [], [1, 2, 3, 4, 5, 6]), - (1720574638.402501, [], [1, 2, 3, 4, 5, 6]), - (1720631962.467861, [], [1, 2, 3, 4, 5, 6]), - (1720632073.350096, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720222983.951049, [], [1, 2, 3, 4, 5, 6]), (1720416117.193584, [], [1, 2, 3, 4, 5, 6])], - [ - (1719971140.695348, [], [1, 2, 3, 4, 5, 6]), - (1720135102.372106, [], [1, 2, 3, 4, 5, 6]), - (1720241528.560118, [], [1, 2, 3, 4, 5, 6]), - (1720494221.442123, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720321074.976216, [], [1, 2, 3, 4, 5, 6]), (1720409891.326419, [], [1, 2, 3, 4, 5, 6])], - [ - (1720048681.074873, [], [1, 2, 3, 4, 5, 6]), - (1720048799.279747, [], [1, 2, 3, 4, 5, 6]), - (1720048801.388645, [], [1, 2, 3, 4, 5, 6]), - (1720078114.283264, [], [1, 2, 3, 4, 5, 6]), - (1720078331.228227, [], [1, 2, 3, 4, 5, 6]), - (1720143813.123392, [], [1, 2, 3, 4, 5, 6]), - (1720143818.771163, [], [1, 2, 3, 4, 5, 6]), - (1720235193.521271, [], [1, 2, 3, 4, 5, 6]), - (1720235195.596896, [], [1, 2, 3, 4, 5, 6]), - (1720325140.395885, [], [1, 2, 3, 4, 5, 6]), - (1720325267.478348, [], [1, 2, 3, 4, 5, 6]), - (1720411316.426439, [], [1, 2, 3, 4, 5, 6]), - (1720411410.991238, [], [1, 2, 3, 4, 5, 6]), - (1720411413.049352, [], [1, 2, 3, 4, 5, 6]), - (1720585972.027756, [], [1, 2, 3, 4, 5, 6]), - (1720586043.355429, [], [1, 2, 3, 4, 5, 6]), - (1720586045.457795, [], [1, 2, 3, 4, 5, 6]), - (1720615162.541609, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719966754.275194, [], [1, 2, 3, 4, 5, 6]), - (1720490216.464205, [], [1, 2, 3, 4, 5, 6]), - (1720553382.681907, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720002644.294528, [], [1, 2, 3, 4, 5, 6]), (1720230090.842106, [], [1, 2, 3, 4, 5, 6])], - [(1720146769.696788, [], [1, 2, 3, 4, 5, 6])], - [ - (1720165756.310512, [], [1, 2, 3, 4, 5, 6]), - (1720410912.566749, [], [1, 2, 3, 4, 5, 6]), - (1720570647.832366, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719976353.576494, [], [1, 2, 3, 4, 5, 6]), - (1720377185.297147, [], [1, 2, 3, 4, 5, 6]), - (1720377289.215042, [], [1, 2, 3, 4, 5, 6]), - (1720377308.468995, [], [1, 2, 3, 4, 5, 6]), - (1720377428.407697, [], [1, 2, 3, 4, 5, 6]), - (1720377485.735576, [], [1, 2, 3, 4, 5, 6]), - (1720377529.508166, [], [1, 2, 3, 4, 5, 6]), - (1720377719.383399, [], [1, 2, 3, 4, 5, 6]), - (1720377809.666048, [], [1, 2, 3, 4, 5, 6]), - (1720377903.918773, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719980213.693083, [], [1, 2, 3, 4, 5, 6]), (1720140384.252931, [], [1, 2, 3, 4, 5, 6])], - [(1720063871.378525, [], [1, 2, 3, 4, 5, 6]), (1720573199.935932, [], [1, 2, 3, 4, 5, 6])], - [(1719985832.719319, [], [1, 2, 3, 4, 5, 6]), (1720406386.34727, [], [1, 2, 3, 4, 5, 6])], - [(1720235695.246766, [], [1, 2, 3, 4, 5, 6]), (1720397629.747797, [], [1, 2, 3, 4, 5, 6])], - [ - (1719968874.938189, [], [1, 2, 3, 4, 5, 6]), - (1719969009.652547, [], [1, 2, 3, 4, 5, 6]), - (1719985700.913806, [], [1, 2, 3, 4, 5, 6]), - (1719985791.512554, [], [1, 2, 3, 4, 5, 6]), - (1720148300.05719, [], [1, 2, 3, 4, 5, 6]), - (1720148304.627225, [], [1, 2, 3, 4, 5, 6]), - (1720148306.685703, [], [1, 2, 3, 4, 5, 6]), - (1720227741.213642, [], [1, 2, 3, 4, 5, 6]), - (1720227808.775173, [], [1, 2, 3, 4, 5, 6]), - (1720295172.811284, [], [1, 2, 3, 4, 5, 6]), - (1720295262.745855, [], [1, 2, 3, 4, 5, 6]), - (1720295264.827116, [], [1, 2, 3, 4, 5, 6]), - (1720295269.130924, [], [1, 2, 3, 4, 5, 6]), - (1720295271.214758, [], [1, 2, 3, 4, 5, 6]), - (1720295276.000757, [], [1, 2, 3, 4, 5, 6]), - (1720295278.050173, [], [1, 2, 3, 4, 5, 6]), - (1720295281.951474, [], [1, 2, 3, 4, 5, 6]), - (1720314791.722567, [], [1, 2, 3, 4, 5, 6]), - (1720314793.809493, [], [1, 2, 3, 4, 5, 6]), - (1720314794.091414, [], [1, 2, 3, 4, 5, 6]), - (1720314796.180282, [], [1, 2, 3, 4, 5, 6]), - (1720334621.693568, [], [1, 2, 3, 4, 5, 6]), - (1720334651.208509, [], [1, 2, 3, 4, 5, 6]), - (1720334653.289286, [], [1, 2, 3, 4, 5, 6]), - (1720405990.429231, [], [1, 2, 3, 4, 5, 6]), - (1720406072.368399, [], [1, 2, 3, 4, 5, 6]), - (1720406074.479232, [], [1, 2, 3, 4, 5, 6]), - (1720502938.778116, [], [1, 2, 3, 4, 5, 6]), - (1720502940.865312, [], [1, 2, 3, 4, 5, 6]), - (1720575136.293441, [], [1, 2, 3, 4, 5, 6]), - (1720575193.500487, [], [1, 2, 3, 4, 5, 6]), - (1720575195.593287, [], [1, 2, 3, 4, 5, 6]), - (1720584066.099625, [], [1, 2, 3, 4, 5, 6]), - (1720584068.181406, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720401234.664166, [], [1, 2, 3, 4, 5, 6])], - [(1720338995.593562, [], [1, 2, 3, 4, 5, 6]), (1720569421.058122, [], [1, 2, 3, 4, 5, 6])], - [ - (1720152780.103003, [], [1, 2, 3, 4, 5, 6]), - (1720318021.21162, [], [1, 2, 3, 4, 5, 6]), - (1720568682.771219, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720153767.474128, [], [1, 2, 3, 4, 5, 6]), - (1720153989.99445, [], [1, 2, 3, 4, 5, 6]), - (1720154117.339532, [], [1, 2, 3, 4, 5, 6]), - (1720154188.47243, [], [1, 2, 3, 4, 5, 6]), - (1720227169.047901, [], [1, 2, 3, 4, 5, 6]), - (1720227171.113334, [], [1, 2, 3, 4, 5, 6]), - (1720290328.719511, [], [1, 2, 3, 4, 5, 6]), - (1720290370.18464, [], [1, 2, 3, 4, 5, 6]), - (1720308949.583306, [], [1, 2, 3, 4, 5, 6]), - (1720309021.205367, [], [1, 2, 3, 4, 5, 6]), - (1720309023.255742, [], [1, 2, 3, 4, 5, 6]), - (1720400031.238045, [], [1, 2, 3, 4, 5, 6]), - (1720400050.534361, [], [1, 2, 3, 4, 5, 6]), - (1720400052.591865, [], [1, 2, 3, 4, 5, 6]), - (1720475517.057002, [], [1, 2, 3, 4, 5, 6]), - (1720475631.221119, [], [1, 2, 3, 4, 5, 6]), - (1720489341.891834, [], [1, 2, 3, 4, 5, 6]), - (1720489520.813888, [], [1, 2, 3, 4, 5, 6]), - (1720522911.41822, [], [1, 2, 3, 4, 5, 6]), - (1720522980.415637, [], [1, 2, 3, 4, 5, 6]), - (1720522982.44762, [], [1, 2, 3, 4, 5, 6]), - (1720590435.585175, [], [1, 2, 3, 4, 5, 6]), - (1720590601.259611, [], [1, 2, 3, 4, 5, 6]), - (1720590603.315457, [], [1, 2, 3, 4, 5, 6]), - (1720590604.638539, [], [1, 2, 3, 4, 5, 6]), - (1720590606.730642, [], [1, 2, 3, 4, 5, 6]), - (1720612053.860624, [], [1, 2, 3, 4, 5, 6]), - (1720612129.921877, [], [1, 2, 3, 4, 5, 6]), - (1720612132.011818, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720426949.173587, [], [1, 2, 3, 4, 5, 6])], - [(1720569102.034496, [], [1, 2, 3, 4, 5, 6])], - [(1720034790.744638, [], [1, 2, 3, 4, 5, 6]), (1720307075.973997, [], [1, 2, 3, 4, 5, 6])], - [ - (1720226287.705985, [], [1, 2, 3, 4, 5, 6]), - (1720397684.607266, [], [1, 2, 3, 4, 5, 6]), - (1720488997.884315, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720228730.18975, [], [1, 2, 3, 4, 5, 6]), - (1720407495.769529, [], [1, 2, 3, 4, 5, 6]), - (1720486995.921451, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720489080.310875, [], [1, 2, 3, 4, 5, 6]), (1720579311.992295, [], [1, 2, 3, 4, 5, 6])], - [(1720138007.433538, [], [1, 2, 3, 4, 5, 6]), (1720243741.609597, [], [1, 2, 3, 4, 5, 6])], - [(1720528666.459324, [], [1, 2, 3, 4, 5, 6])], - [(1719970772.701878, [], [1, 2, 3, 4, 5, 6]), (1720141632.061506, [], [1, 2, 3, 4, 5, 6])], - [(1720068110.038208, [], [1, 2, 3, 4, 5, 6])], - [(1720047191.032235, [], [1, 2, 3, 4, 5, 6])], - [(1719976436.118248, [], [1, 2, 3, 4, 5, 6]), (1720307037.853977, [], [1, 2, 3, 4, 5, 6])], - [(1719972036.639217, [], [1, 2, 3, 4, 5, 6]), (1720057689.829017, [], [1, 2, 3, 4, 5, 6])], - [(1720110461.39165, [], [1, 2, 3, 4, 5, 6]), (1720507249.36072, [], [1, 2, 3, 4, 5, 6])], - [(1719973197.847086, [], [1, 2, 3, 4, 5, 6])], - [ - (1720069338.721539, [], [1, 2, 3, 4, 5, 6]), - (1720524537.017222, [], [1, 2, 3, 4, 5, 6]), - (1720603176.268707, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720142471.621654, [], [1, 2, 3, 4, 5, 6]), (1720221877.173433, [], [1, 2, 3, 4, 5, 6])], - [(1720141771.947396, [], [1, 2, 3, 4, 5, 6])], - [(1720563222.366935, [], [1, 2, 3, 4, 5, 6])], - [(1720237444.558492, [], [1, 2, 3, 4, 5, 6]), (1720586951.821255, [], [1, 2, 3, 4, 5, 6])], - [ - (1720066787.226665, [], [1, 2, 3, 4, 5, 6]), - (1720138501.376918, [], [1, 2, 3, 4, 5, 6]), - (1720332897.490345, [], [1, 2, 3, 4, 5, 6]), - (1720498267.095353, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719979972.999784, [], [1, 2, 3, 4, 5, 6]), (1720326547.318274, [], [1, 2, 3, 4, 5, 6])], - [(1720156142.458551, [], [1, 2, 3, 4, 5, 6]), (1720222674.830191, [], [1, 2, 3, 4, 5, 6])], - [ - (1719976247.399205, [], [1, 2, 3, 4, 5, 6]), - (1719976421.138728, [], [1, 2, 3, 4, 5, 6]), - (1719976457.121053, [], [1, 2, 3, 4, 5, 6]), - (1719976566.522486, [], [1, 2, 3, 4, 5, 6]), - (1720060116.807539, [], [1, 2, 3, 4, 5, 6]), - (1720142201.528128, [], [1, 2, 3, 4, 5, 6]), - (1720142320.790244, [], [1, 2, 3, 4, 5, 6]), - (1720142322.873716, [], [1, 2, 3, 4, 5, 6]), - (1720142323.209429, [], [1, 2, 3, 4, 5, 6]), - (1720142325.32365, [], [1, 2, 3, 4, 5, 6]), - (1720158309.47212, [], [1, 2, 3, 4, 5, 6]), - (1720158464.699924, [], [1, 2, 3, 4, 5, 6]), - (1720158466.803077, [], [1, 2, 3, 4, 5, 6]), - (1720235827.780639, [], [1, 2, 3, 4, 5, 6]), - (1720235829.873017, [], [1, 2, 3, 4, 5, 6]), - (1720235831.516786, [], [1, 2, 3, 4, 5, 6]), - (1720235833.64015, [], [1, 2, 3, 4, 5, 6]), - (1720308111.792929, [], [1, 2, 3, 4, 5, 6]), - (1720308113.917634, [], [1, 2, 3, 4, 5, 6]), - (1720330424.153222, [], [1, 2, 3, 4, 5, 6]), - (1720330426.274619, [], [1, 2, 3, 4, 5, 6]), - (1720397440.529792, [], [1, 2, 3, 4, 5, 6]), - (1720397517.527169, [], [1, 2, 3, 4, 5, 6]), - (1720397519.567891, [], [1, 2, 3, 4, 5, 6]), - (1720489794.692916, [], [1, 2, 3, 4, 5, 6]), - (1720489888.559008, [], [1, 2, 3, 4, 5, 6]), - (1720489890.678539, [], [1, 2, 3, 4, 5, 6]), - (1720577363.385966, [], [1, 2, 3, 4, 5, 6]), - (1720577493.034855, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720042932.668232, [], [1, 2, 3, 4, 5, 6])], - [ - (1720059740.224971, [], [1, 2, 3, 4, 5, 6]), - (1720141837.883794, [], [1, 2, 3, 4, 5, 6]), - (1720405792.751871, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720050443.838424, [], [1, 2, 3, 4, 5, 6]), (1720124719.146549, [], [1, 2, 3, 4, 5, 6])], - [(1719974887.67444, [], [1, 2, 3, 4, 5, 6]), (1720568466.68215, [], [1, 2, 3, 4, 5, 6])], - [(1720065592.314345, [], [1, 2, 3, 4, 5, 6]), (1720205463.888972, [], [1, 2, 3, 4, 5, 6])], - [ - (1720226332.701569, [], [1, 2, 3, 4, 5, 6]), - (1720396620.155135, [], [1, 2, 3, 4, 5, 6]), - (1720492327.218299, [], [1, 2, 3, 4, 5, 6]), - (1720574416.447233, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720045016.618738, [], [1, 2, 3, 4, 5, 6])], - [ - (1720041923.403398, [], [1, 2, 3, 4, 5, 6]), - (1720041925.529856, [], [1, 2, 3, 4, 5, 6]), - (1720041929.344695, [], [1, 2, 3, 4, 5, 6]), - (1720071082.001532, [], [1, 2, 3, 4, 5, 6]), - (1720071201.167727, [], [1, 2, 3, 4, 5, 6]), - (1720071203.259989, [], [1, 2, 3, 4, 5, 6]), - (1720113938.478909, [], [1, 2, 3, 4, 5, 6]), - (1720114019.141219, [], [1, 2, 3, 4, 5, 6]), - (1720147954.114345, [], [1, 2, 3, 4, 5, 6]), - (1720148058.186186, [], [1, 2, 3, 4, 5, 6]), - (1720148060.244565, [], [1, 2, 3, 4, 5, 6]), - (1720230463.28606, [], [1, 2, 3, 4, 5, 6]), - (1720230465.339869, [], [1, 2, 3, 4, 5, 6]), - (1720306384.513301, [], [1, 2, 3, 4, 5, 6]), - (1720306386.603107, [], [1, 2, 3, 4, 5, 6]), - (1720327575.975525, [], [1, 2, 3, 4, 5, 6]), - (1720327821.751969, [], [1, 2, 3, 4, 5, 6]), - (1720327823.781901, [], [1, 2, 3, 4, 5, 6]), - (1720410348.159738, [], [1, 2, 3, 4, 5, 6]), - (1720410448.341114, [], [1, 2, 3, 4, 5, 6]), - (1720457570.237639, [], [1, 2, 3, 4, 5, 6]), - (1720457731.119754, [], [1, 2, 3, 4, 5, 6]), - (1720457733.248545, [], [1, 2, 3, 4, 5, 6]), - (1720499264.385485, [], [1, 2, 3, 4, 5, 6]), - (1720499470.033411, [], [1, 2, 3, 4, 5, 6]), - (1720499472.085357, [], [1, 2, 3, 4, 5, 6]), - (1720543986.94937, [], [1, 2, 3, 4, 5, 6]), - (1720570525.581032, [], [1, 2, 3, 4, 5, 6]), - (1720570749.619565, [], [1, 2, 3, 4, 5, 6]), - (1720585164.854344, [], [1, 2, 3, 4, 5, 6]), - (1720585249.748529, [], [1, 2, 3, 4, 5, 6]), - (1720585251.810485, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720489237.597076, [], [1, 2, 3, 4, 5, 6]), - (1720489298.656835, [], [1, 2, 3, 4, 5, 6]), - (1720489340.853931, [], [1, 2, 3, 4, 5, 6]), - (1720489414.715662, [], [1, 2, 3, 4, 5, 6]), - (1720489578.362748, [], [1, 2, 3, 4, 5, 6]), - (1720489722.080922, [], [1, 2, 3, 4, 5, 6]), - (1720489846.161597, [], [1, 2, 3, 4, 5, 6]), - (1720489902.616032, [], [1, 2, 3, 4, 5, 6]), - (1720489979.179271, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720226541.700259, [], [1, 2, 3, 4, 5, 6]), (1720459357.712795, [], [1, 2, 3, 4, 5, 6])], - [ - (1720137147.179349, [], [1, 2, 3, 4, 5, 6]), - (1720241160.346244, [], [1, 2, 3, 4, 5, 6]), - (1720393844.000636, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720233961.944441, [], [1, 2, 3, 4, 5, 6]), (1720475993.227865, [], [1, 2, 3, 4, 5, 6])], - [(1720339946.523179, [], [1, 2, 3, 4, 5, 6])], - [(1720314335.142585, [], [1, 2, 3, 4, 5, 6])], - [(1720307070.122528, [], [1, 2, 3, 4, 5, 6]), (1720396806.823553, [], [1, 2, 3, 4, 5, 6])], - [(1720134885.254524, [], [1, 2, 3, 4, 5, 6]), (1720582472.172677, [], [1, 2, 3, 4, 5, 6])], - [ - (1720064989.52196, [], [1, 2, 3, 4, 5, 6]), - (1720317471.54011, [], [1, 2, 3, 4, 5, 6]), - (1720405569.646675, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720230240.375249, [], [1, 2, 3, 4, 5, 6])], - [ - (1719981989.67667, [], [1, 2, 3, 4, 5, 6]), - (1719982209.356573, [], [1, 2, 3, 4, 5, 6]), - (1719982211.412871, [], [1, 2, 3, 4, 5, 6]), - (1719990729.511533, [], [1, 2, 3, 4, 5, 6]), - (1719990731.571772, [], [1, 2, 3, 4, 5, 6]), - (1720052018.486278, [], [1, 2, 3, 4, 5, 6]), - (1720052084.895512, [], [1, 2, 3, 4, 5, 6]), - (1720052086.959136, [], [1, 2, 3, 4, 5, 6]), - (1720063752.458627, [], [1, 2, 3, 4, 5, 6]), - (1720063775.402517, [], [1, 2, 3, 4, 5, 6]), - (1720131365.952047, [], [1, 2, 3, 4, 5, 6]), - (1720131512.217778, [], [1, 2, 3, 4, 5, 6]), - (1720131514.344434, [], [1, 2, 3, 4, 5, 6]), - (1720155954.588913, [], [1, 2, 3, 4, 5, 6]), - (1720156049.221823, [], [1, 2, 3, 4, 5, 6]), - (1720326057.45718, [], [1, 2, 3, 4, 5, 6]), - (1720404760.882693, [], [1, 2, 3, 4, 5, 6]), - (1720404806.729924, [], [1, 2, 3, 4, 5, 6]), - (1720404808.834418, [], [1, 2, 3, 4, 5, 6]), - (1720416517.018963, [], [1, 2, 3, 4, 5, 6]), - (1720494367.532053, [], [1, 2, 3, 4, 5, 6]), - (1720500247.551019, [], [1, 2, 3, 4, 5, 6]), - (1720500294.606063, [], [1, 2, 3, 4, 5, 6]), - (1720500296.694825, [], [1, 2, 3, 4, 5, 6]), - (1720500299.259697, [], [1, 2, 3, 4, 5, 6]), - (1720500301.365635, [], [1, 2, 3, 4, 5, 6]), - (1720572338.244531, [], [1, 2, 3, 4, 5, 6]), - (1720572428.794186, [], [1, 2, 3, 4, 5, 6]), - (1720572430.860491, [], [1, 2, 3, 4, 5, 6]), - (1720600877.354363, [], [1, 2, 3, 4, 5, 6]), - (1720601092.109844, [], [1, 2, 3, 4, 5, 6]), - (1720601094.164843, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720230142.987461, [], [1, 2, 3, 4, 5, 6])], - [(1720040834.068992, [], [1, 2, 3, 4, 5, 6])], - [ - (1719984036.646786, [], [1, 2, 3, 4, 5, 6]), - (1720138775.775437, [], [1, 2, 3, 4, 5, 6]), - (1720337436.06649, [], [1, 2, 3, 4, 5, 6]), - (1720567415.54222, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720501680.278938, [], [1, 2, 3, 4, 5, 6]), (1720645969.459141, [], [1, 2, 3, 4, 5, 6])], - [(1719981148.135955, [], [1, 2, 3, 4, 5, 6]), (1720574648.013669, [], [1, 2, 3, 4, 5, 6])], - [ - (1719982323.222591, [], [1, 2, 3, 4, 5, 6]), - (1720101646.380659, [], [1, 2, 3, 4, 5, 6]), - (1720493833.121559, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719977634.84991, [], [1, 2, 3, 4, 5, 6])], - [(1720494761.805988, [], [1, 2, 3, 4, 5, 6]), (1720595943.849674, [], [1, 2, 3, 4, 5, 6])], - [ - (1720155432.624618, [], [1, 2, 3, 4, 5, 6]), - (1720500643.020756, [], [1, 2, 3, 4, 5, 6]), - (1720584683.624928, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720245035.196003, [], [1, 2, 3, 4, 5, 6])], - [(1720313673.855042, [], [1, 2, 3, 4, 5, 6]), (1720381149.495776, [], [1, 2, 3, 4, 5, 6])], - [(1720019520.657419, [], [1, 2, 3, 4, 5, 6])], - [(1719981920.434976, [], [1, 2, 3, 4, 5, 6]), (1720128572.036838, [], [1, 2, 3, 4, 5, 6])], - [(1720136202.220094, [], [1, 2, 3, 4, 5, 6]), (1720279940.922158, [], [1, 2, 3, 4, 5, 6])], - [(1720462395.987553, [], [1, 2, 3, 4, 5, 6])], - [ - (1720653383.244, [], [1, 2, 3, 4, 5, 6]), - (1720653383.325, [], [1, 2, 3, 4, 5, 6]), - (1720653391.627, [], [1, 2, 3, 4, 5, 6]), - (1720653392.102, [], [1, 2, 3, 4, 5, 6]), - (1720653392.298, [], [1, 2, 3, 4, 5, 6]), - (1720653394.934, [], [1, 2, 3, 4, 5, 6]), - (1720653396.411, [], [1, 2, 3, 4, 5, 6]), - (1720653433.093, [], [1, 2, 3, 4, 5, 6]), - (1720653433.236, [], [1, 2, 3, 4, 5, 6]), - (1720653434.991, [], [1, 2, 3, 4, 5, 6]), - (1720653435.037, [], [1, 2, 3, 4, 5, 6]), - (1720653501.654, [], [1, 2, 3, 4, 5, 6]), - (1720653501.71, [], [1, 2, 3, 4, 5, 6]), - (1720653504.799, [], [1, 2, 3, 4, 5, 6]), - (1720653506.446, [], [1, 2, 3, 4, 5, 6]), - (1720653507.872, [], [1, 2, 3, 4, 5, 6]), - (1720654003.023, [], [1, 2, 3, 4, 5, 6]), - (1720654003.148, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719977539.575178, [], [1, 2, 3, 4, 5, 6]), - (1720223726.13705, [], [1, 2, 3, 4, 5, 6]), - (1720396336.894644, [], [1, 2, 3, 4, 5, 6]), - (1720587683.68083, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720327049.710872, [], [1, 2, 3, 4, 5, 6])], - [(1720223003.678086, [], [1, 2, 3, 4, 5, 6]), (1720320656.874702, [], [1, 2, 3, 4, 5, 6])], - [(1720149475.628567, [], [1, 2, 3, 4, 5, 6]), (1720228859.277837, [], [1, 2, 3, 4, 5, 6])], - [(1720049864.230497, [], [1, 2, 3, 4, 5, 6])], - [(1720198432.201466, [], [1, 2, 3, 4, 5, 6])], - [ - (1719975613.439717, [], [1, 2, 3, 4, 5, 6]), - (1719975648.049123, [], [1, 2, 3, 4, 5, 6]), - (1720056914.90013, [], [1, 2, 3, 4, 5, 6]), - (1720057026.542911, [], [1, 2, 3, 4, 5, 6]), - (1720061926.526142, [], [1, 2, 3, 4, 5, 6]), - (1720062007.614611, [], [1, 2, 3, 4, 5, 6]), - (1720147419.43368, [], [1, 2, 3, 4, 5, 6]), - (1720147421.485277, [], [1, 2, 3, 4, 5, 6]), - (1720234139.651394, [], [1, 2, 3, 4, 5, 6]), - (1720234141.758276, [], [1, 2, 3, 4, 5, 6]), - (1720315191.984726, [], [1, 2, 3, 4, 5, 6]), - (1720315194.093018, [], [1, 2, 3, 4, 5, 6]), - (1720315195.836394, [], [1, 2, 3, 4, 5, 6]), - (1720395738.54726, [], [1, 2, 3, 4, 5, 6]), - (1720395740.684533, [], [1, 2, 3, 4, 5, 6]), - (1720410342.218884, [], [1, 2, 3, 4, 5, 6]), - (1720410455.568303, [], [1, 2, 3, 4, 5, 6]), - (1720496479.412713, [], [1, 2, 3, 4, 5, 6]), - (1720496636.329168, [], [1, 2, 3, 4, 5, 6]), - (1720568810.362519, [], [1, 2, 3, 4, 5, 6]), - (1720569040.475975, [], [1, 2, 3, 4, 5, 6]), - (1720652557.884167, [], [1, 2, 3, 4, 5, 6]), - (1720652630.129755, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720233899.203213, [], [1, 2, 3, 4, 5, 6]), - (1720463796.00711, [], [1, 2, 3, 4, 5, 6]), - (1720567454.878169, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720218589.331391, [], [1, 2, 3, 4, 5, 6]), - (1720572246.733219, [], [1, 2, 3, 4, 5, 6]), - (1720585861.133309, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720030179.060188, [], [1, 2, 3, 4, 5, 6]), - (1720330759.17762, [], [1, 2, 3, 4, 5, 6]), - (1720494515.69797, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719979902.797865, [], [1, 2, 3, 4, 5, 6])], - [ - (1720070638.173492, [], [1, 2, 3, 4, 5, 6]), - (1720070819.083453, [], [1, 2, 3, 4, 5, 6]), - (1720070899.802295, [], [1, 2, 3, 4, 5, 6]), - (1720318683.767078, [], [1, 2, 3, 4, 5, 6]), - (1720318886.533145, [], [1, 2, 3, 4, 5, 6]), - (1720318966.57212, [], [1, 2, 3, 4, 5, 6]), - (1720318995.968059, [], [1, 2, 3, 4, 5, 6]), - (1720319139.50433, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720146380.443011, [], [1, 2, 3, 4, 5, 6]), (1720567703.854081, [], [1, 2, 3, 4, 5, 6])], - [(1720485224.936769, [], [1, 2, 3, 4, 5, 6])], - [(1720039180.636756, [], [1, 2, 3, 4, 5, 6]), (1720320703.675688, [], [1, 2, 3, 4, 5, 6])], - [(1720572877.111346, [], [1, 2, 3, 4, 5, 6])], - [(1720149009.624794, [], [1, 2, 3, 4, 5, 6])], - [ - (1720146066.037668, [], [1, 2, 3, 4, 5, 6]), - (1720233446.585623, [], [1, 2, 3, 4, 5, 6]), - (1720397647.223612, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720071535.98588, [], [1, 2, 3, 4, 5, 6]), - (1720222696.675857, [], [1, 2, 3, 4, 5, 6]), - (1720581710.534385, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720320564.297259, [], [1, 2, 3, 4, 5, 6])], - [(1720221596.179609, [], [1, 2, 3, 4, 5, 6]), (1720502714.197196, [], [1, 2, 3, 4, 5, 6])], - [(1720553799.408143, [], [1, 2, 3, 4, 5, 6])], - [ - (1720244362.654861, [], [1, 2, 3, 4, 5, 6]), - (1720412405.21556, [], [1, 2, 3, 4, 5, 6]), - (1720566429.648086, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720225993.003411, [], [1, 2, 3, 4, 5, 6]), (1720576860.143664, [], [1, 2, 3, 4, 5, 6])], - [(1720050680.82043, [], [1, 2, 3, 4, 5, 6])], - [(1719979231.190542, [], [1, 2, 3, 4, 5, 6]), (1720395944.084001, [], [1, 2, 3, 4, 5, 6])], - [(1720483644.896944, [], [1, 2, 3, 4, 5, 6])], - [ - (1720238837.512808, [], [1, 2, 3, 4, 5, 6]), - (1720400917.965225, [], [1, 2, 3, 4, 5, 6]), - (1720499924.896186, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719978670.650293, [], [1, 2, 3, 4, 5, 6]), (1720312908.844749, [], [1, 2, 3, 4, 5, 6])], - [ - (1720064475.615309, [], [1, 2, 3, 4, 5, 6]), - (1720311005.200102, [], [1, 2, 3, 4, 5, 6]), - (1720398033.682041, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1719974842.261587, [], [1, 2, 3, 4, 5, 6]), - (1720070482.809945, [], [1, 2, 3, 4, 5, 6]), - (1720492054.306253, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719985346.86067, [], [1, 2, 3, 4, 5, 6])], - [(1720494586.311794, [], [1, 2, 3, 4, 5, 6])], - [ - (1720151598.2787, [], [1, 2, 3, 4, 5, 6]), - (1720243336.399964, [], [1, 2, 3, 4, 5, 6]), - (1720394460.006175, [], [1, 2, 3, 4, 5, 6]), - (1720584803.786632, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720044414.812182, [], [1, 2, 3, 4, 5, 6]), (1720161374.32603, [], [1, 2, 3, 4, 5, 6])], - [(1720242818.837344, [], [1, 2, 3, 4, 5, 6])], - [(1720407806.948096, [], [1, 2, 3, 4, 5, 6]), (1720511793.967125, [], [1, 2, 3, 4, 5, 6])], - [(1720177465.166679, [], [1, 2, 3, 4, 5, 6])], - [(1720152899.613121, [], [1, 2, 3, 4, 5, 6])], - [(1720144169.768087, [], [1, 2, 3, 4, 5, 6])], - [(1720050900.326256, [], [1, 2, 3, 4, 5, 6]), (1720592651.789908, [], [1, 2, 3, 4, 5, 6])], - [(1720136535.399876, [], [1, 2, 3, 4, 5, 6]), (1720345016.561725, [], [1, 2, 3, 4, 5, 6])], - [(1720148677.039505, [], [1, 2, 3, 4, 5, 6])], - [(1720103982.765975, [], [1, 2, 3, 4, 5, 6]), (1720223275.492349, [], [1, 2, 3, 4, 5, 6])], - [(1719966246.265247, [], [1, 2, 3, 4, 5, 6])], - [(1720048787.249996, [], [1, 2, 3, 4, 5, 6]), (1720588475.186395, [], [1, 2, 3, 4, 5, 6])], - [ - (1720406823.932911, [], [1, 2, 3, 4, 5, 6]), - (1720406854.864424, [], [1, 2, 3, 4, 5, 6]), - (1720406898.943281, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720106989.608336, [], [1, 2, 3, 4, 5, 6])], - [(1719965156.233365, [], [1, 2, 3, 4, 5, 6]), (1720156113.65034, [], [1, 2, 3, 4, 5, 6])], - [(1720237894.767081, [], [1, 2, 3, 4, 5, 6])], - [ - (1720236335.89358, [], [1, 2, 3, 4, 5, 6]), - (1720311377.453215, [], [1, 2, 3, 4, 5, 6]), - (1720406308.416613, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720580297.715893, [], [1, 2, 3, 4, 5, 6])], - [(1719983515.156201, [], [1, 2, 3, 4, 5, 6]), (1720243011.26844, [], [1, 2, 3, 4, 5, 6])], - [(1720412740.206646, [], [1, 2, 3, 4, 5, 6])], - [(1720573676.882026, [], [1, 2, 3, 4, 5, 6])], - [(1720069113.016836, [], [1, 2, 3, 4, 5, 6])], - [(1720065156.88711, [], [1, 2, 3, 4, 5, 6]), (1720342013.62189, [], [1, 2, 3, 4, 5, 6])], - [(1720414414.37316, [], [1, 2, 3, 4, 5, 6]), (1720576057.542994, [], [1, 2, 3, 4, 5, 6])], - [ - (1719965980.977528, [], [1, 2, 3, 4, 5, 6]), - (1720328208.291947, [], [1, 2, 3, 4, 5, 6]), - (1720586256.843288, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719993285.557781, [], [1, 2, 3, 4, 5, 6])], - [ - (1720157474.360894, [], [1, 2, 3, 4, 5, 6]), - (1720317049.692797, [], [1, 2, 3, 4, 5, 6]), - (1720418157.354486, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720492206.117992, [], [1, 2, 3, 4, 5, 6])], - [(1720566094.344648, [], [1, 2, 3, 4, 5, 6])], - [(1719974058.930323, [], [1, 2, 3, 4, 5, 6]), (1720059173.893018, [], [1, 2, 3, 4, 5, 6])], - [(1720517061.661773, [], [1, 2, 3, 4, 5, 6])], - [(1720052300.009912, [], [1, 2, 3, 4, 5, 6]), (1720447926.535749, [], [1, 2, 3, 4, 5, 6])], - [(1720060153.321408, [], [1, 2, 3, 4, 5, 6]), (1720498576.79341, [], [1, 2, 3, 4, 5, 6])], - [(1720415193.154478, [], [1, 2, 3, 4, 5, 6]), (1720494529.74019, [], [1, 2, 3, 4, 5, 6])], - [ - (1719980354.732889, [], [1, 2, 3, 4, 5, 6]), - (1719980356.852338, [], [1, 2, 3, 4, 5, 6]), - (1719980359.805901, [], [1, 2, 3, 4, 5, 6]), - (1719980361.898886, [], [1, 2, 3, 4, 5, 6]), - (1719980364.204398, [], [1, 2, 3, 4, 5, 6]), - (1720063622.210305, [], [1, 2, 3, 4, 5, 6]), - (1720063766.011158, [], [1, 2, 3, 4, 5, 6]), - (1720063768.060823, [], [1, 2, 3, 4, 5, 6]), - (1720134932.57792, [], [1, 2, 3, 4, 5, 6]), - (1720148426.91756, [], [1, 2, 3, 4, 5, 6]), - (1720148428.987966, [], [1, 2, 3, 4, 5, 6]), - (1720237634.85931, [], [1, 2, 3, 4, 5, 6]), - (1720237687.961173, [], [1, 2, 3, 4, 5, 6]), - (1720321596.679301, [], [1, 2, 3, 4, 5, 6]), - (1720394727.592533, [], [1, 2, 3, 4, 5, 6]), - (1720394743.278857, [], [1, 2, 3, 4, 5, 6]), - (1720408408.443408, [], [1, 2, 3, 4, 5, 6]), - (1720419213.527306, [], [1, 2, 3, 4, 5, 6]), - (1720419321.981, [], [1, 2, 3, 4, 5, 6]), - (1720419324.073269, [], [1, 2, 3, 4, 5, 6]), - (1720497577.385151, [], [1, 2, 3, 4, 5, 6]), - (1720497694.789568, [], [1, 2, 3, 4, 5, 6]), - (1720497696.883431, [], [1, 2, 3, 4, 5, 6]), - (1720584999.597212, [], [1, 2, 3, 4, 5, 6]), - (1720585001.687849, [], [1, 2, 3, 4, 5, 6]), - (1720585005.063862, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720134251.830614, [], [1, 2, 3, 4, 5, 6]), - (1720407054.526951, [], [1, 2, 3, 4, 5, 6]), - (1720543564.686466, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720395362.215692, [], [1, 2, 3, 4, 5, 6]), (1720500480.122778, [], [1, 2, 3, 4, 5, 6])], - [ - (1720058412.695383, [], [1, 2, 3, 4, 5, 6]), - (1720228775.865928, [], [1, 2, 3, 4, 5, 6]), - (1720503282.31697, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720614350.980502, [], [1, 2, 3, 4, 5, 6])], - [ - (1719982519.149294, [], [1, 2, 3, 4, 5, 6]), - (1719982589.903207, [], [1, 2, 3, 4, 5, 6]), - (1719982591.966235, [], [1, 2, 3, 4, 5, 6]), - (1719982594.824529, [], [1, 2, 3, 4, 5, 6]), - (1720056059.106374, [], [1, 2, 3, 4, 5, 6]), - (1720056099.353137, [], [1, 2, 3, 4, 5, 6]), - (1720056101.438857, [], [1, 2, 3, 4, 5, 6]), - (1720084141.137944, [], [1, 2, 3, 4, 5, 6]), - (1720084195.578773, [], [1, 2, 3, 4, 5, 6]), - (1720136869.202173, [], [1, 2, 3, 4, 5, 6]), - (1720223064.342828, [], [1, 2, 3, 4, 5, 6]), - (1720223120.591787, [], [1, 2, 3, 4, 5, 6]), - (1720223122.696149, [], [1, 2, 3, 4, 5, 6]), - (1720239556.237398, [], [1, 2, 3, 4, 5, 6]), - (1720239630.045363, [], [1, 2, 3, 4, 5, 6]), - (1720239632.137037, [], [1, 2, 3, 4, 5, 6]), - (1720312988.468776, [], [1, 2, 3, 4, 5, 6]), - (1720313161.594176, [], [1, 2, 3, 4, 5, 6]), - (1720313163.656358, [], [1, 2, 3, 4, 5, 6]), - (1720413652.862676, [], [1, 2, 3, 4, 5, 6]), - (1720413773.395596, [], [1, 2, 3, 4, 5, 6]), - (1720484458.010065, [], [1, 2, 3, 4, 5, 6]), - (1720484503.114542, [], [1, 2, 3, 4, 5, 6]), - (1720484505.173957, [], [1, 2, 3, 4, 5, 6]), - (1720570920.862746, [], [1, 2, 3, 4, 5, 6]), - (1720571065.994777, [], [1, 2, 3, 4, 5, 6]), - (1720571068.086575, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720138634.579801, [], [1, 2, 3, 4, 5, 6]), (1720394701.653755, [], [1, 2, 3, 4, 5, 6])], - [(1720404840.88735, [], [1, 2, 3, 4, 5, 6]), (1720570759.329975, [], [1, 2, 3, 4, 5, 6])], - [(1720474997.255842, [], [1, 2, 3, 4, 5, 6])], - [ - (1719964981.812038, [], [1, 2, 3, 4, 5, 6]), - (1719965079.656724, [], [1, 2, 3, 4, 5, 6]), - (1719965081.766625, [], [1, 2, 3, 4, 5, 6]), - (1720017945.346535, [], [1, 2, 3, 4, 5, 6]), - (1720018196.228851, [], [1, 2, 3, 4, 5, 6]), - (1720018198.332037, [], [1, 2, 3, 4, 5, 6]), - (1720071944.789981, [], [1, 2, 3, 4, 5, 6]), - (1720071989.860765, [], [1, 2, 3, 4, 5, 6]), - (1720071991.963241, [], [1, 2, 3, 4, 5, 6]), - (1720226601.357382, [], [1, 2, 3, 4, 5, 6]), - (1720226662.671017, [], [1, 2, 3, 4, 5, 6]), - (1720226664.720854, [], [1, 2, 3, 4, 5, 6]), - (1720226666.697991, [], [1, 2, 3, 4, 5, 6]), - (1720245432.525672, [], [1, 2, 3, 4, 5, 6]), - (1720245586.690365, [], [1, 2, 3, 4, 5, 6]), - (1720245588.811888, [], [1, 2, 3, 4, 5, 6]), - (1720313288.75101, [], [1, 2, 3, 4, 5, 6]), - (1720313438.935319, [], [1, 2, 3, 4, 5, 6]), - (1720313440.997298, [], [1, 2, 3, 4, 5, 6]), - (1720325185.461926, [], [1, 2, 3, 4, 5, 6]), - (1720325279.708469, [], [1, 2, 3, 4, 5, 6]), - (1720325281.823994, [], [1, 2, 3, 4, 5, 6]), - (1720325284.895173, [], [1, 2, 3, 4, 5, 6]), - (1720325286.963747, [], [1, 2, 3, 4, 5, 6]), - (1720351212.007507, [], [1, 2, 3, 4, 5, 6]), - (1720351417.722923, [], [1, 2, 3, 4, 5, 6]), - (1720351419.786979, [], [1, 2, 3, 4, 5, 6]), - (1720410234.644402, [], [1, 2, 3, 4, 5, 6]), - (1720410236.746729, [], [1, 2, 3, 4, 5, 6]), - (1720484087.598816, [], [1, 2, 3, 4, 5, 6]), - (1720484089.656452, [], [1, 2, 3, 4, 5, 6]), - (1720560975.588946, [], [1, 2, 3, 4, 5, 6]), - (1720561062.767708, [], [1, 2, 3, 4, 5, 6]), - (1720576170.001406, [], [1, 2, 3, 4, 5, 6]), - (1720576274.339938, [], [1, 2, 3, 4, 5, 6]), - (1720634969.318238, [], [1, 2, 3, 4, 5, 6]), - (1720634971.383262, [], [1, 2, 3, 4, 5, 6]), - (1720634973.669218, [], [1, 2, 3, 4, 5, 6]), - (1720634975.727614, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720047138.987663, [], [1, 2, 3, 4, 5, 6]), - (1720239116.860589, [], [1, 2, 3, 4, 5, 6]), - (1720567216.089602, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720062114.160483, [], [1, 2, 3, 4, 5, 6])], - [(1719974901.32474, [], [1, 2, 3, 4, 5, 6]), (1720224712.94567, [], [1, 2, 3, 4, 5, 6])], - [ - (1719985511.407849, [], [1, 2, 3, 4, 5, 6]), - (1720140363.584567, [], [1, 2, 3, 4, 5, 6]), - (1720573348.34834, [], [1, 2, 3, 4, 5, 6]), - (1720649971.95392, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720325668.53657, [], [1, 2, 3, 4, 5, 6]), (1720407800.484026, [], [1, 2, 3, 4, 5, 6])], - [(1720313988.784401, [], [1, 2, 3, 4, 5, 6])], - [ - (1720137608.121513, [], [1, 2, 3, 4, 5, 6]), - (1720230219.916298, [], [1, 2, 3, 4, 5, 6]), - (1720576804.122481, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720060264.40999, [], [1, 2, 3, 4, 5, 6])], - [ - (1719982324.891431, [], [1, 2, 3, 4, 5, 6]), - (1719982326.95929, [], [1, 2, 3, 4, 5, 6]), - (1720072365.964045, [], [1, 2, 3, 4, 5, 6]), - (1720072368.013382, [], [1, 2, 3, 4, 5, 6]), - (1720185779.887725, [], [1, 2, 3, 4, 5, 6]), - (1720185782.013458, [], [1, 2, 3, 4, 5, 6]), - (1720230424.054008, [], [1, 2, 3, 4, 5, 6]), - (1720329463.631365, [], [1, 2, 3, 4, 5, 6]), - (1720329503.210461, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719978020.337077, [], [1, 2, 3, 4, 5, 6]), (1720378278.012205, [], [1, 2, 3, 4, 5, 6])], - [ - (1720198700.302556, [], [1, 2, 3, 4, 5, 6]), - (1720417057.718199, [], [1, 2, 3, 4, 5, 6]), - (1720584860.786802, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720627126.675399, [], [1, 2, 3, 4, 5, 6])], - [(1720072022.286212, [], [1, 2, 3, 4, 5, 6]), (1720366423.980574, [], [1, 2, 3, 4, 5, 6])], - [(1720221042.039954, [], [1, 2, 3, 4, 5, 6])], - [ - (1720060635.731519, [], [1, 2, 3, 4, 5, 6]), - (1720210299.946067, [], [1, 2, 3, 4, 5, 6]), - (1720283154.070272, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720402469.930097, [], [1, 2, 3, 4, 5, 6])], - [ - (1719972611.185894, [], [1, 2, 3, 4, 5, 6]), - (1720227219.185837, [], [1, 2, 3, 4, 5, 6]), - (1720565623.051185, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720060770.015072, [], [1, 2, 3, 4, 5, 6])], - [ - (1719979906.872722, [], [1, 2, 3, 4, 5, 6]), - (1719980094.376717, [], [1, 2, 3, 4, 5, 6]), - (1719980096.476308, [], [1, 2, 3, 4, 5, 6]), - (1720067152.682142, [], [1, 2, 3, 4, 5, 6]), - (1720067174.598435, [], [1, 2, 3, 4, 5, 6]), - (1720137635.975558, [], [1, 2, 3, 4, 5, 6]), - (1720137733.593423, [], [1, 2, 3, 4, 5, 6]), - (1720224539.774939, [], [1, 2, 3, 4, 5, 6]), - (1720323664.982932, [], [1, 2, 3, 4, 5, 6]), - (1720400336.830381, [], [1, 2, 3, 4, 5, 6]), - (1720400497.747426, [], [1, 2, 3, 4, 5, 6]), - (1720400499.843107, [], [1, 2, 3, 4, 5, 6]), - (1720486404.88152, [], [1, 2, 3, 4, 5, 6]), - (1720486460.387837, [], [1, 2, 3, 4, 5, 6]), - (1720486462.465262, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720494960.521565, [], [1, 2, 3, 4, 5, 6])], - [(1720492283.522426, [], [1, 2, 3, 4, 5, 6])], - [(1720588131.39025, [], [1, 2, 3, 4, 5, 6])], - [(1719965172.184078, [], [1, 2, 3, 4, 5, 6])], - [(1720313653.224728, [], [1, 2, 3, 4, 5, 6])], - [(1720133961.331413, [], [1, 2, 3, 4, 5, 6])], - [(1719969914.979558, [], [1, 2, 3, 4, 5, 6])], - [ - (1720051155.959984, [], [1, 2, 3, 4, 5, 6]), - (1720318569.685111, [], [1, 2, 3, 4, 5, 6]), - (1720499729.951734, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720328273.411971, [], [1, 2, 3, 4, 5, 6])], - [ - (1719982315.965122, [], [1, 2, 3, 4, 5, 6]), - (1720423276.150804, [], [1, 2, 3, 4, 5, 6]), - (1720586911.740203, [], [1, 2, 3, 4, 5, 6]), - ], - [(1719968679.211527, [], [1, 2, 3, 4, 5, 6])], - [(1720063388.278848, [], [1, 2, 3, 4, 5, 6]), (1720416336.796001, [], [1, 2, 3, 4, 5, 6])], - [(1720398479.735494, [], [1, 2, 3, 4, 5, 6]), (1720493260.033312, [], [1, 2, 3, 4, 5, 6])], - [ - (1720489609.661573, [], [1, 2, 3, 4, 5, 6]), - (1720489700.750791, [], [1, 2, 3, 4, 5, 6]), - (1720489717.546997, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720500732.208908, [], [1, 2, 3, 4, 5, 6])], - [ - (1720153118.225066, [], [1, 2, 3, 4, 5, 6]), - (1720314031.634943, [], [1, 2, 3, 4, 5, 6]), - (1720590337.724401, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720070140.554847, [], [1, 2, 3, 4, 5, 6]), (1720137932.433554, [], [1, 2, 3, 4, 5, 6])], - [(1719992154.926275, [], [1, 2, 3, 4, 5, 6]), (1720320574.945082, [], [1, 2, 3, 4, 5, 6])], - [(1719984916.520671, [], [1, 2, 3, 4, 5, 6]), (1720569849.178614, [], [1, 2, 3, 4, 5, 6])], - [(1720140614.641046, [], [1, 2, 3, 4, 5, 6]), (1720395184.350061, [], [1, 2, 3, 4, 5, 6])], - [(1720310387.035179, [], [1, 2, 3, 4, 5, 6]), (1720473940.199193, [], [1, 2, 3, 4, 5, 6])], - [(1720062920.051834, [], [1, 2, 3, 4, 5, 6]), (1720226181.474055, [], [1, 2, 3, 4, 5, 6])], - [(1720470329.222623, [], [1, 2, 3, 4, 5, 6])], - [(1720582334.499662, [], [1, 2, 3, 4, 5, 6])], - [(1720443828.896214, [], [1, 2, 3, 4, 5, 6]), (1720580682.756419, [], [1, 2, 3, 4, 5, 6])], - [ - (1720226425.344326, [], [1, 2, 3, 4, 5, 6]), - (1720310598.961662, [], [1, 2, 3, 4, 5, 6]), - (1720589761.631011, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720584698.862455, [], [1, 2, 3, 4, 5, 6])], - [(1720164879.185564, [], [1, 2, 3, 4, 5, 6]), (1720323846.480885, [], [1, 2, 3, 4, 5, 6])], - [(1720051096.071376, [], [1, 2, 3, 4, 5, 6]), (1720157299.452758, [], [1, 2, 3, 4, 5, 6])], - [ - (1720223524.412388, [], [1, 2, 3, 4, 5, 6]), - (1720326592.782923, [], [1, 2, 3, 4, 5, 6]), - (1720578100.065601, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720313647.455076, [], [1, 2, 3, 4, 5, 6]), (1720559337.211802, [], [1, 2, 3, 4, 5, 6])], - [ - (1719981335.449573, [], [1, 2, 3, 4, 5, 6]), - (1720067595.1521, [], [1, 2, 3, 4, 5, 6]), - (1720319132.823969, [], [1, 2, 3, 4, 5, 6]), - (1720491547.165147, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720019886.753884, [], [1, 2, 3, 4, 5, 6]), - (1720153101.556554, [], [1, 2, 3, 4, 5, 6]), - (1720313536.357232, [], [1, 2, 3, 4, 5, 6]), - (1720485395.202604, [], [1, 2, 3, 4, 5, 6]), - (1720568839.562655, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720310841.194378, [], [1, 2, 3, 4, 5, 6]), - (1720310912.574061, [], [1, 2, 3, 4, 5, 6]), - (1720310914.655803, [], [1, 2, 3, 4, 5, 6]), - (1720587828.804404, [], [1, 2, 3, 4, 5, 6]), - (1720588071.078858, [], [1, 2, 3, 4, 5, 6]), - (1720588073.115074, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720055953.618503, [], [1, 2, 3, 4, 5, 6]), - (1720223652.080905, [], [1, 2, 3, 4, 5, 6]), - (1720308372.703732, [], [1, 2, 3, 4, 5, 6]), - (1720624033.359415, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720173756.125859, [], [1, 2, 3, 4, 5, 6]), - (1720315128.683231, [], [1, 2, 3, 4, 5, 6]), - (1720466410.646777, [], [1, 2, 3, 4, 5, 6]), - ], - [(1720157263.810637, [], [1, 2, 3, 4, 5, 6]), (1720235081.49838, [], [1, 2, 3, 4, 5, 6])], - [ - (1720229216.005254, [], [1, 2, 3, 4, 5, 6]), - (1720557735.625871, [], [1, 2, 3, 4, 5, 6]), - (1720627294.521232, [], [1, 2, 3, 4, 5, 6]), - ], - [ - (1720394336.326148, [], [1, 2, 3, 4, 5, 6]), - (1720394382.775033, [], [1, 2, 3, 4, 5, 6]), - (1720394404.054186, [], [1, 2, 3, 4, 5, 6]), - (1720394488.655765, [], [1, 2, 3, 4, 5, 6]), - (1720394583.815862, [], [1, 2, 3, 4, 5, 6]), - (1720394609.744123, [], [1, 2, 3, 4, 5, 6]), - (1720394643.351845, [], [1, 2, 3, 4, 5, 6]), - (1720394661.447752, [], [1, 2, 3, 4, 5, 6]), - (1720394715.354794, [], [1, 2, 3, 4, 5, 6]), - ], - ] - for b in a: - calculate_funnel_from_user_events(6, 1123200, "first_touch", "ordered", [[]], b) diff --git a/posthog/udf_versioner.py b/posthog/udf_versioner.py index a0d96a547b9f6..3be55d183ef42 100644 --- a/posthog/udf_versioner.py +++ b/posthog/udf_versioner.py @@ -1,7 +1,6 @@ import argparse import os import shutil -import glob import datetime import xml.etree.ElementTree as ET from xml import etree @@ -12,7 +11,7 @@ # 3. Copy the `user_defined_function.xml` file in the newly created version folder (e.g. `user_scripts/v4/user_defined_function.xml`) to the `posthog-cloud-infra` repo and deploy it # 4. After that deploy goes out, it is safe to land and deploy the changes to the `posthog` repo # If deploys aren't seamless, look into moving the action that copies the `user_scripts` folder to the clickhouse cluster earlier in the deploy process -UDF_VERSION = 0 # Last modified by: @aspicer, 2024-09-20 +UDF_VERSION = 0 # Last modified by: @aspicer, 2024-10-01 CLICKHOUSE_XML_FILENAME = "user_defined_function.xml" ACTIVE_XML_CONFIG = "../../docker/clickhouse/user_defined_function.xml" @@ -35,8 +34,9 @@ def prepare_version(force=False): raise FileExistsError( f"A directory already exists for this version at posthog/user_scripts/{VERSION_STR}. Did you forget to increment the version? If not, delete the folder and run this again, or run this script with a -f" ) - for file in glob.glob("*.py"): - shutil.copy(file, VERSION_STR) + for file in os.listdir(): + if os.path.isfile(file) and not file.endswith(".xml"): + shutil.copy(file, VERSION_STR) base_xml = ET.parse(ACTIVE_XML_CONFIG) diff --git a/posthog/user_scripts/aggregate_funnel b/posthog/user_scripts/aggregate_funnel new file mode 100755 index 0000000000000..0605040d162eb Binary files /dev/null and b/posthog/user_scripts/aggregate_funnel differ diff --git a/posthog/user_scripts/aggregate_funnel.py b/posthog/user_scripts/aggregate_funnel.py deleted file mode 100755 index 162918a819625..0000000000000 --- a/posthog/user_scripts/aggregate_funnel.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/python3 -import json -import sys -from dataclasses import dataclass, replace -from itertools import groupby, permutations -from typing import Any, cast -from collections.abc import Sequence - - -def parse_args(line): - args = json.loads(line) - return [ - int(args["num_steps"]), - int(args["conversion_window_limit"]), - str(args["breakdown_attribution_type"]), - str(args["funnel_order_type"]), - args["prop_vals"], # Array(Array(String)) - args["value"], # Array(Tuple(Nullable(Float64), Nullable(DateTime), Array(String), Array(Int8))) - ] - - -@dataclass(frozen=True) -class EnteredTimestamp: - timestamp: Any - timings: Any - - -# each one can be multiple steps here -# it only matters when they entered the funnel - you can propagate the time from the previous step when you update -# This function is defined for Clickhouse in user_defined_functions.xml along with types -# num_steps is the total number of steps in the funnel -# conversion_window_limit is in seconds -# events is a array of tuples of (timestamp, breakdown, [steps]) -# steps is an array of integers which represent the steps that this event qualifies for. it looks like [1,3,5,6]. -# negative integers represent an exclusion on that step. each event is either all exclusions or all steps. -def calculate_funnel_from_user_events( - num_steps: int, - conversion_window_limit_seconds: int, - breakdown_attribution_type: str, - funnel_order_type: str, - prop_vals: list[Any], - events: Sequence[tuple[float, list[str] | int | str, list[int]]], -): - default_entered_timestamp = EnteredTimestamp(0, []) - max_step = [0, default_entered_timestamp] - # If the attribution mode is a breakdown step, set this to the integer that represents that step - breakdown_step = int(breakdown_attribution_type[5:]) if breakdown_attribution_type.startswith("step_") else None - - # This function returns an Array. We build up an array of strings to return here. - results: list[tuple[int, Any, list[float]]] = [] - - # Process an event. If this hits an exclusion, return False, else return True. - def process_event(timestamp, breakdown, steps, *, entered_timestamp, prop_val) -> bool: - # iterate the steps in reverse so we don't count this event multiple times - for step in reversed(steps): - exclusion = False - if step < 0: - exclusion = True - step = -step - - in_match_window = timestamp - entered_timestamp[step - 1].timestamp <= conversion_window_limit_seconds - already_reached_this_step_with_same_entered_timestamp = ( - entered_timestamp[step].timestamp == entered_timestamp[step - 1].timestamp - ) - - if in_match_window and not already_reached_this_step_with_same_entered_timestamp: - if exclusion: - results.append((-1, prop_val, [])) - return False - is_unmatched_step_attribution = ( - breakdown_step is not None and step == breakdown_step - 1 and prop_val != breakdown - ) - if not is_unmatched_step_attribution: - entered_timestamp[step] = replace( - entered_timestamp[step - 1], timings=[*entered_timestamp[step - 1].timings, timestamp] - ) - if step > max_step[0]: - max_step[:] = (step, entered_timestamp[step]) - - if funnel_order_type == "strict": - for i in range(len(entered_timestamp)): - if i not in steps: - entered_timestamp[i] = default_entered_timestamp - - return True - - # We call this for each possible breakdown value. - def loop_prop_val(prop_val): - # an array of when the user entered the funnel - # entered_timestamp = [(0, "", [])] * (num_steps + 1) - max_step[:] = [0, default_entered_timestamp] - entered_timestamp: list[EnteredTimestamp] = [default_entered_timestamp] * (num_steps + 1) - - def add_max_step(): - i = cast(int, max_step[0]) - final = cast(EnteredTimestamp, max_step[1]) - results.append((i - 1, prop_val, [final.timings[i] - final.timings[i - 1] for i in range(1, i)])) - - filtered_events = ( - ((timestamp, breakdown, steps) for (timestamp, breakdown, steps) in events if breakdown == prop_val) - if breakdown_attribution_type == "all_events" - else events - ) - for timestamp, events_with_same_timestamp_iterator in groupby(filtered_events, key=lambda x: x[0]): - events_with_same_timestamp = tuple(events_with_same_timestamp_iterator) - entered_timestamp[0] = EnteredTimestamp(timestamp, []) - if len(events_with_same_timestamp) == 1: - if not process_event( - *events_with_same_timestamp[0], entered_timestamp=entered_timestamp, prop_val=prop_val - ): - return - else: - # This is a special case for events with the same timestamp - # We play all of their permutations and most generously take the ones that advanced the furthest - # This has quite bad performance, and can probably be optimized through clever but annoying logic - # but shouldn't be hit too often - entered_timestamps = [] - for events_group_perm in permutations(events_with_same_timestamp): - entered_timestamps.append(list(entered_timestamp)) - for event in events_group_perm: - if not process_event(*event, entered_timestamp=entered_timestamps[-1], prop_val=prop_val): - # If any of the permutations hits an exclusion, we exclude this user. - # This isn't an important implementation detail and we could do something smarter here. - return - for i in range(len(entered_timestamp)): - entered_timestamp[i] = max((x[i] for x in entered_timestamps), key=lambda x: x.timestamp) - - # If we have hit the goal, we can terminate early - if entered_timestamp[num_steps].timestamp > 0: - add_max_step() - return - - # Find the furthest step we have made it to and print it - add_max_step() - return - - [loop_prop_val(prop_val) for prop_val in prop_vals] - print(json.dumps({"result": results}), end="\n") # noqa: T201 - - -if __name__ == "__main__": - for line in sys.stdin: - calculate_funnel_from_user_events(*parse_args(line)) - sys.stdout.flush() diff --git a/posthog/user_scripts/aggregate_funnel_array.py b/posthog/user_scripts/aggregate_funnel_array.py deleted file mode 100755 index 17b053bb7d448..0000000000000 --- a/posthog/user_scripts/aggregate_funnel_array.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/python3 -import sys - -from aggregate_funnel import parse_args, calculate_funnel_from_user_events - -if __name__ == "__main__": - for line in sys.stdin: - calculate_funnel_from_user_events(*parse_args(line)) - sys.stdout.flush() diff --git a/posthog/user_scripts/aggregate_funnel_cohort.py b/posthog/user_scripts/aggregate_funnel_cohort.py deleted file mode 100755 index 17b053bb7d448..0000000000000 --- a/posthog/user_scripts/aggregate_funnel_cohort.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/python3 -import sys - -from aggregate_funnel import parse_args, calculate_funnel_from_user_events - -if __name__ == "__main__": - for line in sys.stdin: - calculate_funnel_from_user_events(*parse_args(line)) - sys.stdout.flush() diff --git a/posthog/user_scripts/aggregate_funnel_test.py b/posthog/user_scripts/aggregate_funnel_test.py index e0689b82af21c..1eae7c9f36914 100755 --- a/posthog/user_scripts/aggregate_funnel_test.py +++ b/posthog/user_scripts/aggregate_funnel_test.py @@ -1,13 +1,14 @@ #!/usr/bin/python3 import json -from aggregate_funnel import calculate_funnel_from_user_events, parse_args import sys +import traceback if __name__ == "__main__": for line in sys.stdin: try: - calculate_funnel_from_user_events(*parse_args(line)) + # calculate_funnel_from_user_events(*parse_args(line)) + print(json.dumps({"result": line})) # noqa: T201 except Exception as e: - print(json.dumps({"result": json.dumps(str(e))}), end="\n") # noqa: T201 + print(json.dumps({"result": json.dumps(str(e) + traceback.format_exc())}), end="\n") # noqa: T201 sys.stdout.flush() diff --git a/posthog/user_scripts/aggregate_funnel_trends.py b/posthog/user_scripts/aggregate_funnel_trends.py index 0aa96b7a19b96..cde3a30928584 100755 --- a/posthog/user_scripts/aggregate_funnel_trends.py +++ b/posthog/user_scripts/aggregate_funnel_trends.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 import sys from dataclasses import dataclass, replace -from typing import Any +from typing import Any, Union from collections.abc import Sequence import json @@ -40,7 +40,7 @@ def calculate_funnel_trends_from_user_events( breakdown_attribution_type: str, funnel_order_type: str, prop_vals: list[Any], - events: Sequence[tuple[float, int, list[str] | int | str, list[int]]], + events: Sequence[tuple[float, int, Union[list[str], int, str], list[int]]], ): default_entered_timestamp = EnteredTimestamp(0, []) # If the attribution mode is a breakdown step, set this to the integer that represents that step diff --git a/posthog/user_scripts/latest_user_defined_function.xml b/posthog/user_scripts/latest_user_defined_function.xml index 6f8f787da15c1..9a0b6001786fa 100644 --- a/posthog/user_scripts/latest_user_defined_function.xml +++ b/posthog/user_scripts/latest_user_defined_function.xml @@ -1,9 +1,9 @@ - executable aggregate_funnel - Array(Tuple(Int8, Nullable(String), Array(Float64))) + Array(Tuple(Int8, Nullable(String), Array(Float64), Array(Array(UUID)))) result UInt8 @@ -26,17 +26,18 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the prop_vals - Array(Tuple(Nullable(Float64), Nullable(String), Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, Nullable(String), Array(Int8))) value JSONEachRow - aggregate_funnel.py + aggregate_funnel + 600 executable aggregate_funnel_cohort - Array(Tuple(Int8, UInt64, Array(Float64))) + Array(Tuple(Int8, UInt64, Array(Float64), Array(Array(UUID)))) result UInt8 @@ -59,17 +60,18 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the prop_vals - Array(Tuple(Nullable(Float64), UInt64, Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, UInt64, Array(Int8))) value JSONEachRow - aggregate_funnel_cohort.py + aggregate_funnel + 600 executable aggregate_funnel_array - Array(Tuple(Int8, Array(String), Array(Float64))) + Array(Tuple(Int8, Array(String), Array(Float64), Array(Array(UUID)))) result UInt8 @@ -92,11 +94,12 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the prop_vals - Array(Tuple(Nullable(Float64), Array(String), Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, Array(String), Array(Int8))) value JSONEachRow - aggregate_funnel_array.py + aggregate_funnel + 600 @@ -125,11 +128,12 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the prop_vals - Array(Tuple(Nullable(Float64), Nullable(String), Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, Nullable(String), Array(Int8))) value JSONEachRow aggregate_funnel_test.py + 600 @@ -171,6 +175,7 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the JSONEachRow aggregate_funnel_trends.py + 600 @@ -209,6 +214,7 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the JSONEachRow aggregate_funnel_array_trends.py + 600 @@ -247,6 +253,7 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the JSONEachRow aggregate_funnel_cohort_trends.py + 600 @@ -284,11 +291,12 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the JSONEachRow aggregate_funnel_array_trends_test.py + 600 executable aggregate_funnel_v0 - Array(Tuple(Int8, Nullable(String), Array(Float64))) + Array(Tuple(Int8, Nullable(String), Array(Float64), Array(Array(UUID)))) result UInt8 @@ -311,17 +319,18 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the prop_vals - Array(Tuple(Nullable(Float64), Nullable(String), Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, Nullable(String), Array(Int8))) value JSONEachRow - v0/aggregate_funnel.py + v0/aggregate_funnel + 600 executable aggregate_funnel_cohort_v0 - Array(Tuple(Int8, UInt64, Array(Float64))) + Array(Tuple(Int8, UInt64, Array(Float64), Array(Array(UUID)))) result UInt8 @@ -344,17 +353,18 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the prop_vals - Array(Tuple(Nullable(Float64), UInt64, Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, UInt64, Array(Int8))) value JSONEachRow - v0/aggregate_funnel_cohort.py + v0/aggregate_funnel + 600 executable aggregate_funnel_array_v0 - Array(Tuple(Int8, Array(String), Array(Float64))) + Array(Tuple(Int8, Array(String), Array(Float64), Array(Array(UUID)))) result UInt8 @@ -377,11 +387,12 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the prop_vals - Array(Tuple(Nullable(Float64), Array(String), Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, Array(String), Array(Int8))) value JSONEachRow - v0/aggregate_funnel_array.py + v0/aggregate_funnel + 600 @@ -410,11 +421,12 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the prop_vals - Array(Tuple(Nullable(Float64), Nullable(String), Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, Nullable(String), Array(Int8))) value JSONEachRow v0/aggregate_funnel_test.py + 600 @@ -456,6 +468,7 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the JSONEachRow v0/aggregate_funnel_trends.py + 600 @@ -494,6 +507,7 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the JSONEachRow v0/aggregate_funnel_array_trends.py + 600 @@ -532,6 +546,7 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the JSONEachRow v0/aggregate_funnel_cohort_trends.py + 600 @@ -569,5 +584,6 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the JSONEachRow v0/aggregate_funnel_array_trends_test.py + 600 \ No newline at end of file diff --git a/posthog/user_scripts/v0/aggregate_funnel b/posthog/user_scripts/v0/aggregate_funnel new file mode 100755 index 0000000000000..0605040d162eb Binary files /dev/null and b/posthog/user_scripts/v0/aggregate_funnel differ diff --git a/posthog/user_scripts/v0/aggregate_funnel.py b/posthog/user_scripts/v0/aggregate_funnel.py deleted file mode 100755 index 162918a819625..0000000000000 --- a/posthog/user_scripts/v0/aggregate_funnel.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/python3 -import json -import sys -from dataclasses import dataclass, replace -from itertools import groupby, permutations -from typing import Any, cast -from collections.abc import Sequence - - -def parse_args(line): - args = json.loads(line) - return [ - int(args["num_steps"]), - int(args["conversion_window_limit"]), - str(args["breakdown_attribution_type"]), - str(args["funnel_order_type"]), - args["prop_vals"], # Array(Array(String)) - args["value"], # Array(Tuple(Nullable(Float64), Nullable(DateTime), Array(String), Array(Int8))) - ] - - -@dataclass(frozen=True) -class EnteredTimestamp: - timestamp: Any - timings: Any - - -# each one can be multiple steps here -# it only matters when they entered the funnel - you can propagate the time from the previous step when you update -# This function is defined for Clickhouse in user_defined_functions.xml along with types -# num_steps is the total number of steps in the funnel -# conversion_window_limit is in seconds -# events is a array of tuples of (timestamp, breakdown, [steps]) -# steps is an array of integers which represent the steps that this event qualifies for. it looks like [1,3,5,6]. -# negative integers represent an exclusion on that step. each event is either all exclusions or all steps. -def calculate_funnel_from_user_events( - num_steps: int, - conversion_window_limit_seconds: int, - breakdown_attribution_type: str, - funnel_order_type: str, - prop_vals: list[Any], - events: Sequence[tuple[float, list[str] | int | str, list[int]]], -): - default_entered_timestamp = EnteredTimestamp(0, []) - max_step = [0, default_entered_timestamp] - # If the attribution mode is a breakdown step, set this to the integer that represents that step - breakdown_step = int(breakdown_attribution_type[5:]) if breakdown_attribution_type.startswith("step_") else None - - # This function returns an Array. We build up an array of strings to return here. - results: list[tuple[int, Any, list[float]]] = [] - - # Process an event. If this hits an exclusion, return False, else return True. - def process_event(timestamp, breakdown, steps, *, entered_timestamp, prop_val) -> bool: - # iterate the steps in reverse so we don't count this event multiple times - for step in reversed(steps): - exclusion = False - if step < 0: - exclusion = True - step = -step - - in_match_window = timestamp - entered_timestamp[step - 1].timestamp <= conversion_window_limit_seconds - already_reached_this_step_with_same_entered_timestamp = ( - entered_timestamp[step].timestamp == entered_timestamp[step - 1].timestamp - ) - - if in_match_window and not already_reached_this_step_with_same_entered_timestamp: - if exclusion: - results.append((-1, prop_val, [])) - return False - is_unmatched_step_attribution = ( - breakdown_step is not None and step == breakdown_step - 1 and prop_val != breakdown - ) - if not is_unmatched_step_attribution: - entered_timestamp[step] = replace( - entered_timestamp[step - 1], timings=[*entered_timestamp[step - 1].timings, timestamp] - ) - if step > max_step[0]: - max_step[:] = (step, entered_timestamp[step]) - - if funnel_order_type == "strict": - for i in range(len(entered_timestamp)): - if i not in steps: - entered_timestamp[i] = default_entered_timestamp - - return True - - # We call this for each possible breakdown value. - def loop_prop_val(prop_val): - # an array of when the user entered the funnel - # entered_timestamp = [(0, "", [])] * (num_steps + 1) - max_step[:] = [0, default_entered_timestamp] - entered_timestamp: list[EnteredTimestamp] = [default_entered_timestamp] * (num_steps + 1) - - def add_max_step(): - i = cast(int, max_step[0]) - final = cast(EnteredTimestamp, max_step[1]) - results.append((i - 1, prop_val, [final.timings[i] - final.timings[i - 1] for i in range(1, i)])) - - filtered_events = ( - ((timestamp, breakdown, steps) for (timestamp, breakdown, steps) in events if breakdown == prop_val) - if breakdown_attribution_type == "all_events" - else events - ) - for timestamp, events_with_same_timestamp_iterator in groupby(filtered_events, key=lambda x: x[0]): - events_with_same_timestamp = tuple(events_with_same_timestamp_iterator) - entered_timestamp[0] = EnteredTimestamp(timestamp, []) - if len(events_with_same_timestamp) == 1: - if not process_event( - *events_with_same_timestamp[0], entered_timestamp=entered_timestamp, prop_val=prop_val - ): - return - else: - # This is a special case for events with the same timestamp - # We play all of their permutations and most generously take the ones that advanced the furthest - # This has quite bad performance, and can probably be optimized through clever but annoying logic - # but shouldn't be hit too often - entered_timestamps = [] - for events_group_perm in permutations(events_with_same_timestamp): - entered_timestamps.append(list(entered_timestamp)) - for event in events_group_perm: - if not process_event(*event, entered_timestamp=entered_timestamps[-1], prop_val=prop_val): - # If any of the permutations hits an exclusion, we exclude this user. - # This isn't an important implementation detail and we could do something smarter here. - return - for i in range(len(entered_timestamp)): - entered_timestamp[i] = max((x[i] for x in entered_timestamps), key=lambda x: x.timestamp) - - # If we have hit the goal, we can terminate early - if entered_timestamp[num_steps].timestamp > 0: - add_max_step() - return - - # Find the furthest step we have made it to and print it - add_max_step() - return - - [loop_prop_val(prop_val) for prop_val in prop_vals] - print(json.dumps({"result": results}), end="\n") # noqa: T201 - - -if __name__ == "__main__": - for line in sys.stdin: - calculate_funnel_from_user_events(*parse_args(line)) - sys.stdout.flush() diff --git a/posthog/user_scripts/v0/aggregate_funnel_array.py b/posthog/user_scripts/v0/aggregate_funnel_array.py deleted file mode 100755 index 17b053bb7d448..0000000000000 --- a/posthog/user_scripts/v0/aggregate_funnel_array.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/python3 -import sys - -from aggregate_funnel import parse_args, calculate_funnel_from_user_events - -if __name__ == "__main__": - for line in sys.stdin: - calculate_funnel_from_user_events(*parse_args(line)) - sys.stdout.flush() diff --git a/posthog/user_scripts/v0/aggregate_funnel_cohort.py b/posthog/user_scripts/v0/aggregate_funnel_cohort.py deleted file mode 100755 index 17b053bb7d448..0000000000000 --- a/posthog/user_scripts/v0/aggregate_funnel_cohort.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/python3 -import sys - -from aggregate_funnel import parse_args, calculate_funnel_from_user_events - -if __name__ == "__main__": - for line in sys.stdin: - calculate_funnel_from_user_events(*parse_args(line)) - sys.stdout.flush() diff --git a/posthog/user_scripts/v0/aggregate_funnel_test.py b/posthog/user_scripts/v0/aggregate_funnel_test.py index e0689b82af21c..1eae7c9f36914 100755 --- a/posthog/user_scripts/v0/aggregate_funnel_test.py +++ b/posthog/user_scripts/v0/aggregate_funnel_test.py @@ -1,13 +1,14 @@ #!/usr/bin/python3 import json -from aggregate_funnel import calculate_funnel_from_user_events, parse_args import sys +import traceback if __name__ == "__main__": for line in sys.stdin: try: - calculate_funnel_from_user_events(*parse_args(line)) + # calculate_funnel_from_user_events(*parse_args(line)) + print(json.dumps({"result": line})) # noqa: T201 except Exception as e: - print(json.dumps({"result": json.dumps(str(e))}), end="\n") # noqa: T201 + print(json.dumps({"result": json.dumps(str(e) + traceback.format_exc())}), end="\n") # noqa: T201 sys.stdout.flush() diff --git a/posthog/user_scripts/v0/aggregate_funnel_trends.py b/posthog/user_scripts/v0/aggregate_funnel_trends.py index 0aa96b7a19b96..cde3a30928584 100755 --- a/posthog/user_scripts/v0/aggregate_funnel_trends.py +++ b/posthog/user_scripts/v0/aggregate_funnel_trends.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 import sys from dataclasses import dataclass, replace -from typing import Any +from typing import Any, Union from collections.abc import Sequence import json @@ -40,7 +40,7 @@ def calculate_funnel_trends_from_user_events( breakdown_attribution_type: str, funnel_order_type: str, prop_vals: list[Any], - events: Sequence[tuple[float, int, list[str] | int | str, list[int]]], + events: Sequence[tuple[float, int, Union[list[str], int, str], list[int]]], ): default_entered_timestamp = EnteredTimestamp(0, []) # If the attribution mode is a breakdown step, set this to the integer that represents that step diff --git a/posthog/user_scripts/v0/user_defined_function.xml b/posthog/user_scripts/v0/user_defined_function.xml index 6f8f787da15c1..9a0b6001786fa 100644 --- a/posthog/user_scripts/v0/user_defined_function.xml +++ b/posthog/user_scripts/v0/user_defined_function.xml @@ -1,9 +1,9 @@ - executable aggregate_funnel - Array(Tuple(Int8, Nullable(String), Array(Float64))) + Array(Tuple(Int8, Nullable(String), Array(Float64), Array(Array(UUID)))) result UInt8 @@ -26,17 +26,18 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the prop_vals - Array(Tuple(Nullable(Float64), Nullable(String), Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, Nullable(String), Array(Int8))) value JSONEachRow - aggregate_funnel.py + aggregate_funnel + 600 executable aggregate_funnel_cohort - Array(Tuple(Int8, UInt64, Array(Float64))) + Array(Tuple(Int8, UInt64, Array(Float64), Array(Array(UUID)))) result UInt8 @@ -59,17 +60,18 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the prop_vals - Array(Tuple(Nullable(Float64), UInt64, Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, UInt64, Array(Int8))) value JSONEachRow - aggregate_funnel_cohort.py + aggregate_funnel + 600 executable aggregate_funnel_array - Array(Tuple(Int8, Array(String), Array(Float64))) + Array(Tuple(Int8, Array(String), Array(Float64), Array(Array(UUID)))) result UInt8 @@ -92,11 +94,12 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the prop_vals - Array(Tuple(Nullable(Float64), Array(String), Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, Array(String), Array(Int8))) value JSONEachRow - aggregate_funnel_array.py + aggregate_funnel + 600 @@ -125,11 +128,12 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the prop_vals - Array(Tuple(Nullable(Float64), Nullable(String), Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, Nullable(String), Array(Int8))) value JSONEachRow aggregate_funnel_test.py + 600 @@ -171,6 +175,7 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the JSONEachRow aggregate_funnel_trends.py + 600 @@ -209,6 +214,7 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the JSONEachRow aggregate_funnel_array_trends.py + 600 @@ -247,6 +253,7 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the JSONEachRow aggregate_funnel_cohort_trends.py + 600 @@ -284,11 +291,12 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the JSONEachRow aggregate_funnel_array_trends_test.py + 600 executable aggregate_funnel_v0 - Array(Tuple(Int8, Nullable(String), Array(Float64))) + Array(Tuple(Int8, Nullable(String), Array(Float64), Array(Array(UUID)))) result UInt8 @@ -311,17 +319,18 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the prop_vals - Array(Tuple(Nullable(Float64), Nullable(String), Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, Nullable(String), Array(Int8))) value JSONEachRow - v0/aggregate_funnel.py + v0/aggregate_funnel + 600 executable aggregate_funnel_cohort_v0 - Array(Tuple(Int8, UInt64, Array(Float64))) + Array(Tuple(Int8, UInt64, Array(Float64), Array(Array(UUID)))) result UInt8 @@ -344,17 +353,18 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the prop_vals - Array(Tuple(Nullable(Float64), UInt64, Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, UInt64, Array(Int8))) value JSONEachRow - v0/aggregate_funnel_cohort.py + v0/aggregate_funnel + 600 executable aggregate_funnel_array_v0 - Array(Tuple(Int8, Array(String), Array(Float64))) + Array(Tuple(Int8, Array(String), Array(Float64), Array(Array(UUID)))) result UInt8 @@ -377,11 +387,12 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the prop_vals - Array(Tuple(Nullable(Float64), Array(String), Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, Array(String), Array(Int8))) value JSONEachRow - v0/aggregate_funnel_array.py + v0/aggregate_funnel + 600 @@ -410,11 +421,12 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the prop_vals - Array(Tuple(Nullable(Float64), Nullable(String), Array(Int8))) + Array(Tuple(Nullable(Float64), UUID, Nullable(String), Array(Int8))) value JSONEachRow v0/aggregate_funnel_test.py + 600 @@ -456,6 +468,7 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the JSONEachRow v0/aggregate_funnel_trends.py + 600 @@ -494,6 +507,7 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the JSONEachRow v0/aggregate_funnel_array_trends.py + 600 @@ -532,6 +546,7 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the JSONEachRow v0/aggregate_funnel_cohort_trends.py + 600 @@ -569,5 +584,6 @@ This file is autogenerated by udf_versioner.py. Do not edit this, only edit the JSONEachRow v0/aggregate_funnel_array_trends_test.py + 600 \ No newline at end of file diff --git a/posthog/user_scripts/v1/aggregate_funnel.py b/posthog/user_scripts/v1/aggregate_funnel.py deleted file mode 100755 index 1c40a993f91df..0000000000000 --- a/posthog/user_scripts/v1/aggregate_funnel.py +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/python3 -import json -import sys -from dataclasses import dataclass, replace -from itertools import groupby, permutations -from typing import Any, cast -from collections.abc import Sequence - - -def parse_args(line): - args = json.loads(line) - return [ - int(args["num_steps"]), - int(args["conversion_window_limit"]), - str(args["breakdown_attribution_type"]), - str(args["funnel_order_type"]), - args["prop_vals"], # Array(Array(String)) - args["value"], # Array(Tuple(Nullable(Float64), UUID, Array(String), Array(Int8))) - ] - - -@dataclass(frozen=True) -class EnteredTimestamp: - timestamp: Any - timings: Any - uuids: list[str] - - -MAX_REPLAY_EVENTS = 10 - - -# each one can be multiple steps here -# it only matters when they entered the funnel - you can propagate the time from the previous step when you update -# This function is defined for Clickhouse in user_defined_functions.xml along with types -# num_steps is the total number of steps in the funnel -# conversion_window_limit is in seconds -# events is an array of tuples of (timestamp, breakdown, [steps]) -# steps is an array of integers which represent the steps that this event qualifies for. it looks like [1,3,5,6]. -# negative integers represent an exclusion on that step. each event is either all exclusions or all steps. -def calculate_funnel_from_user_events( - num_steps: int, - conversion_window_limit_seconds: int, - breakdown_attribution_type: str, - funnel_order_type: str, - prop_vals: list[Any], - events: Sequence[tuple[float, str, list[str] | int | str, list[int]]], -): - default_entered_timestamp = EnteredTimestamp(0, [], []) - max_step = [0, default_entered_timestamp] - # If the attribution mode is a breakdown step, set this to the integer that represents that step - breakdown_step = int(breakdown_attribution_type[5:]) if breakdown_attribution_type.startswith("step_") else None - - # This function returns an Array. We build up an array of strings to return here. - results: list[tuple[int, Any, list[float], list[list[str]]]] = [] - - # Process an event. If this hits an exclusion, return False, else return True. - def process_event(timestamp, uuid, breakdown, steps, *, entered_timestamp, prop_val, event_uuids) -> bool: - # iterate the steps in reverse so we don't count this event multiple times - for step in reversed(steps): - exclusion = False - if step < 0: - exclusion = True - step = -step - - in_match_window = timestamp - entered_timestamp[step - 1].timestamp <= conversion_window_limit_seconds - already_reached_this_step_with_same_entered_timestamp = ( - entered_timestamp[step].timestamp == entered_timestamp[step - 1].timestamp - and entered_timestamp[step].timestamp != 0 - ) - - if in_match_window and not already_reached_this_step_with_same_entered_timestamp: - if exclusion: - results.append((-1, prop_val, [], [])) - return False - is_unmatched_step_attribution = ( - breakdown_step is not None and step == breakdown_step - 1 and prop_val != breakdown - ) - if not is_unmatched_step_attribution: - entered_timestamp[step] = replace( - entered_timestamp[step - 1], - timings=[*entered_timestamp[step - 1].timings, timestamp], - uuids=[*entered_timestamp[step - 1].uuids, uuid], - ) - if len(event_uuids[step - 1]) < MAX_REPLAY_EVENTS - 1: - event_uuids[step - 1].append(uuid) - # TODO: If this is strict, and this didn't match, we should remove this - if step > max_step[0]: - max_step[:] = (step, entered_timestamp[step]) - - if funnel_order_type == "strict": - for i in range(1, len(entered_timestamp)): - if i not in steps: - entered_timestamp[i] = default_entered_timestamp - - return True - - # We call this for each possible breakdown value. - def loop_prop_val(prop_val): - # an array of when the user entered the funnel - # entered_timestamp = [(0, "", [])] * (num_steps + 1) - max_step[:] = [0, default_entered_timestamp] - entered_timestamp: list[EnteredTimestamp] = [default_entered_timestamp] * (num_steps + 1) - event_uuids: list[list[str]] = [[] for _ in range(num_steps)] - - def add_max_step(): - final_index = cast(int, max_step[0]) - final = cast(EnteredTimestamp, max_step[1]) - for i in range(final_index): - # if len(event_uuids[i]) >= MAX_REPLAY_EVENTS and final.uuids[i] not in event_uuids[i]: - # Always put the actual event uuids first, we use it to extract timestamps - # This might create duplicates, but that's fine (we can remove it in clickhouse) - event_uuids[i].insert(0, final.uuids[i]) - results.append( - ( - final_index - 1, - prop_val, - [final.timings[i] - final.timings[i - 1] for i in range(1, final_index)], - event_uuids, - ) - ) - - filtered_events = ( - ( - (timestamp, uuid, breakdown, steps) - for (timestamp, uuid, breakdown, steps) in events - if breakdown == prop_val - ) - if breakdown_attribution_type == "all_events" - else events - ) - for timestamp, events_with_same_timestamp_iterator in groupby(filtered_events, key=lambda x: x[0]): - events_with_same_timestamp = tuple(events_with_same_timestamp_iterator) - entered_timestamp[0] = EnteredTimestamp(timestamp, [], []) - if len(events_with_same_timestamp) == 1: - if not process_event( - *events_with_same_timestamp[0], - entered_timestamp=entered_timestamp, - prop_val=prop_val, - event_uuids=event_uuids, - ): - return - else: - # This is a special case for events with the same timestamp - # We play all of their permutations and most generously take the ones that advanced the furthest - # This has quite bad performance, and can probably be optimized through clever but annoying logic - # but shouldn't be hit too often - # This could potentially cause the same event to be added to the matching events multiple times - entered_timestamps = [] - for events_group_perm in permutations(events_with_same_timestamp): - entered_timestamps.append(list(entered_timestamp)) - for event in events_group_perm: - if not process_event( - *event, entered_timestamp=entered_timestamps[-1], prop_val=prop_val, event_uuids=event_uuids - ): - # If any of the permutations hits an exclusion, we exclude this user. - # This isn't an important implementation detail and we could do something smarter here. - return - for i in range(len(entered_timestamp)): - entered_timestamp[i] = max((x[i] for x in entered_timestamps), key=lambda x: x.timestamp) - - # If we have hit the goal, we can terminate early - if entered_timestamp[num_steps].timestamp > 0: - add_max_step() - return - - # Find the furthest step we have made it to and print it - add_max_step() - return - - [loop_prop_val(prop_val) for prop_val in prop_vals] - print(json.dumps({"result": results}), end="\n") # noqa: T201 - - -if __name__ == "__main__": - for line in sys.stdin: - calculate_funnel_from_user_events(*parse_args(line)) - sys.stdout.flush() diff --git a/posthog/user_scripts/v1/aggregate_funnel_array.py b/posthog/user_scripts/v1/aggregate_funnel_array.py deleted file mode 100755 index 17b053bb7d448..0000000000000 --- a/posthog/user_scripts/v1/aggregate_funnel_array.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/python3 -import sys - -from aggregate_funnel import parse_args, calculate_funnel_from_user_events - -if __name__ == "__main__": - for line in sys.stdin: - calculate_funnel_from_user_events(*parse_args(line)) - sys.stdout.flush() diff --git a/posthog/user_scripts/v1/aggregate_funnel_array_trends.py b/posthog/user_scripts/v1/aggregate_funnel_array_trends.py deleted file mode 100755 index 15e93f5452797..0000000000000 --- a/posthog/user_scripts/v1/aggregate_funnel_array_trends.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/python3 -import sys - -from aggregate_funnel_trends import parse_args, calculate_funnel_trends_from_user_events - -if __name__ == "__main__": - for line in sys.stdin: - calculate_funnel_trends_from_user_events(*parse_args(line)) - sys.stdout.flush() diff --git a/posthog/user_scripts/v1/aggregate_funnel_array_trends_test.py b/posthog/user_scripts/v1/aggregate_funnel_array_trends_test.py deleted file mode 100755 index 44d3cc9b8f059..0000000000000 --- a/posthog/user_scripts/v1/aggregate_funnel_array_trends_test.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/python3 - -from aggregate_funnel_trends import calculate_funnel_trends_from_user_events, parse_args -import sys -import json - -if __name__ == "__main__": - for line in sys.stdin: - try: - calculate_funnel_trends_from_user_events(*parse_args(line)) - except Exception as e: - print(json.dumps({"result": json.dumps(str(e))}), end="\n") # noqa: T201 - sys.stdout.flush() diff --git a/posthog/user_scripts/v1/aggregate_funnel_cohort.py b/posthog/user_scripts/v1/aggregate_funnel_cohort.py deleted file mode 100755 index 17b053bb7d448..0000000000000 --- a/posthog/user_scripts/v1/aggregate_funnel_cohort.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/python3 -import sys - -from aggregate_funnel import parse_args, calculate_funnel_from_user_events - -if __name__ == "__main__": - for line in sys.stdin: - calculate_funnel_from_user_events(*parse_args(line)) - sys.stdout.flush() diff --git a/posthog/user_scripts/v1/aggregate_funnel_cohort_trends.py b/posthog/user_scripts/v1/aggregate_funnel_cohort_trends.py deleted file mode 100755 index 15e93f5452797..0000000000000 --- a/posthog/user_scripts/v1/aggregate_funnel_cohort_trends.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/python3 -import sys - -from aggregate_funnel_trends import parse_args, calculate_funnel_trends_from_user_events - -if __name__ == "__main__": - for line in sys.stdin: - calculate_funnel_trends_from_user_events(*parse_args(line)) - sys.stdout.flush() diff --git a/posthog/user_scripts/v1/aggregate_funnel_test.py b/posthog/user_scripts/v1/aggregate_funnel_test.py deleted file mode 100755 index 3eff43269c58e..0000000000000 --- a/posthog/user_scripts/v1/aggregate_funnel_test.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/python3 -import json - -from aggregate_funnel import calculate_funnel_from_user_events, parse_args -import sys -import traceback - -if __name__ == "__main__": - for line in sys.stdin: - try: - calculate_funnel_from_user_events(*parse_args(line)) - except Exception as e: - print(json.dumps({"result": json.dumps(str(e) + traceback.format_exc())}), end="\n") # noqa: T201 - sys.stdout.flush() diff --git a/posthog/user_scripts/v1/aggregate_funnel_trends.py b/posthog/user_scripts/v1/aggregate_funnel_trends.py deleted file mode 100755 index 0aa96b7a19b96..0000000000000 --- a/posthog/user_scripts/v1/aggregate_funnel_trends.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/python3 -import sys -from dataclasses import dataclass, replace -from typing import Any -from collections.abc import Sequence -import json - - -def parse_args(line): - args = json.loads(line) - return [ - int(args["from_step"]), - int(args["num_steps"]), - int(args["conversion_window_limit"]), - str(args["breakdown_attribution_type"]), - str(args["funnel_order_type"]), - args["prop_vals"], # Array(Array(String)) - args["value"], # Array(Tuple(Nullable(Float64), Nullable(DateTime), Array(String), Array(Int8))) - ] - - -@dataclass(frozen=True) -class EnteredTimestamp: - timestamp: Any - timings: Any - - -# each one can be multiple steps here -# it only matters when they entered the funnel - you can propagate the time from the previous step when you update -# This function is defined for Clickhouse in user_defined_functions.xml along with types -# num_steps is the total number of steps in the funnel -# conversion_window_limit is in seconds -# events is a array of tuples of (timestamp, breakdown, [steps]) -# steps is an array of integers which represent the steps that this event qualifies for. it looks like [1,3,5,6]. -# negative integers represent an exclusion on that step. each event is either all exclusions or all steps. -def calculate_funnel_trends_from_user_events( - from_step: int, - num_steps: int, - conversion_window_limit_seconds: int, - breakdown_attribution_type: str, - funnel_order_type: str, - prop_vals: list[Any], - events: Sequence[tuple[float, int, list[str] | int | str, list[int]]], -): - default_entered_timestamp = EnteredTimestamp(0, []) - # If the attribution mode is a breakdown step, set this to the integer that represents that step - breakdown_step = int(breakdown_attribution_type[5:]) if breakdown_attribution_type.startswith("step_") else None - - # Results is a map of start intervals to success or failure. If an interval isn't here, it means the - # user didn't enter - results = {} - - # We call this for each possible breakdown value. - def loop_prop_val(prop_val): - # we need to track every distinct entry into the funnel through to the end - filtered_events = ( - ( - (timestamp, interval_start, breakdown, steps) - for (timestamp, interval_start, breakdown, steps) in events - if breakdown == prop_val - ) - if breakdown_attribution_type == "all_events" - else events - ) - list_of_entered_timestamps = [] - - for timestamp, interval_start, breakdown, steps in filtered_events: - for step in reversed(steps): - exclusion = False - if step < 0: - exclusion = True - step = -step - # Special code to handle the first step - # Potential Optimization: we could skip tracking here if the user has already completed the funnel for this interval - if step == 1: - entered_timestamp = [default_entered_timestamp] * (num_steps + 1) - # Set the interval start at 0, which is what we want to return if this works. - # For strict funnels, we need to track if the "from_step" has been hit - # Abuse the timings field on the 0th index entered_timestamp to have the elt True if we have - entered_timestamp[0] = EnteredTimestamp(interval_start, [True] if from_step == 0 else []) - entered_timestamp[1] = EnteredTimestamp(timestamp, [timestamp]) - list_of_entered_timestamps.append(entered_timestamp) - else: - for entered_timestamp in list_of_entered_timestamps[:]: - in_match_window = ( - timestamp - entered_timestamp[step - 1].timestamp <= conversion_window_limit_seconds - ) - already_reached_this_step_with_same_entered_timestamp = ( - entered_timestamp[step].timestamp == entered_timestamp[step - 1].timestamp - ) - if in_match_window and not already_reached_this_step_with_same_entered_timestamp: - if exclusion: - # this is a complete failure, exclude this person, don't print anything, don't count - return False - is_unmatched_step_attribution = ( - breakdown_step is not None and step == breakdown_step - 1 and prop_val != breakdown - ) - if not is_unmatched_step_attribution: - entered_timestamp[step] = replace( - entered_timestamp[step - 1], - timings=[*entered_timestamp[step - 1].timings, timestamp], - ) - # check if we have hit the goal. if we have, remove it from the list and add it to the successful_timestamps - if entered_timestamp[num_steps].timestamp > 0: - results[entered_timestamp[0].timestamp] = (1, prop_val) - list_of_entered_timestamps.remove(entered_timestamp) - # If we have hit the from_step threshold, record it (abuse the timings field) - elif step == from_step + 1: - entered_timestamp[0].timings.append(True) - - # At the end of the event, clear all steps that weren't done by that event - if funnel_order_type == "strict": - for entered_timestamp in list_of_entered_timestamps[:]: - for i in range(1, len(entered_timestamp)): - if i not in steps: - entered_timestamp[i] = default_entered_timestamp - - # At this point, everything left in entered_timestamps is a failure, if it has made it to from_step - for entered_timestamp in list_of_entered_timestamps: - if entered_timestamp[0].timestamp not in results and len(entered_timestamp[0].timings) > 0: - results[entered_timestamp[0].timestamp] = (-1, prop_val) - - [loop_prop_val(prop_val) for prop_val in prop_vals] - result = [(interval_start, success_bool, prop_val) for interval_start, (success_bool, prop_val) in results.items()] - print(json.dumps({"result": result}), end="\n") # noqa: T201 - - -if __name__ == "__main__": - for line in sys.stdin: - calculate_funnel_trends_from_user_events(*parse_args(line)) - sys.stdout.flush() diff --git a/posthog/user_scripts/v1/user_defined_function.xml b/posthog/user_scripts/v1/user_defined_function.xml deleted file mode 100644 index cec2d0d340802..0000000000000 --- a/posthog/user_scripts/v1/user_defined_function.xml +++ /dev/null @@ -1,858 +0,0 @@ - - - executable - aggregate_funnel - Array(Tuple(Int8, Nullable(String), Array(Float64))) - result - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(Nullable(String)) - prop_vals - - - Array(Tuple(Nullable(Float64), Nullable(String), Array(Int8))) - value - - JSONEachRow - aggregate_funnel.py - - - - executable - aggregate_funnel_cohort - Array(Tuple(Int8, UInt64, Array(Float64))) - result - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(UInt64) - prop_vals - - - Array(Tuple(Nullable(Float64), UInt64, Array(Int8))) - value - - JSONEachRow - aggregate_funnel_cohort.py - - - - executable - aggregate_funnel_array - Array(Tuple(Int8, Array(String), Array(Float64))) - result - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(Array(String)) - prop_vals - - - Array(Tuple(Nullable(Float64), Array(String), Array(Int8))) - value - - JSONEachRow - aggregate_funnel_array.py - - - - executable - aggregate_funnel_test - String - result - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(Array(String)) - prop_vals - - - Array(Tuple(Nullable(Float64), Nullable(String), Array(Int8))) - value - - JSONEachRow - aggregate_funnel_test.py - - - - executable - aggregate_funnel_trends - Array(Tuple(DateTime, Int8, Nullable(String))) - result - - UInt8 - from_step - - - UInt8 - num_steps - - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(Nullable(String)) - prop_vals - - - Array(Tuple(Nullable(Float64), Nullable(DateTime), Nullable(String), Array(Int8))) - value - - JSONEachRow - aggregate_funnel_trends.py - - - - executable - aggregate_funnel_array_trends - - Array(Tuple(DateTime, Int8, Array(String))) - result - - UInt8 - from_step - - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(Array(String)) - prop_vals - - - Array(Tuple(Nullable(Float64), Nullable(DateTime), Array(String), Array(Int8))) - value - - JSONEachRow - aggregate_funnel_array_trends.py - - - - executable - aggregate_funnel_cohort_trends - - Array(Tuple(DateTime, Int8, UInt64)) - result - - UInt8 - from_step - - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(UInt64) - prop_vals - - - Array(Tuple(Nullable(Float64), Nullable(DateTime), UInt64, Array(Int8))) - value - - JSONEachRow - aggregate_funnel_cohort_trends.py - - - - executable - aggregate_funnel_array_trends_test - String - result - - UInt8 - from_step - - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(Array(String)) - prop_vals - - - Array(Tuple(Nullable(Float64), Nullable(DateTime), Array(String), Array(Int8))) - value - - JSONEachRow - aggregate_funnel_array_trends_test.py - - - executable - aggregate_funnel_v0 - Array(Tuple(Int8, Nullable(String), Array(Float64))) - result - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(Nullable(String)) - prop_vals - - - Array(Tuple(Nullable(Float64), Nullable(String), Array(Int8))) - value - - JSONEachRow - v0/aggregate_funnel.py - - - - executable - aggregate_funnel_cohort_v0 - Array(Tuple(Int8, UInt64, Array(Float64))) - result - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(UInt64) - prop_vals - - - Array(Tuple(Nullable(Float64), UInt64, Array(Int8))) - value - - JSONEachRow - v0/aggregate_funnel_cohort.py - - - - executable - aggregate_funnel_array_v0 - Array(Tuple(Int8, Array(String), Array(Float64))) - result - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(Array(String)) - prop_vals - - - Array(Tuple(Nullable(Float64), Array(String), Array(Int8))) - value - - JSONEachRow - v0/aggregate_funnel_array.py - - - - executable - aggregate_funnel_test_v0 - String - result - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(Array(String)) - prop_vals - - - Array(Tuple(Nullable(Float64), Nullable(String), Array(Int8))) - value - - JSONEachRow - v0/aggregate_funnel_test.py - - - - executable - aggregate_funnel_trends_v0 - Array(Tuple(DateTime, Int8, Nullable(String))) - result - - UInt8 - from_step - - - UInt8 - num_steps - - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(Nullable(String)) - prop_vals - - - Array(Tuple(Nullable(Float64), Nullable(DateTime), Nullable(String), Array(Int8))) - value - - JSONEachRow - v0/aggregate_funnel_trends.py - - - - executable - aggregate_funnel_array_trends_v0 - - Array(Tuple(DateTime, Int8, Array(String))) - result - - UInt8 - from_step - - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(Array(String)) - prop_vals - - - Array(Tuple(Nullable(Float64), Nullable(DateTime), Array(String), Array(Int8))) - value - - JSONEachRow - v0/aggregate_funnel_array_trends.py - - - - executable - aggregate_funnel_cohort_trends_v0 - - Array(Tuple(DateTime, Int8, UInt64)) - result - - UInt8 - from_step - - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(UInt64) - prop_vals - - - Array(Tuple(Nullable(Float64), Nullable(DateTime), UInt64, Array(Int8))) - value - - JSONEachRow - v0/aggregate_funnel_cohort_trends.py - - - - executable - aggregate_funnel_array_trends_test_v0 - String - result - - UInt8 - from_step - - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(Array(String)) - prop_vals - - - Array(Tuple(Nullable(Float64), Nullable(DateTime), Array(String), Array(Int8))) - value - - JSONEachRow - v0/aggregate_funnel_array_trends_test.py - - - executable - aggregate_funnel_v1 - Array(Tuple(Int8, Nullable(String), Array(Float64), Array(Array(UUID)))) - result - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(Nullable(String)) - prop_vals - - - Array(Tuple(Nullable(Float64), UUID, Nullable(String), Array(Int8))) - value - - JSONEachRow - v1/aggregate_funnel.py - - - - executable - aggregate_funnel_cohort_v1 - Array(Tuple(Int8, UInt64, Array(Float64), Array(Array(UUID)))) - result - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(UInt64) - prop_vals - - - Array(Tuple(Nullable(Float64), UUID, UInt64, Array(Int8))) - value - - JSONEachRow - v1/aggregate_funnel_cohort.py - - - - executable - aggregate_funnel_array_v1 - Array(Tuple(Int8, Array(String), Array(Float64), Array(Array(UUID)))) - result - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(Array(String)) - prop_vals - - - Array(Tuple(Nullable(Float64), UUID, Array(String), Array(Int8))) - value - - JSONEachRow - v1/aggregate_funnel_array.py - - - - executable - aggregate_funnel_test_v1 - String - result - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(Array(String)) - prop_vals - - - Array(Tuple(Nullable(Float64), UUID, Nullable(String), Array(Int8))) - value - - JSONEachRow - v1/aggregate_funnel_test.py - - - - executable - aggregate_funnel_trends_v1 - Array(Tuple(DateTime, Int8, Nullable(String))) - result - - UInt8 - from_step - - - UInt8 - num_steps - - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(Nullable(String)) - prop_vals - - - Array(Tuple(Nullable(Float64), Nullable(DateTime), Nullable(String), Array(Int8))) - value - - JSONEachRow - v1/aggregate_funnel_trends.py - - - - executable - aggregate_funnel_array_trends_v1 - - Array(Tuple(DateTime, Int8, Array(String))) - result - - UInt8 - from_step - - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(Array(String)) - prop_vals - - - Array(Tuple(Nullable(Float64), Nullable(DateTime), Array(String), Array(Int8))) - value - - JSONEachRow - v1/aggregate_funnel_array_trends.py - - - - executable - aggregate_funnel_cohort_trends_v1 - - Array(Tuple(DateTime, Int8, UInt64)) - result - - UInt8 - from_step - - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(UInt64) - prop_vals - - - Array(Tuple(Nullable(Float64), Nullable(DateTime), UInt64, Array(Int8))) - value - - JSONEachRow - v1/aggregate_funnel_cohort_trends.py - - - - executable - aggregate_funnel_array_trends_test_v1 - String - result - - UInt8 - from_step - - - UInt8 - num_steps - - - UInt64 - conversion_window_limit - - - String - breakdown_attribution_type - - - String - funnel_order_type - - - Array(Array(String)) - prop_vals - - - Array(Tuple(Nullable(Float64), Nullable(DateTime), Array(String), Array(Int8))) - value - - JSONEachRow - v1/aggregate_funnel_array_trends_test.py - - \ No newline at end of file