From bc283b429d76ae66f9ca3ff2b571b551bb20dde7 Mon Sep 17 00:00:00 2001 From: Phani Raj Date: Sat, 12 Oct 2024 12:07:52 -0500 Subject: [PATCH 01/52] fix(surveys): Exclude nullable fields from django admin (#25550) By default, Django admin assumes that all fields are required, even if they're defined as NULL=true, blank=true in the django model specification. This is an issue if we want to change just one property start_date of a Survey, which is a common request. --- posthog/admin/admins/survey_admin.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/posthog/admin/admins/survey_admin.py b/posthog/admin/admins/survey_admin.py index cefbf87ff3aa7..dcfffce666a57 100644 --- a/posthog/admin/admins/survey_admin.py +++ b/posthog/admin/admins/survey_admin.py @@ -18,6 +18,22 @@ class SurveyAdmin(admin.ModelAdmin): autocomplete_fields = ("team", "created_by") ordering = ("-created_at",) + def get_form(self, request, obj=None, change=False, **kwargs): + form = super().get_form(request, obj, **kwargs) + for field in [ + "start_date", + "end_date", + "responses_limit", + "iteration_count", + "iteration_frequency_days", + "iteration_start_dates", + "current_iteration", + "current_iteration_start_date", + "actions", + ]: + form.base_fields[field].required = False + return form + def team_link(self, experiment: Experiment): return format_html( '{}', From c941cc1de216e03d0d6b796cc668421264be8927 Mon Sep 17 00:00:00 2001 From: Kyle Yannelli Date: Sat, 12 Oct 2024 19:58:56 +0000 Subject: [PATCH 02/52] fix: improper builds from typo in deploy-hobby (#25551) --- bin/deploy-hobby | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/deploy-hobby b/bin/deploy-hobby index 93df1b1aac3a6..a3f9e05371f27 100755 --- a/bin/deploy-hobby +++ b/bin/deploy-hobby @@ -12,7 +12,7 @@ export SENTRY_DSN="${SENTRY_DSN:-'https://public@sentry.example.com/1'}" POSTHOG_SECRET=$(head -c 28 /dev/urandom | sha224sum -b | head -c 56) export POSTHOG_SECRET -ENCRYPTION_KEY=$(openssl rand -hex 16) +ENCRYPTION_SALT_KEYS=$(openssl rand -hex 16) export ENCRYPTION_SALT_KEYS # Talk to the user From 266ba3aa16c3e69d6e5870547570ffce9183540a Mon Sep 17 00:00:00 2001 From: Daesgar Date: Mon, 14 Oct 2024 09:35:46 +0200 Subject: [PATCH 03/52] feat: copy UDFs when user_scripts folder is updated (#25361) --- .github/workflows/copy-clickhouse-udfs.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/copy-clickhouse-udfs.yml diff --git a/.github/workflows/copy-clickhouse-udfs.yml b/.github/workflows/copy-clickhouse-udfs.yml new file mode 100644 index 0000000000000..c6862b0345c67 --- /dev/null +++ b/.github/workflows/copy-clickhouse-udfs.yml @@ -0,0 +1,21 @@ +name: Trigger UDFs Workflow + +on: + push: + branches: + - master + paths: + - 'posthog/user_scripts/**' + +jobs: + trigger_udfs_workflow: + runs-on: ubuntu-latest + steps: + - name: Trigger UDFs Workflow + uses: benc-uk/workflow-dispatch@v1 + with: + workflow: .github/workflows/clickhouse-udfs.yml + repo: posthog/posthog-cloud-infra + token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }} + ref: refs/heads/main + From 7a04e96bbbf6ff9aec87f6443f51e5ce8a043f2b Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Mon, 14 Oct 2024 10:02:14 +0100 Subject: [PATCH 04/52] fix(hogql): toString boolean values when transforming them (#25546) Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- .../schema/util/test/test_person_where_clause_extractor.py | 2 +- posthog/hogql/test/test_printer.py | 6 +++--- posthog/hogql/transforms/property_types.py | 2 +- .../transforms/test/__snapshots__/test_property_types.ambr | 6 +++--- posthog/hogql/transforms/test/test_property_types.py | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/posthog/hogql/database/schema/util/test/test_person_where_clause_extractor.py b/posthog/hogql/database/schema/util/test/test_person_where_clause_extractor.py index 8c095a33b7ec6..252382d545b40 100644 --- a/posthog/hogql/database/schema/util/test/test_person_where_clause_extractor.py +++ b/posthog/hogql/database/schema/util/test/test_person_where_clause_extractor.py @@ -179,6 +179,6 @@ def test_boolean(self): ) actual = self.print_query("SELECT * FROM events WHERE person.properties.person_boolean = false") assert ( - f"FROM person WHERE and(equals(person.team_id, {self.team.id}), ifNull(equals(transform(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties" + f"FROM person WHERE and(equals(person.team_id, {self.team.id}), ifNull(equals(transform(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties" in actual ) diff --git a/posthog/hogql/test/test_printer.py b/posthog/hogql/test/test_printer.py index c1eeeaf75a90e..fd735a8b87ce4 100644 --- a/posthog/hogql/test/test_printer.py +++ b/posthog/hogql/test/test_printer.py @@ -1470,9 +1470,9 @@ def test_field_nullable_boolean(self): ) assert generated_sql_statements1 == ( f"SELECT " - "ifNull(equals(transform(nullIf(nullIf(events.mat_is_boolean, ''), 'null'), %(hogql_val_0)s, %(hogql_val_1)s, NULL), 1), 0), " - "ifNull(equals(transform(nullIf(nullIf(events.mat_is_boolean, ''), 'null'), %(hogql_val_2)s, %(hogql_val_3)s, NULL), 0), 0), " - "isNull(transform(nullIf(nullIf(events.mat_is_boolean, ''), 'null'), %(hogql_val_4)s, %(hogql_val_5)s, NULL)) " + "ifNull(equals(transform(toString(nullIf(nullIf(events.mat_is_boolean, ''), 'null')), %(hogql_val_0)s, %(hogql_val_1)s, NULL), 1), 0), " + "ifNull(equals(transform(toString(nullIf(nullIf(events.mat_is_boolean, ''), 'null')), %(hogql_val_2)s, %(hogql_val_3)s, NULL), 0), 0), " + "isNull(transform(toString(nullIf(nullIf(events.mat_is_boolean, ''), 'null')), %(hogql_val_4)s, %(hogql_val_5)s, NULL)) " f"FROM events WHERE equals(events.team_id, {self.team.pk}) LIMIT {MAX_SELECT_RETURNED_ROWS}" ) assert context.values == { diff --git a/posthog/hogql/transforms/property_types.py b/posthog/hogql/transforms/property_types.py index e248b4ad41203..c1bfcc622ff75 100644 --- a/posthog/hogql/transforms/property_types.py +++ b/posthog/hogql/transforms/property_types.py @@ -222,7 +222,7 @@ def _field_type_to_property_call(self, node: ast.Field, field_type: str): return ast.Call( name="transform", args=[ - node, + ast.Call(name="toString", args=[node]), ast.Constant(value=["true", "false"]), ast.Constant(value=[True, False]), ast.Constant(value=None), diff --git a/posthog/hogql/transforms/test/__snapshots__/test_property_types.ambr b/posthog/hogql/transforms/test/__snapshots__/test_property_types.ambr index db95037a881ea..c0c2e1a610370 100644 --- a/posthog/hogql/transforms/test/__snapshots__/test_property_types.ambr +++ b/posthog/hogql/transforms/test/__snapshots__/test_property_types.ambr @@ -2,7 +2,7 @@ # name: TestPropertyTypes.test_data_warehouse_person_property_types ''' - SELECT persons__extended_properties.string_prop AS string_prop, persons__extended_properties.int_prop AS int_prop, transform(persons__extended_properties.bool_prop, %(hogql_val_8)s, %(hogql_val_9)s, NULL) AS bool_prop + SELECT persons__extended_properties.string_prop AS string_prop, persons__extended_properties.int_prop AS int_prop, transform(toString(persons__extended_properties.bool_prop), %(hogql_val_8)s, %(hogql_val_9)s, NULL) AS bool_prop FROM ( SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', ''), person.version) AS persons___properties___email, person.id AS id FROM person @@ -19,7 +19,7 @@ # name: TestPropertyTypes.test_group_boolean_property_types ''' - SELECT ifNull(equals(transform(events__group_0.properties___group_boolean, %(hogql_val_2)s, %(hogql_val_3)s, NULL), 1), 0), ifNull(equals(transform(events__group_0.properties___group_boolean, %(hogql_val_4)s, %(hogql_val_5)s, NULL), 0), 0), isNull(transform(events__group_0.properties___group_boolean, %(hogql_val_6)s, %(hogql_val_7)s, NULL)) + SELECT ifNull(equals(transform(toString(events__group_0.properties___group_boolean), %(hogql_val_2)s, %(hogql_val_3)s, NULL), 1), 0), ifNull(equals(transform(toString(events__group_0.properties___group_boolean), %(hogql_val_4)s, %(hogql_val_5)s, NULL), 0), 0), isNull(transform(toString(events__group_0.properties___group_boolean), %(hogql_val_6)s, %(hogql_val_7)s, NULL)) FROM events LEFT JOIN ( SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, %(hogql_val_1)s)) AS properties___group_boolean, groups.group_type_index AS index, groups.group_key AS key FROM groups @@ -66,7 +66,7 @@ # name: TestPropertyTypes.test_resolve_property_types_event ''' - SELECT multiply(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', ''), %(hogql_val_1)s), accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_2)s), ''), 'null'), '^"|"$', ''), %(hogql_val_3)s)), transform(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_4)s), ''), 'null'), '^"|"$', ''), %(hogql_val_5)s, %(hogql_val_6)s, NULL) AS bool + SELECT multiply(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', ''), %(hogql_val_1)s), accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_2)s), ''), 'null'), '^"|"$', ''), %(hogql_val_3)s)), transform(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, %(hogql_val_4)s), ''), 'null'), '^"|"$', '')), %(hogql_val_5)s, %(hogql_val_6)s, NULL) AS bool FROM events WHERE equals(events.team_id, 420) LIMIT 50000 diff --git a/posthog/hogql/transforms/test/test_property_types.py b/posthog/hogql/transforms/test/test_property_types.py index b551d85db2589..b4a24a79fe1aa 100644 --- a/posthog/hogql/transforms/test/test_property_types.py +++ b/posthog/hogql/transforms/test/test_property_types.py @@ -134,7 +134,7 @@ def test_group_boolean_property_types(self): ) assert printed == self.snapshot assert ( - "SELECT ifNull(equals(transform(events__group_0.properties___group_boolean, hogvar, hogvar, NULL), 1), 0), ifNull(equals(transform(events__group_0.properties___group_boolean, hogvar, hogvar, NULL), 0), 0), isNull(transform(events__group_0.properties___group_boolean, hogvar, hogvar, NULL))" + "SELECT ifNull(equals(transform(toString(events__group_0.properties___group_boolean), hogvar, hogvar, NULL), 1), 0), ifNull(equals(transform(toString(events__group_0.properties___group_boolean), hogvar, hogvar, NULL), 0), 0), isNull(transform(toString(events__group_0.properties___group_boolean), hogvar, hogvar, NULL))" in re.sub(r"%\(hogql_val_\d+\)s", "hogvar", printed) ) From b2070a7d909a98c1ea534def23dec858c128da00 Mon Sep 17 00:00:00 2001 From: Juraj Majerik Date: Mon, 14 Oct 2024 11:27:13 +0200 Subject: [PATCH 05/52] chore(experiment): relocate trend statistical methods (#25472) --- ee/clickhouse/queries/experiments/__init__.py | 3 + .../experiments/funnel_experiment_result.py | 219 ------------------ .../test_trend_experiment_result.py | 93 ++++---- .../experiments/trend_experiment_result.py | 203 +--------------- frontend/src/queries/schema.json | 8 +- frontend/src/queries/schema.ts | 2 + .../experiment_trend_query_runner.py | 7 +- .../test_experiment_trend_query_runner.py | 12 +- .../experiments/trend_statistics.py | 196 ++++++++++++++++ posthog/schema.py | 2 + 10 files changed, 273 insertions(+), 472 deletions(-) create mode 100644 posthog/hogql_queries/experiments/trend_statistics.py diff --git a/ee/clickhouse/queries/experiments/__init__.py b/ee/clickhouse/queries/experiments/__init__.py index 1e6c17112c535..5d14ceca45be5 100644 --- a/ee/clickhouse/queries/experiments/__init__.py +++ b/ee/clickhouse/queries/experiments/__init__.py @@ -8,3 +8,6 @@ # If probability of a variant is below this threshold, it will be considered # insignificant MIN_PROBABILITY_FOR_SIGNIFICANCE = 0.9 + +# Trends only: If p-value is below this threshold, the results are considered significant +P_VALUE_SIGNIFICANCE_LEVEL = 0.05 diff --git a/ee/clickhouse/queries/experiments/funnel_experiment_result.py b/ee/clickhouse/queries/experiments/funnel_experiment_result.py index 35be4bc49686c..0ec82f553d6d2 100644 --- a/ee/clickhouse/queries/experiments/funnel_experiment_result.py +++ b/ee/clickhouse/queries/experiments/funnel_experiment_result.py @@ -145,225 +145,6 @@ def get_variants(self, funnel_results): return control_variant, test_variants - # @staticmethod - # def calculate_probabilities( - # control_variant: Variant, - # test_variants: list[Variant], - # priors: tuple[int, int] = (1, 1), - # ) -> list[Probability]: - # """ - # Calculates the probability that each variant outperforms the others. - - # Supports up to 10 variants (1 control + 9 test). - - # Method: - # 1. For each variant, create a Beta distribution of conversion rates: - # α (alpha) = success count of variant + prior success - # β (beta) = failure count + variant + prior failures - # 2. Use Monte Carlo simulation to estimate winning probabilities. - - # The prior represents our initial belief about conversion rates. - # We use a non-informative prior (1, 1) by default, assuming equal - # likelihood of success and failure. - - # Returns: List of probabilities, where index 0 is control. - # """ - - # if len(test_variants) >= 10: - # raise ValidationError( - # "Can't calculate experiment results for more than 10 variants", - # code="too_much_data", - # ) - - # if len(test_variants) < 1: - # raise ValidationError( - # "Can't calculate experiment results for less than 2 variants", - # code="no_data", - # ) - - # variants = [control_variant, *test_variants] - # probabilities = [] - - # # simulate winning for each test variant - # for index, variant in enumerate(variants): - # probabilities.append(simulate_winning_variant_for_conversion(variant, variants[:index] + variants[index + 1 :])) - - # total_test_probabilities = sum(probabilities[1:]) - - # return [max(0, 1 - total_test_probabilities), *probabilities[1:]] - - # @staticmethod - # def are_results_significant( - # control_variant: Variant, - # test_variants: list[Variant], - # probabilities: list[Probability], - # ) -> tuple[ExperimentSignificanceCode, Probability]: - # def get_conversion_rate(variant: Variant): - # return variant.success_count / (variant.success_count + variant.failure_count) - - # control_sample_size = control_variant.success_count + control_variant.failure_count - - # for variant in test_variants: - # # We need a feature flag distribution threshold because distribution of people - # # can skew wildly when there are few people in the experiment - # if variant.success_count + variant.failure_count < FF_DISTRIBUTION_THRESHOLD: - # return ExperimentSignificanceCode.NOT_ENOUGH_EXPOSURE, 1 - - # if control_sample_size < FF_DISTRIBUTION_THRESHOLD: - # return ExperimentSignificanceCode.NOT_ENOUGH_EXPOSURE, 1 - - # if ( - # probabilities[0] < MIN_PROBABILITY_FOR_SIGNIFICANCE - # and sum(probabilities[1:]) < MIN_PROBABILITY_FOR_SIGNIFICANCE - # ): - # # Sum of probability of winning for all variants except control is less than 90% - # return ExperimentSignificanceCode.LOW_WIN_PROBABILITY, 1 - - # best_test_variant = max( - # test_variants, - # key=lambda variant: get_conversion_rate(variant), - # ) - - # if get_conversion_rate(best_test_variant) > get_conversion_rate(control_variant): - # expected_loss = calculate_expected_loss(best_test_variant, [control_variant]) - # else: - # expected_loss = calculate_expected_loss(control_variant, [best_test_variant]) - - # if expected_loss >= EXPECTED_LOSS_SIGNIFICANCE_LEVEL: - # return ExperimentSignificanceCode.HIGH_LOSS, expected_loss - - # return ExperimentSignificanceCode.SIGNIFICANT, expected_loss - - -# def calculate_expected_loss(target_variant: Variant, variants: list[Variant]) -> float: -# """ -# Calculates expected loss in conversion rate for a given variant. -# Loss calculation comes from VWO's SmartStats technical paper: -# https://cdn2.hubspot.net/hubfs/310840/VWO_SmartStats_technical_whitepaper.pdf (pg 12) - -# > The loss function is the amount of uplift that one can expect to -# be lost by choosing a given variant, given particular values of λA and λB - -# The unit of the return value is conversion rate values - -# """ -# random_sampler = default_rng() -# prior_success = 1 -# prior_failure = 1 -# simulations_count = 100_000 - -# variant_samples = [] -# for variant in variants: -# # Get `N=simulations` samples from a Beta distribution with alpha = prior_success + variant_sucess, -# # and beta = prior_failure + variant_failure -# samples = random_sampler.beta( -# variant.success_count + prior_success, -# variant.failure_count + prior_failure, -# simulations_count, -# ) -# variant_samples.append(samples) - -# target_variant_samples = random_sampler.beta( -# target_variant.success_count + prior_success, -# target_variant.failure_count + prior_failure, -# simulations_count, -# ) - -# loss = 0 -# variant_conversions = list(zip(*variant_samples)) -# for i in range(simulations_count): -# loss += max(0, max(variant_conversions[i]) - target_variant_samples[i]) - -# return loss / simulations_count - - -# def simulate_winning_variant_for_conversion(target_variant: Variant, variants: list[Variant]) -> Probability: -# random_sampler = default_rng() -# prior_success = 1 -# prior_failure = 1 -# simulations_count = 100_000 - -# variant_samples = [] -# for variant in variants: -# # Get `N=simulations` samples from a Beta distribution with alpha = prior_success + variant_sucess, -# # and beta = prior_failure + variant_failure -# samples = random_sampler.beta( -# variant.success_count + prior_success, -# variant.failure_count + prior_failure, -# simulations_count, -# ) -# variant_samples.append(samples) - -# target_variant_samples = random_sampler.beta( -# target_variant.success_count + prior_success, -# target_variant.failure_count + prior_failure, -# simulations_count, -# ) - -# winnings = 0 -# variant_conversions = list(zip(*variant_samples)) -# for i in range(simulations_count): -# if target_variant_samples[i] > max(variant_conversions[i]): -# winnings += 1 - -# return winnings / simulations_count - - -# def calculate_probability_of_winning_for_each(variants: list[Variant]) -> list[Probability]: -# """ -# Calculates the probability of winning for each variant. -# """ -# if len(variants) > 10: -# raise ValidationError( -# "Can't calculate experiment results for more than 10 variants", -# code="too_much_data", -# ) - -# probabilities = [] -# # simulate winning for each test variant -# for index, variant in enumerate(variants): -# probabilities.append(simulate_winning_variant_for_conversion(variant, variants[:index] + variants[index + 1 :])) - -# total_test_probabilities = sum(probabilities[1:]) - -# return [max(0, 1 - total_test_probabilities), *probabilities[1:]] - -# def calculate_credible_intervals(variants, lower_bound=0.025, upper_bound=0.975): -# """ -# Calculate the Bayesian credible intervals for a list of variants. -# If no lower/upper bound provided, the function calculates the 95% credible interval. -# """ -# intervals = {} - -# for variant in variants: -# try: -# if variant.success_count < 0 or variant.failure_count < 0: -# capture_exception( -# Exception("Invalid variant success/failure count"), -# { -# "variant": variant.key, -# "success_count": variant.success_count, -# "failure_count": variant.failure_count, -# }, -# ) -# return {} - -# # Calculate the credible interval -# # Laplace smoothing: we add 1 to alpha and beta to avoid division errors if either is zero -# alpha = variant.success_count + 1 -# beta = variant.failure_count + 1 -# credible_interval = stats.beta.ppf([lower_bound, upper_bound], alpha, beta) - -# intervals[variant.key] = (credible_interval[0], credible_interval[1]) -# except Exception as e: -# capture_exception( -# Exception(f"Error calculating credible interval for variant {variant.key}"), -# {"error": str(e)}, -# ) -# return {} - -# return intervals - def validate_event_variants(funnel_results, variants): errors = { diff --git a/ee/clickhouse/queries/experiments/test_trend_experiment_result.py b/ee/clickhouse/queries/experiments/test_trend_experiment_result.py index fd7d823f56ce2..e2ee634812e65 100644 --- a/ee/clickhouse/queries/experiments/test_trend_experiment_result.py +++ b/ee/clickhouse/queries/experiments/test_trend_experiment_result.py @@ -1,29 +1,30 @@ import unittest from functools import lru_cache -from math import exp, lgamma, log +from math import exp, lgamma, log, ceil from flaky import flaky -from ee.clickhouse.queries.experiments.trend_experiment_result import ( - ClickhouseTrendExperimentResult, - Variant as CountVariant, - calculate_credible_intervals as calculate_trend_credible_intervals, -) -from ee.clickhouse.queries.experiments.trend_experiment_result import calculate_p_value from posthog.constants import ExperimentSignificanceCode +from posthog.hogql_queries.experiments.trend_statistics import ( + are_results_significant, + calculate_credible_intervals, + calculate_p_value, + calculate_probabilities, +) +from posthog.schema import ExperimentVariantTrendResult Probability = float @lru_cache(maxsize=100000) -def logbeta(x: int, y: int) -> float: +def logbeta(x: float, y: float) -> float: return lgamma(x) + lgamma(y) - lgamma(x + y) # Helper function to calculate probability using a different method than the one used in actual code # calculation: https://www.evanmiller.org/bayesian-ab-testing.html#count_ab def calculate_probability_of_winning_for_target_count_data( - target_variant: CountVariant, other_variants: list[CountVariant] + target_variant: ExperimentVariantTrendResult, other_variants: list[ExperimentVariantTrendResult] ) -> Probability: """ Calculates the probability of winning for target variant. @@ -48,9 +49,11 @@ def calculate_probability_of_winning_for_target_count_data( return 0 -def probability_B_beats_A_count_data(A_count: int, A_exposure: float, B_count: int, B_exposure: float) -> Probability: +def probability_B_beats_A_count_data( + A_count: float, A_exposure: float, B_count: float, B_exposure: float +) -> Probability: total: Probability = 0 - for i in range(B_count): + for i in range(ceil(B_count)): total += exp( i * log(B_exposure) + A_count * log(A_exposure) @@ -63,17 +66,17 @@ def probability_B_beats_A_count_data(A_count: int, A_exposure: float, B_count: i def probability_C_beats_A_and_B_count_data( - A_count: int, + A_count: float, A_exposure: float, - B_count: int, + B_count: float, B_exposure: float, - C_count: int, + C_count: float, C_exposure: float, ) -> Probability: total: Probability = 0 - for i in range(B_count): - for j in range(A_count): + for i in range(ceil(B_count)): + for j in range(ceil(A_count)): total += exp( i * log(B_exposure) + j * log(A_exposure) @@ -95,10 +98,10 @@ def probability_C_beats_A_and_B_count_data( @flaky(max_runs=10, min_passes=1) class TestTrendExperimentCalculator(unittest.TestCase): def test_calculate_results(self): - variant_control = CountVariant("A", 20, 1, 200) - variant_test = CountVariant("B", 30, 1, 200) + variant_control = ExperimentVariantTrendResult(key="A", count=20, exposure=1, absolute_exposure=200) + variant_test = ExperimentVariantTrendResult(key="B", count=30, exposure=1, absolute_exposure=200) - probabilities = ClickhouseTrendExperimentResult.calculate_results(variant_control, [variant_test]) + probabilities = calculate_probabilities(variant_control, [variant_test]) self.assertAlmostEqual(probabilities[1], 0.92, places=1) computed_probability = calculate_probability_of_winning_for_target_count_data(variant_test, [variant_control]) @@ -108,17 +111,17 @@ def test_calculate_results(self): p_value = calculate_p_value(variant_control, [variant_test]) self.assertAlmostEqual(p_value, 0.20, places=2) - credible_intervals = calculate_trend_credible_intervals([variant_control, variant_test]) + credible_intervals = calculate_credible_intervals([variant_control, variant_test]) self.assertAlmostEqual(credible_intervals[variant_control.key][0], 0.0650, places=3) self.assertAlmostEqual(credible_intervals[variant_control.key][1], 0.1544, places=3) self.assertAlmostEqual(credible_intervals[variant_test.key][0], 0.1053, places=3) self.assertAlmostEqual(credible_intervals[variant_test.key][1], 0.2141, places=3) def test_calculate_results_small_numbers(self): - variant_control = CountVariant("A", 2, 1, 200) - variant_test = CountVariant("B", 1, 1, 200) + variant_control = ExperimentVariantTrendResult(key="A", count=2, exposure=1, absolute_exposure=200) + variant_test = ExperimentVariantTrendResult(key="B", count=1, exposure=1, absolute_exposure=200) - probabilities = ClickhouseTrendExperimentResult.calculate_results(variant_control, [variant_test]) + probabilities = calculate_probabilities(variant_control, [variant_test]) self.assertAlmostEqual(probabilities[1], 0.31, places=1) computed_probability = calculate_probability_of_winning_for_target_count_data(variant_test, [variant_control]) @@ -127,7 +130,7 @@ def test_calculate_results_small_numbers(self): p_value = calculate_p_value(variant_control, [variant_test]) self.assertAlmostEqual(p_value, 1, places=2) - credible_intervals = calculate_trend_credible_intervals([variant_control, variant_test]) + credible_intervals = calculate_credible_intervals([variant_control, variant_test]) self.assertAlmostEqual(credible_intervals[variant_control.key][0], 0.0031, places=3) self.assertAlmostEqual(credible_intervals[variant_control.key][1], 0.0361, places=3) self.assertAlmostEqual(credible_intervals[variant_test.key][0], 0.0012, places=3) @@ -143,13 +146,11 @@ def test_calculate_count_data_probability(self): self.assertAlmostEqual(probability, probability2) def test_calculate_results_with_three_variants(self): - variant_control = CountVariant("A", 20, 1, 200) - variant_test_1 = CountVariant("B", 26, 1, 200) - variant_test_2 = CountVariant("C", 19, 1, 200) + variant_control = ExperimentVariantTrendResult(key="A", count=20, exposure=1, absolute_exposure=200) + variant_test_1 = ExperimentVariantTrendResult(key="B", count=26, exposure=1, absolute_exposure=200) + variant_test_2 = ExperimentVariantTrendResult(key="C", count=19, exposure=1, absolute_exposure=200) - probabilities = ClickhouseTrendExperimentResult.calculate_results( - variant_control, [variant_test_1, variant_test_2] - ) + probabilities = calculate_probabilities(variant_control, [variant_test_1, variant_test_2]) self.assertAlmostEqual(probabilities[0], 0.16, places=1) self.assertAlmostEqual(probabilities[1], 0.72, places=1) self.assertAlmostEqual(probabilities[2], 0.12, places=1) @@ -162,7 +163,7 @@ def test_calculate_results_with_three_variants(self): p_value = calculate_p_value(variant_control, [variant_test_1, variant_test_2]) self.assertAlmostEqual(p_value, 0.46, places=2) - credible_intervals = calculate_trend_credible_intervals([variant_control, variant_test_1, variant_test_2]) + credible_intervals = calculate_credible_intervals([variant_control, variant_test_1, variant_test_2]) self.assertAlmostEqual(credible_intervals[variant_control.key][0], 0.0650, places=3) self.assertAlmostEqual(credible_intervals[variant_control.key][1], 0.1544, places=3) self.assertAlmostEqual(credible_intervals[variant_test_1.key][0], 0.0890, places=3) @@ -171,31 +172,31 @@ def test_calculate_results_with_three_variants(self): self.assertAlmostEqual(credible_intervals[variant_test_2.key][1], 0.1484, places=3) def test_calculate_significance_when_target_variants_underperform(self): - variant_control = CountVariant("A", 250, 1, 200) - variant_test_1 = CountVariant("B", 180, 1, 200) - variant_test_2 = CountVariant("C", 50, 1, 200) + variant_control = ExperimentVariantTrendResult(key="A", count=250, exposure=1, absolute_exposure=200) + variant_test_1 = ExperimentVariantTrendResult(key="B", count=180, exposure=1, absolute_exposure=200) + variant_test_2 = ExperimentVariantTrendResult(key="C", count=50, exposure=1, absolute_exposure=200) # in this case, should choose B as best test variant p_value = calculate_p_value(variant_control, [variant_test_1, variant_test_2]) self.assertAlmostEqual(p_value, 0.001, places=3) # manually assign probabilities to control test case - significant, p_value = ClickhouseTrendExperimentResult.are_results_significant( + significant, p_value = are_results_significant( variant_control, [variant_test_1, variant_test_2], [0.5, 0.4, 0.1] ) self.assertAlmostEqual(p_value, 1, places=3) self.assertEqual(significant, ExperimentSignificanceCode.LOW_WIN_PROBABILITY) # new B variant is worse, such that control probability ought to be high enough - variant_test_1 = CountVariant("B", 100, 1, 200) + variant_test_1 = ExperimentVariantTrendResult(key="B", count=100, exposure=1, absolute_exposure=200) - significant, p_value = ClickhouseTrendExperimentResult.are_results_significant( + significant, p_value = are_results_significant( variant_control, [variant_test_1, variant_test_2], [0.95, 0.03, 0.02] ) self.assertAlmostEqual(p_value, 0, places=3) self.assertEqual(significant, ExperimentSignificanceCode.SIGNIFICANT) - credible_intervals = calculate_trend_credible_intervals([variant_control, variant_test_1, variant_test_2]) + credible_intervals = calculate_credible_intervals([variant_control, variant_test_1, variant_test_2]) self.assertAlmostEqual(credible_intervals[variant_control.key][0], 1.1045, places=3) self.assertAlmostEqual(credible_intervals[variant_control.key][1], 1.4149, places=3) self.assertAlmostEqual(credible_intervals[variant_test_1.key][0], 0.4113, places=3) @@ -204,13 +205,11 @@ def test_calculate_significance_when_target_variants_underperform(self): self.assertAlmostEqual(credible_intervals[variant_test_2.key][1], 0.3295, places=3) def test_results_with_different_exposures(self): - variant_control = CountVariant("A", 50, 1.3, 260) - variant_test_1 = CountVariant("B", 30, 1.8, 360) - variant_test_2 = CountVariant("C", 20, 0.7, 140) + variant_control = ExperimentVariantTrendResult(key="A", count=50, exposure=1.3, absolute_exposure=260) + variant_test_1 = ExperimentVariantTrendResult(key="B", count=30, exposure=1.8, absolute_exposure=360) + variant_test_2 = ExperimentVariantTrendResult(key="C", count=20, exposure=0.7, absolute_exposure=140) - probabilities = ClickhouseTrendExperimentResult.calculate_results( - variant_control, [variant_test_1, variant_test_2] - ) # a is control + probabilities = calculate_probabilities(variant_control, [variant_test_1, variant_test_2]) # a is control self.assertAlmostEqual(probabilities[0], 0.86, places=1) self.assertAlmostEqual(probabilities[1], 0, places=1) self.assertAlmostEqual(probabilities[2], 0.13, places=1) @@ -228,14 +227,12 @@ def test_results_with_different_exposures(self): p_value = calculate_p_value(variant_control, [variant_test_1, variant_test_2]) self.assertAlmostEqual(p_value, 0, places=3) - significant, p_value = ClickhouseTrendExperimentResult.are_results_significant( - variant_control, [variant_test_1, variant_test_2], probabilities - ) + significant, p_value = are_results_significant(variant_control, [variant_test_1, variant_test_2], probabilities) self.assertAlmostEqual(p_value, 1, places=3) # False because max probability is less than 0.9 self.assertEqual(significant, ExperimentSignificanceCode.LOW_WIN_PROBABILITY) - credible_intervals = calculate_trend_credible_intervals([variant_control, variant_test_1, variant_test_2]) + credible_intervals = calculate_credible_intervals([variant_control, variant_test_1, variant_test_2]) self.assertAlmostEqual(credible_intervals[variant_control.key][0], 0.1460, places=3) self.assertAlmostEqual(credible_intervals[variant_control.key][1], 0.2535, places=3) self.assertAlmostEqual(credible_intervals[variant_test_1.key][0], 0.0585, places=3) diff --git a/ee/clickhouse/queries/experiments/trend_experiment_result.py b/ee/clickhouse/queries/experiments/trend_experiment_result.py index 489520fd02596..74dcd89ccf48d 100644 --- a/ee/clickhouse/queries/experiments/trend_experiment_result.py +++ b/ee/clickhouse/queries/experiments/trend_experiment_result.py @@ -1,20 +1,13 @@ import json from dataclasses import asdict, dataclass from datetime import datetime -from functools import lru_cache -from math import exp, lgamma, log from typing import Optional from zoneinfo import ZoneInfo -from numpy.random import default_rng from rest_framework.exceptions import ValidationError -import scipy.stats as stats -from sentry_sdk import capture_exception from ee.clickhouse.queries.experiments import ( CONTROL_VARIANT_KEY, - FF_DISTRIBUTION_THRESHOLD, - MIN_PROBABILITY_FOR_SIGNIFICANCE, ) from posthog.constants import ( ACTIONS, @@ -25,6 +18,11 @@ ExperimentSignificanceCode, ExperimentNoResultsErrorKeys, ) +from posthog.hogql_queries.experiments.trend_statistics import ( + are_results_significant, + calculate_credible_intervals, + calculate_probabilities, +) from posthog.models.feature_flag import FeatureFlag from posthog.models.filters.filter import Filter from posthog.models.team import Team @@ -33,8 +31,6 @@ Probability = float -P_VALUE_SIGNIFICANCE_LEVEL = 0.05 - @dataclass(frozen=True) class Variant: @@ -240,14 +236,14 @@ def get_results(self, validate: bool = True): control_variant, test_variants = self.get_variants(insight_results, exposure_results) - probabilities = self.calculate_results(control_variant, test_variants) + probabilities = calculate_probabilities(control_variant, test_variants) mapping = { variant.key: probability for variant, probability in zip([control_variant, *test_variants], probabilities) } - significance_code, p_value = self.are_results_significant(control_variant, test_variants, probabilities) + significance_code, p_value = are_results_significant(control_variant, test_variants, probabilities) credible_intervals = calculate_credible_intervals([control_variant, *test_variants]) except ValidationError: @@ -322,191 +318,6 @@ def get_variants(self, insight_results, exposure_results): return control_variant, test_variants - @staticmethod - def calculate_results(control_variant: Variant, test_variants: list[Variant]) -> list[Probability]: - """ - Calculates probability that A is better than B. First variant is control, rest are test variants. - - Supports maximum 10 variants today - - For each variant, we create a Gamma distribution of arrival rates, - where alpha (shape parameter) = count of variant + 1 - beta (exposure parameter) = 1 - """ - if not control_variant: - raise ValidationError("No control variant data found", code="no_data") - - if len(test_variants) >= 10: - raise ValidationError( - "Can't calculate experiment results for more than 10 variants", - code="too_much_data", - ) - - if len(test_variants) < 1: - raise ValidationError( - "Can't calculate experiment results for less than 2 variants", - code="no_data", - ) - - return calculate_probability_of_winning_for_each([control_variant, *test_variants]) - - @staticmethod - def are_results_significant( - control_variant: Variant, - test_variants: list[Variant], - probabilities: list[Probability], - ) -> tuple[ExperimentSignificanceCode, Probability]: - # TODO: Experiment with Expected Loss calculations for trend experiments - - for variant in test_variants: - # We need a feature flag distribution threshold because distribution of people - # can skew wildly when there are few people in the experiment - if variant.absolute_exposure < FF_DISTRIBUTION_THRESHOLD: - return ExperimentSignificanceCode.NOT_ENOUGH_EXPOSURE, 1 - - if control_variant.absolute_exposure < FF_DISTRIBUTION_THRESHOLD: - return ExperimentSignificanceCode.NOT_ENOUGH_EXPOSURE, 1 - - if ( - probabilities[0] < MIN_PROBABILITY_FOR_SIGNIFICANCE - and sum(probabilities[1:]) < MIN_PROBABILITY_FOR_SIGNIFICANCE - ): - # Sum of probability of winning for all variants except control is less than 90% - return ExperimentSignificanceCode.LOW_WIN_PROBABILITY, 1 - - p_value = calculate_p_value(control_variant, test_variants) - - if p_value >= P_VALUE_SIGNIFICANCE_LEVEL: - return ExperimentSignificanceCode.HIGH_P_VALUE, p_value - - return ExperimentSignificanceCode.SIGNIFICANT, p_value - - -def simulate_winning_variant_for_arrival_rates(target_variant: Variant, variants: list[Variant]) -> float: - random_sampler = default_rng() - simulations_count = 100_000 - - variant_samples = [] - for variant in variants: - # Get `N=simulations` samples from a Gamma distribution with alpha = variant_sucess + 1, - # and exposure = relative exposure of variant - samples = random_sampler.gamma(variant.count + 1, 1 / variant.exposure, simulations_count) - variant_samples.append(samples) - - target_variant_samples = random_sampler.gamma( - target_variant.count + 1, 1 / target_variant.exposure, simulations_count - ) - - winnings = 0 - variant_conversions = list(zip(*variant_samples)) - for i in range(simulations_count): - if target_variant_samples[i] > max(variant_conversions[i]): - winnings += 1 - - return winnings / simulations_count - - -def calculate_probability_of_winning_for_each(variants: list[Variant]) -> list[Probability]: - """ - Calculates the probability of winning for each variant. - """ - - if len(variants) > 10: - raise ValidationError( - "Can't calculate experiment results for more than 10 variants", - code="too_much_data", - ) - - probabilities = [] - # simulate winning for each test variant - for index, variant in enumerate(variants): - probabilities.append( - simulate_winning_variant_for_arrival_rates(variant, variants[:index] + variants[index + 1 :]) - ) - - total_test_probabilities = sum(probabilities[1:]) - - return [max(0, 1 - total_test_probabilities), *probabilities[1:]] - - -@lru_cache(maxsize=100_000) -def combinationln(n: int, k: int) -> float: - """ - Returns the log of the binomial coefficient. - """ - return lgamma(n + 1) - lgamma(k + 1) - lgamma(n - k + 1) - - -def intermediate_poisson_term(count: int, iterator: int, relative_exposure: float): - return exp( - combinationln(count, iterator) - + iterator * log(relative_exposure) - + (count - iterator) * log(1 - relative_exposure) - ) - - -def poisson_p_value(control_count, control_exposure, test_count, test_exposure): - """ - Calculates the p-value of the experiment. - Calculations from: https://www.evanmiller.org/statistical-formulas-for-programmers.html#count_test - """ - relative_exposure = test_exposure / (control_exposure + test_exposure) - total_count = control_count + test_count - - low_p_value = 0.0 - high_p_value = 0.0 - - for i in range(test_count + 1): - low_p_value += intermediate_poisson_term(total_count, i, relative_exposure) - - for i in range(test_count, total_count + 1): - high_p_value += intermediate_poisson_term(total_count, i, relative_exposure) - - return min(1, 2 * min(low_p_value, high_p_value)) - - -def calculate_p_value(control_variant: Variant, test_variants: list[Variant]) -> Probability: - best_test_variant = max(test_variants, key=lambda variant: variant.count) - - return poisson_p_value( - control_variant.count, - control_variant.exposure, - best_test_variant.count, - best_test_variant.exposure, - ) - - -def calculate_credible_intervals(variants, lower_bound=0.025, upper_bound=0.975): - """ - Calculate the Bayesian credible intervals for the mean (average events per unit) - for a list of variants in a Trend experiment. - If no lower/upper bound is provided, the function calculates the 95% credible interval. - """ - intervals = {} - - for variant in variants: - try: - # Alpha (shape parameter) is count + 1, assuming a Gamma distribution for counts - alpha = variant.count + 1 - - # Beta (scale parameter) is the inverse of absolute_exposure, - # representing the average rate of events per user - beta = 1 / variant.absolute_exposure - - # Calculate the credible interval for the mean using Gamma distribution - credible_interval = stats.gamma.ppf([lower_bound, upper_bound], a=alpha, scale=beta) - - intervals[variant.key] = (credible_interval[0], credible_interval[1]) - - except Exception as e: - capture_exception( - Exception(f"Error calculating credible interval for variant {variant.key}"), - {"error": str(e)}, - ) - return {} - - return intervals - def validate_event_variants(trend_results, variants): errors = { diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index 9f35a775afd6b..c950088374fb4 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -4908,14 +4908,20 @@ "ExperimentVariantTrendResult": { "additionalProperties": false, "properties": { + "absolute_exposure": { + "type": "number" + }, "count": { "type": "number" }, "exposure": { "type": "number" + }, + "key": { + "type": "string" } }, - "required": ["count", "exposure"], + "required": ["key", "count", "exposure", "absolute_exposure"], "type": "object" }, "ExperimentalAITrendsQuery": { diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index 9bbffba136dab..e92c4860a0d8c 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -1591,8 +1591,10 @@ export type InsightQueryNode = | LifecycleQuery export interface ExperimentVariantTrendResult { + key: string count: number exposure: number + absolute_exposure: number } export interface ExperimentVariantFunnelResult { diff --git a/posthog/hogql_queries/experiments/experiment_trend_query_runner.py b/posthog/hogql_queries/experiments/experiment_trend_query_runner.py index efd2a6a189c0d..b63ee09bd2c09 100644 --- a/posthog/hogql_queries/experiments/experiment_trend_query_runner.py +++ b/posthog/hogql_queries/experiments/experiment_trend_query_runner.py @@ -241,7 +241,10 @@ def _process_results( self, count_results: list[dict[str, Any]], exposure_results: list[dict[str, Any]] ) -> dict[str, ExperimentVariantTrendResult]: variants = self.feature_flag.variants - processed_results = {variant["key"]: ExperimentVariantTrendResult(count=0, exposure=0) for variant in variants} + processed_results = { + variant["key"]: ExperimentVariantTrendResult(key=variant["key"], count=0, exposure=0, absolute_exposure=0) + for variant in variants + } for result in count_results: variant = result.get("breakdown_value") @@ -251,7 +254,7 @@ def _process_results( for result in exposure_results: variant = result.get("breakdown_value") if variant in processed_results: - processed_results[variant].exposure += result.get("count", 0) + processed_results[variant].absolute_exposure += result.get("count", 0) return processed_results diff --git a/posthog/hogql_queries/experiments/test/test_experiment_trend_query_runner.py b/posthog/hogql_queries/experiments/test/test_experiment_trend_query_runner.py index 77496a299f042..b92e58280f7a9 100644 --- a/posthog/hogql_queries/experiments/test/test_experiment_trend_query_runner.py +++ b/posthog/hogql_queries/experiments/test/test_experiment_trend_query_runner.py @@ -114,8 +114,8 @@ def test_query_runner(self): self.assertEqual(control_result.count, 11) self.assertEqual(test_result.count, 15) - self.assertEqual(control_result.exposure, 7) - self.assertEqual(test_result.exposure, 9) + self.assertEqual(control_result.absolute_exposure, 7) + self.assertEqual(test_result.absolute_exposure, 9) @freeze_time("2020-01-01T12:00:00Z") def test_query_runner_with_custom_exposure(self): @@ -216,8 +216,8 @@ def test_query_runner_with_custom_exposure(self): self.assertEqual(control_result.count, 3) self.assertEqual(test_result.count, 5) - self.assertEqual(control_result.exposure, 2) - self.assertEqual(test_result.exposure, 2) + self.assertEqual(control_result.absolute_exposure, 2) + self.assertEqual(test_result.absolute_exposure, 2) @freeze_time("2020-01-01T12:00:00Z") def test_query_runner_with_default_exposure(self): @@ -311,8 +311,8 @@ def test_query_runner_with_default_exposure(self): self.assertEqual(control_result.count, 3) self.assertEqual(test_result.count, 5) - self.assertEqual(control_result.exposure, 2) - self.assertEqual(test_result.exposure, 2) + self.assertEqual(control_result.absolute_exposure, 2) + self.assertEqual(test_result.absolute_exposure, 2) @freeze_time("2020-01-01T12:00:00Z") def test_query_runner_with_avg_math(self): diff --git a/posthog/hogql_queries/experiments/trend_statistics.py b/posthog/hogql_queries/experiments/trend_statistics.py new file mode 100644 index 0000000000000..f35a1b42136b3 --- /dev/null +++ b/posthog/hogql_queries/experiments/trend_statistics.py @@ -0,0 +1,196 @@ +from functools import lru_cache +from math import exp, lgamma, log, ceil + +from numpy.random import default_rng +from rest_framework.exceptions import ValidationError +import scipy.stats as stats +from sentry_sdk import capture_exception + +from ee.clickhouse.queries.experiments import ( + FF_DISTRIBUTION_THRESHOLD, + MIN_PROBABILITY_FOR_SIGNIFICANCE, + P_VALUE_SIGNIFICANCE_LEVEL, +) +from posthog.constants import ExperimentSignificanceCode +from posthog.schema import ExperimentVariantTrendResult + +Probability = float + + +def calculate_probabilities( + control_variant: ExperimentVariantTrendResult, test_variants: list[ExperimentVariantTrendResult] +) -> list[Probability]: + """ + Calculates probability that A is better than B. First variant is control, rest are test variants. + + Supports maximum 10 variants today + + For each variant, we create a Gamma distribution of arrival rates, + where alpha (shape parameter) = count of variant + 1 + beta (exposure parameter) = 1 + """ + if not control_variant: + raise ValidationError("No control variant data found", code="no_data") + + if len(test_variants) >= 10: + raise ValidationError( + "Can't calculate experiment results for more than 10 variants", + code="too_much_data", + ) + + if len(test_variants) < 1: + raise ValidationError( + "Can't calculate experiment results for less than 2 variants", + code="no_data", + ) + + variants = [control_variant, *test_variants] + probabilities = [] + + # simulate winning for each test variant + for index, variant in enumerate(variants): + probabilities.append( + simulate_winning_variant_for_arrival_rates(variant, variants[:index] + variants[index + 1 :]) + ) + + total_test_probabilities = sum(probabilities[1:]) + + return [max(0, 1 - total_test_probabilities), *probabilities[1:]] + + +def simulate_winning_variant_for_arrival_rates( + target_variant: ExperimentVariantTrendResult, variants: list[ExperimentVariantTrendResult] +) -> float: + random_sampler = default_rng() + simulations_count = 100_000 + + variant_samples = [] + for variant in variants: + # Get `N=simulations` samples from a Gamma distribution with alpha = variant_sucess + 1, + # and exposure = relative exposure of variant + samples = random_sampler.gamma(variant.count + 1, 1 / variant.exposure, simulations_count) + variant_samples.append(samples) + + target_variant_samples = random_sampler.gamma( + target_variant.count + 1, 1 / target_variant.exposure, simulations_count + ) + + winnings = 0 + variant_conversions = list(zip(*variant_samples)) + for i in range(ceil(simulations_count)): + if target_variant_samples[i] > max(variant_conversions[i]): + winnings += 1 + + return winnings / simulations_count + + +def are_results_significant( + control_variant: ExperimentVariantTrendResult, + test_variants: list[ExperimentVariantTrendResult], + probabilities: list[Probability], +) -> tuple[ExperimentSignificanceCode, Probability]: + # TODO: Experiment with Expected Loss calculations for trend experiments + + for variant in test_variants: + # We need a feature flag distribution threshold because distribution of people + # can skew wildly when there are few people in the experiment + if variant.absolute_exposure < FF_DISTRIBUTION_THRESHOLD: + return ExperimentSignificanceCode.NOT_ENOUGH_EXPOSURE, 1 + + if control_variant.absolute_exposure < FF_DISTRIBUTION_THRESHOLD: + return ExperimentSignificanceCode.NOT_ENOUGH_EXPOSURE, 1 + + if ( + probabilities[0] < MIN_PROBABILITY_FOR_SIGNIFICANCE + and sum(probabilities[1:]) < MIN_PROBABILITY_FOR_SIGNIFICANCE + ): + # Sum of probability of winning for all variants except control is less than 90% + return ExperimentSignificanceCode.LOW_WIN_PROBABILITY, 1 + + p_value = calculate_p_value(control_variant, test_variants) + + if p_value >= P_VALUE_SIGNIFICANCE_LEVEL: + return ExperimentSignificanceCode.HIGH_P_VALUE, p_value + + return ExperimentSignificanceCode.SIGNIFICANT, p_value + + +@lru_cache(maxsize=100_000) +def combinationln(n: float, k: float) -> float: + """ + Returns the log of the binomial coefficient. + """ + return lgamma(n + 1) - lgamma(k + 1) - lgamma(n - k + 1) + + +def intermediate_poisson_term(count: float, iterator: float, relative_exposure: float): + return exp( + combinationln(count, iterator) + + iterator * log(relative_exposure) + + (count - iterator) * log(1 - relative_exposure) + ) + + +def poisson_p_value(control_count, control_exposure, test_count, test_exposure): + """ + Calculates the p-value of the experiment. + Calculations from: https://www.evanmiller.org/statistical-formulas-for-programmers.html#count_test + """ + relative_exposure = test_exposure / (control_exposure + test_exposure) + total_count = control_count + test_count + + low_p_value = 0.0 + high_p_value = 0.0 + + for i in range(ceil(test_count) + 1): + low_p_value += intermediate_poisson_term(total_count, i, relative_exposure) + + for i in range(ceil(test_count), ceil(total_count) + 1): + high_p_value += intermediate_poisson_term(total_count, i, relative_exposure) + + return min(1, 2 * min(low_p_value, high_p_value)) + + +def calculate_p_value( + control_variant: ExperimentVariantTrendResult, test_variants: list[ExperimentVariantTrendResult] +) -> Probability: + best_test_variant = max(test_variants, key=lambda variant: variant.count) + + return poisson_p_value( + control_variant.count, + control_variant.exposure, + best_test_variant.count, + best_test_variant.exposure, + ) + + +def calculate_credible_intervals(variants, lower_bound=0.025, upper_bound=0.975): + """ + Calculate the Bayesian credible intervals for the mean (average events per unit) + for a list of variants in a Trend experiment. + If no lower/upper bound is provided, the function calculates the 95% credible interval. + """ + intervals = {} + + for variant in variants: + try: + # Alpha (shape parameter) is count + 1, assuming a Gamma distribution for counts + alpha = variant.count + 1 + + # Beta (scale parameter) is the inverse of absolute_exposure, + # representing the average rate of events per user + beta = 1 / variant.absolute_exposure + + # Calculate the credible interval for the mean using Gamma distribution + credible_interval = stats.gamma.ppf([lower_bound, upper_bound], a=alpha, scale=beta) + + intervals[variant.key] = (credible_interval[0], credible_interval[1]) + + except Exception as e: + capture_exception( + Exception(f"Error calculating credible interval for variant {variant.key}"), + {"error": str(e)}, + ) + return {} + + return intervals diff --git a/posthog/schema.py b/posthog/schema.py index d78dc0d8ac2f8..de474b9007222 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -522,8 +522,10 @@ class ExperimentVariantTrendResult(BaseModel): model_config = ConfigDict( extra="forbid", ) + absolute_exposure: float count: float exposure: float + key: str class FilterLogicalOperator(StrEnum): From 27364a3218598db9235149d8cc16513672347dff Mon Sep 17 00:00:00 2001 From: Michael Matloka Date: Mon, 14 Oct 2024 15:14:26 +0200 Subject: [PATCH 06/52] feat(max): Smart question suggestions (#25556) Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- .../scenes-app-max-ai--welcome--dark.png | Bin 21953 -> 22519 bytes .../scenes-app-max-ai--welcome--light.png | Bin 22103 -> 22689 bytes ...-ai--welcome-loading-suggestions--dark.png | Bin 0 -> 19329 bytes ...ai--welcome-loading-suggestions--light.png | Bin 0 -> 19701 bytes .../lib/lemon-ui/LemonButton/LemonButton.tsx | 4 +- frontend/src/lib/lemon-ui/icons/categories.ts | 3 + frontend/src/queries/schema.json | 97 ++++++++++++++++++ frontend/src/queries/schema.ts | 16 ++- frontend/src/scenes/max/Max.stories.tsx | 32 ++++++ frontend/src/scenes/max/QuestionInput.tsx | 2 +- .../src/scenes/max/QuestionSuggestions.tsx | 67 +++++++++--- frontend/src/scenes/max/maxLogic.ts | 47 ++++++++- .../scenes/saved-insights/SavedInsights.tsx | 7 +- package.json | 2 +- pnpm-lock.yaml | 8 +- posthog/hogql/ai.py | 33 +++--- .../ai/suggested_questions_query_runner.py | 92 +++++++++++++++++ ...test_suggested_questions_query_runner.ambr | 18 ++++ .../test_suggested_questions_query_runner.py | 36 +++++++ posthog/hogql_queries/query_runner.py | 11 ++ posthog/schema.py | 50 +++++++++ 21 files changed, 483 insertions(+), 42 deletions(-) create mode 100644 frontend/__snapshots__/scenes-app-max-ai--welcome-loading-suggestions--dark.png create mode 100644 frontend/__snapshots__/scenes-app-max-ai--welcome-loading-suggestions--light.png create mode 100644 posthog/hogql_queries/ai/suggested_questions_query_runner.py create mode 100644 posthog/hogql_queries/ai/test/__snapshots__/test_suggested_questions_query_runner.ambr create mode 100644 posthog/hogql_queries/ai/test/test_suggested_questions_query_runner.py diff --git a/frontend/__snapshots__/scenes-app-max-ai--welcome--dark.png b/frontend/__snapshots__/scenes-app-max-ai--welcome--dark.png index 023ee3bb7c7cbff56189ca507b35713ed5e2450f..a439dd9822a1933bb9c7eab17af0ef1c7ead5bbf 100644 GIT binary patch delta 4939 zcmb_gc{G&o+aF|!ELoHArR+->OA%@$`!=>FYh({!#;);@Ela6n&mbX*EQ5*}Tg4db z7?gdV$uhEUG4FW4zu)_w-#PD}?{l7Wp8GuKzOT=9U!QZWZ51?oA8D%fZ_@JR)EVB) zJp;Uv6xNQ`j%=0hDrfQ(;t>ij{BEXe0Pg*OLB@w8hHIPYH~tvD3u9zRZ81N5xdxA=y-hlKEl!1kM+PDJYZZ}fw~1&9zK_mC$~ zpDunj04ZFzr(pn5pOgR@r!j2#Y)Qxt(5K2Q^>H-XM?gQv@RhBLZseMps%psEp~<5L zM$l69S%qkNOG$1>k$L%7OLY*akO!MJ*kh6RY-!b2lDnISIbDSn)Z8IA0OI{ng_#;- zP9sqPJf(B%rsv*BjM&)h1n(@7xj`4$xtu{PHZTjf%flxxIjMVoLwt4x1ONdaA6H8Y zm=`&UCVE4GhN0TIzoH^Tx+FFh{aJ$+1RB7T`l1>W*IvqS4-cy>kuQQSm<@dPF@x{X zfG(CeG&Mkq3QJ2HFhw-ejMU4`WSL8%8ft56UPyCyM${}65ofu zt7~hOVk0gux+W%RsEPs>3(UF*2ZXMhSk*Bt3X~0^L!Ifb@IlU7@}HV2ng>I7?sNeH zRnm&Lo_?u1Njua2Km7#$Poa;0ObCKJYt)1Oh6e_6Cf%luX`MjO6c$0txtVpn%0EmPBuRNQM+a()~C#W7H&=JPSvQ6R$TkG}uc6t3PTwHr=9Z zY-~(^`fP!Z-7f}l_jHX%4*0hqvFN-L%XVxO648rioU&v^?N25qV9J#3^p>~r6X-iZ zv0gl(p^Zi@`HM|;W4WHqDo173@GiL~EQU+M^nwrB5rh(@DxKY4f|GJ|JT3=EYoQ~r ztE;mK&Y7TY^nqL z<6DUX$s>Er!dG?J#(gHuXGi@d$1&lBh&ZqBeZ;$wrgCcXDj4D&WUFMg=i^z%xlpX`V8B9SK{{f z&tbc}YwCQ4Ds0V`vxZ=>>r}U~!Unib8#E!rc9u(CTy`qU+$JA)?@bG~!=g+_dPkjv?=S@f1*J|kN@#4|3B7D>gytA4fSgall^Zq;r997 zn$j*FAxlr~)S!>G9K?82$``rBQ#C0#vrxDn<<;M{q3gDHRpA3SKQ$9LxrYyy-*Otj^##|aEPWol^GuYLpB+_-*zQ^>H-Pc0L3C7&u5i8= z2(9(E;CH>KdGY%D>PH_)r74joUYGSW4B-c>3Udg8ai9^?^_@v}8k|k&-=UPSK3VUi zyRwb~+XYC?FI+StaZffJoIsXsXD+mI8yewif!e@kG~;>5qesnL{jbr%D`o{nQ*j*Y zZFK4K5*0=f;jG!}#~tI1Lk+$^Z_2@1~|Y-IW|YK zH_mE{d#k>+%f_Q48CA3ecins~cqb&O;bj+ELn8zrDY%WTK7al@1DEw)W>8ZZ6U=)9qNpRAn~y2dD8n?E12aZ4-1*;e5A=aJ@lI)wTGAz66=9E!?% zoZg+)Rl-3$R=8)PRwYP}lzIOn>*BSbfnB+XX+LVbkh1h(^q)|T967f$3-~=)4m*%` zb%l1K_6QgGrN=~4BKL<&P4aJe^tB(b{Fy>{^yE@L3`AsjZF?@fYgd}Wmm2YPdc&U4 z9(g1NtQWh#w5y+teDgu%9X4vVwqw*7u%h`O zBD8c$e$Zx)M4AES=a1V6FfT9uxaU9U7}d@&DhlSVzSj5>nIOu|+@ck=Kt>^n_*##rbR$#Mf$Zn^ zr86F<(5Gcs2ZSguAqbZOD|orMBsiExsvNdHe7Ytrop2A&%}N8}GLEJ(z9J>%Hq!G8 zaQwJyE6D?#OcB{Td{_8)e{7kUEJK1Ao{+(aw4~{aB@MliN&z3rKq>&lFTeA z@7Mz~ZL*mwDAKW!jB1ds1E;7rLY7jzDNvPrqp}~YB;O73L7qV5%j5$&A)NiMBjJQ6 z9eU3&iY{T!=GCTrKWxhG*%nwYOaZ{m9Ntuxmq^rh{PJ+8oxb1<;)}YRrl#zpbz#8X zZMq;$=(w`#D;8@Y%L}&dvqGYy z-w&V~IrPC=jdE8~_qMLjLwh+PQ4`4JO6;iUK?6XbtDot0Rc7tnsw1i z=K8}C3_8W#mt9y3Z~qN%S=O{i#fac>gp|Zn9a343(}M_{vq>#xG%HS)9}5~Zjs9$f z(=uE+UR5~!Z)1D)=JF`zq~;z&&0^%`mx$jlfo5o_RWH})%m@cr-mBRZ1H*2f5@=3l zW)W1181-kCYGxofIM~=HnEa=`^r?@8Sx)%c*v|RO37zahUHCIu?Yg@COd=;V6-r7< zD!9`#^`eSXlKu++)CCOy*-y=C8qf$@>OOgb}=xs8}P#6c$gtiZo4=*I864H^b_mxTI^71VED1;xsymFg)yJgjRI#$;*-#$2X1+_rtF4 zI0enEq!U69l<-gqlS>pxCmS2vF6#~}0micC4t+ezxVtuVq*5Sq5|v__e;5Ol1eR`w zhXdHf&3Kljh}|{t-aLE#GVxzIN)I5Gm3Tb-iJGeO7xrJjmBM#29XNK%#JdWEs1&~H zQ0!!{N*JPWqaNaMQlwi=&os=i7mN9`-c zNP<^KM^N>4NbI7d&wayB1(f8+W%1KM{tezse|{xA9haA&v;|PP$Sa_9d-Nx_9E(S} z{ynfO*rcrCJXr3^FTBspgM0q1h5^z=3`lf!6?JhH?MI?HIKNn~SeFhKZxl0I5`%}_ z%unjyyJ<7aUfk01X8XK@0Psjn4Q3XW@}F0Cn(}(>bW)O&%l*#MUz1=|IDZQuR%!N5 zGDh=$8>%ylh$#IWIfxQBy~Njm@S)fyjt~ghn4Md6Xw&=~yN3Q(Cna1rpi^fYIGN6f zzIgIw!~g4(StlW_fl57XbR$tkC6m;UU^S%VPq3qV+h=ZEthaT^`kr(Yyy@YuxyU`? z*IFhN4Y%g=ps>VsYAa_Z%ZmjTnc;hH%FppD?WbR_Gm}7md)4a6IR9pmb+hZ;g(7{@ zdgJe<%5isXv*9DF>dbuadru-Pb(T|Gl_QipEJ{k|G#dvM19o<{)$YN=9WR3}FgMzP zew!V4g6|&xsiNE!4~faktDsIWT6uVEGX=z7c+Uv_)2t7V^MWquEsz#m$XXy2 zAQ=)8lCfA~GXmAqi_A!tJgsKD$fh&{94X(Z9qQV66UUqPQJ@sS(o!fNUu3_bM TmNqCn9ljXon%=I0I6wX`t=o_S delta 4368 zcmZu!XE>Z&*M4+C#1TPAhy;l~(dLL6y^B#2qlc)$(M21H5eG+)j1ax#Fa$A5^b$Qr z8KX0aZggg%#PE*feSdx5{)^z^i*artVwY!B}jtpJH$jl?`@CSnOZ3$B$49HcvLj7e05@x|f?G zYtqzct__suKhdChqxP>#Zw*Xe+2SpKDqP)J@HkBQjIT2jyfgT7FTIDn2d5U{Wd&>= z%u>(-W+V=H4>ndgvXcz0-_?!{LdyCoB$vfHrE;2#P=T ziy(W)p*37a>*QbMA&_$C_>%=`YtsI4Xpt^)l=(7X9*t0;`YXMuYUCkY%a5GZbkE>% ztI+_j+~4Z)`!ZIw5gz7@Yd9R!_z>w0P!!Ns1Ln z6ZbJs&yV_GY5-^(U;7yDQ2X4Z)YRXfs1LpjKoeGHs^~Bt01$`roQoY=#ZG{wh1r6X zK>!fpPligD+&f~N4=E85L8f~$15c>0v{Qa-ijCEEzN_{4@ngEG$Va#A8sx|rWHn$V z5Se~%cQ<$M9ga3yI|?Q{#_S2Ka*x+D@@D{m23;Xny;zepO|?zgiCfjS=Kd{8|?!VgI-0!(KlsppBuaS+NLG zqH^ooUXXqrjmnjH%>qb?un2OXzh8frrli)4p6V|{MpZ_jtm|NFGICuOh|p;8-^d?H zavmymcXL}NH$IapH~Puux0My)w0>zWf8!DBg5_j4?$;n-HJuv@*hz`PA}H$1E$hT} zi$2vKfs45*?NZQLw%-qq36GPsYqC^0BYgEQi2;>2=Bb$4>}_na;Vf2Ml+m-zA&EVU zFJ8Rh6pk>RxdNy7v&Y4SB7#Id%Jrn1%M+jgw(Y z;HNzWxf12e+~VT5>*g>Z{X=oF`7IXC|0hX-=Sm`+0-ZdtEIqWZQM$#@0lZh^CF52m z-^Fu-&6W6`gd2rtLyLPp;mw}{hnVqtT3S7g8lM+?5-&0Bw!$ig`}+FcFocCRA;d0U zjZv5Qg#8+FJSXkW#>vLUc64CW2h*tYG0f7(uTCJ6WuGWw%R2IS?;A$94-6bE3^_BI zx$B0Ht7|MdIeW%TxtK@FNw=8yyl$!HLji5VM>p&MOW*r4iU9|j$=ZdF?wt^}m$$c- zf*!FD%OjMVg<0rFN5%cO39;+fY7h2oYNbW(aTHOidmYe01^M;j&xMe6Sf0dTjy%lX zer;5PabmyZej`mCryHHoR|Meojg33=HO;qnpwRx~cT&aRxh&yR zG(m`5)OJw+x~%k3=W?8HwN!=G$&0a#JM?^d2)NCkIr`TE0%5))EvZ@_VDCWun*3rd z`;zrQ!&{!J;HXDOJvpb{iM+nsJtlPwUhf~g8*2tOig&YhTmk6!y`&M+R zS0J?crPH{DafFqWd2WN^_xc3}qvddx(OQp8*v(1HbQozYP{-th;V2JuXLv39x>8Qv zcB(;6S)%y8df4$`9qu?NWbc?3=B*G0trC8Endo)CN*AS?r!E@c?|)zHv!AF=O&({u z|GuqNihs4UIVwX&4eZe1P+BM7U|rh`qNY8r)qz_;=fT~}w#EAI3d^m) z%5NS<(q`+NUEy=Y%^&rp3UbB5M}$mwK=0DQ&1xmjU<)dioKEh8$YN(~hikk*9t~jL zPRZFeZfb76yvjjKL&K{2Dl*dWZcbnSiA>O6EPVVfb}G7V8MTG2=Mg0sXIy=9vp0#$ zO>e0O$)hNIQz~Gzwe=+U@Z%OdX{K&R)S@AUWxspP`);fJ96_ZaN@%XZKMUkNjYUpe z;*(#5JylegHq4g1<1|p)n|Qw0{crLhRNhXdRS4r$RAjBRd(0D~uCV8Y5x4D|B@Fq) zla`jMtsDIClkEr=R#r(=OUw7Cx~1!lS5+5kz1F8|2F6^g)jAOdFG5E5pM8fXF}G|j z27cj@|ADQ)WU{}{XdYS*Z^@*5X*d2gkX_sgCK+h6mV*nu^OVSt(eK)iky>`-@<^H2 zT?l+m-20I&=`mhxP{k|$6E}EyE^j2niOhh^HClab%_`mQ89j3r1YV+y*1*1kRsa)4>)zy2s?7%EH>{uVv`T!{?|qyIrKmu2DYqZT)** zosZgbJoCGeR9BQvu!whHAR{w#a0veE=BAaZc1O$5gt`VNSH`Q3%m(Mz9jgb4yox~^ zKeCp2KT1v`er$z}Yig9?SWHY|emySsm>N+mBv5Ej+c@38r+U)@3=PJ7tdoz4(aH?4 znA3F%u^;w4BoH{2p7-Jen^Q9pIhly_HDlM~R38Z=qvq<{HE~w+lh4x$njOC{Zsas5 zLuPnt>hoLpRO~^ikacA!w4kB;HF^AmT+H}L2xQb5oxXkU27{$Ri5nGiT=7oxvxg*- ztpME7(FVhgEG-p;OG`_$s8Un%@x|{n%lmG>4>S`tv#^l!A5rmTQ;VwMwOm-<(xI0- zTol6JhTLOwl$8s{bhxF(Hb8H~|9zv2cjDk%^4p)9(0U!}y*upA*J-YyzFzy%iOhA= zBI)_^fiozWs~8)Do-U8LO@mMS>6P2&6<^VPQ>=}PqeUWw)q^t<-L!5Yq~y^(Y17Yn z`vhX9k7T^8PKhbrd|{ag+iJ4*L!2be=jF*s%{2xR3527KNgYjGua>pE=ParVKan7& zSCf|v5l)-7o=kES^DV%l?lce*gbV^v*0p^g7g(K%GDp|pwy~{EW3~o;9(!9ub^bgp z=Ywb`gS^4v)%frQ$*^28rs`{}SSKc|-FK8`SlneYOZ_0LoHp8LjyBgAq7GKWlCr}s z%FUgfSKXxXayvul-X=lBz~!ywjTz6m=|n-Jcge}#GW?ED7TSOewRA{*)9EEX{?NWy z*zBx=n_gI0q{U4gLnZk@bck9s(_8H?9O72e?zVUvU4(d6379%yeYPY&KLslGUNPis zsc$Fi{rk6~HYf?c7Tfb%@tSHTW*;8gQKYLu|3}FVIm^MGuI__Do6wdBmeGQcx-$|?3SJSQFds%r0k%Pk)AkTk6?=}FYX zU@DrxA+Pm?-33xT;^KK^MJLM=mX=o15Xv#9L0BUW@eCj98B+_3V~N(nGRp&F<<^$= z_J?tu96a*b>dmeou>1k6{L_J2Jt6tG9}~AQA=cXsB(yAc9K!qk&}MLP?W>qBK+a{AJw$e9U#kRHZqFHw?^(` zb(1V)Re0q>6<+n{OHWA|Ck%|4HXu83;f<|A4+5q}E3M^~NhN&-rLSIP#zZG{_jDO@ zDtEhlhdHTsBGS?jcf*gTvK8fheD)2qrT1^JSu3CZRqH&o{lVzmZaV>G?il5)1D5tRo&G0M*q zNN1O|ZK#vy@mgUDU>_*I};WojDzncN(d|mq4bT1yJq15daW>l1YZRkb!U>EHR z_%+pD{nu7Su~_lH%-cN&;bX%ZqSHTukJg zHRH?T9yKKek*g~y$=)`+s>1y+{2v&Pkb$3v43=H+SU^2aD1ei;l4TLYm2#=g^}%SjMnJ-(S{o?+K9`A<4^J6OOKoMpj`EtHg7-03=yyn>E#K!h(CGtqku>evi zubn%oeJ(b{*x&9ZT8S*E!^wiW&7L7DWoY$9a4?9RXJYJK^rHDT#6$VQ-4tBV;?;V+ z8PFUp;KIkjftYqqyG<9<*of3r8*{6HmOC{V1HtrBs(0^xsP&j($uFsZA+IsqEJfz5 zi^Z7EC|vj`!i7irE-(z-SXo)YFYz18Dsj*YpwS8YGvu`1^qvo!T>U*W^`eeAs+UEk!Zi>Lb!PFi_@% diff --git a/frontend/__snapshots__/scenes-app-max-ai--welcome--light.png b/frontend/__snapshots__/scenes-app-max-ai--welcome--light.png index 38686893a504a5eeecdcbb64152c5594aaeaaaa7..e46c5b1b9ded0a56919cf4c8ddc446646de39e19 100644 GIT binary patch delta 4914 zcmcIo`9GB1`yV9PvQ#Rv3?jR%*+z}*j10+=v1S<=dv-Tv3GtxpYgvaZ8I!$HJ;*kS zK^T;M8#@ydzSHx3zpu~dANbxs-LG@beXi?zU*|sW>wS&ArFs31rcUoR?G-JI!EIZ5 zAXd?bZmQ@d8uqSk`G)he+1Jy!bpQ|n8AizhS@5ZWKBrLmA| z<^<1o6CPe8)(_0BFTJfENeFO?XEuCu7MezJm>?3JUF4#{;H?ur&1G+2Jm4~YZ81xq z7Gy6U_Bqy^C=ujM`GheDu7-bf8gvW=?(~h~QM@6uSi49O7G`@8j|i6`5?M)Fx3+9$ z2Z2C+KHIE*+pL)H-bYn4IXTGqyN>T((?kX~(Q|jis@%GDqu@)mEjTMcvCShs6SytL&AbapOrKx$cQA$R2=5M$$aa zs;XiDPuMh~{;?TZ))lmG{Ym=O%+10(_A% zWI8$!XsfEM%=&dopK~Ye>yGa7d~6?0WXMO_D@9^AWPsnlwu{S6E{Ui2PGG4XD2Q>! z|NZe}&(zdZCw=WP?RP+(YM(j*0u@kR}h9vx!k$NA8)M_Q{LTvl?80ldthRVHeFPo~6M zgf22pHj=SVz@R*Uc(PldQ?*SMk=9ZW3N#dZ@uAUxKpxWhk9ajlAH|@VsKH_Hv^*^JR1)Xl`OWFGw#d4v~ zPJ;vN6N3h;jYQ_EBtSoCsZkha$*Z}~Kfg(sQRFt(^KKL~n?(bV;GMr~wU&rPy2Hym zg&t$3uYdSshL0&^dAZS10ztufWYqO=>g}*q-5bB)B}yBy)~p+lu00=7TvYT`Tyd)a zS=70C^X;0S0pMSP7e4;%p>x%UO-x#PgLmpd)yR(R=Bazp;pi%^np+Ds9^QOfRf~iC zi6n=EdPpR};;K7CrG11;9V1mZ3k!>dh5X_gmxzLa<9p)b{ z^|=B2zkCjx0H4jXXU<+zf3>!>bXhY%)zB&1C?hZL4P_UvwrLXNB`9Nu?hUQ{NZTI) z0|z`*`1)#rL<;}aszn!fRSbu}+s1y-H+ zWP=#K^`o(kNs63ZP+Xo^70|;eg@`aqJf_F8};_ z2mAYA^7Cu3OmyI8!1{MKLPpgCfOluHZb^+{e^dF!DHMWzwEQ(SI;E)Nt41qkpFs{~ z)Lv5Oa9kEC{ZaEI=qWoKfw-xN3bDVfcei#~1~P*_^{(8m$#2!tLIkaJ7KoY@SYyS7 znmjBA0lrk-G`)FKuN4X))w<)%icJcYgk%LuY%IwOeGoX>oNz$R z?eU4q)*=25HEnaNBX(9#gnm6W_HHz}<5#}g2zom>-_5lz>O zUS7W%2(@)-CvQ+_Y@af8e<)~otwp&_m;p{<&JEng$kjOJ%Xl^>rZB%RYn`3fk=Puu zP&nFuq4yG@F#dulq0)RrQ9L+3&&GG0AUe_&YF%EWlvcqL*Zs@chiEmy!0i-rq#m&s zG*r?=1wrGJP(6lxlUHzV=^_a%E+J98b7*jym6$1Q#gwfS-y_8`m(X}#L08D)Oh97zu5ty~D zjudXb9?ilJk@T3FNJ|?sX$;r#G)0^DW!APGP*_(1Z={Du@YQo5qffL~Iyi|M#Ai=w z2MY(wQQZkiNx?rmQLclFrk6c|_E6n5YeZX+Sm@?P9qKhR#HjRriHz0a!a~6Ey^Spn zt+}}ijNp7VO6@?uKy(-|KU3R&Qh9Q_D@kmu)^1Rv4i`$!JO_?vlz+tLtU|%OOj{}` zmpMJ56rUYINZ%l~dYL#o*OEnzYAh^({rj-Nf}gu)T;2l1 z^;YyZH^^z&QkjX;zmdkhv|}TURMV@EVj3jeGgn^EVI(`Uiq#P+>^~1JUzF!8B0qag_21y zo3u(Bxe?`nA*D9rN1H;zOg{1HAf#ANb&v7_P(1Q50Lb(CH z{k;2@i#8MfSLeP<^;y}UPB~2PR;12n2(q7pWNBbptb>n_k|ri5_OAEk*y6yL^HE^%lZ)vU!_Q4yh3HIRz&#z}==DL!knjU6l{lewe zb8?m)9v!{l2Tl(ron}nM(+>^~#zO<$vDVEGH#TR`Z{M!3440ojf1VT;eh&QM!v`1) zc2s;tx$M2@VD1`;WPDCcN=n0Lc0RQ@uGN2q^BQr#Kf6tffsOUbS3BHGD-36Qdwchb ztNFPUu(}^o@P?1Vq4Uev@KMoG2AGR9pezldBmS$EC+dTF4($n+S)2qwO%q%Y#C4uF# zRNXw*CKP2)KbcT)M8hOG_np6En7LMvvT6Y$0FR&RlU2(_1_KQ(ZM8=}jJP|JOZ~?` z0D1kz7VX5#s}bgKI8-fE)^iC`B|FgAup%xYr3jIESmtwJm2(`Z8(yHB1a7<-b(>w& zv(ma6(c<5%%kx)sv^#-ont>L?WeK9Q)HC;=zihYO;=i4C@5b_|l#u}YnESy7tT)J`^9=Kyt|o#IP1rZdH@@WNQB(4=hi{o&1{f-Kj+(iK z?FnyZCr{qoH)A;1!;h{O+rYI-}73PeIiD5eP)N}HETwI{n zc>VR^;tgVRYtXl}yu5Xw;B?vN_piS;Ulex8oyqKiyYoQxMq3m=+P1=NtW{-`VG>G8 zE|a4qDy5+;;+BROLptB!5V?55!>^U{R$V078k@GPZM zWO6wIxnCh4^M?z^t&$I24Vuc!U$MjecWZwg?+p(UI{m}Xr`9y)DktvoJGP>o3i&0u z#HBTs6WV-$IkwB0qet4*tM6Z^J2g(vPUWaG2dJWmh5sCFD$*hMW2+MT2)~-1F|gf# zFvh}jFiNc^)B+<;M#da@-0p#7Bgxcr{=B_CJSd1PG{aBg<<&mN5r3B>l|%P~CikU* zgX~g*gejG=aQ-VhT}g$@@6}XQIG{Vg4t@f-`+Glefmk>74Pz>hF>w{%)WyX_@hkN z@qQE)mHYSabH?-T2bVXr>RaV}_}!as2P=eNP*brES~HF-dFb}S!a_Y^;K+G*^wTGa zNyrrHt7hgi_j=(8*hC{BEi4>linMn4nNBTLl|7F<)JSLtRk>((tTo2rv&cc3fq}tL zrlfpRfydg)_|x*u3)UnbY$0=a1)zjzh2R|NJyp}3ZvL)_%eXZ`nbd(hQ^It%pTU6P$i+}FvaYu$- z0}G3|d^I`LbakomBYU5&V|agEbOR+)#aV}M#TsTArwNo@R~1iV=j0tY=uwS`;9VTZ z%}q^J3Ac8f`4H|uNk>OF&8Mt%(_vAW;e5{H$M&DQB+=!2`P&y}XUiBE0u_I?U66Kk zJZsx}5W4>3PC&Q-SXkb}xEz+RlXQ)Tzfd;p*H-s%@d(3q!gm?Y_m6X}P8O=mYl5*udJ344~sn#NT6+#Jsg@gF?E!%)z z+?~dC*C_3O*?}qzT!ryzAtCT8JLq#uNko>aop?IDu%}}TAWP9i>i?gr6vNrIj$bc& ziLG+8Y;I;qTP%CnlDE0M>~{EVS7x4#S9ND>hABz&YGURA*A4yRZxtiR)h^9dmp)At zLOwVYq2&~h^3zk4TGMG)xxKoY;C|qihcPhg^g@lj?b#!f0TT9OSim{Z`W7EmiH?kp zCIFqr2R-STfhbD3nxhkggTDQHYOTm=D9p$>OhE1*?eo}$tY}x+mFN6s>MhGhT0alb z(pv5BPzY7{WL1iu)oAqbd6|uH?aHL1Q#eDUW$nNrA0=T8-EOe)L#}CVXuPw`Q}?RD zdc1RBHoIJbGL}O=R#ElPqZPu3)M~ zaj<~=Z)@I29Yc{&DoWad9A4ZHKNz$gB%9tS#b z_4RR%QTK|lPk2E@w=>pB783h=_U_Nj>2N8kb=n#IQkj212x6mZa&-+2Bo>)&A;q3} z1{y_ev=oW5Go5iXP0|@y;ATdjMF>!px@7jGY7#xjr9+|v#9CTufVeOAK-|N7l1~9f zb!&PW-;9+fQ`Sccw#Z(OtjYc(`;vVb`&#xT87ah!J#K1{WQj!CD*H}`VXRS>pRHmH zBU_eX7-VNG@9lZs_w)Ssp1;m@-{*5a=X=h%&UL;YYbkIQ6g37K7uXK#pc+n>fH*mS z;YbJocT?24cu`55-=0wk#pcvhR=(Ten_NHrX{=0yj#zeM`URfn(zkp}RdOX;GU-$O zWYrj1SAhr{#cFF8Qu|>w>SHYSy%>*w@y?~w0nZ5LzAU8Wc_5hAP~ha zh2O21xNphpPLa-14__M&-m5bo0O~eo*YNec>f}o`?&A>3Cz<9Vsr^}NYj`^h9|QuC zN>1=YCis0dYIack_(TWCK>U>Ml?e{0HhiUs@a;P>r=EA z>FHCo2yM`=B5`jWjarjW?pYoGf&FPh8PzavSWy4gmlBe>FxN#^{i{7K9CovK??7D| z0*MOkTQspOEhs8_)Kmb8dRQresEYudTy$|3;BTovS9tWl4sR`{o+x4zym2m7}BkYxzty zY(?is-v5}!ef;!^Uor^z04@A!SIKv1S051QHT)vx^2jT1>9{4_+W7Zi`SW~S0$bq` zD=UwaH6k4|*BE)C7ZIVY`9+tQyFtjS7PAgep<2qn+w z1~q^3Jy?J1bAMJcRJT2 zp!!#}53bdJerYJDS)II=w)3r2FQ+&CFye~%bnx4M#009QQ|vN5&qLb@Uo^1Y_BKW6mx zYXP8#Sxf6#*M4Lg?nnzCQ<`iZpx{GGN3Rx0yvX2 zyzNw2wPJs@dRGO$v_{B-#q7OW#wi?)*UlqTcI_r8spyO7a9C{c?yBi^g-ryJ2=^Rj zV5D1R`vIH6Cue14mChd?#!;PwE|;rm0n^-nhIUsKI5Bo1O*Q2%`@Fz`8N%Ap;7N$L zci8>=0@{m+MF*gZt=DwP-dz@Nd2kZ7)$+n7JFKK>U>1J+Zn!CUGr3*y;R>I88$mCA=-nH#pt>OEY zW+MlG?yq!zFA=h)gikLQrMi#mWU+X{E6efA7}wvQyMfg27a-*{5>pU}bbDjtcmjvi zV7A=+)?(+vn+asfWrCETps`4seQN4fJ9S5D8jO5&Wc>cN|8HcU%sXRob^>WK-?CJY z#c+R}0*l2aCMGg6GCDntpy6w4Ycn-9t*rj!l&PHC5b_$0R^TfrD4C`QlYs)ge$K&8(IPVS<9UXCO@a$`R znpLr>lK=6-2Ti~iXS$6QpfASZaCxes-e;$+SFX$eMCfAN*T&=3!VqWtt5&TMp-;{1P~^7-@D8ZZH; zK5LQ^7r!koKHA^kKQdB>DT5Nq408C{6Xc2vVS=yhwdAA;iE}+8CGm_*Omad4o{o1J zR-Uc&b7SSsC-gWaIHAR^vhti@?}opuy87p^#krL~M0I+Pq*CR0iyx_@w`ait_g~SV zspUwZ<&%(*K&IrV1pAx?#F+ykJDWQn)cxHX>73kPFc^_3J>3SGqCm}ruB*$JyfT4K z))TUJi;gm*xe|{=GJn+2n5t&hie=yp`+d-N%St#$Ibvx(dCVIfJqM1usql;DLX;OU zYN1;N;EG|yF|(Fk;p2d=b|Cxf=p1zF0tqjP!>YNFk<}$)rRC**fUj>+d3Kd5{3MpX zXj(X4ho1xmUt0%sUHJZ;(+_ZOPQ4`%2Nr8Hb4F&i$^G)2%w+q(qe#k7mfU)O($Yl1 zivP7i1RYOmh38|GeXvf}kkW(YBuu|87uBvmk?_eu0U%b7-WsLKT))bIxbHQ}QWmY` zH*h&)j@8FKa}a2(^8Qf%-=>m@{>iog@9QwZBBY4 z24gwQZW*&N6RweCsvw6Dn4h0-C6_yeZCvl~7X$MqY4=(2#)qtFNzQVgQqSC5?11pZ zxL2uJLI2-_-&8j=M$f21irzan_b+71&t`yr2{unPf@cnXuHw$c#xeDqxQyLn0e2pzr_`Jsbn)MQ_x7syB%NO6;L)bJLUYmLhsF7c zwST9`k(ggLiT2}QbDC-i1a^V3-Q3O>wQuI|hKVDla9=_v-z?vRdp){E=U?VyPLg=<(^T-@#E@fW-6d6gq0`wqZ&Gkj#+Kfe^cgpAoc z#kDiqyt#Y(_U#usL8A_CfkA1Xh2c~6kJ__Uf+Odm^rhfziFk7dmrx&7KR~tw| zLb#ygh+@5edU2)V>+nW;P0jaoF&QYd%1OKt^`M`(C(x8(s+U>oD%}I$b@nA;8(Uk? zD(7Yl@0Q36vvU@sO20=qdQBhH~_k{(8gp#@C z0=_&uM5Zh+mo&lCaFr(^2b=AVXNyM}lW21VZ{=n~E?LjforkilFu_F5EA1kI6z6-2 zlaq?1OD$p;H#1;t%&m-gp?5;XC=5?q7M<8zWQV~B`}#?CweHvfyLjR}-b!0R1_`7P zhI#y^TD~2t!VLe^@9bD7nwkF0DBvOklAI_$?11dIRKZ6c!zEs&D+8QU*+f!OeXORB zm2VKNdPv|*IV`Xn|LPT6Y+gx`T5PQN#{nZS5_(zpHJvO@&k>wWSDp9KXT9q%LobRb zp%9jbm4%ZKh|kVyWplJ9#arKRKtg&RAQEU1_5-V>uiuja?1GD-fPPYwf1S<-k0ajp zROq&0skP`;IwF@Yk*jlNeflR^WHzCj+3Ar3;vFFJB;q zn2}tnaSv4&q^NG~bCCiH(h(4HkvnSgy+?qN!kljC@xh-0Bok804gXvBt?%VH376Um zyJ=Oor=7?@X^xJL@{684Pktr{jd0kXyBg>i>x6i^O*}C11EDYeO;=CCR;%D~d(O_? zt?gGkK8uQZa|7Ih^k%L6*ak{B$dDW3IkQm*`Hyt8#s^DLUhMLz6E{TM+G_h=p(X9b z3z!T4OHg)V!FPvgrkrB7CdgeyzqYl)&e_IpestPUyrC;&F)GL2-rhr|sX&vTndaIz z_d-j}yDX&*VdqqC4y1B>T>{_F>C=r+U|NLr&qWE*mhb+ZR6K@d*8KC4?cqD*GN zu`&kriRT=O{(S4a22$D%pWNJa2~piT=oePHkf*tYS(G1ml85JMF9j5KTBL;Q7vM2` zAp6scIcI-)=e#+V%22yiM>uW%P{Z|Rw8jngNu}T?^2be@1$g^&Qy&`8_t+W5FkFxe T8-MWaoa`Fvn(9<(xjy?JqT*5d diff --git a/frontend/__snapshots__/scenes-app-max-ai--welcome-loading-suggestions--dark.png b/frontend/__snapshots__/scenes-app-max-ai--welcome-loading-suggestions--dark.png new file mode 100644 index 0000000000000000000000000000000000000000..8fd19ae835a64f83723f6dd6cf0ea88d372a5570 GIT binary patch literal 19329 zcmd43XHZjL^fnq46j7u~6%dgoy?0Q0M~d_&z4y?oBE2XeMF`cc?}6X^@7(+0ez`O6JNHax&Pj5z&t7}4@~mgCo#;2}3ixveZ=Z zZK<9LXOg2aiI{hi%8GD}Hs@gG+Ww01N`0FOXPOJM_tSv{Nb{haPMRV(`dhKO(k_IQ zO>q*QflP!xi+b@YG+$BkJeXE8m@L*{clS4Ycuo7TR``;zVT>Kg6 z-tBd0ZYYT2zfa%axnus{6)qlbDAw)eGnW6IH|Az+J^~(tO{2_enMJtYqCv=lSQyX* zK5~N)g6@Jqarc35;>N>R(zK*;PgT1`x{k=e%)l3+JmK`CzCi3nzQgbc7#|853w_@cSp+Z^f2Y5lq44sHr?#dmURLwZ z7?)SZO$w`3qAzXwVI#^gjl?feJKBym2>HmA>=xMy8V?y6GyQYa)X==4p-F9i#CL}# zpd%n-=C4VD*7?UbGM2e3R3w@D%>R0C8F@Ao1lJ(5qHEDN`;`1bOC6`?TPxU9UK|=u z;<2kWv75rup*}!?X>Ko^g>rQ1lTfFXPdQL_&Bcq4r2#6(-I8}7m`3xsmescvxx%8E z)+z|*LWSAO*4mYV@%&!?e0{BnY>Ibr1O(6&+A0V2IBrIHiUz~InqP$}ES}!Re|ClM zCPQ>p^i1ZhJLq*t1x|kT7yzB*cDPlAep;wbD#WJcQpK^*r$Gq8)NjT3gBzw?W<=5x z|A0tdSuUebjxSn^EAx z?gmblO1UY7&xh1Iu03?)+;ts{ZKNL|uJo<7wZd5?#dQen?A27Hn|WNC_kQ4N2l}!w zw|^^T6nM!l#N(i(F0Ei|TNaP|ep95G3TW%j$G5q_JoQJ29%Z<54#8%Zcg+sceQZq= zR8k4d?EYqr3`N;yrRg?t_aBdw2-7n=7>Y3a*}I2-I+_z4aaP`JkMsRllT>P>Nhmm? z=6UL~jo%vGDGR9DApWfiQMmd1-o_W5Uh=!#C5>l!TIDdV^!5FMl@ddCvEz)x4>=K~ zBzf`RJj2`0^r zSjUK>iR58^24I!;fk5*Szn;V8{`%%(drd7hN1y%^O7N*#TupSRF(81_gIfVkiLrl) z2yKGPbUF7+B%4I$Q<6!UhhQPQT&_(1k&C{w-{IO46=T>fnQn}X<*&%?tiH-!SoSg$ z3&0#_mn11-jD1ghBhk@no*`!omU;dazbWX$J@cocf&>S;FzCJ3>1@`z4TZBwITjP| zNJqy(s2G`=z;;8{r}6F7_7h)$9~@=&!zzi|6v_NwP1eadPwmOdXRDnRh(yC~1`+F4 zxXqGxrQdU_TD`nYOk1CAaWS>_u5~Xp zOy|KF@?KeN52oBBcdJhgoKU*Wkkegm z(ZFLl7SrT8{ob0127JapSj&#vdnGoM!LC2AXjWHO#%WwX{M+ojL2B2<;eeQBuc5{2 zIe@p_l3 zn4(l!M;tjDSrTG-oaJA?m%`S!CuUJ4p_0N?Iz!q{}kC zl0u2XSKkgt6BRaBzj>X@I8_l%)ZhH?P1yL#F<2q9;z4vYXPPQE{>_9E-g-FsM6F*U zBZG?BA|VE50lgi5JQ8r4DGrGaK}JC)LsEfkmACHF*p*VglsTVIB{qvaZST&rhwDafMnDpt2Ec&I)_A7=F1(2kQ#=v!1^Ac#fF-+kpCy zF5=Zqd`lATnI*K)Z?S-Ou3HFiEoXHEjh{jEV%W(Z9(4WASm9C5Anw->5nKMfk?)#a zeZA@0*KsYFp-BU-RCMzOn{Z0LEhDlI_P@#%>9!W&2>;Z6A`J7BQ$fNf^0~e|t(}9U z-vh!RA>pmRo%u-olFxfnYY3VoYN<2sgE?l-(6;59RmWw}`J2VvFM^WC#!}D-p*&DOA zyER3Yl8!3fveWK_3hlVc-+o)E35A&RkR|_sk=OCluO86HXfxm?y})~)TZuLNfLve> zz2E2>xJs??m}cBgMkJY)F&yt&9lEPk@XFrQ|0!dAed|X;US^I|t^BilBTBK1;VCl> zBGwty1<+5U(VT$d{Lh2Ic_~>H#d87Qr0J;&e0cx z%F>upDIX`g|3ZVl;GS`|CSjCaP#A=QhU4BtS^LL zNXY4cea8%eU;nMw)G@Z(#BFVOx-s6-AnJvT(cPqyM9ME67lUsxoLFN52 zCf~;xrjpP+Il8!!(h!cYZ_#{_&pA0hd(?XnbPVn4q*s<_60=TIc^2~wH|g2$FN$H` z*dIEo_0jHFO#N-A>B|o@zJv-kW z+fZEfUh7N)>8FPA&+LQ|af5MUkc1Y^X=}bDNxuQDOl{68rv|YnMdi&_asYTWUWwz` z*P&qiVdF!ny;2gPu6%wKhaL7<>VCSGme$nsH?`bsJdESz1bzJGz*K_FaRGnR%~C>u zl|Z-~TT3#c)5PDr7VlY?e4@qt#EJJZ_k40|PF;zVOQ3$H(aVOhPrn>i2T zSIqM_^s-z${UXdM#E@64^l>B^gqgh;D~%Idb!3r^Z)wGw1JH3>&!L7Dg*64XeUqg9 zEi9u++o@M4zQ{8GY^3#Izijtw?Wk2DdN6P!J1FGVHBjISY)n$0_&Xl&QY3TmUy+m* zU0eFt8&wIwp60WtRwc7A*7oq{cZ<}BncRUo6xDoim&aw4<-&_&9fs)EiL_T0lnhn= zNRL<5d|75lr_ZC!fC~w_#t~7%TfGCd7O3#sa^Ef;E83p7h5op*pGWHk6+Xkg^?_Gm zH@Sd$%84jE=1Ht1=Rzt8eMOt+alU@!ZmpQLIxBsaThD_Qs+3Z(s`h8$#(qlCuUMhR zq;fscEO>&EJ5Y}MlH!5yQ;2=v3#r zduuSDScA79=vDV5NS{o=J>hsM1f~5>{zLB;f5c`Hv`;a=o1!3vQ2N1>DhbsfhtIYc zF8Xk8^@J?v^Qsvip0wCl6nDW%-({*u)=?s+T|kP}!O&we`s?&n-1+z8FjlyhE_kjG z8q+QEAGC`Ycm{wI>^QB`8GE5}gH&PbXn9i7ewhvh?V_YTMrGBEJb&iQ#3zQtlHa&lfO{M?x#d# z@$`t;0$}4sM_$Fi%C_?ZqK2*j%qyd`zGpFf+Lj#Nh#HmQlu;3T_kG(tBWj5Z;ZIxt zOe*0*jpGNITW>Vm#rwY)D?=|kuMHu$pNNs&kR8Vj=TpY{PN5Sf_eL3?ULN0F^B1G0IUqyyGp(1zum`ri9Nv8o z=)+@lQm`%A^X@F`>BXe-zgI8Pvnn#*x%#~~R7ZPb}ClQXe*CBxKU>6+i?apiy2ZTGsge5kR6ZW#yl|0Dd#&tr_kHiP6EW+)tt5zgn|e~QB93ye8og1XNeu5t?w%pX`329= zp`AJ5WgKlj?PlxCN|VQtG>~p!EE%M442O_Hxg?^taXO+in9Wi(SBC-b?g8%mru#I0 z!QHRsMk$_00inG3mAzKrT4VCmDI_TTP0y29x!|D7ngK!1bV0Vg%?zpv`K)BJ0Ptfs z!>5QVJL{o^6uap+Y(u=aVi3`TZem>;>V%DJh1BBsNVHoOzVXr2w8i|H;&eRW*M zEY`b5;f{=Br1@$w50=1%uy5p^4Jfyc7g}4)RW6pY)n2Fa5g%bn@z9i~Wlq^S8Qjbq z>ZIYVLlS)27Mw=AXViYn!0-w)G^u1TF|6|oQ@y@<1#A`SyrTGsAd5-$iTpV$h z&F}@Vf{Tm?Hz}oVT8~yi?}&h@OM`HK0WR=hCaGIjhxTYqDH%M>#5*#UsF0AL$qIN3 zE(~{rrK8#z4sz%TpSx{X1UbvP6G=~<8;s-nSI9HlYppUg<&>55KT-O6vVr`Ys~SM%l?yZ|S0N2oTmnt<}BR8!&vMPzo|yi7`Lm>FL-k&^yZl zYQv3a(*QHq%&8VHmqr%cd_|J{xO^73{;1vDq--(1gh&y3Rsl0K#4TuyQz zBVxf9+P>Va++9nTv2|Qgqt}$@_)OE1AyTy`?)_K4Zv#5vI3G|?6>rLRDq_s+V5D{r zGMg)AvSd1D^MEJf;d{4+hc_vD#0Oe)pjV$Yl|>M;gq9X04~N)7>JFT`R46IXzwbwg zYb!tST}uxYMYv9|sE4hkqi3@bgJ9u}>0G+xt6? zY{hbzgtOgKVi%cTR8f2zpkb&=v3h0U8L9G4>9dWpBG?Ey`D0VIF4zJO;7QDDUlRbS zYN**H?Con(hUd@0_mY*gra68>!=v&)kvi*&d6b|Yy{L36`Xa`@FslJ2?4jIoTu-3g z-(@!Lto=77u#g-f7O>Y+;*Q@DR7D0Ej^PzSwf-Qq*=pJ->KF7j%vYoT%ybu8U)#X< zi`0VGZbYy?N8$FFA-6dccBm&zBrhfXg|%AnW9a*5qv`($E9BdgWNEH`<08Vj_jeI6 z$sX&Ti!Y--=@f6_jBlvkjf+_e3UH`H*z`ZMknt$uynnsPH?8B!dBcM{iUxXG^>{MFf8 zdGXWv^5uW`{>Gz?^UP2F;a1Q&vhiCjKET`ry@xa5VAzUjx%`pOsuT}T{rN_we)iRa z61)8V%vNIZ_*mNBMVgfUTWLjryhr`) zdh6>Mz8GW6UgqTOz=C;gmFJ}=hQrGneNyycyWUmv&)N?a$n6=Z9bMtpy{{43<6 zV^ftJcGw)`sARjbKS8ocr7+x%QLU&C<-a_}Uy#=_L*>jboAD@RJf<+gPFwNY*fV>6 z#wX-l!uU;9MoWMGJZHnZ89mU_f7&4*&qwpvHb}edmBIIVS%c34`NiDhb5kKLMN&Oh zf5TbuY_r(n`1-%Fh5WXZxz;A_f8Z|srLJL*tSp(26Bvb$i@^gqjZcNOO$C_6Sf9&| zJgBnUjuNB<)Dq~zMkV(LLBWMd4#%>cd1r4iI{uSe!^~y1*JctW0J@Q!BwsLF?Y%Wy zTd*v;?&-rp%0iB#Nm#5)jPtU1MAn1rsRX|`bo-l`G5`hvBg;!6hOt5ceGt>?Hsqho_vg#GDM)Y{UruX9uEWWR5_bzFV_ zE~~^(E%R7sa(GB_)IqkWJY}l>_mHZ2++NG5d_#of%L>B=)M(x+naWbLnJA!{-)?Q; zufI}K1n$=_Lu5KRl8f`+O0IFsG1^5$fINheI5{r2_ZCO0xqAa_Y3_4lnxeO_!`>J$zsMv$Am- zWvi=3f<3mL6rvZ%DJc1tlt0U}LVd?7i}1PT3e`~xgHpkm1K2y#@_pqER<9jb^|~2L zrlzo|Y#aw4VOdQe<*lE?F^Kk?9W;#ZXB|2&`(&1nDQKd=hMSYqzb%cic6j*d2zrA?X&FU?Fq$F|AY&06fnEaK&R zDUK=baQv(dT>H6PAQxcWj|;_N)8#NVeiq{}!=;&?KeHt!CT4a%i17l0x4cCADYW&? zLNJPcSb&>f{;&I{4;~yYUBZM^nUYn!HI-TO?tL(Q2QYIF#*~x3o4=7U)5!A79NzG1 zVaW@3+nTd-_*BlKY->xt8pl^TWq)~Dfw}0-5fl{E`0*CV`&Iq_^Q=mQ*!}v7QOoy7 zLVWhKX5)Ck3%m&9RO?d7ziEKFvJY)5+Ji@HTVJzI+41kYkL_D+tN2j)=#;a>^jVaY zR9q$=beT8IF`bM9U1egtMF#~%{l$le?wB)phyYc5?entlrLU?C4je|0^jsW%?=ny# zj5SI7)77hXc6U)@52c#)QCaHb*}>QVBl+K=P5&!9yJENDy5J6Ryat@_Qd8yWuFF}p_Q-EHI2%@Pkf`g7L9 zfjR4NE&BEXzIl8Hw7`4g72?ci_B;0X`eMl*`TJZPG3qs?ahwLKBz_!#+%mjN4v@b+i0nhkb6 zb~Nl6sC(^Z(9E$pI4W#Mvh|wyfCR)MAMR?;Pc`z%_oD*0DsG0DZK`^ac2jH&a;R3^ z)ZkvI(J)D()Y(57g5vqOnwXe}r>F7TDoRP{9R$~!m93iHJuDE@{TuycINrrJt#|7T z5R(dV0rc#0J%zO;d--ss?;(7?&F6qRE$dEq#a4|1osQ7c$B5px+Z?J}h9K=A|bLxywO?s!4hwf!s}t zIdf^#*9=}8U8EDoba2=Cz4^MiS?n6Jvors&DNDk{1MRd`38A}2VbBA7_1=m9yt>-= z8XG>GJz`@P+abpsPgp9%aO!3))LefUpbh$|VDXx8zw@$k#x!W_XI=+6n>wvXJ+dmq z&;58q4Cnhy7lVd4Iz8ZIE0_MO&tlEk^)#OuDlnJ%NU-=6DuwR9o~}0VUt89A^z}=H z|LGTU_~nF__QEA{k5$K~6SJSW4?*k2e3glpKi<{LUTiA*XCZen)_aLQzKSDnK7#fp9T|S{A`mgUl z-jj0MT-P&|2s|(Tx|_&l3Mqwr`t+B94+P5nF90xw)z%)tV3j|A;!6L_@6#I)=Q0!I z-Y`hDj+d9gHE47iY&!^gg-ayozSu5)3QuO%Fza04Ik5~uW|}Q}&l`fp=;_fD8i5`c zwNM|I)+L9AQZe%1BbeiykORsm2rJulD_iG9uYszNx@*wzlNRIYH2MO^C@Y1U^MsM~KNM8T;gg{JsY0sAxjJhy% zOGI*?32PnJk3u@H7pESTx3`-AN#&q3K}28a=>&FnbeIRD^JEkz34fXxgnyEYC7+(2 zB9VU1z`)BpJ_YH!Bxp^cEs#&M65&?y+$9fo~7`T^)cqB^%E+INscX!r# zFsZKz-Utx=DuC@FvdYBjYI(EKD~L(wy4jPg>qI%27A#WEQlakr$2g`uzGJcs+rz zwXH2NEv2>KJ);WQ^}bNox0J^L`3VWIyLz=XffrgMR{KxtR8%x3r-Y1w0iF5tN9XO^ z3Ky>QoGO>!>5{`%ysZ<>2=E9psx2nw?7j}Gi%J@2Ctx{Go{)>Pi83(2O~%4>a}G{` zF&rDyTg^V;5JowlypI==hyDYV{q zs>di$r><9B;GKgx-vNo7%d4rSQJ;m**KqyTZ?BR7D3mq?Ub;?P3`TTh?xTMs=UluO zTQ~7>(h;H!+8tD&3mESRxjZugtpDg|FF`~kftOLo@qX%iM#ue*gU?UO1o;I72-E@( zBr?f3(-G*kkFk1iA51|kp6(N}{Wm^?aC;^!1g_ zA9P@r)3bht?7Q?|U5N{<>DSwJ+P*KNpZ-EoURBlg?L_v~mlZN9D%65rS4(j3#_K0R zm;LjT@vq88d{8{>@)f%NhlyS4%E~%=Ca5F5&fx8njJ!el_U_PHFNp?EnW2}tZT%SU z6%78$pcOBAzp3l04|6qX&Rv{$iIBvoIy=LwKLQar`^I&iTbqqdS!YfCorcf0IU-HV zru^4_c9)BvzEj*c+uPeq)5sQ;IJwLAlJjac=b&_czUe^Jb0e!2l9g5L;>58a8L+<+ zS5E17oSKr7S|Tr8^Frjt{vzofPR)M5X6!_mLu<{_U#|miNqMuJ}>z=`w(B^D~rGA_gX7=In z(S3h|kdvN~#)Cf+PPIf_+_~%~8hh9;bK`KlX3m$?uex4~U$1n9_#?g$`U4SlN62Zh zt;v^^5n#u{Z-)rlG^3@(`%gv}-v!QQzPHdYb8U7y*qt(tqvxM`wo4g&jd4iOU-HM^&vr$!1QDF?%ub7sJd`u$ky+3#S0nt@-MlkYKpts|En3Id( z)k>V%n$&092CyxU!A2O4~>gure(~}ebeYz8^CzO;WE?(aFrXnR} zZ2@n-zU#oC6}IjEhYIX4iCLGbnKxG86J#%AjI8DTx(S|g!+ic?V&9UUtL1w{2hf^Qg>^_IT_ z^BXK{_S`n6T<;YG{z~c>$nqk3Py;iN_z(vaS~hz35$G=$lHdF_VdrU;831l#2}%nX zsi~;gIE{6fk---#cayJ;IrR?FnrKl>|4h#H_yi^&9V!`kzUkA2yt+#e?4fQNMi-gT z4G(!jNwnM&GOF@+=F1=R%{C&vLRoM8Xgcw|xsCygE$xE$0A67?YLrcSAwc`bu=!e4 zL-wQNv?sA_7uv*AgWsguJUamYTYmoCfrKw}PJ+@u0erN;-~{x>(ZOMAk6Mq_avinf zSX^CQorBp`SWSg$&cN`8^QAzD&cQ@ zZ5;^Nu8K2&@L0AYjt$c|s5kDTKE|dkL{D`JzXhCJ9OcB+9sv6!)e&yOP0mXtE>*@Y z3lcC-WOh*YN0QO#1&DnK>tHs*gTUc*7;c-Mp8hu&?Vve^1$g7lf11*YhKAb`K1%}C zORKZuLX}$mzfNb;=27^W8d<_XV@7SLF$LPEIqdPR-Y$0iK*YV{z#P2z%*EPzJ;#5= zbKC4MIX;MR_r_+i&5_xTw01v4kjBe?2$;l&2?8YOh15s0i_bK~7H^pEIW~+Z2A|U{ zj<==lMJFa6Uk~mN@Y*+wA1`4lTT{W((pNKGXS<_64myxwVuu>0=V78X+C>xS^N)yz zxiGpD*RHxgBqdx0UTN46xcBv~p5(>XWe66exTZ!4p3cjN%!}OD3mEA<9j?NUiR=&o zqv8T!w!vs)>7N(Jx zRAh*x(br3zknC)-p91l`Dg~7^uKnL`z{VYCX7s`{7bT@=BPsu;lk+gJqh6x#9+1D` z0&y}F)U4O41rZ0<1+Qx_t(EKz^z@o+@3@kLUN7%wE32qTc$p)mmi+hreeVF!<5BA3 zHE3$K&emlfh2{!5eH+b;fnmpJG=?q7r(6%<5fP@!OLMhR&trSsH6 zl~Vq`OY@k(YoM9^0Rb8b|G=U9bQhhe*OvsPe2SKq7L$lJefFRaXjkc1%ya(huLZ(C znEcoBz9tlhXpe`E*I~12mM`c~9NG zF#YrAf#0vLii^YOt`!n#F+CF_y#0Y@vH2`?K{}f)wvNl*r-Oq9Vq>l$NQIH2YYQb_O|uSG2$f=cWQpGY4~S-1B1PNQ(X+5g2utr+k`v+H_h-v9S>E}iu=mKVMCe6 z>$aWT<3u0gpQ{v<{Xca=&m`ka((aBp2z0x4^$PU#agpuPWTnZ6HaqYAc2xPK$qWzL z0D^Ts%Am13ueUHUzvI;k%a*`Q+IF6gG%X#c1_QGwb-32GE9F*P)HJi~CMY0KVxrXh zl6}D)fK5Qy55Dk->|GZ3+*sfHkhM z$E;k`LAUvb&d!D&l2zaq(-Hrjj-bW{nFrn=caRJFP4?rrT{195hjd7M8oY3~*zE&Q zcf&kTD)PMq;WHNEK#`hy-7u2Drr-n7HO;3QH2mM|Waki`*2`cAeRo8L2Tp zN~x6b^77vd3W>JX*8V$4fC$4C{2C1dG^L}s_@@Fb`p3S+rEaDEhMTa|m7AlDZh!k# z%k}kikF%Jep0C5b(bxa8n&W@6=d%dF0jBT;Fhbry6YrtVIn311;n(&Qn;Mfg0M(WoX~&{5FZAQpS*yZBV(X#W-%-7)!z_w%}6OCJS1U7q% z);7wGTdoqh&;~*Ff|=R=M~8aoqo)0R7gv$En3%vT?L_oST9Nnu3dpwzFXCD{7#k>DFAn651$i| zx3tsN_FQg+Twh;za#^8);qCX#y3DDBb=F zL?@I%c4ry*PwG}dJibwNSn6Ok{(R zX>t>gKoqlFUFcFCE}p=HBb<9nL-!xk;Rr+^|5iJj^EPI^bCexM+^2Ifsq=@9**Uqm za4t0VUUz+xzsjd*@;K9dyg1jAgBd;VR0El7$q;aNR}>T!Ok9h-BJErI_CsjNzpbWZ zD?32Yo^h+8t7CRy{D#$dmnZ?=_EPt5R4;mKR`~73$A}0W9b+%%=Acy+pj61F!1Hah zB{vZhAdoYKpPZaL6*H~%=;M`Ew6KtNa8TA93nMxf5pq=r6d@$!m@c2C(x6TmE)sC| zdwnSFsj%-RR~5RV`TTDZV=Uq~WmJa%m72HrbihGSwd+#w5-Ys4)NwKRQbPJCP=$sF z0vHUl@M8qxVAkklyDUUZGx(@dObF-~{@|QlErLmD{Oc58TWsi|vZ^nskU!1uf&f^T z%m3x=lwn0mN=k9(??Ir*%26+b2$dt_I$dGZ0X*h(ms+rV=%*sVrkeo4L$crVdi9tu zUG2Kw`GHg0PkNcC8+m-k9iRdGi(dIxYi4GbZO|%>c&VTqpZx^`V5WmltE!qB%!2>r zT{0>i)!fW^z+xQ&)1fA$SJUvTGODQeizI`hq zGP?~UDzkyly%o%K9)L;6NV)xReG=kAva~l~ME&?=|DWa?%K(DM4tKMeG=~m#8S+^+ zh9PwgZkJ0-SCR4s2Ecq{TBw>lXj7ZK_YqS3VZk4;y4T**nui)CKf}FGK(n>8Bc z0pw4>o6pu4va^F4fh&veCT6CDPLN0qh$yuEij>}JBaRYCV4mYj;r%k zX9woDu2t3pF!)d!mnNSKE4(A1s*QPCL0L#n6IG!r9rcw?dF$%pqRo3x^ODjZb^_Qi z;HYP2giL_~YksQ3?(e+~4dC!kKyvyqBAgu{d^)|7ty^kqMVXn)YHFehlaiCcoh#D0 zA}2PZvxt$Qasu@z1 zLPEL_4d>`^z{gg6oPR)1&h^&0*%@954-i3gme~NXY}P3l*+CEiVU<{EijiuHJ~-oZ zqNeSG^K4RgPyAW@@SzhTu9ly=S|N}9n&su9)aK>rh;h7;2*Hq%81k5os%~}-BSZmG z6$RF2Tnx7L1q?n@TGp}yrF$&?Dj}$oj10cQw6vV@Ew9;G1I#%>Ec@VBZBj-?c6vI6 zjbLAYf3uW=W{N~WtRNg503@D+$wk+L=_t<_!g1GzQW}4oI#zyutK*F}VY>7$%B+lQ z+pdq12iZj0kB6*ysU86=EfX^P>({yVmb!tL0WR5=KX9&9 z_#^uJ`hx#0PrvfqjF+-Sz8;nd3i=SB5hT-F&DtSwuL%ea*49&&i4tDRJy4ajm(&uK zLpjH#T?)fC$xYXbr3|^ps1b$ltwx23R3}~L@b)j!7%lB2oUqcIH+p(pG1AH&S|&js z5}*J2{Zqf)_j@c?2e8`Ilz4hrv4U}4DY$-QG->JK<4ZLkFsh&#y1wv&90HEt%XhtV>U=k%T+STh&Fy6%`Xv7Jz3n zHg=McDM&1_?xo!?;IbHVzYcCpm*$FcuVM2>y?+nwrm=!q_%UeOp2k6N%OJS`jDj?yY10;rkoZR$YM zvG+K~xb%P-uy9E5Y#*aMGtcd$(Dm=$=K4eyQTq^}6#e*`o`&q((Y3ETEd28OBh%jC zDTTSfJ;}rdF%bc^zM)vUWeugJ{u{ry>TF=b;O#hs(Ytr=faL9<6Zsm*&cNV!`J@Hc z==^5?DWI=+lU@vGOL%$}HR_|vbu-88c3a*PK9A`Qe0`DQF{&w;DZ$G6y@Q#9qoJZA z=yzROQqu0Cx`u|MYnM1bzUA&p->PPp$A+pUgkqwUnyfFlN zT)=WpZwE8D88Lved*nhD&m<~ZnwM8oeBRx-CuR%F+Gk+yAd7{&XsFTr6o&%PIfI^_ ze(~(GIN{xhSQG|lrOiAtHM@!GMDr$gMj3#MMqn^ew}1Z}Ljq*3#e6pvy}ikxlL|x~ z$FNJfV4Y?s`|qPIJLrGMf`>rlG7L_8zyXL2X+bSQafUZScPFM?`Sh)V4>9jcSgAU?$*!QaRY%x*E*CiyNd4 z{rUx9I>W~LZp_kVwlG2=k){hEq5(_;Se*dl#bSWq`Y^;(g0(`I8X?Zd2Ov)mSRxIZ zL5IJ!ynHJ_$n#jvUHSO|zoR?e2O#P3QrW4__$Vy_7yM8BvSt;ErmNgoK0h39aMZQ2u}L2; z1UO*TOi}CR_os6d3;E9(=={z}C@Htz*?79Bd;9n#CT6+8RFpN!4LW!^1Q^A|1J{39 z1FdN`>Z)h~2ieJ;=P}^lZa`+SeDV}Q*8{iG_$m{G&#BACM=F@Gp=k}@(Y2t zPQ{XQIk>t?`JbFe%uf?Z_x#?#IYx9`E|S6goK1p&m5^Ugh9<#2cz9%t%#3J&OGf!^ z*VcNZO*Ax{zw_1iH8WdkIexO3uOOv8=hO&qLspnI6*g8@R=X)E*;B4Mot8e?{vafY z@-=$3qjKRleshOW+ng0I7z_?NIc#i2Ef@6*@bkZnSr{Gt>t<|X^IPGu6v}tL6&2k2vev|hXQ&&3|7qst=_mYT+|B0}z(=|%qo{XZS(R`+BM+Mpn z*$6-nU$KK{X!59EBMzsUA0+VvM|mkHs|%hpL_VeiaA`?J#c^Ol)6jZpMcL5s@NA1? z;=yxK_Z2H0t2KD$f&c1(t6!r=$oZ|Ck(~R-vY+r8M}UP|XodFawvCGjdNN7~9jF-x z1g(Imrm8zqzzoqy?(W?KB|lbEI!0jt5=h>;Hv%kJ#^Jl6hVTD$Ju3U^q$mbSM}f*U zZ!W<$tl*Ab7!4rg7{FfwgWT$FYS|VDH9vp852MT8G+v17>FxFOtd-R)F2(yxDF@=# zbt6b1NZ|u5WC7_=H~u@`3~@TFJ1ztZAx-RTbgpn1MdoHZ2%BNy!|L7jM<5~QM`1U` zBtRKJZs_fHI^gnu?-tPigG>B3a{Yf2=60KEHw7;K1D=dHQDARXUo`9?u&~7;r%4() zF+1;AW9Uo*AqO>^0^ozGW3yOMTFERV#MVd1(kZ_V*yiJ=LWKRAO(`#r{$593L7nH# zhch_9=0Bz;;O2eJpOw(Z<$v*mdyEn*bW8^Hh&wt%RX!6gIo1MvjY}rj26y6B^Wp@Vp8zwj zY9U%Nz2$N99t3*-abkUZyPgE{5ZJ*2`03NP+cyW_n14Hnw#y(YmU6$JB(S%|WOPsZBc@JsY z(38>wmFVgqUn({etc9=K*f%w1Af`yQx^nkr`Wj75QAn^;fthp^OO!uEQt+jyWKXqy zD%eTtDaeKRwzdXjPMn_RkjU*YTv$yV^t_IheY|*B<+~>t$mR9_+@P4Kqe`E*>d9%T zXeXb-D~Y%hdU#vl1w0BX{wJe-7VYa7N^{gTKgmFiyRn=_$T>KVn)1j95-_|<3Qcmz@!k$)_T^Geq3L(QuC4r+U|d=RUdr4 z5G8{M7@20ccMtbwbM1Q^C_fwD@n+4cK}JT(cr@%W&{(r_!gLjc#=ZCs5BIA#6E`(p z=2_pqz3}W$pyPsw*?KE0dzhGXv`UGH*aIYX|LADpM(UoJ_ZiYiB9Rz-xidh>#|OE) z&|o`yfBO{0h!b1O%U5sc3?R+yjy|$j>$!T?NtW87k^&C9l%pC?6?T{ZgZgv}UXxLkc@e0_E+ z&%OqAqw5Q@H^%TWYnL6OjyxfkEb1f`umE6CP|#iawcf-=fa0;S`nKeL8R7Ya{Omxg zxG+OXDuU#)AGWD@VY0tj zTvn!`rZlaXB#NFL2m?sheEH;bLON>l`O#5+J3(H+M!*g`-ejrZ4W!$9RZD~$DtOqm6@4204UCDL++Bu@rbjR*VL2%ug*&J*}SXg{qBR#CdMD; zf9khjz%j?hX4CTVV@7fkv031jb=j0W?%+>FpRTF#ab;FWnv|YIX=P`g`JuE@(qmfj z-OE?~f1oN~fP50D*aKKOnzULhP^kIX_&PiL-cO}R2I7oP`9RSZ$-P)}tOXQ+d0)gA z?#lWBD>^7WU2kbNC16@VeL^QyX-!GLSCp2Ats>- z+R~cBzuEqS`#he;TL5Juia*Bn>J8wP+;)bB2P-Sed|Ir4icS^aYGim2Aa<63g>3*4 z(_z*a-fdI^%;r5BBY@2A`KXe)-zo#t7wr4AtobZmnr~0CY6NbNS*zeB&~P0FHWe18 zY8mR8Y2a#-LQ7g&@@h0w*NF(6oSd|*T3QRIx4ah9djLpbt~0UmGfs#d60=GE?9W(A zJu3<1K8OPPJFdPpqCXneM-i}cIB-TMtYZ*;vr(YA|HDP<+WDHhn2C%^uMxB`?@~KC z86dR=IH*C~jobji??Se`{0_(y@>z8CW>f(c0FJ+8H&MWa3<(ctVjueWah|f5gbEAz z!h9E?^FuTHa?FCk`E|etLR`SVOQ;g|TRx!fr+}`y(Sj&k^|66S2t<(kX^vT`ie4>Q z;m_{eYsR^)BLy~^hI&p<%m_&$wzqbevl2&IfPd2XpRc<2iBY@CiO_+>`Wa|ZQ_%?L zys;2U9K?-v)6<(a!#Fel>hS0P{qzO)MkeZoQvS1qWdxh5dy8?nR^zS4=*I?)b8W1exRmpnzfwrXwuzlMHJ^EaBl!#I9)pB%{wKIi`Y37U zc`H-U=MOUydlcN!$^0;%^(`;pAHvMVF(yH+{WUXA=waCHE`1QUNS58`yh-^8laEof z4DBj`&(Cl3$|#!J;gT%7NdkYQhtUMa^}r>SS|FeMk&KLxH1V00)5uG0ZG&OC!rGs_ zUrKkuZ(G8h3TvZ67!%Kn#A2iQiVsCVu47ELI`SB3~Z{uRLg-3;l0Xjz(aI6da-n!5@Bp}Ikj;`Ar@rEx1-RPKY! zr8~v<(ls>@>nCjC#hDvUAylE@KpZNHntb*zYh*MaP}Z-ZE#!Vq*OwG)%`+3y&|V(U zU52yv3#t)Qvu{9{D)$w*uGB5_d+M5)Dn~oNPBZm$G&HNbMAh@eG9Gtg}JHO1#sAdN8LB)P$hHFEj(AfbNBzHiSX)! zJ0N{GzliA7uvzN}XGB#9ff@L5lt*#G+uwtQL&~&4B*0 z(7CvXkEk5;4;>Y{pg8Xj;PpJq?mPB#ififUm~WH75*84>)AadYM<){a8Sg!F^99L; z1v~fxTIawWT_q5@%QYk(K3^O&c%xl9^Pe51mboxzWXTd z&${)>#iG?xl<>j+{vz@^HujtI-8*hJUf3>eZ9)Hn4pxrxoj+iDq}Ia^Ot3+Z=6gVQvoq9ufGcyF2OueJ%Sz z8(!9%Y=m?@=Op&uD?sGbb-&S!f8sg*J#FMzpxyzKT!32-s#?AGLi1aS);q$O(2%LS zP-+t)ut6%Y8+swPawj3PoEWqN{Prw0JOJ{b3dtn?NI`@<^>gbxFK(Q*VkAkzLcfq zEC-Z;aS_S6yvXWd-hQiuSs+=K#0%w|789o46J+QH`YcU#&ZU^Vsl)6Os(}Jt>iD4A5UHyE6Bjz zbJ4#)+k5vSWeJ(zoY&W>-pTm;CFMCVF^ZVazs=^;u3xh_{{E81pj;U&qPa9gqRr_; z;B|-0wXb*So!+ETHb3RYZJ?{gWgBnboATk@Z#&M1ujg{sm#92EuFu54*2gFzVfmS^T5rEf;)V@GOgb8>MqUc{^%mKFCCc7Dr!1ddv52Lz240- zCqU-?OMONQV_;Yx{dnfR4YSScqt92TP57nsV18d@vt>51+)--U7l3u%4p~poa$L#!FY2gOr1wb#KZdrd_rKbIT6fkqc z%h)xMGUuOfoONdV#XHwbcCA^Jv-|A5nn{feb3}mgH4;S+9d7#`2 z$ishMvi=`^F*A<~Ota`>rX6Q4jb5qqN+8mO^ty{10%^kvf)GL=8A9C^%rMRzDDKZA z-^R=mYMF2jS%w_+Pbd&b1Nj{z2qcgsj6)iIiGKiZg!o}+XQ#9*I%UvGJ~Nkeetcf= zxPonSEFKHk)(g#-$UQlsYA&xx;(cO?ibg^`z1AkKEtiHEqPv>~~d#Ph6Sl6kw?N~?D(N4V9OQfQN-O@{2(>Ym_ z*3Q#JpMXv1qTKGK#B!slxK6KfGub?OqSUrQuiQ+2_Umkz>GH3!J6#7q4}SI<4`)L0 zq`S+n4u3qlzB35pyreA3pZ(tQozG#miuJqF=kb1hFHb_MEI^?W{M%7kbA^xP7>XGi zk1IPo3K+E6pRrTrS^Zx06i0ux@wKVKc$kvYlr*R7L2q!^v2pl;aKQHrUo*P|YM90F zm4wAhsmjUkrv~*lUV3`pnPo^X-XVv^`OnY~byyi0{m;wq=Sg{yi5ha#(7eNY-upY` zcBSFCvfUv+Lq~aGLFM>Tu~aKR%{nDjrrn+o{#iY`)`VqcJ?CH{0Ni#LM>1z9mRjY{ zz0Calzf<>j-T8QC1$cYx2xw$fy}ivlJtN84zQ2xmP+cj{AYh=(!OEcuHJ9P0DCTau zx~6;d=n)|%7T~YQtq{9m!CYIVq&jZA4|E-`Iloz}%eq-nx8C^GT>_C?!Nw?&2GXkuKIr-(;JkFG1r z_Jm}~iocdMbDA-iwYG7~*M5wnPd|=#Iu^e$J~rk`N(XaMN@oQ^TKZ4wF}uSbiEnO_ zw484WO)--SA&!&6WEKwgWFDT% z{QV1&Tm&{vkQsqGr!+IHqOTJtw3&@&{pvc^(D$Hv%5Ifkaf1R4Y$VSJ9jR(3cv333 zceU~2SrLEQBUxg8sY6wV?ezQ>t1Qz$6NH>w-Sv2~@Vp#`6+wyeLnTBaqP|OHHb|K- zNn1}qFq6VGoc%3zshgIY&>#i{Q+@(_MtVuh(xv#(8dF>_+I^E}V2&4b&mdsN^*a_Z znHd>>Bd0uCxoKtnm5nD-Z37or%2W~a;?;^akA1c+7@IuP&E~!SH80&yHAmH0)e3CR zlJa&&VVVcUEqYw`y~oBXEwmxf_;Er)3OoXC+l$frBPM<#JmJdX=rodED_{E(hRtlo zNv<#Y2VkS6tFz8HSX`lKFSyyAM9fz{&ybr)qr7B)$-1bE< zRRLSM_bsu{4xD0=*}DGP#)Tha?2qPqDMp)#e?DBEFA9L-neZm-KK%5|uvAP`)Wb@T zdQ9`uGrxJZ+CexInk$L+Mm&}%Gg$JYvaI4`btfqH1DwZqW7FeGwCZznq)H_hMf8yZ z)r@g#6%xkypT$eZYA#4lL zP{G5GQEy|_CtRSsRS8lu1B?YGL0AzFD1*$NXmUt@l4gX(lIGN6g(_>tVKT!2y>$<&?jqh1q|n<4&0oFucNOCZ;xgiDByUrn+bHCm~l@ zmYsu_=QqdEHsjWtU9FrFO4hz~D(W0VcA8^JW2vGuRGbhnDQRE5nu!Su3mUxM1Z>HP zGz|qQxzuo?#W(_q&LxTR51;r9S@qee%UViu%d%QBr5<2E_~Z4%yO2>Dy^FfC8-14assc4?$<53O`@2I6dSLTFu#b1i0|i| zF*HRgz0mH8Sv`e>S(98QPTlK6dv6KXF#0!GJUJ&EW8(&jV*1Sc+&qj`V_a=?VT|aM z64~N}rUJ1iNB8fNPgj(elwOwi9 z@rX7{Ru#rN$aUP6(glHfuzk37?J>~#iPhXy)R|gp&m_}NQat0(=IdUV?OTe+qYhBq z=ZSaI-V}I7u0RDch>V*u97>A3H*(@5Nd>9%MG}}*WzpeD3cq`rR3m?9s!;BTqNd;x z$)oo$7I21SsjTQv%w`2Gi0C83m}azFsocs^#av^3yer`H^7(t_khQZ?!Y9vh#U&G+ z99q0T6lgoLC-=+8edg3lR8`^3D?Jw-3VYNG_DUJV!84fC`Kc#7mA=p@2n%?Qf3{V% zloekb?bY2w=-nx@-rHD%0|4uxp!ex;_nZT<|)?!Rd1wL9dXfrwPaX zI^6g3LFBYAnwGYxfpYG+72T`L?Www!l7bN2_egTKy+jWLsxK(}3J^D1W$V3f3w+(4;0dFTY37IozZ0UFCtR&T3)LZEqp`2Rd@8nayDy&Qoj z)d;HhP1QEz|Fo&COW zfC&9z+_Dy0BpaTaD6#yrnvo2yBk5&&v9WEE8oGAGs{T}S-Q?E?;*(rX{g@KPy)ME* zXKCqy(Q38PE!!>P2{ULXwtU3N7aG%}^UwzLfKn$hHu29VK7pkyZ=(HqTSMw#Pzp|N zDcqnmRvNLMs%+qON|0SxI?3hhv zRjw#Yd1)Wv;PA@FCj}~*ac~gk;ZX&B#X47z)BOJ6@XS0slL8qJOZ&kXmbsJ^r5FuMV7%loXDIOUmRojFojtT zF4$aK>-a`NyZGgU!&4{?Lu-bMvrcrIb3^Vb$H7u`AHS>;fjV}?1C`t+!8bZw^uk7S z<&_KDFgFhm&4z5BhSz`$4sQv$L6S2>Bs@r>^AOge`_*Tv?Y<3N6zSHUsju6BRZKTM zP0%B0Ldpqm^VnETFWmGrfocd9aAUEG2?xSAs?@X-$sZM$K1i{a zJBFw2_U1^_RE1{V{^b30Z+Aai3l-iZDPXD+Lf4XY!VM8llj*A2^tm+BuftWul^$!* zv)#1_Pi@c=VefN1Nldc#n7iqViL$ODcDsX&^U;x(FmMEF>3|61bet+zi-z-8b@z_lk8mces%)xd zS)6R%6{^42p#e&)Lt~{TocXP#W;c@ZK&+LeSao9b8=t4uSbv3EMOjmpLHa1$XVqwR zJ!>yICkqvt95)LW>H>PcL~|P2y$w}hOzNg?V<@LcW?0adxYgC-8IMRJffRgh#>j_} z7~}c(Wa5bOveGC=nZiAT*n7FeiQt5pobV7!`V(~Xl+Z_t7M!N?aTwr-h*C1gy{OQU zn@_FoX!E8-Oj&(qz4J&0@ty=LH@wM3)=JjwGd#6TB1;^dGS~%HXf9KvzGUA9XcBpw zi!PGVj6NoEoiHTbT*z7t>HaLch#tcL=N~lcEux1roW-E@KyhUSdNG?c(bwjJT?(1F zO!dj|S-5^re$SKl-#(R58@)OYQ($~K>9eG2Vcb!cpe)|glO@~v`H*lQe2RDHPl|r^ zPwQv-M9zo-cyEAqD*~?2ADuSUz5V@fo>|ByhOU_(hy!_d(E*QLF#80bN*8LfXU{RhIcOjy1;ZGm956|meVO4OjA@*`&g4Hd zz1?@PUIVg14>j|;8}R71Hj@>y%s9hnV_s&fs-(Fb zoO75fMg7yXY1B_roIVVa0Q*4j?$#kvhI2%&goepk45oD@C3Rwl>bYE673x;4R-iBO!8ufTL1qQ@> z2AZx1C4K1qu6jN2i*GG2-|gPoV$xf7WdMXgwxFfndVcnvDbQ?#5h|1v(6&^1)9}ZVX#HiDz~p`m=!w_NfGO{UGHQI<&RQf zei44^MOAmi*DDv<_5B3VpfBH-Wh<%AFpotvz3VH6C&{Fc=Hl{Sj+1ccX&?6E57SV+ z>8DUEb$9o8qDt~jc?@IuxY#qYsd@ZY=v`)&vuaY*T$E}d>FRg#)G>);huB5~W;U{x zH@#`Yh~I6k?8#wpg#2#ldv8R6Q%OzHUEfY9w@IR>Gvn5>?V`GnVZ^4u2L-dXu|RJ5 z*##}puWp-=o2{K0iAsf;LTkOl*r=eK+R@2HiDoU+sI!a@H3WmrX_Mi(HqR5G{E4d1 zxjJ3-#1|l9jD}p~j2yLknyMp&e|EI>M(Xyi zq++PMKK)H)YStSMJVPQ>{CEG&PoERc!A6n3H&G(2YkReAl^Nos`AMvj356{O+D%(u zu4nVzsN!{k{N&ivBhEzj#Gb5CElR#P{3-eM>VWsLFQ)XfWkVkv9ynp1p6S#J3DsuB zvG*u+P>AWyy?Fi7-OubxRG*fESA=%-FGMo~m7Ra{I+ zSjVS}Bo&74t}gv>hGs*Re>xxc&b-5*k*BSYarpj)ST?8g0j-?uEEA7jGzrsZy9R;m)M)6Z!t4j(RROeKWd>7EhpSDMCDyQ>BK1K4yWMD()hb-vYECIaOtaakzM7 z+di2OZ*zjkTfAiPZsk-TW?@Fgi^vrWsvzag&)U#BTdtRH;8HD;ai8ttNMP{J2hWp3 znDJ@#&ug&EpA0&qK1coa!8c8h-C;QB-NN?Z_`x4GrG7F2c9*Ve*K$gK=``gT>Pq=s_%N{kyE;$BO zt-3(0ly4CNC*mBn9imASZWky>2R9Wg!wP8K=ZnjD+Tj!mtzR@)_Z z_qX0LY?)2_s;QE1`}yJV62Eum_fN{xtx6EeZyALS%90uBqXP*xU~sG3WHAt*KawiQ z)7v{TRn!BX8CD(94^@>78DKnLOHz8OLDY|D zED!y&*+6T-EB2Mw$;#}tv3hEsmbezgn7UT{6?zEvX%BtDlb^pl-?g3+AhX7@Ti6si zpJ}*tRZ;#( z^-^I3_11I$V)B2{iz}3WWCV89a=B-3q7y?;ByaI<4U3k6tCW+e^`{XY(r3t7sBng8 zUfW)5RGe?7_b`V>f9FdWe5}s5sbEnl$pH5%Y3bQN@BEObxa!qlz5E_w*2xMT!|lZT zb%&k?#Qa&*IH3?Go8)4ep74B>ZgWF?Hq+y)0yY}4Z65@4q zqPj>QBr4E2xWlF_JIZJfXC_|S@73WfY+H#h`t zUql`q4#`&duBx77e16u6oEiNuDHE~TG?~_ux&{;uVx^jV91v3>A_9?@; zH4$_nS>1$1<71^dEhd+_Cn!2zJ@yh+HuF|-gr-5t@VPc9UgXMjq@qPoHwe^?gXm+d zRb=m`jga)GI|>u#E;ozJ;pov55zM_}*{4+z)K=qQ&W&P*HM$n&Pe*v3v+7i>55FQm zK0V$1I&vR?1n#%w*JF$6(Rwpechv~*I(6XTfgb8CG$g?oBVC}q;fdy}J6lXnP0X;~ zed|ubF6U_Gv5Bz){=#JeZ3~vHKkyZ%{U&Xw;;LpRnrDLKhDmc~Y1Ftr%X-9e2b#pU zRyu1djx!A1CaYmTu}0X$#H$?DljA916_1Jkr3QO86l^p>ggOsTuugP}Ybq*iI;&=O zN${37mgYxiuXYL!DQfLD@d?IfXQ|QO|90PATGBSq;B_5Oxs1N7tFPzb*@#bNovE>x zb9()G*lNh8nihmSYS7nNqnjD;+ia^HbolIXQjwsZBh9fJ687=tpgE2;PvppTA7Ol1 zAX`jML?Oz}`g3NaGP8i8_N$cCE9+_<3JS+9dmiGG^~oX1a=$Hk`QmZvGoDvnl0{=S z_ivSUh4-g<( zm_@JehspNj?%4MIbOF!f&zV(UonF??iPY!nOlVC|_e8^qjM7#VEJmt?C}t}yu%eK84jGr28iL{p`@jYg&&1$SY6^E-- zoe4c=oQeItOp$8uGsbKf>_u?6P5X@46QsW4$)e-?CvEpiYDa%-pMt{VPnw>pKzn7G?!E+p3*L85fS-SZ{w~@RR5eKW=4&n^!SLkn4^H3dX!&z1 znU&SmT*;1TYNCf|5z=JCwK%s}z2L1eda=g9Q3w{sRLmas{8g!RrMyOqs%Xv5b+w%X zqI}N4M3%nGxq=zvJtMPl(3(Kxiro$_Sj6^i_&sz_QsqMC^`>25Z$iHUls=I|yx>{J z%2SY$9@ke3t2|9>y8xA7SU%#=EYAD$h;N^34o zN0h{BeH(2jb~P5}z1BqH9C65nCDF@$eUFE`8V{&R-D50q9Ju9B>(0_HeFWK>r9BLI zHa0HP{Z`{mrK6(46y%AaoTeAw#aa&5qNBt3F6tnV+J{KEi7bKBy~6ywdx@&JWiGvC z!R9>hM3Xj#;m9nmVjB+UF=^i?Nm6PrX-=vf3*LM?Tdbl{!F;o+*;>B`rkpG*ud3=Z z3&*ftQD;>5FeidI@EH1^pM=7hVnNwMTwGjIvhGmjaYwwtmHWWQ z6!QLOkQaMMHfNZMZQK=gVObV&zA|%lt%R?9#C-bwm+97|XD`&&2QYrzV}VtZVASU1 z@X3P1LOFaf^VkrvDP>M$`Y;vj{5D4%Dld4Grxk}o(ZKt_+tAy)2zq%&QN^qq+#5ru zWY?%0({nuNu^^YsE+FdAmj89d|7?28-TkEax0&f_ng?u`X(+d|9@!wDet~9TU zr@`1{HHz5}_o{qnTkKyI@$B^0e>s|EGbH5jmDjUdt}5VaJ-^Tu3ZaqFsdYdg&P2Fv z{?ye?wyRKei8$T-+8Qs8r|uch07aKr^JD9x#v=t1KHZj!#RUpMpJ%!sbSV6eone|g z78{L@@zvgpx(AD>a?D#{p`k2Wbh!#D8UyLILlwUdWK~?Oc$05V2G78^AyFjkF71;O z{+E~2(+wI7EEGcDMPL2)C~2{qYVor)Yt@3&bL9@&*x9*l-X$AUjUOGrVqwZFNy459 zdGqwOq~v6svuRfZ$z?YcP81QBj?C%gAVs;Is)mLG9jL`-;NI4Vn_t*$DvJfQ>gcmj z{^|bu5atb&XcZo0s)b#SoU!TEy&!^mUj4;@&p@GnGBPr18vQm4ENdNSGMVG!;{(p& z6C0N}cGFb@BO`CL6P>wDd!oCDiHTW~|G-z_@KyWSx{VFXw5xtc|G5pW>1x}v5hX;r zL8+$b>YqPN7n^(2pB6$lH#hUryifDTLcoyV+M8>1obq52z1m+ud55y&z;8LQE!K4X zN;?jN(&ucu@n}FQuJm+!&Qd!~$Z^1(yxZ9te}oxE^eQTXOh_8z9doP{x`6NB*J^6i zl;2ie6pW3(R=8XRIcwIsG^nQe9umn(X{xFDo{aGUW#h9q*n`8}Q}aejN?1Bw*9LBu zZ_-RbP44}N4`MgVF*Q@W`*+FuexFQ4R;Dll@)EzA2ITEtGxVK4rEm1uDQA9l9I>uP z2{4q$#~^2M3ty>|(RA#0b#Zay5fp#z=O>{Nc=Grf0{PbqFwS5OXY)V!USYI>BI+kN z!ztJ>cQ6=?O+F7K&)2sJj$#VJasM9yH4^b8mF+wljr95U`>lYWVgmS<2G_5Yim z6ITL?8QtD%_7hq~B_;m5@oZYVD<3|52*#3gcXjnSn+vE`V3zp}G}|-;*VoVQ6As1d z+MWlT8Jo->@Gn;AZ_OQ4)UK#jyu@uerCP-rkHOf>%hA!yuU`=*M=v77KBGs4glNP? zkqMPBrwF)eY>cUC!oytKmY#bbqQy`(g-N*k)DDnMU3Dqsq47j;EVq<_2N35p#?Aq`F zH=!i~%EOCa3PwsI-ivy3f53IwZ+i*<#3L-cy%r@_s8JNVTRNMfmy_NbJ%91YBrG>K z*W>!2Hnp+FVea%-7X@4O_nW0)ir9~+mpGrZ5^QvYgy}UA&>jk>W zgb9lasI)uhi#r*yl(((UjWu5=E_2hJbMCs52p*Xz(QUTei`?HYs|lGX(NA|iPB0$~ zjw1c*zf<@t_-ujK^;5;+{t^`luU(bgMkXulbLS z&C28!%&=iAZqe(bJXtvyR6G6}u@^u?hvSg^{TIF)quj(BA-<~bDGfHHj&zZR&&Gki z0jW-Wnif!ylOu&OCw8$AJUpzjLjXdo|By2TpYBk&l$oh^apSTyJRnWbJv1+GZGAW! z8r6G|t+_Xiii$deMS_Qix7%P_Ug0~_M}&GGTRBdSEtV0p(lOnA^SI|{)@)2-7jvmH zadfoFlmx&Y)&|VGB2#&s#&H+8*cG6oJadKQAIH!z=CW?1TfZ-@AX# z8MDEl%*ip2XN}bf*2?Y*$Ge;98XNuPOPSR$S#O2AlwpJC#>HSxF)MSYVe@fCPW(x2 zX*1_a&BhMx*8bhp-rpyqKxumu-e^Xw?6eZ`*pnAbrO6qa3LWjqGnig%Ug262M?HiZB&9w)*>wZ^qoA*+~hF;nFU0$2fTThPp zQ-Anj)I4v*o+9jZ2o&uSv3I@S9oW+?awZF{U`!Pa8k$d5-l5)~D4xk+zJLF| z*!5r28r8R+?-k8AE^ND;70vq!ilmob3proiDbLT(FB5idHu}*t*!s6hDP1&`5d>VC zK{90!2q}+wuNmC=N2TSUxn(*7D=TGCU1KBDn-lqD-jkDs*37Xu{i+HkoWfqvA*1rwD(@!kpsrKqT=J^sY#C_U!2W_m=`{?UsV|sJ% z%BrdZe7YPK{R=Ug_RWdX(jGHa-tutRj%1FT*0BDT^yBPdt2*Cv7b7F1#)W@7_qOTM zzTJ6e!Q17Bz=3?djQ>BOeFlx0;$7^`y57sk~B&SXy!Lv|)bSGzbZRi!&QVCXX#*7(xlgu5WN@2fH+GA1D>w@6If=v0^0tiQgnDQan1(qwC% zwb$Ns+nF5-7jPfrA5$c4dcjjGjry#4-aU+hd2*d`co! zQY=nFjbQZa_PVu9m{hMK%Vzn6Y)=aJVn4RG2gNeQi|j8VvZbTAlRLKfKQLXKY!xw^ zS*Dl!8y20&g|jTLtT2bzvKnp)jE=2WR1VU^Ffb51q>4pj0|^`o+VhQ;>2L%q5abG0 z(mZUl4f#gT^3mvM4hy1Dn}wh%CcjVS=H|BJ-__$res_X25RO&U(Lu9SiVdmp=NpVD zf|j@-PRz^gFL5M3>3P`!(=xZibmNPSi@RP)^_ZbGaO(1vTV7~7)TN+(NX~>n_->ZpPI`Gac%)D?$ z#OHKpy8f~^ZSBU)Gw;p)3i)QArJo<+Fb(<-iVOaidr!%dwQcqnsbMn?iYfdBcNr-u zDaVH=>Rs1jKXMm6su|D-T_G(Mz1(R0)&iwx;@~)#3uwuf7m-3gA4#8Qrl)sXO%Y7Y zQJjcm(O1<_)aMzf!GFby8zvVoYdQb+a)^?u#&Kb4sU8*bf(gmm6e9#DHE1)Eb#Zbk zgRkr_{qjZl7USy;j((5ZEm&TTE;s#6z|vrQoPQwXv0D)<%F8RQ%@3@2_(rAwdDxnL zPScsChmVgB1I%`!go4ZaCmAMzQlfBnZf@$26wIfrgrdF)6-_5y_Sw=Ax^?>{!`YpV z^%o7fkEntNlli)?>*W*J4TxlQ^!1CI7CK@t5>(V+hYYXKV!3UhWYkIR!k z@kSgCHs{~P3ua!b)9F46Tl+wDf6MSlG^SxqSaN_(vKDU84J)Dzk8X5q zoD9Zq;kE1B-n}Fjf8BNXo?iFv%R0oV_PDLZK#K8(&UX8|H2;g9?r#4$dv`&735gXl zk3I?xmmoxnmevKHu3vaqRHc8Vt*mT?yYh}*4*$RbGIEmJgiTXL34ebmOdFIUoOr%> zcC+#)(VbQ4Dh)srV5&O zgznKQ${(GcsurlBqEY)_Om%t8*LQ1NoFA?EU3bu}bVi(jhecycOUALW>NR+SNg_v= z)kPKq4J^-|y(reeA>{{I%$4oI^2Z%oYzz$TdYh@n8(+(-zs^9$8Dkl_xw%>Ff40v1 zc))wj?F~nMm)sLe9USLet#rMAczSA3>(JjU)=03F|*KgIp{&!6JoO5 zZtEAKQ~dUu`XAvRg~{dW)T>Bh6#R!6?^Q2$#coRaeoLZz-5_uj)B4;_w^RC6*KE4t zN*I~CN|O|;kXq6KysxOyi>%Vp!G-JVII29*mN9N+`erK;2L<$wrqG9?;@xN3`-JuI zVITk7?gIbUfx0BDxRAizcgS9Yx^n1wkSOBV6~3B=%jq~zIy;L>@AM8-hUxw_t=4j? zJPyu@1hsxIx$HZ^WOaw*ZYn6`Q79&+_wI^ zPthSS_>d7TgiqPaOA08fV&2idlah>#*HuW@Z~qHg5{Tv}%)5|`WMe#L;D*8FgpqI; z^WW|Ni+ITT2_qw;fft-->!YGwi{;>ajiKKOoDa0bHL~{7Zn$ALD$rGU#H6UuRb)(A;5My3HcXv)Zc21*AKtQ0h=}^Z((INQU3m!XB zSXkIVDR+7_^nqyoiNk>0;cDL#NFWQadx5y(kz_&xgp~#(Lqks2KR>3yhEIMXs{D3$ zcL#+p)s)tu5kYbCY0Kp$mH%UAT6Pq?4-oMY4PfdYx z=Z_6=aaH&-{m$Cr4+)UuelIP;!QA4av=?ub8g($W(hc7l zceKiP{2U&t*W~#%DykNNKUK?y0B`pz^dp{MvgmjQ2Lu9{swvFgagHJtZE!%KiFX&3 zUb9kW-t;AMkuob?j!iqjszy4eS+p!+-{wBq%lR9Kb~)qI<&p_{);()|#wt3X@krZ| zM&@^3y8rIpUaRN9PYtQ1?o!#{iCo0#hy#fRzq|V+Lq2O0+KfGFb2+?9mr7Q2ukmIc zi-Z@T`b@{>Fv5|x&j?@fBu-sy;m<9S5sF< z#%Ha6;g!VI-g6BnL0*QX>nLM2R3b7nN=bGP0w=wzePZQ~|fIy)7~{<)Q0Y!Vymho@AXZ>I{zHCRRzu>`fs$l_#X zKG}B_j7)`~;@x}sn)Hhm(Y-n2vG4kg^K-Q)x_TG~l z1?t8yS8j~tonKsplL@W<>FFyN$;}_zEH~})Jv_+U$&Ct&IyyZV=oRF#=(94vI$Hem zgSfw%a_DT{hmR6eNbtlv!^w`$Ee+|n=oDmOeaK7DZGR}7Xr#HRNw9AQh%^9L(J4pr zl->x^5Sli=-Dx?SZ;VxB3ih_rrgh1{a`)x_;py`r`tz^m zUJLWb=a{QBw6u8QSEyH8o1165a<#`H`bwwFk}^roGga4!OMLMwL@*C8@9tc3iEov1 z(HOu2_+3}!<>k34|Hz*v{(=4&SRIPfc(qXL*jmANpTSG3ZXG*^uXACcG&9q9YkbH4 z&9u!!0&l-C?Zo&%(TtUQGscOYK0dnXVF-~FA)non#;5q=Ad@aJI6=I> zcf=oAX^4c)ZEbDinZ5Go=yG;$3i&+~CmqFhYL*oB$cp_@zPs=6Sk0m7Y)G+mX+(5$ zCi867XtV}nuNKbq*6Z)oxOUxS4sY0y-P}VBG`Nq#-o_$FR78s|oTFB(!33kNteq^J6=0zAqh^#yLw(Ha}{as0YlL&c#>B|YPoI^!6y-Yp0VZ6~Mw z{9!7x9xdSbD;D89=G=+sl-1Sl5efg3X6&y$G2jgr$et1 zD>&5?d}H<7+LEtc>BgCwngYr>CesSgcTv);jU6&CtR7>N0iONTzPKFVr$q5_)3-KaCp8u- zYUIZ*LB&wu+o30^d^Efs>z=)eW-HzmmVRtBFqg*0%>^+7ybh4?$i=gEbas0B%=lJ) zpX=X|yey8KH%`K?Zj8Yu`LKL_f0v1#zOVjErSDW|rbKX`4VTk_jv}G0m}$*8zDb^- zpm0gQ9e-S>Ms*ELqc_sFYqwr#d33|>!|>luiW1?+jIzHswz*_WGm4*2*Yc&RCVh3b z`x$hQD(Vo7v>Gd~Nm?o_=}NI}`F4q`Vh(fd1x{_-zY5Rvs@KN4HYQKntYKdtEp4-5 zK#re#q5)FuA085aN$>;_z)F|mLkf<*3nu-DlCrY(B^sRE9EI2N@>RRiyvh9lVoVb8 z$sINmcKK_yhP|`3wX(4^JT!#s8GHD5>g1bUb!>2>kF#@#xq^d(ftIIMyyc)b?M_O{ ztf;ZybncRuzw+;l?d|Q6yh@;nLVcM!#VQ7WvcI?b)^}=%&WJRuBs1lC?ao%y1s@$w zd$A8HnwEj?=qFE}l$Uc#a*k>dBTXPISbD3B30X4MkV%m?#)0AuXSyz(6!ud zHA00j^s zkAWRf&ySeBl&5)nweFumU8FBY8*UB2^IZ#?#%@bM>b2C&K{z5NfH94hDrYzV8d3yR%-YC8zBn zF^P%V_Ayh|4+%gHc7xc@K3W^xvT*}JtBM1+Jv?Zo&UvM)#ITivkM8Uc4ol{{DYKn$ zsMfK!Y%bokI;$(@H+V)6saBLqT=k}GAWo*}lLESSxyjh+PO%cD7~6swX9Ni!@83co zYeW&9@2~C%Nqti-03er`?P(uCF*GzaJ+zqZ<=Vc&+7&O$$36*pxK}aeYdEy?Opf=@ zx{wxpeJk)+Vps%jXB+c&rhcXMPYr9#TQ)w>2f==q7*Y}v6?Jxc8W|dX3@~$p(qSud zgTtYQ>Maku1tNlR%irHZiZqH<8n91y=Faw`gZU%F_9!UG$W8zj>#^Yal+tXbhODEb zL+s`}8=KsFwyxYJ)@ChIpNEG>v_+JGfdQ2Dn$n;K1_l5*)W1Xs!cz(GX~O}VRyzB8 z!}xX)Q*(;_Z1BaGYyHWM4s#a`(+!|l<+IzYOXNJQ0sIylI>xYygM&j{Y>mrnhGyVw z$NgJ(5oe;G*GZgb?73RbR~u_CAA1@vW!)>%C@Cl@!G7?d;i#ko0|O({vE;Q}a+~vP zXGAjbCdtL=mJf&oz!56LBJLXE57nPP?`v0DmgqOWT$r6@t5}#jo3$3B14~}!cj!N} zO-@ewMfH;+V3{LA+VsCOO#Rt|`$c1lB$euAsq`@#SdE1hg**GhBU6?DYJ(of>4Zf^ ziMa~`nCaeWm=&PN*w`3IwEEvDi7f5zI<%TiJIpMZo8%!TZGNRT3hF#jE3%vYY+F{G zTo310rK;gS{KqbQU9o%E*U>Z6O>;0Th&S1P4mAeq4)w+sH4BUAuhkc5lx9~&ahwha z?F}UHRPKbmU&>ups}^T-$u50aMS9PF3)t@gFGdkGFOD|F@^x3VS8Gu5+OQ>z;tjS#S=XHgFH(22SY) z8ptnXogE)uhbq|!3O5@_d4*vVo$Nh}44SB8rm|0{TwTYMtf659uHu1#5LsI#e z_9<7ZBRwpxk42u-eaV5Lbal)@bgAc24`zc?EyU@~?ZIBqq98qxcE(Nq&M$awXB9v9 z_glE@3kwNTIlqFEb*tTZoMk%EnU6kbfywtjy#W1Ep!eWQ*^Gp}H3H~V8b zbQSC7CJDT3X_0jz+?d9*4<&N~kXh>6J{)jvN!i<`c$%wjuq~1A;)P$dh3zb&SJA$1 z`2L;1T|flnV!hurRM_3L6wJ9YFSHc6o98vGX+efk|7(P2L?=4wWxE1#$&Pha?OTx$ zhkwU*AQ?}Mq6=W@m(@}^C%e-OxhVgFBjk|(n+BLgg80-_;6|p9G&~jtU=!;f5ZHq( z8~C87+jLB9YynFjpE5ErH8L535%}Uj`DDCUx3Z$bTaXGe^b4$~?#uV@%#kacTwFfj zD+pu`fE6~^SLXm42DK|QrL}tyZ@fE35KT&$%iFi7;Y3^&m6hh&yBpi!Ww)Y)gaogn z)t&if5)Sr2CV*xwGP=0B3PdJznm?eVW@8(y5T&DA>*rhWJ%`7~v+C;@8ZOu{$wXft zrQa|cwlwWvCnqIo6ziN^Y?ZcN9l|IAuF4hjsDjMQ%$g9VGqQJ@4Q`gh*mmcebP5 zNUU!>1W;Q~Umq#A)t6`Jz!~@U^=&V-icz*3vHMoZo2?;71yz28i|SC>_UY zn=wWYkF&Qu9}9Q z-*chGucbv`P&xzxc^HrUy8tZ*2M2SpJdiqV&-PWYhW` zKvQY~F9HshJFrm~LC588?<2^p8&ZIRnwo@VqL-JK(J2v^XHTU#`1trJDSs?1h%qp1 zP~80O`a&szhC}}SZh9!7v%^fy&dG$~)1cD;`nxareNOFER8$lc6s)a5cc{iV4k7y@ z5MEhGlUYwRs5Lx(@+2f{9S(m&Ol)KM3q3z{hU73L_K6t7@y|+ z{CsU7^>VYQ6f{AIdhF`a^aEE-p97ki(zvZvXUI4^MK9K=KK%HSltldWDII8({91Ik zEj{4cOF?$9~JFkS*x*$Nfv#ml#~=$9N-_Q-_EIn_SZ$c-rnBv z@7gg?D7W#?Af(YgM)qpeifYG!BC7}}egQPcTmdZ2+xr5rl18Smq$CnlC%_dIbpsa2 zjIxifBs@GidJKH#r%#_~X=!VwEdk;D{jW(Jo0i~kDfFI!fg0Z@L`0x!u{+lo3uz3v zxd!E4Q2k61@u@*zd8H{SDh~AaGBPj_iFmu!LP2|BT`0(oiVd0t85m>|6YoIu@Gah< z)t$gsSCiy(tE#HL6wxZCSaA~{9UcP9Fz>V14461RI%;Na4&++92GkBfUI}_*5I}=x z20B59S-;umRNuUMtIG?? zuw5bo@sT#LQK_PSI?>c{_aDGgfj=OC>_(quj$_p?R4pL)Ttv}jGq{6%P)A+|yBnC? zu?PBm(gwg+vrY7+yd=cxI;9T$aZ^Jtywv$f(IE{)v_FvV2O0#r=*jt6pOF^jlt5k9 z>F?CtyFC#Cp1dH4sP1^xxTXdEHLJ|@iM$7yyFG^k9O|9-oiRMIajEA^(iHj3KKR3$ z=sJTWe$OE5{)E$DH^7$$j?*!>>6aArF9_Of z)~M2N*y}%2QQH*mAbWO^Bcpt!N@i3Z|7a|T9JJ$u zemNCQibs%!C$~qk!4V9~=kQ1icdKmLNY4&sb_Ca04?u-+ua<5R0AN$jgAcKbPLS73Co$&@;8g+wWqtx|ZV1{j727dsx=~ zX*SAnq;AGLl;nPkFex%Y5Yt7UxSw4D?dixWDWnjmySI3575EONvF`7|8SLgG5Q;!< zzYzrRD5nUxCs^bx4-y&it+Ri6T+B<{eP+Sk)>h_M*VJ3Fy?w>)_I!O| zY^>VV#hGJoGWgDZ2pGRa%)jFK9M*g=jLEd`{Ho8?;L+3?HRPF#?|nkTvWDyL&}CI2 z^`rAde0SG3jx+D>?1zKj8W>QDbv@ZJnZd5*tCcMne%pGPau?Dbg$8~HK37p#IHbM! zxwlZSbbQ7j2XbX9^`)kQc`vTWzLZrnxq=a4@i=ui1`_yp7=nsQ`+UTOSP+v`MO#sj zoJ#`bZ&&Cu`w^FEl_fd_vtniNi0@jL%bxt`a5zO?NQEQ{M zGA7Hir#~gyc@h8+IT~OfLC(3hZAD!0D+%JxC}Kslh~#;$b@!ELUj?9o&pPH9Vnv94 zB9(va3LkLa`w*QI=d?&8VtPD80OG+d&_IH?)E-E#dshzbobked5%7(RY4*NjI#-phkk_9%cD=N{*{uN7h3&v=9>B~#(y z=W?WRo_7VLhwlIbsS$bnE>0Xq6meb(fNw(X8=a>WA^M3_{!uF2xs-43cl5tNa`*%o zNR8IvuNa8Wi@iK;Wsd>~KhwcPPb)(76N4uZU+km8olE&Q3qAaX06=7DfPvK9U(Y)TGT->OeXR)Fezqe?BcknID%=i4#{G;(m{KAYe*aI!HOGU;dpkq` zqQWP@z<@cWM8ueHf0Jd5o=ggBMZ`L%o!j;|Du{uj)<$b(Oup?yhVDN>`P>57cC|+g zwIW16k;>nv!go9v+FF}s-`*9hwVqBV)>;wgL|g^&64KOfZ zTB)XKr_;%kuV$Iy-mRf)nzrLr3cx>B1ZiuVX4ekww$3oiw!M;+?vnt3`0-8*4BL9O z4xIvyXtis7DFDw{5u|OoTI*dqcuOWuJ5K@tB1Z!Z445%S#MSlZ(23wEZf|Sl)Z_bU zA5Z|Eu_8!YT`#_r@wOd|G0M66cHuX#i0nKG0Ei#AKm!Ao7x|>fTWfDV-A>DCQRJhz z|15RZ+NNn&tFkTH*Vj1OUX3 z1{fHyvMe{7MroBPlNA%i-)l&VQ%V_CZNJi@06b$wkT$24GO8>)8{npKN@aSa6~6N% z03d#RhXEM2larU$iFKllR=WGyB^U^;wb9mzC^ { @@ -41,8 +43,6 @@ export interface LemonButtonPropsBase disableClientSideRouting?: boolean /** If set clicking this button will open the page in a new tab. */ targetBlank?: boolean - /** External URL to link to. */ - className?: string /** Icon displayed on the left. */ icon?: React.ReactElement | null diff --git a/frontend/src/lib/lemon-ui/icons/categories.ts b/frontend/src/lib/lemon-ui/icons/categories.ts index c57ef8d09c6ef..1b7b97b39abe5 100644 --- a/frontend/src/lib/lemon-ui/icons/categories.ts +++ b/frontend/src/lib/lemon-ui/icons/categories.ts @@ -52,6 +52,7 @@ export const OBJECTS = { 'IconGearFilled', 'IconStack', 'IconSparkles', + 'IconPuzzle', ], People: ['IconPeople', 'IconPeopleFilled', 'IconPerson', 'IconProfile', 'IconUser', 'IconGroups'], 'Business & Finance': ['IconStore', 'IconCart', 'IconReceipt', 'IconPiggyBank', 'IconHandMoney'], @@ -72,6 +73,7 @@ export const TECHNOLOGY = { 'IconDatabase', 'IconHardDrive', 'IconMouse', + 'IconCdCase', ], Software: ['IconBrowser', 'IconCode', 'IconCodeInsert', 'IconTerminal', 'IconApp'], UI: [ @@ -187,6 +189,7 @@ export const TEAMS_AND_COMPANIES = { 'IconPlay', 'IconPlayFilled', 'IconPlaylist', + 'IconShuffle', 'IconPause', 'IconFastForward', 'IconPauseFilled', diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index c950088374fb4..82efdf3b31478 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -2038,6 +2038,55 @@ ], "type": "object" }, + "CachedSuggestedQuestionsQueryResponse": { + "additionalProperties": false, + "properties": { + "cache_key": { + "type": "string" + }, + "cache_target_age": { + "format": "date-time", + "type": "string" + }, + "calculation_trigger": { + "description": "What triggered the calculation of the query, leave empty if user/immediate", + "type": "string" + }, + "is_cached": { + "type": "boolean" + }, + "last_refresh": { + "format": "date-time", + "type": "string" + }, + "next_allowed_client_refresh": { + "format": "date-time", + "type": "string" + }, + "query_status": { + "$ref": "#/definitions/QueryStatus", + "description": "Query status indicates whether next to the provided data, a query is still running." + }, + "questions": { + "items": { + "type": "string" + }, + "type": "array" + }, + "timezone": { + "type": "string" + } + }, + "required": [ + "cache_key", + "is_cached", + "last_refresh", + "next_allowed_client_refresh", + "questions", + "timezone" + ], + "type": "object" + }, "CachedTeamTaxonomyQueryResponse": { "additionalProperties": false, "properties": { @@ -7216,6 +7265,7 @@ "ExperimentFunnelQuery", "ExperimentTrendQuery", "DatabaseSchemaQuery", + "SuggestedQuestionsQuery", "TeamTaxonomyQuery", "EventTaxonomyQuery" ], @@ -9377,6 +9427,19 @@ }, "required": ["tables"], "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "questions": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": ["questions"], + "type": "object" } ] }, @@ -9486,6 +9549,9 @@ }, { "$ref": "#/definitions/DatabaseSchemaQuery" + }, + { + "$ref": "#/definitions/SuggestedQuestionsQuery" } ], "required": ["kind"], @@ -10586,6 +10652,37 @@ "required": ["results"], "type": "object" }, + "SuggestedQuestionsQuery": { + "additionalProperties": false, + "properties": { + "kind": { + "const": "SuggestedQuestionsQuery", + "type": "string" + }, + "modifiers": { + "$ref": "#/definitions/HogQLQueryModifiers", + "description": "Modifiers used when performing the query" + }, + "response": { + "$ref": "#/definitions/SuggestedQuestionsQueryResponse" + } + }, + "required": ["kind"], + "type": "object" + }, + "SuggestedQuestionsQueryResponse": { + "additionalProperties": false, + "properties": { + "questions": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": ["questions"], + "type": "object" + }, "TableSettings": { "additionalProperties": false, "properties": { diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index e92c4860a0d8c..8762e320251c0 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -107,6 +107,7 @@ export enum NodeKind { DatabaseSchemaQuery = 'DatabaseSchemaQuery', // AI queries + SuggestedQuestionsQuery = 'SuggestedQuestionsQuery', TeamTaxonomyQuery = 'TeamTaxonomyQuery', EventTaxonomyQuery = 'EventTaxonomyQuery', } @@ -168,7 +169,7 @@ export type QuerySchema = | SavedInsightNode | InsightVizNode - // New queries, not yet implemented + // Classic insights | TrendsQuery | FunnelsQuery | RetentionQuery @@ -180,6 +181,9 @@ export type QuerySchema = // Misc | DatabaseSchemaQuery + // AI + | SuggestedQuestionsQuery + // Keep this, because QuerySchema itself will be collapsed as it is used in other models export type QuerySchemaRoot = QuerySchema @@ -1993,6 +1997,16 @@ export interface HogCompileResponse { bytecode: any[] } +export interface SuggestedQuestionsQuery extends DataNode { + kind: NodeKind.SuggestedQuestionsQuery +} + +export interface SuggestedQuestionsQueryResponse { + questions: string[] +} + +export type CachedSuggestedQuestionsQueryResponse = CachedQueryResponse + export interface TeamTaxonomyItem { event: string count: integer diff --git a/frontend/src/scenes/max/Max.stories.tsx b/frontend/src/scenes/max/Max.stories.tsx index 812c87f0fd423..1e9761f352370 100644 --- a/frontend/src/scenes/max/Max.stories.tsx +++ b/frontend/src/scenes/max/Max.stories.tsx @@ -36,9 +36,41 @@ const Template = ({ sessionId }: { sessionId: string }): JSX.Element => { } export const Welcome: StoryFn = () => { + useStorybookMocks({ + post: { + '/api/projects/:team_id/query/': () => [ + 200, + { + questions: [ + 'What are my most popular pages?', + 'Where are my users located?', + 'Who are the biggest customers?', + 'Which feature drives most usage?', + ], + }, + ], + }, + }) + + const sessionId = 'd210b263-8521-4c5b-b3c4-8e0348df574b' + return