From 323106db64ac0c1e22dee56d2e37cd6a706972a3 Mon Sep 17 00:00:00 2001 From: Anders Asheim Hennum Date: Mon, 9 Dec 2024 19:27:09 +0100 Subject: [PATCH 01/10] WIP --- .../commands/generate_experiment_data.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 posthog/management/commands/generate_experiment_data.py diff --git a/posthog/management/commands/generate_experiment_data.py b/posthog/management/commands/generate_experiment_data.py new file mode 100644 index 0000000000000..cddbe05d8c2c4 --- /dev/null +++ b/posthog/management/commands/generate_experiment_data.py @@ -0,0 +1,83 @@ +from datetime import datetime, timedelta +import logging +import random +import secrets +import uuid + +from django.conf import settings +from django.core.management.base import BaseCommand +from posthoganalytics.client import Client + + +logging.getLogger("kafka").setLevel(logging.ERROR) # Hide kafka-python's logspam + + +class Command(BaseCommand): + help = "Generate experiment data" + + def add_arguments(self, parser): + parser.add_argument("--experiment-id", type=str, help="Experiment ID") + parser.add_argument("--seed", type=str, help="Simulation seed for deterministic output") + + def handle(self, *args, **options): + # Make sure this runs in development environment only + if not settings.DEBUG: + raise ValueError("This command should only be run in development! DEBUG must be True.") + + experiment_id = options.get("experiment_id") + seed = options.get("seed") or secrets.token_hex(16) + + if not experiment_id: + raise ValueError("Experiment ID is required") + + # Create a new Posthog Client + posthog_client = Client(api_key=settings.POSTHOG_API_KEY) + + experiment_config = { + "experiment_id": experiment_id, + "seed": seed, + "number_of_users": 1000, + "start_timestamp": datetime.now() - timedelta(days=7), + "end_timestamp": datetime.now(), + "variants": { + "control": { + "weight": 0.5, + "actions": [ + {"event": "$pageview", "probability": 0.75}, + ], + }, + "test": { + "weight": 0.5, + "actions": [ + {"event": "$pageview", "probability": 1}, + ], + }, + }, + } + + for _ in range(experiment_config["number_of_users"]): + variant = random.choices( + list(experiment_config["variants"].keys()), + weights=[v["weight"] for v in experiment_config["variants"].values()], + )[0] + distinct_id = uuid.uuid4() + random_timestamp = random.uniform( + experiment_config["start_timestamp"], experiment_config["end_timestamp"] - timedelta(hours=1) + ) + posthog_client.capture( + distinct_id=distinct_id, + event="$feature_flag_called", + timestamp=random_timestamp, + properties={ + "feature_flag": experiment_config["experiment_id"], + f"feature/{experiment_config['experiment_id']}": variant, + }, + ) + + for action in experiment_config["variants"][variant]["actions"]: + if random.random() < action["probability"]: + posthog_client.capture( + distinct_id=distinct_id, + event=action["event"], + timestamp=random_timestamp + timedelta(minutes=1), + ) From 57bb4f34606c5b2fcdeca2567f861a866c252c5d Mon Sep 17 00:00:00 2001 From: Anders Asheim Hennum Date: Mon, 9 Dec 2024 20:06:01 +0100 Subject: [PATCH 02/10] WIP --- .../management/commands/generate_experiment_data.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/posthog/management/commands/generate_experiment_data.py b/posthog/management/commands/generate_experiment_data.py index cddbe05d8c2c4..a9fe35c836702 100644 --- a/posthog/management/commands/generate_experiment_data.py +++ b/posthog/management/commands/generate_experiment_data.py @@ -6,7 +6,7 @@ from django.conf import settings from django.core.management.base import BaseCommand -from posthoganalytics.client import Client +import posthoganalytics logging.getLogger("kafka").setLevel(logging.ERROR) # Hide kafka-python's logspam @@ -30,13 +30,10 @@ def handle(self, *args, **options): if not experiment_id: raise ValueError("Experiment ID is required") - # Create a new Posthog Client - posthog_client = Client(api_key=settings.POSTHOG_API_KEY) - experiment_config = { "experiment_id": experiment_id, "seed": seed, - "number_of_users": 1000, + "number_of_users": 10, "start_timestamp": datetime.now() - timedelta(days=7), "end_timestamp": datetime.now(), "variants": { @@ -64,7 +61,8 @@ def handle(self, *args, **options): random_timestamp = random.uniform( experiment_config["start_timestamp"], experiment_config["end_timestamp"] - timedelta(hours=1) ) - posthog_client.capture( + print(f"Generating data for user {distinct_id} at {random_timestamp} with variant {variant}") + posthoganalytics.capture( distinct_id=distinct_id, event="$feature_flag_called", timestamp=random_timestamp, @@ -76,7 +74,8 @@ def handle(self, *args, **options): for action in experiment_config["variants"][variant]["actions"]: if random.random() < action["probability"]: - posthog_client.capture( + print(f"Generating data for user {distinct_id} at {random_timestamp + timedelta(minutes=1)} with event {action['event']}") + posthoganalytics.capture( distinct_id=distinct_id, event=action["event"], timestamp=random_timestamp + timedelta(minutes=1), From 5fe827889ffaaf1c8b1ba6ba21c4dd1d628e4968 Mon Sep 17 00:00:00 2001 From: Anders Asheim Hennum Date: Mon, 9 Dec 2024 21:16:42 +0100 Subject: [PATCH 03/10] WIP --- .../commands/generate_experiment_data.py | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/posthog/management/commands/generate_experiment_data.py b/posthog/management/commands/generate_experiment_data.py index a9fe35c836702..1daf23366e080 100644 --- a/posthog/management/commands/generate_experiment_data.py +++ b/posthog/management/commands/generate_experiment_data.py @@ -2,6 +2,7 @@ import logging import random import secrets +import time import uuid from django.conf import settings @@ -25,15 +26,18 @@ def handle(self, *args, **options): raise ValueError("This command should only be run in development! DEBUG must be True.") experiment_id = options.get("experiment_id") + + # TODO: actually implement a seed seed = options.get("seed") or secrets.token_hex(16) if not experiment_id: raise ValueError("Experiment ID is required") + # TODO: this can be a config file taken as an argument experiment_config = { "experiment_id": experiment_id, "seed": seed, - "number_of_users": 10, + "number_of_users": 1000, "start_timestamp": datetime.now() - timedelta(days=7), "end_timestamp": datetime.now(), "variants": { @@ -52,31 +56,39 @@ def handle(self, *args, **options): }, } + variants = list(experiment_config["variants"].keys()) + variant_counts = {variant: 0 for variant in variants} for _ in range(experiment_config["number_of_users"]): variant = random.choices( - list(experiment_config["variants"].keys()), + variants, weights=[v["weight"] for v in experiment_config["variants"].values()], )[0] + variant_counts[variant] += 1 distinct_id = uuid.uuid4() random_timestamp = random.uniform( experiment_config["start_timestamp"], experiment_config["end_timestamp"] - timedelta(hours=1) ) - print(f"Generating data for user {distinct_id} at {random_timestamp} with variant {variant}") posthoganalytics.capture( distinct_id=distinct_id, event="$feature_flag_called", timestamp=random_timestamp, properties={ - "feature_flag": experiment_config["experiment_id"], - f"feature/{experiment_config['experiment_id']}": variant, + "$feature_flag": experiment_config["experiment_id"], + f"$feature/{experiment_config['experiment_id']}": variant, }, ) for action in experiment_config["variants"][variant]["actions"]: if random.random() < action["probability"]: - print(f"Generating data for user {distinct_id} at {random_timestamp + timedelta(minutes=1)} with event {action['event']}") posthoganalytics.capture( distinct_id=distinct_id, event=action["event"], timestamp=random_timestamp + timedelta(minutes=1), ) + + logging.info(f"Generated data for {experiment_config['experiment_id']} with seed {seed}") + logging.info(f"Variant counts: {variant_counts}") + + # TODO: need to figure out how to wait for the data to be flushed. shutdown() doesn't work as expected. + time.sleep(10) + posthoganalytics.shutdown() From 52637e3caa76da2b893dcdec1eebbb7c5146c74d Mon Sep 17 00:00:00 2001 From: Anders Asheim Hennum Date: Tue, 10 Dec 2024 08:35:30 +0100 Subject: [PATCH 04/10] use pydantic classes to get validation on config --- .../commands/generate_experiment_data.py | 140 +++++++++++------- 1 file changed, 89 insertions(+), 51 deletions(-) diff --git a/posthog/management/commands/generate_experiment_data.py b/posthog/management/commands/generate_experiment_data.py index 1daf23366e080..7d724ba63dc17 100644 --- a/posthog/management/commands/generate_experiment_data.py +++ b/posthog/management/commands/generate_experiment_data.py @@ -1,94 +1,132 @@ from datetime import datetime, timedelta import logging import random -import secrets import time import uuid +import json from django.conf import settings from django.core.management.base import BaseCommand import posthoganalytics +from pydantic import BaseModel, ValidationError -logging.getLogger("kafka").setLevel(logging.ERROR) # Hide kafka-python's logspam +class ActionConfig(BaseModel): + event: str + count: int + probability: float + + +class VariantConfig(BaseModel): + weight: float + actions: list[ActionConfig] + + +class ExperimentConfig(BaseModel): + number_of_users: int + start_timestamp: datetime + end_timestamp: datetime + variants: dict[str, VariantConfig] + + +def get_default_experiment_config() -> ExperimentConfig: + return ExperimentConfig( + number_of_users=1000, + start_timestamp=datetime.now() - timedelta(days=7), + end_timestamp=datetime.now(), + variants={ + "control": VariantConfig( + weight=0.5, + actions=[ActionConfig(event="$pageview", count=1, probability=0.75)], + ), + "test": VariantConfig( + weight=0.5, + actions=[ActionConfig(event="$pageview", count=1, probability=1)], + ), + }, + ) class Command(BaseCommand): - help = "Generate experiment data" + help = "Generate experiment test data" def add_arguments(self, parser): - parser.add_argument("--experiment-id", type=str, help="Experiment ID") - parser.add_argument("--seed", type=str, help="Simulation seed for deterministic output") + group = parser.add_mutually_exclusive_group(required=False) + group.add_argument( + "--init-config", type=str, help="Initialize a new experiment configuration file at the specified path" + ) + + experiment_group = parser.add_argument_group("experiment arguments") + experiment_group.add_argument("--experiment-id", type=str, help="Experiment ID (feature flag name)") + experiment_group.add_argument("--config", type=str, help="Path to experiment config file") + experiment_group.add_argument( + "--seed", type=str, required=False, help="Simulation seed for deterministic output" + ) def handle(self, *args, **options): # Make sure this runs in development environment only if not settings.DEBUG: raise ValueError("This command should only be run in development! DEBUG must be True.") + if config_path := options.get("init_config"): + with open(config_path, "w") as f: + f.write(get_default_experiment_config().model_dump_json(indent=2)) + logging.info(f"Created example configuration file at: {config_path}") + return + experiment_id = options.get("experiment_id") + config_path = options.get("config") - # TODO: actually implement a seed - seed = options.get("seed") or secrets.token_hex(16) - - if not experiment_id: - raise ValueError("Experiment ID is required") - - # TODO: this can be a config file taken as an argument - experiment_config = { - "experiment_id": experiment_id, - "seed": seed, - "number_of_users": 1000, - "start_timestamp": datetime.now() - timedelta(days=7), - "end_timestamp": datetime.now(), - "variants": { - "control": { - "weight": 0.5, - "actions": [ - {"event": "$pageview", "probability": 0.75}, - ], - }, - "test": { - "weight": 0.5, - "actions": [ - {"event": "$pageview", "probability": 1}, - ], - }, - }, - } + if not experiment_id or not config_path: + raise ValueError("Both --experiment-id and --config are required when not using --init-config") + + with open(config_path) as config_file: + config_data = json.load(config_file) + + try: + # Use the ExperimentConfig model to parse and validate the JSON data + experiment_config = ExperimentConfig(**config_data) + except ValidationError as e: + raise ValueError(f"Invalid configuration: {e}") - variants = list(experiment_config["variants"].keys()) + variants = list(experiment_config.variants.keys()) variant_counts = {variant: 0 for variant in variants} - for _ in range(experiment_config["number_of_users"]): + + for _ in range(experiment_config.number_of_users): variant = random.choices( variants, - weights=[v["weight"] for v in experiment_config["variants"].values()], + weights=[v.weight for v in experiment_config.variants.values()], )[0] variant_counts[variant] += 1 - distinct_id = uuid.uuid4() + distinct_id = str(uuid.uuid4()) random_timestamp = random.uniform( - experiment_config["start_timestamp"], experiment_config["end_timestamp"] - timedelta(hours=1) + experiment_config.start_timestamp.timestamp(), + experiment_config.end_timestamp.timestamp() - 3600, ) + random_timestamp = datetime.fromtimestamp(random_timestamp) + posthoganalytics.capture( distinct_id=distinct_id, event="$feature_flag_called", timestamp=random_timestamp, properties={ - "$feature_flag": experiment_config["experiment_id"], - f"$feature/{experiment_config['experiment_id']}": variant, + "$feature_flag": experiment_id, + f"$feature/{experiment_id}": variant, }, ) - for action in experiment_config["variants"][variant]["actions"]: - if random.random() < action["probability"]: - posthoganalytics.capture( - distinct_id=distinct_id, - event=action["event"], - timestamp=random_timestamp + timedelta(minutes=1), - ) - - logging.info(f"Generated data for {experiment_config['experiment_id']} with seed {seed}") - logging.info(f"Variant counts: {variant_counts}") + for action in experiment_config.variants[variant].actions: + for _ in range(action.count): + if random.random() < action.probability: + posthoganalytics.capture( + distinct_id=distinct_id, + event=action.event, + timestamp=random_timestamp + timedelta(minutes=1), + ) # TODO: need to figure out how to wait for the data to be flushed. shutdown() doesn't work as expected. - time.sleep(10) + time.sleep(2) posthoganalytics.shutdown() + + logging.info(f"Generated data for {experiment_id}") + logging.info(f"Variant counts: {variant_counts}") From 6ccf402693561afbca1690218df26c84bfbe9d6a Mon Sep 17 00:00:00 2001 From: Anders Asheim Hennum Date: Tue, 17 Dec 2024 12:40:00 +0100 Subject: [PATCH 05/10] lint: typing --- posthog/management/commands/generate_experiment_data.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/posthog/management/commands/generate_experiment_data.py b/posthog/management/commands/generate_experiment_data.py index 7d724ba63dc17..d2dedf5d1cd69 100644 --- a/posthog/management/commands/generate_experiment_data.py +++ b/posthog/management/commands/generate_experiment_data.py @@ -99,11 +99,12 @@ def handle(self, *args, **options): )[0] variant_counts[variant] += 1 distinct_id = str(uuid.uuid4()) - random_timestamp = random.uniform( - experiment_config.start_timestamp.timestamp(), - experiment_config.end_timestamp.timestamp() - 3600, + random_timestamp = datetime.fromtimestamp( + random.uniform( + experiment_config.start_timestamp.timestamp(), + experiment_config.end_timestamp.timestamp() - 3600, + ) ) - random_timestamp = datetime.fromtimestamp(random_timestamp) posthoganalytics.capture( distinct_id=distinct_id, From c7eea54008876149a9d530987fcc211f5534fdd9 Mon Sep 17 00:00:00 2001 From: Anders Asheim Hennum Date: Tue, 17 Dec 2024 12:42:19 +0100 Subject: [PATCH 06/10] include feature flag data on metric events --- posthog/management/commands/generate_experiment_data.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/posthog/management/commands/generate_experiment_data.py b/posthog/management/commands/generate_experiment_data.py index d2dedf5d1cd69..58601a74daaf7 100644 --- a/posthog/management/commands/generate_experiment_data.py +++ b/posthog/management/commands/generate_experiment_data.py @@ -123,6 +123,9 @@ def handle(self, *args, **options): distinct_id=distinct_id, event=action.event, timestamp=random_timestamp + timedelta(minutes=1), + properties={ + f"$feature/{experiment_id}": variant, + }, ) # TODO: need to figure out how to wait for the data to be flushed. shutdown() doesn't work as expected. From e38e184110f067e04e2b11e5ca6e3e039d8172a6 Mon Sep 17 00:00:00 2001 From: Anders Asheim Hennum Date: Wed, 18 Dec 2024 10:41:17 +0100 Subject: [PATCH 07/10] add funnel and trends default configs --- .../commands/generate_experiment_data.py | 92 ++++++++++++++----- 1 file changed, 71 insertions(+), 21 deletions(-) diff --git a/posthog/management/commands/generate_experiment_data.py b/posthog/management/commands/generate_experiment_data.py index 58601a74daaf7..291e78ad6ad58 100644 --- a/posthog/management/commands/generate_experiment_data.py +++ b/posthog/management/commands/generate_experiment_data.py @@ -2,6 +2,7 @@ import logging import random import time +from typing import Literal import uuid import json @@ -29,65 +30,114 @@ class ExperimentConfig(BaseModel): variants: dict[str, VariantConfig] -def get_default_experiment_config() -> ExperimentConfig: +def get_default_funnel_experiment_config() -> ExperimentConfig: return ExperimentConfig( - number_of_users=1000, + number_of_users=2000, start_timestamp=datetime.now() - timedelta(days=7), end_timestamp=datetime.now(), variants={ "control": VariantConfig( weight=0.5, - actions=[ActionConfig(event="$pageview", count=1, probability=0.75)], + actions=[ + ActionConfig(event="signup started", count=1, probability=1), + ActionConfig(event="signup completed", count=1, probability=0.25), + ], ), "test": VariantConfig( weight=0.5, - actions=[ActionConfig(event="$pageview", count=1, probability=1)], + actions=[ + ActionConfig(event="signup started", count=1, probability=1), + ActionConfig(event="signup completed", count=1, probability=0.35), + ], ), }, ) +def get_default_trends_experiment_config() -> ExperimentConfig: + return ExperimentConfig( + number_of_users=2000, + start_timestamp=datetime.now() - timedelta(days=7), + end_timestamp=datetime.now(), + variants={ + "control": VariantConfig( + weight=0.5, + actions=[ActionConfig(event="$pageview", count=5, probability=0.25)], + ), + "test": VariantConfig( + weight=0.5, + actions=[ActionConfig(event="$pageview", count=5, probability=0.35)], + ), + }, + ) + + +def get_default_config(type: Literal["funnel", "trends"]) -> ExperimentConfig: + match type: + case "funnel": + return get_default_funnel_experiment_config() + case "trends": + return get_default_trends_experiment_config() + case _: + raise ValueError(f"Invalid experiment type: {type}") + + class Command(BaseCommand): help = "Generate experiment test data" def add_arguments(self, parser): group = parser.add_mutually_exclusive_group(required=False) group.add_argument( - "--init-config", type=str, help="Initialize a new experiment configuration file at the specified path" + "--init-config", + type=str, + help="Initialize a new experiment configuration file at the specified path. Does not generate data.", + ) + group.add_argument( + "--type", + type=str, + choices=["trends", "funnel"], + default="trends", + help="Type of experiment data to generate or configuration to initialize.", ) experiment_group = parser.add_argument_group("experiment arguments") experiment_group.add_argument("--experiment-id", type=str, help="Experiment ID (feature flag name)") experiment_group.add_argument("--config", type=str, help="Path to experiment config file") - experiment_group.add_argument( - "--seed", type=str, required=False, help="Simulation seed for deterministic output" - ) def handle(self, *args, **options): # Make sure this runs in development environment only if not settings.DEBUG: raise ValueError("This command should only be run in development! DEBUG must be True.") - if config_path := options.get("init_config"): - with open(config_path, "w") as f: - f.write(get_default_experiment_config().model_dump_json(indent=2)) - logging.info(f"Created example configuration file at: {config_path}") + experiment_type = options.get("type") + + if init_config_path := options.get("init_config"): + with open(init_config_path, "w") as f: + f.write(get_default_config(experiment_type).model_dump_json(indent=2)) + logging.info(f"Created example {experiment_type} configuration file at: {init_config_path}") return experiment_id = options.get("experiment_id") config_path = options.get("config") - if not experiment_id or not config_path: - raise ValueError("Both --experiment-id and --config are required when not using --init-config") + # Validate required arguments + if not experiment_id: + raise ValueError("--experiment-id is missing!") + + if config_path is None and experiment_type is None: + raise ValueError("--config or --type trends|funnel is missing!") - with open(config_path) as config_file: - config_data = json.load(config_file) + if config_path: + with open(config_path) as config_file: + config_data = json.load(config_file) - try: - # Use the ExperimentConfig model to parse and validate the JSON data - experiment_config = ExperimentConfig(**config_data) - except ValidationError as e: - raise ValueError(f"Invalid configuration: {e}") + try: + # Use the ExperimentConfig model to parse and validate the JSON data + experiment_config = ExperimentConfig(**config_data) + except ValidationError as e: + raise ValueError(f"Invalid configuration: {e}") + else: + experiment_config = get_default_config(experiment_type) variants = list(experiment_config.variants.keys()) variant_counts = {variant: 0 for variant in variants} From 1aaa5b59b363683821ca17b3db4dd29ea079e579 Mon Sep 17 00:00:00 2001 From: Anders Asheim Hennum Date: Wed, 18 Dec 2024 14:54:39 +0100 Subject: [PATCH 08/10] add "required_for_next" option on actions --- .../commands/generate_experiment_data.py | 52 +++++++++++-------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/posthog/management/commands/generate_experiment_data.py b/posthog/management/commands/generate_experiment_data.py index 291e78ad6ad58..1bd5eeeeae7c0 100644 --- a/posthog/management/commands/generate_experiment_data.py +++ b/posthog/management/commands/generate_experiment_data.py @@ -14,8 +14,13 @@ class ActionConfig(BaseModel): event: str - count: int probability: float + count: int = 1 + required_for_next: bool = False + + def model_post_init(self, __context) -> None: + if self.required_for_next and self.count > 1: + raise ValueError("'required_for_next' cannot be used with 'count' greater than 1") class VariantConfig(BaseModel): @@ -39,22 +44,22 @@ def get_default_funnel_experiment_config() -> ExperimentConfig: "control": VariantConfig( weight=0.5, actions=[ - ActionConfig(event="signup started", count=1, probability=1), - ActionConfig(event="signup completed", count=1, probability=0.25), + ActionConfig(event="signup started", probability=1, required_for_next=True), + ActionConfig(event="signup completed", probability=0.25, required_for_next=True), ], ), "test": VariantConfig( weight=0.5, actions=[ - ActionConfig(event="signup started", count=1, probability=1), - ActionConfig(event="signup completed", count=1, probability=0.35), + ActionConfig(event="signup started", probability=1, required_for_next=True), + ActionConfig(event="signup completed", probability=0.35, required_for_next=True), ], ), }, ) -def get_default_trends_experiment_config() -> ExperimentConfig: +def get_default_trend_experiment_config() -> ExperimentConfig: return ExperimentConfig( number_of_users=2000, start_timestamp=datetime.now() - timedelta(days=7), @@ -72,12 +77,12 @@ def get_default_trends_experiment_config() -> ExperimentConfig: ) -def get_default_config(type: Literal["funnel", "trends"]) -> ExperimentConfig: +def get_default_config(type: Literal["funnel", "trend"]) -> ExperimentConfig: match type: case "funnel": return get_default_funnel_experiment_config() - case "trends": - return get_default_trends_experiment_config() + case "trend": + return get_default_trend_experiment_config() case _: raise ValueError(f"Invalid experiment type: {type}") @@ -86,23 +91,21 @@ class Command(BaseCommand): help = "Generate experiment test data" def add_arguments(self, parser): - group = parser.add_mutually_exclusive_group(required=False) - group.add_argument( - "--init-config", - type=str, - help="Initialize a new experiment configuration file at the specified path. Does not generate data.", - ) - group.add_argument( + parser.add_argument( "--type", type=str, - choices=["trends", "funnel"], - default="trends", + choices=["trend", "funnel"], + default="trend", help="Type of experiment data to generate or configuration to initialize.", ) - experiment_group = parser.add_argument_group("experiment arguments") - experiment_group.add_argument("--experiment-id", type=str, help="Experiment ID (feature flag name)") - experiment_group.add_argument("--config", type=str, help="Path to experiment config file") + parser.add_argument( + "--init-config", + type=str, + help="Initialize a new experiment configuration file at the specified path. Does not generate data.", + ) + parser.add_argument("--experiment-id", type=str, help="Experiment ID (feature flag name)") + parser.add_argument("--config", type=str, help="Path to experiment config file") def handle(self, *args, **options): # Make sure this runs in development environment only @@ -166,6 +169,7 @@ def handle(self, *args, **options): }, ) + should_stop = False for action in experiment_config.variants[variant].actions: for _ in range(action.count): if random.random() < action.probability: @@ -177,6 +181,12 @@ def handle(self, *args, **options): f"$feature/{experiment_id}": variant, }, ) + else: + if action.required_for_next: + should_stop = True + break + if should_stop: + break # TODO: need to figure out how to wait for the data to be flushed. shutdown() doesn't work as expected. time.sleep(2) From 8da095e9db5ff311dd776e5100248fba0ce40feb Mon Sep 17 00:00:00 2001 From: Anders Asheim Hennum Date: Wed, 18 Dec 2024 15:16:48 +0100 Subject: [PATCH 09/10] make mypy happy --- posthog/management/commands/generate_experiment_data.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/posthog/management/commands/generate_experiment_data.py b/posthog/management/commands/generate_experiment_data.py index 1bd5eeeeae7c0..2969b742c8747 100644 --- a/posthog/management/commands/generate_experiment_data.py +++ b/posthog/management/commands/generate_experiment_data.py @@ -2,7 +2,6 @@ import logging import random import time -from typing import Literal import uuid import json @@ -77,7 +76,7 @@ def get_default_trend_experiment_config() -> ExperimentConfig: ) -def get_default_config(type: Literal["funnel", "trend"]) -> ExperimentConfig: +def get_default_config(type) -> ExperimentConfig: match type: case "funnel": return get_default_funnel_experiment_config() From a669c955d7fa147d6a5e58f60a2e07b5fa8f0729 Mon Sep 17 00:00:00 2001 From: Anders Asheim Hennum Date: Thu, 19 Dec 2024 15:14:00 +0100 Subject: [PATCH 10/10] correctly send feature_flag_called events --- posthog/management/commands/generate_experiment_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/posthog/management/commands/generate_experiment_data.py b/posthog/management/commands/generate_experiment_data.py index 2969b742c8747..5116d786be76e 100644 --- a/posthog/management/commands/generate_experiment_data.py +++ b/posthog/management/commands/generate_experiment_data.py @@ -163,8 +163,8 @@ def handle(self, *args, **options): event="$feature_flag_called", timestamp=random_timestamp, properties={ + "$feature_flag_response": variant, "$feature_flag": experiment_id, - f"$feature/{experiment_id}": variant, }, )