From d38333f6f409c35b03c5e2f4f42345fab66dbda3 Mon Sep 17 00:00:00 2001 From: Daniel Bachhuber Date: Thu, 19 Dec 2024 15:08:13 -0800 Subject: [PATCH] `delta` is better than `places` for statistical variability --- .../test_experiment_trends_query_runner.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py b/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py index d1f6c4905dd93..b52bb3bf39727 100644 --- a/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py +++ b/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py @@ -1373,15 +1373,15 @@ def test_query_runner_standard_flow(self): self.assertEqual(control_variant.absolute_exposure, 2) self.assertEqual(test_variant.absolute_exposure, 2) - self.assertAlmostEqual(result.credible_intervals["control"][0], 0.5449, places=3) - self.assertAlmostEqual(result.credible_intervals["control"][1], 4.3836, places=3) - self.assertAlmostEqual(result.credible_intervals["test"][0], 1.1009, places=3) - self.assertAlmostEqual(result.credible_intervals["test"][1], 5.8342, places=3) + self.assertAlmostEqual(result.credible_intervals["control"][0], 0.5449, delta=0.1) + self.assertAlmostEqual(result.credible_intervals["control"][1], 4.3836, delta=0.1) + self.assertAlmostEqual(result.credible_intervals["test"][0], 1.1009, delta=0.1) + self.assertAlmostEqual(result.credible_intervals["test"][1], 5.8342, delta=0.1) - self.assertAlmostEqual(result.p_value, 1.0, places=3) + self.assertAlmostEqual(result.p_value, 1.0, delta=0.1) - self.assertAlmostEqual(result.probability["control"], 0.2549, places=2) - self.assertAlmostEqual(result.probability["test"], 0.7453, places=2) + self.assertAlmostEqual(result.probability["control"], 0.2549, delta=0.1) + self.assertAlmostEqual(result.probability["test"], 0.7453, delta=0.1) self.assertEqual(result.significance_code, ExperimentSignificanceCode.NOT_ENOUGH_EXPOSURE) @@ -1496,15 +1496,15 @@ def test_query_runner_standard_flow_v2_stats(self): self.assertEqual(control_variant.absolute_exposure, 2) self.assertEqual(test_variant.absolute_exposure, 2) - self.assertAlmostEqual(result.credible_intervals["control"][0], 0.3633, places=3) - self.assertAlmostEqual(result.credible_intervals["control"][1], 2.9224, places=3) - self.assertAlmostEqual(result.credible_intervals["test"][0], 0.7339, places=3) - self.assertAlmostEqual(result.credible_intervals["test"][1], 3.8894, places=3) + self.assertAlmostEqual(result.credible_intervals["control"][0], 0.3633, delta=0.1) + self.assertAlmostEqual(result.credible_intervals["control"][1], 2.9224, delta=0.1) + self.assertAlmostEqual(result.credible_intervals["test"][0], 0.7339, delta=0.1) + self.assertAlmostEqual(result.credible_intervals["test"][1], 3.8894, delta=0.1) - self.assertAlmostEqual(result.p_value, 1.0, places=3) + self.assertAlmostEqual(result.p_value, 1.0, delta=0.1) - self.assertAlmostEqual(result.probability["control"], 0.2549, places=2) - self.assertAlmostEqual(result.probability["test"], 0.7453, places=2) + self.assertAlmostEqual(result.probability["control"], 0.2549, delta=0.1) + self.assertAlmostEqual(result.probability["test"], 0.7453, delta=0.1) self.assertEqual(result.significance_code, ExperimentSignificanceCode.NOT_ENOUGH_EXPOSURE)