delta is better than places for statistical variability

PostHog · Dec 19, 2024 · d38333f · d38333f
1 parent 3843cc0
commit d38333f
Showing 1 changed file with 14 additions and 14 deletions.
diff --git a/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py b/posthog/hogql_queries/experiments/test/test_experiment_trends_query_runner.py
@@ -1373,15 +1373,15 @@ def test_query_runner_standard_flow(self):
         self.assertEqual(control_variant.absolute_exposure, 2)
         self.assertEqual(test_variant.absolute_exposure, 2)
 
-        self.assertAlmostEqual(result.credible_intervals["control"][0], 0.5449, places=3)
-        self.assertAlmostEqual(result.credible_intervals["control"][1], 4.3836, places=3)
-        self.assertAlmostEqual(result.credible_intervals["test"][0], 1.1009, places=3)
-        self.assertAlmostEqual(result.credible_intervals["test"][1], 5.8342, places=3)
+        self.assertAlmostEqual(result.credible_intervals["control"][0], 0.5449, delta=0.1)
+        self.assertAlmostEqual(result.credible_intervals["control"][1], 4.3836, delta=0.1)
+        self.assertAlmostEqual(result.credible_intervals["test"][0], 1.1009, delta=0.1)
+        self.assertAlmostEqual(result.credible_intervals["test"][1], 5.8342, delta=0.1)
 
-        self.assertAlmostEqual(result.p_value, 1.0, places=3)
+        self.assertAlmostEqual(result.p_value, 1.0, delta=0.1)
 
-        self.assertAlmostEqual(result.probability["control"], 0.2549, places=2)
-        self.assertAlmostEqual(result.probability["test"], 0.7453, places=2)
+        self.assertAlmostEqual(result.probability["control"], 0.2549, delta=0.1)
+        self.assertAlmostEqual(result.probability["test"], 0.7453, delta=0.1)
 
         self.assertEqual(result.significance_code, ExperimentSignificanceCode.NOT_ENOUGH_EXPOSURE)
 
@@ -1496,15 +1496,15 @@ def test_query_runner_standard_flow_v2_stats(self):
         self.assertEqual(control_variant.absolute_exposure, 2)
         self.assertEqual(test_variant.absolute_exposure, 2)
 
-        self.assertAlmostEqual(result.credible_intervals["control"][0], 0.3633, places=3)
-        self.assertAlmostEqual(result.credible_intervals["control"][1], 2.9224, places=3)
-        self.assertAlmostEqual(result.credible_intervals["test"][0], 0.7339, places=3)
-        self.assertAlmostEqual(result.credible_intervals["test"][1], 3.8894, places=3)
+        self.assertAlmostEqual(result.credible_intervals["control"][0], 0.3633, delta=0.1)
+        self.assertAlmostEqual(result.credible_intervals["control"][1], 2.9224, delta=0.1)
+        self.assertAlmostEqual(result.credible_intervals["test"][0], 0.7339, delta=0.1)
+        self.assertAlmostEqual(result.credible_intervals["test"][1], 3.8894, delta=0.1)
 
-        self.assertAlmostEqual(result.p_value, 1.0, places=3)
+        self.assertAlmostEqual(result.p_value, 1.0, delta=0.1)
 
-        self.assertAlmostEqual(result.probability["control"], 0.2549, places=2)
-        self.assertAlmostEqual(result.probability["test"], 0.7453, places=2)
+        self.assertAlmostEqual(result.probability["control"], 0.2549, delta=0.1)
+        self.assertAlmostEqual(result.probability["test"], 0.7453, delta=0.1)
 
         self.assertEqual(result.significance_code, ExperimentSignificanceCode.NOT_ENOUGH_EXPOSURE)