From 733124c1a8bad039a2f829c50832fc751375e3ea Mon Sep 17 00:00:00 2001
From: Georgiy Tarasov <gtarasov.work@gmail.com>
Date: Thu, 26 Sep 2024 12:42:12 +0200
Subject: [PATCH] chore: improve observability when bootstrapping scheduled
 tasks (#25210)

---
 posthog/celery.py                    | 20 +++++++++++++++++++-
 posthog/tasks/test/test_scheduled.py | 15 +++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)
 create mode 100644 posthog/tasks/test/test_scheduled.py

diff --git a/posthog/celery.py b/posthog/celery.py
index f7626f6b297de..ac7f5c90138cf 100644
--- a/posthog/celery.py
+++ b/posthog/celery.py
@@ -1,6 +1,7 @@
 import os
 import time
 
+import structlog
 from celery import Celery
 from celery.signals import (
     setup_logging,
@@ -16,6 +17,11 @@
 from django_structlog.celery.steps import DjangoStructLogInitStep
 from prometheus_client import Counter, Histogram
 
+from posthog.cloud_utils import is_cloud
+
+logger = structlog.get_logger(__name__)
+
+
 # set the default Django settings module for the 'celery' program.
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "posthog.settings")
 
@@ -153,6 +159,18 @@ def retry_signal_handler(sender, reason, **kwargs):
 
 @app.on_after_finalize.connect
 def setup_periodic_tasks(sender: Celery, **kwargs):
+    from posthog.pagerduty.pd import create_incident
     from posthog.tasks.scheduled import setup_periodic_tasks
 
-    setup_periodic_tasks(sender)
+    try:
+        setup_periodic_tasks(sender)
+    except Exception as exc:
+        # Setup fails silently. Alert the team if a configuration error is detected in periodic tasks.
+        if is_cloud():
+            create_incident(
+                f"Periodic tasks setup failed: {exc}",
+                "posthog.celery.setup_periodic_tasks",
+                "critical",
+            )
+        else:
+            logger.exception("Periodic tasks setup failed", exception=exc)
diff --git a/posthog/tasks/test/test_scheduled.py b/posthog/tasks/test/test_scheduled.py
new file mode 100644
index 0000000000000..687aad922d05b
--- /dev/null
+++ b/posthog/tasks/test/test_scheduled.py
@@ -0,0 +1,15 @@
+from django.test import TestCase
+
+from posthog.celery import app
+from posthog.tasks.scheduled import setup_periodic_tasks
+
+
+class TestScheduledTasks(TestCase):
+    def test_scheduled_tasks(self) -> None:
+        """
+        `setup_periodic_tasks` may fail silently. This test ensures that it doesn't.
+        """
+        try:
+            setup_periodic_tasks(app)
+        except Exception as exc:
+            assert exc is None, exc