chore: improve observability when bootstrapping scheduled tasks (#25210)

posthog/celery.py

@@ -1,6 +1,7 @@
 import os
 import time
 
+import structlog
 from celery import Celery
 from celery.signals import (
     setup_logging,
@@ -16,6 +17,11 @@
 from django_structlog.celery.steps import DjangoStructLogInitStep
 from prometheus_client import Counter, Histogram
 
+from posthog.cloud_utils import is_cloud
+
+logger = structlog.get_logger(__name__)
+
+
 # set the default Django settings module for the 'celery' program.
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "posthog.settings")
 
@@ -153,6 +159,18 @@ def retry_signal_handler(sender, reason, **kwargs):
 
 @app.on_after_finalize.connect
 def setup_periodic_tasks(sender: Celery, **kwargs):
+    from posthog.pagerduty.pd import create_incident
     from posthog.tasks.scheduled import setup_periodic_tasks
 
-    setup_periodic_tasks(sender)
+    try:
+        setup_periodic_tasks(sender)
+    except Exception as exc:
+        # Setup fails silently. Alert the team if a configuration error is detected in periodic tasks.
+        if is_cloud():
+            create_incident(
+                f"Periodic tasks setup failed: {exc}",
+                "posthog.celery.setup_periodic_tasks",
+                "critical",
+            )
+        else:
+            logger.exception("Periodic tasks setup failed", exception=exc)

posthog/tasks/test/test_scheduled.py

@@ -0,0 +1,15 @@
+from django.test import TestCase
+
+from posthog.celery import app
+from posthog.tasks.scheduled import setup_periodic_tasks
+
+
+class TestScheduledTasks(TestCase):
+    def test_scheduled_tasks(self) -> None:
+        """
+        `setup_periodic_tasks` may fail silently. This test ensures that it doesn't.
+        """
+        try:
+            setup_periodic_tasks(app)
+        except Exception as exc:
+            assert exc is None, exc