From 0e81cf7b065c01183baf0419a7c459b92141412c Mon Sep 17 00:00:00 2001 From: Ben White Date: Mon, 16 Dec 2024 13:38:34 +0100 Subject: [PATCH 1/3] Added support for cdn purging --- posthog/models/remote_config.py | 28 +++++++++++++++++++++++ posthog/models/test/test_remote_config.py | 23 +++++++++++++++++++ posthog/settings/web.py | 5 ++++ 3 files changed, 56 insertions(+) diff --git a/posthog/models/remote_config.py b/posthog/models/remote_config.py index 3f127a710c3e4..58dd3f9a55a09 100644 --- a/posthog/models/remote_config.py +++ b/posthog/models/remote_config.py @@ -7,6 +7,7 @@ from django.http import HttpRequest from django.utils import timezone from prometheus_client import Counter +import requests from sentry_sdk import capture_exception import structlog @@ -354,6 +355,8 @@ def sync(self): cache.set(cache_key_for_team_token(self.team.api_token, "config"), config, timeout=CACHE_TIMEOUT) + self._purge_cdn() + # TODO: Invalidate caches - in particular this will be the Cloudflare CDN cache self.synced_at = timezone.now() self.save() @@ -365,6 +368,31 @@ def sync(self): CELERY_TASK_REMOTE_CONFIG_SYNC.labels(result="failure").inc() raise + def _purge_cdn(self): + if ( + not settings.REMOTE_CONFIG_CDN_PURGE_ENDPOINT + or not settings.REMOTE_CONFIG_CDN_PURGE_TOKEN + or not settings.REMOTE_CONFIG_CDN_PURGE_DOMAINS + ): + return + + logger.info(f"Purging CDN for team {self.team_id}") + + data = {"files": []} + + for domain in settings.REMOTE_CONFIG_CDN_PURGE_DOMAINS: + # Check if the domain starts with https:// and if not add it + full_domain = domain if domain.startswith("https://") else f"https://{domain}" + data["files"].append({"url": f"{full_domain}/array/{self.team.api_token}/config"}) + data["files"].append({"url": f"{full_domain}/array/{self.team.api_token}/config.js"}) + data["files"].append({"url": f"{full_domain}/array/{self.team.api_token}/array.js"}) + + requests.post( + settings.REMOTE_CONFIG_CDN_PURGE_ENDPOINT, + headers={"Authorization": f"Bearer {settings.REMOTE_CONFIG_CDN_PURGE_TOKEN}"}, + data=data, + ) + def __str__(self): return f"RemoteConfig {self.team_id}" diff --git a/posthog/models/test/test_remote_config.py b/posthog/models/test/test_remote_config.py index fa03badeca141..7bb985b78de6c 100644 --- a/posthog/models/test/test_remote_config.py +++ b/posthog/models/test/test_remote_config.py @@ -440,6 +440,29 @@ def test_only_includes_recording_for_approved_domains(self): config = self.remote_config.get_config_via_token(self.team.api_token, request=mock_request) assert not config["sessionRecording"] + @patch("posthog.models.remote_config.requests.post") + def test_purges_cdn_cache_on_sync(self, mock_post): + with self.settings( + REMOTE_CONFIG_CDN_PURGE_ENDPOINT="https://api.cloudflare.com/client/v4/zones/MY_ZONE_ID/purge_cache", + REMOTE_CONFIG_CDN_PURGE_TOKEN="MY_TOKEN", + REMOTE_CONFIG_CDN_PURGE_DOMAINS=["cdn.posthog.com", "https://cdn2.posthog.com"], + ): + self.remote_config.sync() + mock_post.assert_called_once_with( + "https://api.cloudflare.com/client/v4/zones/MY_ZONE_ID/purge_cache", + headers={"Authorization": "Bearer MY_TOKEN"}, + data={ + "files": [ + {"url": "https://cdn.posthog.com/array/phc_12345/config"}, + {"url": "https://cdn.posthog.com/array/phc_12345/config.js"}, + {"url": "https://cdn.posthog.com/array/phc_12345/array.js"}, + {"url": "https://cdn2.posthog.com/array/phc_12345/config"}, + {"url": "https://cdn2.posthog.com/array/phc_12345/config.js"}, + {"url": "https://cdn2.posthog.com/array/phc_12345/array.js"}, + ] + }, + ) + class TestRemoteConfigJS(_RemoteConfigBase): def test_renders_js_including_config(self): diff --git a/posthog/settings/web.py b/posthog/settings/web.py index cca19f6221a50..49c68b0adb978 100644 --- a/posthog/settings/web.py +++ b/posthog/settings/web.py @@ -398,3 +398,8 @@ # disables frontend side navigation hooks to make hot-reload work seamlessly DEV_DISABLE_NAVIGATION_HOOKS = get_from_env("DEV_DISABLE_NAVIGATION_HOOKS", False, type_cast=bool) + + +REMOTE_CONFIG_CDN_PURGE_ENDPOINT = get_from_env("REMOTE_CONFIG_CDN_PURGE_ENDPOINT", "") +REMOTE_CONFIG_CDN_PURGE_TOKEN = get_from_env("REMOTE_CONFIG_CDN_PURGE_TOKEN", "") +REMOTE_CONFIG_CDN_PURGE_DOMAINS = get_list(os.getenv("REMOTE_CONFIG_CDN_PURGE_DOMAINS", "")) From 84ffbe92accaaf999019400e327fce6a2ce5de51 Mon Sep 17 00:00:00 2001 From: Ben White Date: Mon, 16 Dec 2024 13:39:13 +0100 Subject: [PATCH 2/3] Fixes --- posthog/models/remote_config.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/posthog/models/remote_config.py b/posthog/models/remote_config.py index 58dd3f9a55a09..a0dc225724792 100644 --- a/posthog/models/remote_config.py +++ b/posthog/models/remote_config.py @@ -39,6 +39,12 @@ labelnames=["result"], ) +REMOTE_CONFIG_CDN_PURGE_COUNTER = Counter( + "posthog_remote_config_cdn_purge", + "Number of times the remote config CDN purge task has been run", + labelnames=["result"], +) + logger = structlog.get_logger(__name__) @@ -387,11 +393,17 @@ def _purge_cdn(self): data["files"].append({"url": f"{full_domain}/array/{self.team.api_token}/config.js"}) data["files"].append({"url": f"{full_domain}/array/{self.team.api_token}/array.js"}) - requests.post( - settings.REMOTE_CONFIG_CDN_PURGE_ENDPOINT, - headers={"Authorization": f"Bearer {settings.REMOTE_CONFIG_CDN_PURGE_TOKEN}"}, - data=data, - ) + try: + requests.post( + settings.REMOTE_CONFIG_CDN_PURGE_ENDPOINT, + headers={"Authorization": f"Bearer {settings.REMOTE_CONFIG_CDN_PURGE_TOKEN}"}, + data=data, + ) + except Exception: + logger.exception(f"Failed to purge CDN for team {self.team_id}") + REMOTE_CONFIG_CDN_PURGE_COUNTER.labels(result="failure").inc() + else: + REMOTE_CONFIG_CDN_PURGE_COUNTER.labels(result="success").inc() def __str__(self): return f"RemoteConfig {self.team_id}" From 13c5b8a1d3746b5f7851be1114e44b4b9deb416c Mon Sep 17 00:00:00 2001 From: Ben White Date: Mon, 16 Dec 2024 13:46:36 +0100 Subject: [PATCH 3/3] fix --- posthog/models/remote_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/posthog/models/remote_config.py b/posthog/models/remote_config.py index a0dc225724792..4410392ffda54 100644 --- a/posthog/models/remote_config.py +++ b/posthog/models/remote_config.py @@ -384,7 +384,7 @@ def _purge_cdn(self): logger.info(f"Purging CDN for team {self.team_id}") - data = {"files": []} + data: dict[str, Any] = {"files": []} for domain in settings.REMOTE_CONFIG_CDN_PURGE_DOMAINS: # Check if the domain starts with https:// and if not add it