fix(plugin-server): jitter and retry reloadPlugins

plugin-server/src/worker/tasks.ts

@@ -1,4 +1,7 @@
 import { PluginEvent } from '@posthog/plugin-scaffold/src/types'
+import { retryIfRetriable } from 'utils/retries'
+import { status } from 'utils/status'
+import { sleep } from 'utils/utils'
 
 import { EnqueuedPluginJob, Hub, PluginTaskType } from '../types'
 import { loadSchedule } from './plugins/loadSchedule'
@@ -8,6 +11,8 @@ import { teardownPlugins } from './plugins/teardown'
 
 type TaskRunner = (hub: Hub, args: any) => Promise<any> | any
 
+const RELOAD_PLUGIN_JITTER_MAX_MS = 60000
+
 export const workerTasks: Record<string, TaskRunner> = {
     runPluginJob: (hub, { job }: { job: EnqueuedPluginJob }) => {
         return runPluginTask(hub, job.type, PluginTaskType.Job, job.pluginConfigId, job.payload)
@@ -28,7 +33,11 @@ export const workerTasks: Record<string, TaskRunner> = {
         return hub.pluginSchedule !== null
     },
     reloadPlugins: async (hub) => {
-        await setupPlugins(hub)
+        // Jitter the reload time to avoid all workers reloading at the same time.
+        const jitterMs = Math.random() * RELOAD_PLUGIN_JITTER_MAX_MS
+        status.info('💤', `Sleeping for ${jitterMs}ms to jitter reloadPlugins`)
+        await sleep(jitterMs)
+        await retryIfRetriable(async () => await setupPlugins(hub))
     },
     reloadSchedule: async (hub) => {
         await loadSchedule(hub)