diff --git a/latest_migrations.manifest b/latest_migrations.manifest index a446e72c61968..6338f39708702 100644 --- a/latest_migrations.manifest +++ b/latest_migrations.manifest @@ -5,7 +5,7 @@ contenttypes: 0002_remove_content_type_name ee: 0016_rolemembership_organization_member otp_static: 0002_throttling otp_totp: 0002_auto_20190420_0723 -posthog: 0448_add_mysql_externaldatasource_source_type +posthog: 0449_alter_plugin_organization_alter_plugin_plugin_type_and_more sessions: 0001_initial social_django: 0010_uid_db_index two_factor: 0007_auto_20201201_1019 diff --git a/plugin-server/functional_tests/plugins.test.ts b/plugin-server/functional_tests/plugins.test.ts index db56c3f2cefdb..e9129c0ae5188 100644 --- a/plugin-server/functional_tests/plugins.test.ts +++ b/plugin-server/functional_tests/plugins.test.ts @@ -583,7 +583,7 @@ test.concurrent('plugins can use attachements', async () => { key: 'testAttachment', contents: 'test', }) - await enablePluginConfig(teamId, plugin.id) + await enablePluginConfig(teamId, pluginConfig.id) await reloadPlugins() diff --git a/plugin-server/src/capabilities.ts b/plugin-server/src/capabilities.ts index 9bfe5a642155e..11158a284b951 100644 --- a/plugin-server/src/capabilities.ts +++ b/plugin-server/src/capabilities.ts @@ -26,6 +26,7 @@ export function getPluginServerCapabilities(config: PluginsServerConfig): Plugin cdpProcessedEvents: true, cdpFunctionCallbacks: true, cdpFunctionOverflow: true, + syncInlinePlugins: true, ...sharedCapabilities, } case PluginServerMode.ingestion: @@ -89,6 +90,7 @@ export function getPluginServerCapabilities(config: PluginsServerConfig): Plugin return { pluginScheduledTasks: true, appManagementSingleton: true, + syncInlinePlugins: true, ...sharedCapabilities, } case PluginServerMode.cdp_processed_events: @@ -121,6 +123,7 @@ export function getPluginServerCapabilities(config: PluginsServerConfig): Plugin sessionRecordingBlobIngestion: true, appManagementSingleton: true, preflightSchedules: true, + syncInlinePlugins: true, ...sharedCapabilities, } } diff --git a/plugin-server/src/main/pluginsServer.ts b/plugin-server/src/main/pluginsServer.ts index d8d619be7e7b3..d12a2f4362fe1 100644 --- a/plugin-server/src/main/pluginsServer.ts +++ b/plugin-server/src/main/pluginsServer.ts @@ -28,6 +28,7 @@ import { OrganizationManager } from '../worker/ingestion/organization-manager' import { TeamManager } from '../worker/ingestion/team-manager' import Piscina, { makePiscina as defaultMakePiscina } from '../worker/piscina' import { RustyHook } from '../worker/rusty-hook' +import { syncInlinePlugins } from '../worker/vm/inline/inline' import { GraphileWorker } from './graphile-worker/graphile-worker' import { loadPluginSchedule } from './graphile-worker/schedule' import { startGraphileWorker } from './graphile-worker/worker-setup' @@ -439,6 +440,13 @@ export async function startPluginsServer( healthChecks['webhooks-ingestion'] = isWebhooksIngestionHealthy } + if (capabilities.syncInlinePlugins) { + ;[hub, closeHub] = hub ? [hub, closeHub] : await createHub(serverConfig, capabilities) + serverInstance = serverInstance ? serverInstance : { hub } + + await syncInlinePlugins(hub) + } + if (hub && serverInstance) { pubSub = new PubSub(hub, { [hub.PLUGINS_RELOAD_PUBSUB_CHANNEL]: async () => { diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts index c4df28fa9e798..92ec13670deed 100644 --- a/plugin-server/src/types.ts +++ b/plugin-server/src/types.ts @@ -33,7 +33,7 @@ import { TeamManager } from './worker/ingestion/team-manager' import { RustyHook } from './worker/rusty-hook' import { PluginsApiKeyManager } from './worker/vm/extensions/helpers/api-key-manager' import { RootAccessManager } from './worker/vm/extensions/helpers/root-acess-manager' -import { LazyPluginVM } from './worker/vm/lazy' +import { PluginInstance } from './worker/vm/lazy' export { Element } from '@posthog/plugin-scaffold' // Re-export Element from scaffolding, for backwards compat. @@ -314,7 +314,7 @@ export interface Hub extends PluginsServerConfig { // diagnostics lastActivity: number lastActivityType: string - statelessVms: StatelessVmMap + statelessVms: StatelessInstanceMap conversionBufferEnabledTeams: Set // functions enqueuePluginJob: (job: EnqueuedPluginJob) => Promise @@ -344,6 +344,7 @@ export interface PluginServerCapabilities { preflightSchedules?: boolean // Used for instance health checks on hobby deploy, not useful on cloud http?: boolean mmdb?: boolean + syncInlinePlugins?: boolean } export type EnqueuedJob = EnqueuedPluginJob | GraphileWorkerCronScheduleJob @@ -394,9 +395,9 @@ export interface JobSpec { export interface Plugin { id: number - organization_id: string + organization_id?: string name: string - plugin_type: 'local' | 'respository' | 'custom' | 'source' + plugin_type: 'local' | 'respository' | 'custom' | 'source' | 'inline' description?: string is_global: boolean is_preinstalled?: boolean @@ -443,7 +444,7 @@ export interface PluginConfig { order: number config: Record attachments?: Record - vm?: LazyPluginVM | null + instance?: PluginInstance | null created_at: string updated_at?: string // We're migrating to a new functions that take PostHogEvent instead of PluginEvent @@ -528,7 +529,7 @@ export interface PluginTask { __ignoreForAppMetrics?: boolean } -export type VMMethods = { +export type PluginMethods = { setupPlugin?: () => Promise teardownPlugin?: () => Promise getSettings?: () => PluginSettings @@ -538,7 +539,7 @@ export type VMMethods = { } // Helper when ensuring that a required method is implemented -export type VMMethodsConcrete = Required +export type PluginMethodsConcrete = Required export enum AlertLevel { P0 = 0, @@ -565,7 +566,7 @@ export interface Alert { } export interface PluginConfigVMResponse { vm: VM - methods: VMMethods + methods: PluginMethods tasks: Record> vmResponseVariable: string usedImports: Set @@ -1150,7 +1151,7 @@ export enum PropertyUpdateOperation { SetOnce = 'set_once', } -export type StatelessVmMap = Record +export type StatelessInstanceMap = Record export enum OrganizationPluginsAccessLevel { NONE = 0, diff --git a/plugin-server/src/utils/db/sql.ts b/plugin-server/src/utils/db/sql.ts index 6aab87a5f9ceb..37f2bfeff4384 100644 --- a/plugin-server/src/utils/db/sql.ts +++ b/plugin-server/src/utils/db/sql.ts @@ -1,4 +1,5 @@ import { Hub, Plugin, PluginAttachmentDB, PluginCapabilities, PluginConfig, PluginConfigId } from '../../types' +import { InlinePluginDescription } from '../../worker/vm/inline/inline' import { PostgresUse } from './postgres' function pluginConfigsInForceQuery(specificField?: keyof PluginConfig): string { @@ -58,6 +59,49 @@ const PLUGIN_SELECT = `SELECT LEFT JOIN posthog_pluginsourcefile psf__site_ts ON (psf__site_ts.plugin_id = posthog_plugin.id AND psf__site_ts.filename = 'site.ts')` +const PLUGIN_UPSERT_RETURNING = `INSERT INTO posthog_plugin + ( + name, + url, + tag, + from_json, + from_web, + error, + plugin_type, + organization_id, + is_global, + capabilities, + public_jobs, + is_stateless, + log_level, + description, + is_preinstalled, + config_schema, + updated_at, + created_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, NOW(), NOW()) + ON CONFLICT (url) + DO UPDATE SET + name = $1, + tag = $3, + from_json = $4, + from_web = $5, + error = $6, + plugin_type = $7, + organization_id = $8, + is_global = $9, + capabilities = $10, + public_jobs = $11, + is_stateless = $12, + log_level = $13, + description = $14, + is_preinstalled = $15, + config_schema = $16, + updated_at = NOW() + RETURNING * +` + export async function getPlugin(hub: Hub, pluginId: number): Promise { const result = await hub.db.postgres.query( PostgresUse.COMMON_READ, @@ -68,14 +112,14 @@ export async function getPlugin(hub: Hub, pluginId: number): Promise { +export async function getActivePluginRows(hub: Hub): Promise { const { rows }: { rows: Plugin[] } = await hub.db.postgres.query( PostgresUse.COMMON_READ, `${PLUGIN_SELECT} WHERE posthog_plugin.id IN (${pluginConfigsInForceQuery('plugin_id')} GROUP BY posthog_pluginconfig.plugin_id)`, undefined, - 'getPluginRows' + 'getActivePluginRows' ) return rows @@ -124,3 +168,53 @@ export async function disablePlugin(hub: Hub, pluginConfigId: PluginConfigId): P ) await hub.db.redisPublish(hub.PLUGINS_RELOAD_PUBSUB_CHANNEL, 'reload!') } + +// Given an inline plugin description, upsert it into the known plugins table, returning the full +// Plugin object. Matching is done based on plugin url, not id, since that varies by region. +export async function upsertInlinePlugin(hub: Hub, inline: InlinePluginDescription): Promise { + const fullPlugin: Plugin = { + id: 0, + name: inline.name, + url: inline.url, + tag: inline.tag, + from_json: false, + from_web: false, + error: undefined, + plugin_type: 'inline', + organization_id: undefined, + is_global: inline.is_global, + capabilities: inline.capabilities, + public_jobs: undefined, + is_stateless: inline.is_stateless, + log_level: inline.log_level, + description: inline.description, + is_preinstalled: inline.is_preinstalled, + config_schema: inline.config_schema, + } + + const { rows }: { rows: Plugin[] } = await hub.db.postgres.query( + PostgresUse.COMMON_WRITE, + `${PLUGIN_UPSERT_RETURNING}`, + [ + fullPlugin.name, + fullPlugin.url, + fullPlugin.tag, + fullPlugin.from_json, + fullPlugin.from_web, + fullPlugin.error, + fullPlugin.plugin_type, + fullPlugin.organization_id, + fullPlugin.is_global, + fullPlugin.capabilities, + fullPlugin.public_jobs, + fullPlugin.is_stateless, + fullPlugin.log_level, + fullPlugin.description, + fullPlugin.is_preinstalled, + JSON.stringify(fullPlugin.config_schema), + ], + 'upsertInlinePlugin' + ) + + return rows[0] +} diff --git a/plugin-server/src/worker/plugins/loadPlugin.ts b/plugin-server/src/worker/plugins/loadPlugin.ts index 26a7d45f97e62..a264961ad8c6c 100644 --- a/plugin-server/src/worker/plugins/loadPlugin.ts +++ b/plugin-server/src/worker/plugins/loadPlugin.ts @@ -18,10 +18,16 @@ export async function loadPlugin(hub: Hub, pluginConfig: PluginConfig): Promise< const isLocalPlugin = plugin?.plugin_type === 'local' if (!plugin) { - pluginConfig.vm?.failInitialization!() + pluginConfig.instance?.failInitialization!() return false } + // Inline plugins don't need "loading", and have no source files. + if (plugin.plugin_type === 'inline') { + await pluginConfig.instance?.initialize!('', pluginDigest(plugin)) + return true + } + try { // load config json const configJson = isLocalPlugin @@ -32,7 +38,7 @@ export async function loadPlugin(hub: Hub, pluginConfig: PluginConfig): Promise< try { config = JSON.parse(configJson) } catch (e) { - pluginConfig.vm?.failInitialization!() + pluginConfig.instance?.failInitialization!() await processError(hub, pluginConfig, `Could not load "plugin.json" for ${pluginDigest(plugin)}`) return false } @@ -46,11 +52,11 @@ export async function loadPlugin(hub: Hub, pluginConfig: PluginConfig): Promise< readFileIfExists(hub.BASE_DIR, plugin, 'index.ts') : plugin.source__index_ts if (pluginSource) { - void pluginConfig.vm?.initialize!(pluginSource, pluginDigest(plugin)) + void pluginConfig.instance?.initialize!(pluginSource, pluginDigest(plugin)) return true } else { // always call this if no backend app present, will signal that the VM is done - pluginConfig.vm?.failInitialization!() + pluginConfig.instance?.failInitialization!() // if there is a frontend or site app, don't save an error if no backend app const hasFrontend = isLocalPlugin @@ -72,7 +78,7 @@ export async function loadPlugin(hub: Hub, pluginConfig: PluginConfig): Promise< } } } catch (error) { - pluginConfig.vm?.failInitialization!() + pluginConfig.instance?.failInitialization!() await processError(hub, pluginConfig, error) } return false diff --git a/plugin-server/src/worker/plugins/loadPluginsFromDB.ts b/plugin-server/src/worker/plugins/loadPluginsFromDB.ts index 81282e0646794..b36fb0e251141 100644 --- a/plugin-server/src/worker/plugins/loadPluginsFromDB.ts +++ b/plugin-server/src/worker/plugins/loadPluginsFromDB.ts @@ -2,7 +2,7 @@ import { PluginAttachment } from '@posthog/plugin-scaffold' import { Summary } from 'prom-client' import { Hub, Plugin, PluginConfig, PluginConfigId, PluginId, PluginMethod, TeamId } from '../../types' -import { getPluginAttachmentRows, getPluginConfigRows, getPluginRows } from '../../utils/db/sql' +import { getActivePluginRows, getPluginAttachmentRows, getPluginConfigRows } from '../../utils/db/sql' const loadPluginsMsSummary = new Summary({ name: 'load_plugins_ms', @@ -29,7 +29,7 @@ export async function loadPluginsFromDB( hub: Hub ): Promise> { const startTimer = new Date() - const pluginRows = await getPluginRows(hub) + const pluginRows = await getActivePluginRows(hub) const plugins = new Map() for (const row of pluginRows) { @@ -78,7 +78,7 @@ export async function loadPluginsFromDB( ...row, plugin: plugin, attachments: attachmentsPerConfig.get(row.id) || {}, - vm: null, + instance: null, method, } pluginConfigs.set(row.id, pluginConfig) diff --git a/plugin-server/src/worker/plugins/loadSchedule.ts b/plugin-server/src/worker/plugins/loadSchedule.ts index 6c5c4684d7390..ff54dae570aa1 100644 --- a/plugin-server/src/worker/plugins/loadSchedule.ts +++ b/plugin-server/src/worker/plugins/loadSchedule.ts @@ -18,7 +18,7 @@ export async function loadSchedule(server: Hub): Promise { let count = 0 for (const [id, pluginConfig] of server.pluginConfigs) { - const tasks = (await pluginConfig.vm?.getScheduledTasks()) ?? {} + const tasks = (await pluginConfig.instance?.getScheduledTasks()) ?? {} for (const [taskName, task] of Object.entries(tasks)) { if (task && taskName in pluginSchedule) { pluginSchedule[taskName].push(id) diff --git a/plugin-server/src/worker/plugins/run.ts b/plugin-server/src/worker/plugins/run.ts index 7b24bc10a4a0e..4fb0635994aaf 100644 --- a/plugin-server/src/worker/plugins/run.ts +++ b/plugin-server/src/worker/plugins/run.ts @@ -1,6 +1,6 @@ import { PluginEvent, Webhook } from '@posthog/plugin-scaffold' -import { Hub, PluginConfig, PluginTaskType, PostIngestionEvent, VMMethodsConcrete } from '../../types' +import { Hub, PluginConfig, PluginMethodsConcrete, PluginTaskType, PostIngestionEvent } from '../../types' import { processError } from '../../utils/db/error' import { convertToOnEventPayload, @@ -19,7 +19,7 @@ async function runSingleTeamPluginOnEvent( hub: Hub, event: PostIngestionEvent, pluginConfig: PluginConfig, - onEvent: VMMethodsConcrete['onEvent'] + onEvent: PluginMethodsConcrete['onEvent'] ): Promise { const timeout = setTimeout(() => { status.warn('⌛', `Still running single onEvent plugin for team ${event.teamId} for plugin ${pluginConfig.id}`) @@ -85,7 +85,7 @@ async function runSingleTeamPluginComposeWebhook( hub: Hub, postIngestionEvent: PostIngestionEvent, pluginConfig: PluginConfig, - composeWebhook: VMMethodsConcrete['composeWebhook'] + composeWebhook: PluginMethodsConcrete['composeWebhook'] ): Promise { // 1. Calls `composeWebhook` for the plugin, send `composeWebhook` appmetric success/fail if applicable. // 2. Send via Rusty-Hook if enabled. @@ -329,7 +329,7 @@ export async function runPluginTask( let shouldQueueAppMetric = false try { - const task = await pluginConfig?.vm?.getTask(taskName, taskType) + const task = await pluginConfig?.instance?.getTask(taskName, taskType) if (!task) { throw new Error( `Task "${taskName}" not found for plugin "${pluginConfig?.plugin?.name}" with config id ${pluginConfigId}` @@ -381,23 +381,23 @@ export async function runPluginTask( return response } -async function getPluginMethodsForTeam( +async function getPluginMethodsForTeam( hub: Hub, teamId: number, method: M -): Promise<[PluginConfig, VMMethodsConcrete[M]][]> { +): Promise<[PluginConfig, PluginMethodsConcrete[M]][]> { const pluginConfigs = hub.pluginConfigsPerTeam.get(teamId) || [] if (pluginConfigs.length === 0) { return [] } const methodsObtained = await Promise.all( - pluginConfigs.map(async (pluginConfig) => [pluginConfig, await pluginConfig?.vm?.getVmMethod(method)]) + pluginConfigs.map(async (pluginConfig) => [pluginConfig, await pluginConfig?.instance?.getPluginMethod(method)]) ) const methodsObtainedFiltered = methodsObtained.filter(([_, method]) => !!method) as [ PluginConfig, - VMMethodsConcrete[M] + PluginMethodsConcrete[M] ][] return methodsObtainedFiltered diff --git a/plugin-server/src/worker/plugins/setup.ts b/plugin-server/src/worker/plugins/setup.ts index b2e4e0bdd0f0c..161309f76877a 100644 --- a/plugin-server/src/worker/plugins/setup.ts +++ b/plugin-server/src/worker/plugins/setup.ts @@ -1,8 +1,8 @@ import { Gauge, Summary } from 'prom-client' -import { Hub, StatelessVmMap } from '../../types' +import { Hub, StatelessInstanceMap } from '../../types' import { status } from '../../utils/status' -import { LazyPluginVM } from '../vm/lazy' +import { constructPluginInstance } from '../vm/lazy' import { loadPlugin } from './loadPlugin' import { loadPluginsFromDB } from './loadPluginsFromDB' import { loadSchedule } from './loadSchedule' @@ -24,7 +24,7 @@ export async function setupPlugins(hub: Hub): Promise { status.info('🔁', `Loading plugin configs...`) const { plugins, pluginConfigs, pluginConfigsPerTeam } = await loadPluginsFromDB(hub) const pluginVMLoadPromises: Array> = [] - const statelessVms = {} as StatelessVmMap + const statelessInstances = {} as StatelessInstanceMap const timer = new Date() @@ -37,11 +37,11 @@ export async function setupPlugins(hub: Hub): Promise { const pluginChanged = plugin?.updated_at !== prevPlugin?.updated_at if (!pluginConfigChanged && !pluginChanged) { - pluginConfig.vm = prevConfig.vm - } else if (plugin?.is_stateless && statelessVms[plugin.id]) { - pluginConfig.vm = statelessVms[plugin.id] + pluginConfig.instance = prevConfig.instance + } else if (plugin?.is_stateless && statelessInstances[plugin.id]) { + pluginConfig.instance = statelessInstances[plugin.id] } else { - pluginConfig.vm = new LazyPluginVM(hub, pluginConfig) + pluginConfig.instance = constructPluginInstance(hub, pluginConfig) if (hub.PLUGIN_LOAD_SEQUENTIALLY) { await loadPlugin(hub, pluginConfig) } else { @@ -52,7 +52,7 @@ export async function setupPlugins(hub: Hub): Promise { } if (plugin?.is_stateless) { - statelessVms[plugin.id] = pluginConfig.vm + statelessInstances[plugin.id] = pluginConfig.instance } } } @@ -67,7 +67,7 @@ export async function setupPlugins(hub: Hub): Promise { importUsedGauge.reset() const seenPlugins = new Set() for (const pluginConfig of pluginConfigs.values()) { - const usedImports = pluginConfig.vm?.usedImports + const usedImports = pluginConfig.instance?.usedImports if (usedImports && !seenPlugins.has(pluginConfig.plugin_id)) { seenPlugins.add(pluginConfig.plugin_id) for (const importName of usedImports) { diff --git a/plugin-server/src/worker/plugins/teardown.ts b/plugin-server/src/worker/plugins/teardown.ts index 8d465a7644369..4fe1a4f52c19e 100644 --- a/plugin-server/src/worker/plugins/teardown.ts +++ b/plugin-server/src/worker/plugins/teardown.ts @@ -6,9 +6,9 @@ export async function teardownPlugins(server: Hub, pluginConfig?: PluginConfig): const teardownPromises: Promise[] = [] for (const pluginConfig of pluginConfigs) { - if (pluginConfig.vm) { - pluginConfig.vm.clearRetryTimeoutIfExists() - const teardownPlugin = await pluginConfig.vm.getTeardownPlugin() + if (pluginConfig.instance) { + pluginConfig.instance.clearRetryTimeoutIfExists() + const teardownPlugin = await pluginConfig.instance.getTeardown() if (teardownPlugin) { teardownPromises.push( (async () => { diff --git a/plugin-server/src/worker/vm/capabilities.ts b/plugin-server/src/worker/vm/capabilities.ts index 5c4fa2e90386e..daa12444eb9be 100644 --- a/plugin-server/src/worker/vm/capabilities.ts +++ b/plugin-server/src/worker/vm/capabilities.ts @@ -1,17 +1,17 @@ -import { PluginCapabilities, PluginTask, PluginTaskType, VMMethods } from '../../types' +import { PluginCapabilities, PluginMethods, PluginTask, PluginTaskType } from '../../types' import { PluginServerCapabilities } from './../../types' const PROCESS_EVENT_CAPABILITIES = new Set(['ingestion', 'ingestionOverflow', 'ingestionHistorical']) export function getVMPluginCapabilities( - methods: VMMethods, + methods: PluginMethods, tasks: Record> ): PluginCapabilities { const capabilities: Required = { scheduled_tasks: [], jobs: [], methods: [] } if (methods) { for (const [key, value] of Object.entries(methods)) { - if (value as VMMethods[keyof VMMethods] | undefined) { + if (value as PluginMethods[keyof PluginMethods] | undefined) { capabilities.methods.push(key) } } diff --git a/plugin-server/src/worker/vm/extensions/jobs.ts b/plugin-server/src/worker/vm/extensions/jobs.ts index cdeaa9c1ff45b..3d9ffac9a35b9 100644 --- a/plugin-server/src/worker/vm/extensions/jobs.ts +++ b/plugin-server/src/worker/vm/extensions/jobs.ts @@ -64,7 +64,7 @@ export function createJobs(server: Hub, pluginConfig: PluginConfig): Jobs { pluginJobEnqueueCounter.labels(String(pluginConfig.plugin?.id)).inc() await server.enqueuePluginJob(job) } catch (e) { - await pluginConfig.vm?.createLogEntry( + await pluginConfig.instance?.createLogEntry( `Failed to enqueue job ${type} with error: ${e.message}`, PluginLogEntryType.Error ) diff --git a/plugin-server/src/worker/vm/inline/inline.ts b/plugin-server/src/worker/vm/inline/inline.ts new file mode 100644 index 0000000000000..42a90248c5c4b --- /dev/null +++ b/plugin-server/src/worker/vm/inline/inline.ts @@ -0,0 +1,92 @@ +import { PluginConfigSchema } from '@posthog/plugin-scaffold' + +import { Hub, PluginCapabilities, PluginConfig, PluginLogLevel } from '../../../types' +import { upsertInlinePlugin } from '../../../utils/db/sql' +import { status } from '../../../utils/status' +import { PluginInstance } from '../lazy' +import { NoopInlinePlugin } from './noop' +import { SEMVER_FLATTENER_CONFIG_SCHEMA, SemverFlattener } from './semver-flattener' + +export function constructInlinePluginInstance(hub: Hub, pluginConfig: PluginConfig): PluginInstance { + const url = pluginConfig.plugin?.url + + if (!INLINE_PLUGIN_URLS.includes(url as InlinePluginId)) { + throw new Error(`Invalid inline plugin URL: ${url}`) + } + const plugin = INLINE_PLUGIN_MAP[url as InlinePluginId] + + return plugin.constructor(hub, pluginConfig) +} + +export interface RegisteredInlinePlugin { + constructor: (hub: Hub, config: PluginConfig) => PluginInstance + description: Readonly +} + +export const INLINE_PLUGIN_URLS = ['inline://noop', 'inline://semver-flattener'] as const +type InlinePluginId = (typeof INLINE_PLUGIN_URLS)[number] + +// TODO - add all inline plugins here +export const INLINE_PLUGIN_MAP: Record = { + 'inline://noop': { + constructor: (hub: Hub, config: PluginConfig) => new NoopInlinePlugin(hub, config), + description: { + name: 'Noop Plugin', + description: 'A plugin that does nothing', + is_global: false, + is_preinstalled: false, + url: 'inline://noop', + config_schema: {}, + tag: 'noop', + capabilities: {}, + is_stateless: true, + log_level: PluginLogLevel.Info, + }, + }, + + 'inline://semver-flattener': { + constructor: (hub: Hub, config: PluginConfig) => new SemverFlattener(hub, config), + description: { + name: 'posthog-semver-flattener', + description: + 'Processes specified properties to flatten sematic versions. Assumes any property contains a string which matches [the SemVer specification](https://semver.org/#backusnaur-form-grammar-for-valid-semver-versions)', + is_global: false, + is_preinstalled: false, + url: 'inline://semver-flattener', + config_schema: SEMVER_FLATTENER_CONFIG_SCHEMA, + tag: 'semver-flattener', + capabilities: { + jobs: [], + scheduled_tasks: [], + methods: ['processEvent'], + }, + is_stateless: false, // TODO - this plugin /could/ be stateless, but right now we cache config parsing, which is stateful + log_level: PluginLogLevel.Info, + }, + }, +} + +// Inline plugins are uniquely identified by their /url/, not their ID, and do +// not have most of the standard plugin properties. This reduced interface is +// the "canonical" description of an inline plugin, but can be mapped to a region +// specific Plugin object by url. +export interface InlinePluginDescription { + name: string + description: string + is_global: boolean + is_preinstalled: boolean + url: string + config_schema: Record | PluginConfigSchema[] + tag: string + capabilities: PluginCapabilities + is_stateless: boolean + log_level: PluginLogLevel +} + +export async function syncInlinePlugins(hub: Hub): Promise { + status.info('⚡', 'Syncing inline plugins') + for (const url of INLINE_PLUGIN_URLS) { + const plugin = INLINE_PLUGIN_MAP[url] + await upsertInlinePlugin(hub, plugin.description) + } +} diff --git a/plugin-server/src/worker/vm/inline/noop.ts b/plugin-server/src/worker/vm/inline/noop.ts new file mode 100644 index 0000000000000..aaa80d8b1007f --- /dev/null +++ b/plugin-server/src/worker/vm/inline/noop.ts @@ -0,0 +1,68 @@ +import { PluginEvent } from '@posthog/plugin-scaffold' + +import { + Hub, + PluginConfig, + PluginLogEntrySource, + PluginLogEntryType, + PluginMethods, + PluginTask, + PluginTaskType, +} from '../../../types' +import { PluginInstance } from '../lazy' + +export class NoopInlinePlugin implements PluginInstance { + // The noop plugin has no initialization behavior, or imports + initialize = async () => {} + failInitialization = () => {} + usedImports: Set | undefined + methods: PluginMethods + + hub: Hub + config: PluginConfig + + constructor(hub: Hub, pluginConfig: PluginConfig) { + this.hub = hub + this.config = pluginConfig + this.usedImports = new Set() + + this.methods = { + processEvent: (event: PluginEvent) => { + return Promise.resolve(event) + }, + } + } + + public getTeardown(): Promise { + return Promise.resolve(null) + } + + public getTask(_name: string, _type: PluginTaskType): Promise { + return Promise.resolve(null) + } + + public getScheduledTasks(): Promise> { + return Promise.resolve({}) + } + + public getPluginMethod(method_name: T): Promise { + return Promise.resolve(this.methods[method_name] as PluginMethods[T]) + } + + public clearRetryTimeoutIfExists = () => {} + + public setupPluginIfNeeded(): Promise { + return Promise.resolve(true) + } + + public async createLogEntry(message: string, logType = PluginLogEntryType.Info): Promise { + // TODO - this will be identical across all plugins, so figure out a better place to put it. + await this.hub.db.queuePluginLogEntry({ + message, + pluginConfig: this.config, + source: PluginLogEntrySource.System, + type: logType, + instanceId: this.hub.instanceId, + }) + } +} diff --git a/plugin-server/src/worker/vm/inline/semver-flattener.ts b/plugin-server/src/worker/vm/inline/semver-flattener.ts new file mode 100644 index 0000000000000..50290c6f5066e --- /dev/null +++ b/plugin-server/src/worker/vm/inline/semver-flattener.ts @@ -0,0 +1,135 @@ +import { PluginEvent } from '@posthog/plugin-scaffold' + +import { + Hub, + PluginConfig, + PluginLogEntrySource, + PluginLogEntryType, + PluginMethods, + PluginTask, + PluginTaskType, +} from '../../../types' +import { PluginInstance } from '../lazy' + +export class SemverFlattener implements PluginInstance { + initialize = async () => {} + failInitialization = async () => {} + clearRetryTimeoutIfExists = () => {} + usedImports: Set | undefined + methods: PluginMethods + + hub: Hub + config: PluginConfig + targetProps: string[] + + constructor(hub: Hub, pluginConfig: PluginConfig) { + this.hub = hub + this.config = pluginConfig + this.usedImports = new Set() + + this.targetProps = (this.config.config.properties as string)?.split(',').map((s) => s.trim()) + if (!this.targetProps) { + this.targetProps = [] + } + + this.methods = { + processEvent: (event: PluginEvent) => { + return Promise.resolve(this.flattenSemver(event)) + }, + } + } + + public getTeardown(): Promise { + return Promise.resolve(null) + } + + public getTask(_name: string, _type: PluginTaskType): Promise { + return Promise.resolve(null) + } + + public getScheduledTasks(): Promise> { + return Promise.resolve({}) + } + + public getPluginMethod(method_name: T): Promise { + return Promise.resolve(this.methods[method_name] as PluginMethods[T]) + } + + public setupPluginIfNeeded(): Promise { + return Promise.resolve(true) + } + + public async createLogEntry(message: string, logType = PluginLogEntryType.Info): Promise { + // TODO - this will be identical across all plugins, so figure out a better place to put it. + await this.hub.db.queuePluginLogEntry({ + message, + pluginConfig: this.config, + source: PluginLogEntrySource.System, + type: logType, + instanceId: this.hub.instanceId, + }) + } + + flattenSemver(event: PluginEvent): PluginEvent { + if (!event.properties) { + return event + } + + for (const target of this.targetProps) { + const candidate = event.properties[target] + + if (candidate) { + const { major, minor, patch, preRelease, build } = splitVersion(candidate) + event.properties[`${target}__major`] = major + event.properties[`${target}__minor`] = minor + if (patch !== undefined) { + event.properties[`${target}__patch`] = patch + } + if (preRelease !== undefined) { + event.properties[`${target}__preRelease`] = preRelease + } + if (build !== undefined) { + event.properties[`${target}__build`] = build + } + } + } + + return event + } +} + +export interface VersionParts { + major: number + minor: number + patch?: number + preRelease?: string + build?: string +} + +const splitVersion = (candidate: string): VersionParts => { + const [head, build] = candidate.split('+') + const [version, ...preRelease] = head.split('-') + const [major, minor, patch] = version.split('.') + return { + major: Number(major), + minor: Number(minor), + patch: patch ? Number(patch) : undefined, + preRelease: preRelease.join('-') || undefined, + build, + } +} + +export const SEMVER_FLATTENER_CONFIG_SCHEMA = [ + { + markdown: + 'Processes specified properties to flatten sematic versions. Assumes any property contains a string which matches [the SemVer specification](https://semver.org/#backusnaur-form-grammar-for-valid-semver-versions)', + }, + { + key: 'properties', + name: 'comma separated properties to explode version number from', + type: 'string' as const, + hint: 'my_version_number,app_version', + default: '', + required: true, + }, +] diff --git a/plugin-server/src/worker/vm/lazy.ts b/plugin-server/src/worker/vm/lazy.ts index 9c1964a792269..c873c4a437c7e 100644 --- a/plugin-server/src/worker/vm/lazy.ts +++ b/plugin-server/src/worker/vm/lazy.ts @@ -9,9 +9,9 @@ import { PluginConfigVMResponse, PluginLogEntrySource, PluginLogEntryType, + PluginMethods, PluginTask, PluginTaskType, - VMMethods, } from '../../types' import { processError } from '../../utils/db/error' import { disablePlugin, getPlugin, setPluginCapabilities } from '../../utils/db/sql' @@ -20,6 +20,7 @@ import { getNextRetryMs } from '../../utils/retries' import { status } from '../../utils/status' import { pluginDigest } from '../../utils/utils' import { getVMPluginCapabilities, shouldSetupPluginInServer } from '../vm/capabilities' +import { constructInlinePluginInstance } from './inline/inline' import { createPluginConfigVM } from './vm' export const VM_INIT_MAX_RETRIES = 5 @@ -44,7 +45,33 @@ const pluginDisabledBySystemCounter = new Counter({ labelNames: ['plugin_id'], }) -export class LazyPluginVM { +export function constructPluginInstance(hub: Hub, pluginConfig: PluginConfig): PluginInstance { + if (pluginConfig.plugin?.plugin_type == 'inline') { + return constructInlinePluginInstance(hub, pluginConfig) + } + return new LazyPluginVM(hub, pluginConfig) +} + +export interface PluginInstance { + // These are "optional", but if they're not set, loadPlugin will fail + initialize?: (indexJs: string, logInfo: string) => Promise + failInitialization?: () => void + + getTeardown: () => Promise + getTask: (name: string, type: PluginTaskType) => Promise + getScheduledTasks: () => Promise> + getPluginMethod: (method_name: T) => Promise + clearRetryTimeoutIfExists: () => void + setupPluginIfNeeded: () => Promise + + createLogEntry: (message: string, logType?: PluginLogEntryType) => Promise + + // This is only used for metrics, and can probably be dropped as we start to care less about + // what imports are used by plugins (or as inlining more plugins makes imports irrelevant) + usedImports: Set | undefined +} + +export class LazyPluginVM implements PluginInstance { initialize?: (indexJs: string, logInfo: string) => Promise failInitialization?: () => void resolveInternalVm!: Promise @@ -68,15 +95,7 @@ export class LazyPluginVM { this.initVm() } - public async getOnEvent(): Promise { - return await this.getVmMethod('onEvent') - } - - public async getProcessEvent(): Promise { - return await this.getVmMethod('processEvent') - } - - public async getTeardownPlugin(): Promise { + public async getTeardown(): Promise { // if we never ran `setupPlugin`, there's no reason to run `teardownPlugin` - it's essentially "tore down" already if (!this.ready) { return null @@ -112,15 +131,15 @@ export class LazyPluginVM { return tasks || {} } - public async getVmMethod(method: T): Promise { - let vmMethod = (await this.resolveInternalVm)?.methods[method] || null - if (!this.ready && vmMethod) { + public async getPluginMethod(method_name: T): Promise { + let method = (await this.resolveInternalVm)?.methods[method_name] || null + if (!this.ready && method) { const pluginReady = await this.setupPluginIfNeeded() if (!pluginReady) { - vmMethod = null + method = null } } - return vmMethod + return method } public clearRetryTimeoutIfExists(): void { @@ -207,6 +226,7 @@ export class LazyPluginVM { return true } + // TODO - this is only called in tests, try to remove at some point. public async _setupPlugin(vm?: VM): Promise { const logInfo = this.pluginConfig.plugin ? pluginDigest(this.pluginConfig.plugin) diff --git a/plugin-server/tests/helpers/sqlMock.ts b/plugin-server/tests/helpers/sqlMock.ts index 378c6bf6273e9..a323d0fd18cb7 100644 --- a/plugin-server/tests/helpers/sqlMock.ts +++ b/plugin-server/tests/helpers/sqlMock.ts @@ -2,7 +2,9 @@ import * as s from '../../src/utils/db/sql' // mock functions that get data from postgres and give them the right types type UnPromisify = F extends (...args: infer A) => Promise ? (...args: A) => T : never -export const getPluginRows = s.getPluginRows as unknown as jest.MockedFunction> +export const getPluginRows = s.getActivePluginRows as unknown as jest.MockedFunction< + UnPromisify +> export const getPluginAttachmentRows = s.getPluginAttachmentRows as unknown as jest.MockedFunction< UnPromisify > diff --git a/plugin-server/tests/server.test.ts b/plugin-server/tests/server.test.ts index 3f497be03703c..52fe0b989bf40 100644 --- a/plugin-server/tests/server.test.ts +++ b/plugin-server/tests/server.test.ts @@ -58,6 +58,7 @@ describe('server', () => { ingestionHistorical: true, appManagementSingleton: true, preflightSchedules: true, + syncInlinePlugins: true, } ) }) @@ -73,6 +74,7 @@ describe('server', () => { { http: true, eventsIngestionPipelines: true, + syncInlinePlugins: true, } ) }) @@ -95,6 +97,7 @@ describe('server', () => { cdpProcessedEvents: true, cdpFunctionCallbacks: true, cdpFunctionOverflow: true, + syncInlinePlugins: true, } ) }) @@ -112,6 +115,7 @@ describe('server', () => { http: true, sessionRecordingBlobIngestion: true, sessionRecordingBlobOverflowIngestion: true, + syncInlinePlugins: true, } ) }) @@ -126,6 +130,7 @@ describe('server', () => { pluginScheduledTasks: true, processAsyncWebhooksHandlers: true, preflightSchedules: true, + syncInlinePlugins: true, } ) @@ -141,7 +146,7 @@ describe('server', () => { test('starts graphile for scheduled tasks capability', async () => { pluginsServer = await createPluginServer( {}, - { ingestion: true, pluginScheduledTasks: true, processPluginJobs: true } + { ingestion: true, pluginScheduledTasks: true, processPluginJobs: true, syncInlinePlugins: true } ) expect(startGraphileWorker).toHaveBeenCalled() diff --git a/plugin-server/tests/sql.test.ts b/plugin-server/tests/sql.test.ts index 24c294a6a97c2..d23b133b4c5bf 100644 --- a/plugin-server/tests/sql.test.ts +++ b/plugin-server/tests/sql.test.ts @@ -1,7 +1,7 @@ import { Hub } from '../src/types' import { createHub } from '../src/utils/db/hub' import { PostgresUse } from '../src/utils/db/postgres' -import { disablePlugin, getPluginAttachmentRows, getPluginConfigRows, getPluginRows } from '../src/utils/db/sql' +import { disablePlugin, getActivePluginRows, getPluginAttachmentRows, getPluginConfigRows } from '../src/utils/db/sql' import { commonOrganizationId } from './helpers/plugins' import { resetTestDatabase } from './helpers/sql' @@ -66,7 +66,7 @@ describe('sql', () => { expect(rows1).toEqual([expectedRow]) }) - test('getPluginRows', async () => { + test('getActivePluginRows', async () => { const rowsExpected = [ { error: null, @@ -92,7 +92,7 @@ describe('sql', () => { }, ] - const rows1 = await getPluginRows(hub) + const rows1 = await getActivePluginRows(hub) expect(rows1).toEqual(rowsExpected) await hub.db.postgres.query( PostgresUse.COMMON_WRITE, @@ -100,7 +100,7 @@ describe('sql', () => { undefined, 'testTag' ) - const rows2 = await getPluginRows(hub) + const rows2 = await getActivePluginRows(hub) expect(rows2).toEqual(rowsExpected) }) diff --git a/plugin-server/tests/worker/plugins.test.ts b/plugin-server/tests/worker/plugins.test.ts index e43dd0a628ec0..286f289e46cd4 100644 --- a/plugin-server/tests/worker/plugins.test.ts +++ b/plugin-server/tests/worker/plugins.test.ts @@ -8,6 +8,7 @@ import { loadPlugin } from '../../src/worker/plugins/loadPlugin' import { loadSchedule } from '../../src/worker/plugins/loadSchedule' import { runProcessEvent } from '../../src/worker/plugins/run' import { setupPlugins } from '../../src/worker/plugins/setup' +import { LazyPluginVM } from '../../src/worker/vm/lazy' import { commonOrganizationId, mockPluginSourceCode, @@ -64,7 +65,6 @@ describe('plugins', () => { expect(pluginConfig.enabled).toEqual(pluginConfig39.enabled) expect(pluginConfig.order).toEqual(pluginConfig39.order) expect(pluginConfig.config).toEqual(pluginConfig39.config) - expect(pluginConfig.error).toEqual(pluginConfig39.error) expect(pluginConfig.plugin).toEqual({ ...plugin60, @@ -78,16 +78,15 @@ describe('plugins', () => { contents: pluginAttachment1.contents, }, }) - expect(pluginConfig.vm).toBeDefined() - const vm = await pluginConfig.vm!.resolveInternalVm - expect(Object.keys(vm!.methods).sort()).toEqual([ - 'composeWebhook', - 'getSettings', - 'onEvent', - 'processEvent', - 'setupPlugin', - 'teardownPlugin', - ]) + expect(pluginConfig.instance).toBeDefined() + const instance = pluginConfig.instance! + + expect(instance.getPluginMethod('composeWebhook')).toBeDefined() + expect(instance.getPluginMethod('getSettings')).toBeDefined() + expect(instance.getPluginMethod('onEvent')).toBeDefined() + expect(instance.getPluginMethod('processEvent')).toBeDefined() + expect(instance.getPluginMethod('setupPlugin')).toBeDefined() + expect(instance.getPluginMethod('teardownPlugin')).toBeDefined() // async loading of capabilities expect(setPluginCapabilities).toHaveBeenCalled() @@ -101,7 +100,7 @@ describe('plugins', () => { ], ]) - const processEvent = vm!.methods['processEvent']! + const processEvent = await instance.getPluginMethod('processEvent') const event = { event: '$test', properties: {}, team_id: 2 } as PluginEvent await processEvent(event) @@ -135,10 +134,10 @@ describe('plugins', () => { expect(pluginConfigTeam1.plugin).toEqual(plugin) expect(pluginConfigTeam2.plugin).toEqual(plugin) - expect(pluginConfigTeam1.vm).toBeDefined() - expect(pluginConfigTeam2.vm).toBeDefined() + expect(pluginConfigTeam1.instance).toBeDefined() + expect(pluginConfigTeam2.instance).toBeDefined() - expect(pluginConfigTeam1.vm).toEqual(pluginConfigTeam2.vm) + expect(pluginConfigTeam1.instance).toEqual(pluginConfigTeam2.instance) }) test('plugin returns null', async () => { @@ -211,9 +210,11 @@ describe('plugins', () => { const { pluginConfigs } = hub const pluginConfig = pluginConfigs.get(39)! - pluginConfig.vm!.totalInitAttemptsCounter = 20 // prevent more retries + expect(pluginConfig.instance).toBeInstanceOf(LazyPluginVM) + const vm = pluginConfig.instance as LazyPluginVM + vm.totalInitAttemptsCounter = 20 // prevent more retries await delay(4000) // processError is called at end of retries - expect(await pluginConfig.vm!.getScheduledTasks()).toEqual({}) + expect(await pluginConfig.instance!.getScheduledTasks()).toEqual({}) const event = { event: '$test', properties: {}, team_id: 2 } as PluginEvent const returnedEvent = await runProcessEvent(hub, { ...event }) @@ -238,9 +239,11 @@ describe('plugins', () => { const { pluginConfigs } = hub const pluginConfig = pluginConfigs.get(39)! - pluginConfig.vm!.totalInitAttemptsCounter = 20 // prevent more retries + expect(pluginConfig.instance).toBeInstanceOf(LazyPluginVM) + const vm = pluginConfig.instance as LazyPluginVM + vm!.totalInitAttemptsCounter = 20 // prevent more retries await delay(4000) // processError is called at end of retries - expect(await pluginConfig.vm!.getScheduledTasks()).toEqual({}) + expect(await pluginConfig.instance!.getScheduledTasks()).toEqual({}) const event = { event: '$test', properties: {}, team_id: 2 } as PluginEvent const returnedEvent = await runProcessEvent(hub, { ...event }) @@ -308,7 +311,7 @@ describe('plugins', () => { await setupPlugins(hub) const { pluginConfigs } = hub - expect(await pluginConfigs.get(39)!.vm!.getScheduledTasks()).toEqual({}) + expect(await pluginConfigs.get(39)!.instance!.getScheduledTasks()).toEqual({}) const event = { event: '$test', properties: {}, team_id: 2 } as PluginEvent const returnedEvent = await runProcessEvent(hub, { ...event }) @@ -341,7 +344,7 @@ describe('plugins', () => { await setupPlugins(hub) const { pluginConfigs } = hub - expect(await pluginConfigs.get(39)!.vm!.getScheduledTasks()).toEqual({}) + expect(await pluginConfigs.get(39)!.instance!.getScheduledTasks()).toEqual({}) const event = { event: '$test', properties: {}, team_id: 2 } as PluginEvent const returnedEvent = await runProcessEvent(hub, { ...event }) @@ -379,7 +382,7 @@ describe('plugins', () => { `Could not load "plugin.json" for plugin test-maxmind-plugin ID ${plugin60.id} (organization ID ${commonOrganizationId})` ) - expect(await pluginConfigs.get(39)!.vm!.getScheduledTasks()).toEqual({}) + expect(await pluginConfigs.get(39)!.instance!.getScheduledTasks()).toEqual({}) }) test('local plugin with broken plugin.json does not do much', async () => { @@ -403,7 +406,7 @@ describe('plugins', () => { pluginConfigs.get(39)!, expect.stringContaining('Could not load "plugin.json" for plugin ') ) - expect(await pluginConfigs.get(39)!.vm!.getScheduledTasks()).toEqual({}) + expect(await pluginConfigs.get(39)!.instance!.getScheduledTasks()).toEqual({}) unlink() }) @@ -426,7 +429,7 @@ describe('plugins', () => { pluginConfigs.get(39)!, `Could not load source code for plugin test-maxmind-plugin ID 60 (organization ID ${commonOrganizationId}). Tried: index.js` ) - expect(await pluginConfigs.get(39)!.vm!.getScheduledTasks()).toEqual({}) + expect(await pluginConfigs.get(39)!.instance!.getScheduledTasks()).toEqual({}) }) test('plugin config order', async () => { @@ -499,7 +502,7 @@ describe('plugins', () => { const pluginConfig = pluginConfigs.get(39)! - await pluginConfig.vm?.resolveInternalVm + await (pluginConfig.instance as LazyPluginVM)?.resolveInternalVm // async loading of capabilities expect(pluginConfig.plugin!.capabilities!.methods!.sort()).toEqual(['processEvent', 'setupPlugin']) @@ -529,7 +532,7 @@ describe('plugins', () => { const pluginConfig = pluginConfigs.get(39)! - await pluginConfig.vm?.resolveInternalVm + await (pluginConfig.instance as LazyPluginVM)?.resolveInternalVm // async loading of capabilities expect(pluginConfig.plugin!.capabilities!.methods!.sort()).toEqual(['onEvent', 'processEvent']) @@ -553,7 +556,7 @@ describe('plugins', () => { const pluginConfig = pluginConfigs.get(39)! - await pluginConfig.vm?.resolveInternalVm + await (pluginConfig.instance as LazyPluginVM)?.resolveInternalVm // async loading of capabilities expect(pluginConfig.plugin!.capabilities!.methods!.sort()).toEqual(['onEvent', 'processEvent']) @@ -581,7 +584,7 @@ describe('plugins', () => { const pluginConfig = pluginConfigs.get(39)! - await pluginConfig.vm?.resolveInternalVm + await (pluginConfig.instance as LazyPluginVM)?.resolveInternalVm // async loading of capabilities expect(pluginConfig.plugin!.capabilities!.methods!.sort()).toEqual(['onEvent', 'processEvent']) @@ -675,7 +678,7 @@ describe('plugins', () => { await setupPlugins(hub) const pluginConfig = hub.pluginConfigs.get(39)! - await pluginConfig.vm?.resolveInternalVm + await (pluginConfig.instance as LazyPluginVM)?.resolveInternalVm // async loading of capabilities expect(setPluginCapabilities.mock.calls.length).toBe(1) @@ -685,7 +688,7 @@ describe('plugins', () => { await setupPlugins(hub) const newPluginConfig = hub.pluginConfigs.get(39)! - await newPluginConfig.vm?.resolveInternalVm + await (newPluginConfig.instance as LazyPluginVM)?.resolveInternalVm // async loading of capabilities expect(newPluginConfig.plugin).not.toBe(pluginConfig.plugin) @@ -694,7 +697,7 @@ describe('plugins', () => { }) describe('loadSchedule()', () => { - const mockConfig = (tasks: any) => ({ vm: { getScheduledTasks: () => Promise.resolve(tasks) } }) + const mockConfig = (tasks: any) => ({ instance: { getScheduledTasks: () => Promise.resolve(tasks) } }) const hub = { pluginConfigs: new Map( diff --git a/plugin-server/tests/worker/plugins/inline.test.ts b/plugin-server/tests/worker/plugins/inline.test.ts new file mode 100644 index 0000000000000..d03d66b357552 --- /dev/null +++ b/plugin-server/tests/worker/plugins/inline.test.ts @@ -0,0 +1,167 @@ +import { PluginEvent } from '@posthog/plugin-scaffold' + +import { Hub, LogLevel, Plugin, PluginConfig } from '../../../src/types' +import { createHub } from '../../../src/utils/db/hub' +import { PostgresUse } from '../../../src/utils/db/postgres' +import { + constructInlinePluginInstance, + INLINE_PLUGIN_MAP, + INLINE_PLUGIN_URLS, + syncInlinePlugins, +} from '../../../src/worker/vm/inline/inline' +import { VersionParts } from '../../../src/worker/vm/inline/semver-flattener' +import { PluginInstance } from '../../../src/worker/vm/lazy' +import { resetTestDatabase } from '../../helpers/sql' + +describe('Inline plugin', () => { + let hub: Hub + let closeHub: () => Promise + + beforeAll(async () => { + console.info = jest.fn() as any + console.warn = jest.fn() as any + ;[hub, closeHub] = await createHub({ LOG_LEVEL: LogLevel.Log }) + await resetTestDatabase() + }) + + afterAll(async () => { + await closeHub() + }) + + // Sync all the inline plugins, then assert that for each plugin URL, a + // plugin exists in the database with the correct properties. + test('syncInlinePlugins', async () => { + await syncInlinePlugins(hub) + + const { rows }: { rows: Plugin[] } = await hub.postgres.query( + PostgresUse.COMMON_WRITE, + 'SELECT * FROM posthog_plugin', + undefined, + 'getPluginRows' + ) + for (const url of INLINE_PLUGIN_URLS) { + const plugin = INLINE_PLUGIN_MAP[url] + const row = rows.find((row) => row.url === url)! + // All the inline plugin properties should align + expect(row).not.toBeUndefined() + expect(row.name).toEqual(plugin.description.name) + expect(row.description).toEqual(plugin.description.description) + expect(row.is_global).toEqual(plugin.description.is_global) + expect(row.is_preinstalled).toEqual(plugin.description.is_preinstalled) + expect(row.config_schema).toEqual(plugin.description.config_schema) + expect(row.tag).toEqual(plugin.description.tag) + expect(row.capabilities).toEqual(plugin.description.capabilities) + expect(row.is_stateless).toEqual(plugin.description.is_stateless) + expect(row.log_level).toEqual(plugin.description.log_level) + + // These non-inline plugin properties should be fixed across all inline plugins + // (in true deployments some of these would not be the case, as they're leftovers from + // before inlining, but in tests the inline plugins are always newly created) + expect(row.plugin_type).toEqual('inline') + expect(row.from_json).toEqual(false) + expect(row.from_web).toEqual(false) + expect(row.source__plugin_json).toBeUndefined() + expect(row.source__index_ts).toBeUndefined() + expect(row.source__frontend_tsx).toBeUndefined() + expect(row.source__site_ts).toBeUndefined() + expect(row.error).toBeNull() + expect(row.organization_id).toBeNull() + expect(row.metrics).toBeNull() + expect(row.public_jobs).toBeNull() + } + }) + + test('semver-flattener', async () => { + interface SemanticVersionTestCase { + versionString: string + expected: VersionParts + } + + const config: PluginConfig = { + plugin: { + id: null, + organization_id: null, + plugin_type: null, + name: null, + is_global: null, + url: 'inline://semver-flattener', + }, + config: { + properties: 'version,version2', + }, + id: null, + plugin_id: null, + enabled: null, + team_id: null, + order: null, + created_at: null, + } + + const instance: PluginInstance = constructInlinePluginInstance(hub, config) + + const versionExamples: SemanticVersionTestCase[] = [ + { + versionString: '1.2.3', + expected: { major: 1, minor: 2, patch: 3, build: undefined }, + }, + { + versionString: '22.7', + expected: { major: 22, minor: 7, preRelease: undefined, build: undefined }, + }, + { + versionString: '22.7-pre-release', + expected: { major: 22, minor: 7, patch: undefined, preRelease: 'pre-release', build: undefined }, + }, + { + versionString: '1.0.0-alpha+001', + expected: { major: 1, minor: 0, patch: 0, preRelease: 'alpha', build: '001' }, + }, + { + versionString: '1.0.0+20130313144700', + expected: { major: 1, minor: 0, patch: 0, build: '20130313144700' }, + }, + { + versionString: '1.2.3-beta+exp.sha.5114f85', + expected: { major: 1, minor: 2, patch: 3, preRelease: 'beta', build: 'exp.sha.5114f85' }, + }, + { + versionString: '1.0.0+21AF26D3—-117B344092BD', + expected: { major: 1, minor: 0, patch: 0, preRelease: undefined, build: '21AF26D3—-117B344092BD' }, + }, + ] + + const test_event: PluginEvent = { + distinct_id: '', + ip: null, + site_url: '', + team_id: 0, + now: '', + event: '', + uuid: '', + properties: {}, + } + + const method = await instance.getPluginMethod('processEvent') + + for (const { versionString, expected } of versionExamples) { + test_event.properties.version = versionString + test_event.properties.version2 = versionString + const flattened = await method(test_event) + + expect(flattened.properties.version__major).toEqual(expected.major) + expect(flattened.properties.version__minor).toEqual(expected.minor) + expect(flattened.properties.version__patch).toEqual(expected.patch) + expect(flattened.properties.version__preRelease).toEqual(expected.preRelease) + expect(flattened.properties.version__build).toEqual(expected.build) + + expect(flattened.properties.version2__major).toEqual(expected.major) + expect(flattened.properties.version2__minor).toEqual(expected.minor) + expect(flattened.properties.version2__patch).toEqual(expected.patch) + expect(flattened.properties.version2__preRelease).toEqual(expected.preRelease) + expect(flattened.properties.version2__build).toEqual(expected.build) + + // reset the event for the next iteration + test_event.properties = {} + } + }) +}) diff --git a/plugin-server/tests/worker/plugins/run.test.ts b/plugin-server/tests/worker/plugins/run.test.ts index aa48e0b8451a1..928b31ee7ab00 100644 --- a/plugin-server/tests/worker/plugins/run.test.ts +++ b/plugin-server/tests/worker/plugins/run.test.ts @@ -20,7 +20,7 @@ describe('runPluginTask()', () => { { team_id: 2, enabled: true, - vm: { + instance: { getTask, }, }, @@ -30,7 +30,7 @@ describe('runPluginTask()', () => { { team_id: 2, enabled: false, - vm: { + instance: { getTask, }, }, @@ -142,8 +142,8 @@ describe('runOnEvent', () => { plugin_id: 100, team_id: 2, enabled: false, - vm: { - getVmMethod: () => onEvent, + instance: { + getPluginMethod: () => onEvent, }, }, @@ -151,8 +151,8 @@ describe('runOnEvent', () => { plugin_id: 101, team_id: 2, enabled: false, - vm: { - getVmMethod: () => onEvent, + instance: { + getPluginMethod: () => onEvent, }, }, ], @@ -264,8 +264,8 @@ describe('runComposeWebhook', () => { plugin_id: 100, team_id: 2, enabled: false, - vm: { - getVmMethod: () => composeWebhook, + instance: { + getPluginMethod: () => composeWebhook, } as any, } mockActionManager = { diff --git a/plugin-server/tests/worker/vm.extra-lazy.test.ts b/plugin-server/tests/worker/vm.extra-lazy.test.ts index e571b2f809b59..78bcc0da60f6c 100644 --- a/plugin-server/tests/worker/vm.extra-lazy.test.ts +++ b/plugin-server/tests/worker/vm.extra-lazy.test.ts @@ -33,7 +33,7 @@ describe('VMs are extra lazy 💤', () => { const pluginConfig = { ...pluginConfig39, plugin: plugin60 } const lazyVm = new LazyPluginVM(hub, pluginConfig) - pluginConfig.vm = lazyVm + pluginConfig.instance = lazyVm jest.spyOn(lazyVm, 'setupPluginIfNeeded') await lazyVm.initialize!(indexJs, pluginDigest(plugin60)) @@ -58,7 +58,7 @@ describe('VMs are extra lazy 💤', () => { const pluginConfig = { ...pluginConfig39, plugin: plugin60 } const lazyVm = new LazyPluginVM(hub, pluginConfig) - pluginConfig.vm = lazyVm + pluginConfig.instance = lazyVm jest.spyOn(lazyVm, 'setupPluginIfNeeded') await lazyVm.initialize!(indexJs, pluginDigest(plugin60)) @@ -80,7 +80,7 @@ describe('VMs are extra lazy 💤', () => { await resetTestDatabase(indexJs) const pluginConfig = { ...pluginConfig39, plugin: plugin60 } const lazyVm = new LazyPluginVM(hub, pluginConfig) - pluginConfig.vm = lazyVm + pluginConfig.instance = lazyVm jest.spyOn(lazyVm, 'setupPluginIfNeeded') await lazyVm.initialize!(indexJs, pluginDigest(plugin60)) @@ -88,7 +88,7 @@ describe('VMs are extra lazy 💤', () => { expect(lazyVm.setupPluginIfNeeded).not.toHaveBeenCalled() expect(fetch).not.toHaveBeenCalled() - await lazyVm.getOnEvent() + await lazyVm.getPluginMethod('onEvent') expect(lazyVm.ready).toEqual(true) expect(lazyVm.setupPluginIfNeeded).toHaveBeenCalled() expect(fetch).toHaveBeenCalledWith('https://onevent.com/', undefined) @@ -107,14 +107,14 @@ describe('VMs are extra lazy 💤', () => { await resetTestDatabase(indexJs) const pluginConfig = { ...pluginConfig39, plugin: plugin60 } const lazyVm = new LazyPluginVM(hub, pluginConfig) - pluginConfig.vm = lazyVm + pluginConfig.instance = lazyVm jest.spyOn(lazyVm, 'setupPluginIfNeeded') await lazyVm.initialize!(indexJs, pluginDigest(plugin60)) lazyVm.ready = false lazyVm.inErroredState = true - const onEvent = await lazyVm.getOnEvent() + const onEvent = await lazyVm.getPluginMethod('onEvent') expect(onEvent).toBeNull() expect(lazyVm.ready).toEqual(false) expect(lazyVm.setupPluginIfNeeded).toHaveBeenCalled() diff --git a/plugin-server/tests/worker/vm.lazy.test.ts b/plugin-server/tests/worker/vm.lazy.test.ts index fc77c5c9f3582..cfe13bc628902 100644 --- a/plugin-server/tests/worker/vm.lazy.test.ts +++ b/plugin-server/tests/worker/vm.lazy.test.ts @@ -65,7 +65,7 @@ describe('LazyPluginVM', () => { const vm = createVM() void initializeVm(vm) - expect(await vm.getProcessEvent()).toEqual('processEvent') + expect(await vm.getPluginMethod('processEvent')).toEqual('processEvent') expect(await vm.getTask('someTask', PluginTaskType.Schedule)).toEqual(null) expect(await vm.getTask('runEveryMinute', PluginTaskType.Schedule)).toEqual('runEveryMinute') expect(await vm.getScheduledTasks()).toEqual(mockVM.tasks.schedule) @@ -109,7 +109,7 @@ describe('LazyPluginVM', () => { void initializeVm(vm) - expect(await vm.getProcessEvent()).toEqual(null) + expect(await vm.getPluginMethod('processEvent')).toEqual(null) expect(await vm.getTask('runEveryMinute', PluginTaskType.Schedule)).toEqual(null) expect(await vm.getScheduledTasks()).toEqual({}) }) diff --git a/posthog/api/plugin.py b/posthog/api/plugin.py index 481b63476f10e..04578f5e64eba 100644 --- a/posthog/api/plugin.py +++ b/posthog/api/plugin.py @@ -290,7 +290,10 @@ def get_latest_tag(self, plugin: Plugin) -> Optional[str]: return None def get_organization_name(self, plugin: Plugin) -> str: - return plugin.organization.name + if plugin.organization: + return plugin.organization.name + else: + return "posthog-inline" def create(self, validated_data: dict, *args: Any, **kwargs: Any) -> Plugin: validated_data["url"] = self.initial_data.get("url", None) diff --git a/posthog/api/test/__snapshots__/test_plugin.ambr b/posthog/api/test/__snapshots__/test_plugin.ambr index d658a166f5858..e424770da1794 100644 --- a/posthog/api/test/__snapshots__/test_plugin.ambr +++ b/posthog/api/test/__snapshots__/test_plugin.ambr @@ -141,7 +141,7 @@ "posthog_organization"."personalization", "posthog_organization"."domain_whitelist" FROM "posthog_plugin" - INNER JOIN "posthog_organization" ON ("posthog_plugin"."organization_id" = "posthog_organization"."id") + LEFT OUTER JOIN "posthog_organization" ON ("posthog_plugin"."organization_id" = "posthog_organization"."id") WHERE ("posthog_plugin"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid OR "posthog_plugin"."is_global" OR "posthog_plugin"."id" IN @@ -329,7 +329,7 @@ "posthog_organization"."personalization", "posthog_organization"."domain_whitelist" FROM "posthog_plugin" - INNER JOIN "posthog_organization" ON ("posthog_plugin"."organization_id" = "posthog_organization"."id") + LEFT OUTER JOIN "posthog_organization" ON ("posthog_plugin"."organization_id" = "posthog_organization"."id") WHERE ("posthog_plugin"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid OR "posthog_plugin"."is_global" OR "posthog_plugin"."id" IN @@ -542,7 +542,7 @@ "posthog_organization"."personalization", "posthog_organization"."domain_whitelist" FROM "posthog_plugin" - INNER JOIN "posthog_organization" ON ("posthog_plugin"."organization_id" = "posthog_organization"."id") + LEFT OUTER JOIN "posthog_organization" ON ("posthog_plugin"."organization_id" = "posthog_organization"."id") WHERE ("posthog_plugin"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid OR "posthog_plugin"."is_global" OR "posthog_plugin"."id" IN diff --git a/posthog/api/test/test_plugin.py b/posthog/api/test/test_plugin.py index 968a18faa8b98..0176cc6077739 100644 --- a/posthog/api/test/test_plugin.py +++ b/posthog/api/test/test_plugin.py @@ -885,6 +885,8 @@ def test_plugin_unused(self, mock_get, mock_reload): ) def test_install_plugin_on_multiple_orgs(self, mock_get, mock_reload): + # Expectation: since plugins are url-unique, installing the same plugin on a second orgs should + # return a 400 response, as the plugin is already installed on the first org my_org = self.organization other_org = Organization.objects.create( name="FooBar2", plugins_access_level=Organization.PluginsAccessLevel.INSTALL @@ -914,6 +916,7 @@ def test_install_plugin_on_multiple_orgs(self, mock_get, mock_reload): f"/api/organizations/{other_org.id}/plugins/", {"url": "https://github.com/PostHog/helloworldplugin"}, ) + # Fails due to org membership self.assertEqual(response.status_code, 403) self.assertEqual(Plugin.objects.count(), 1) @@ -923,14 +926,9 @@ def test_install_plugin_on_multiple_orgs(self, mock_get, mock_reload): f"/api/organizations/{other_org.id}/plugins/", {"url": "https://github.com/PostHog/helloworldplugin"}, ) - self.assertEqual(response.status_code, 201) - self.assertEqual(Plugin.objects.count(), 2) - response = self.client.post( - f"/api/organizations/{other_org.id}/plugins/", - {"url": "https://github.com/PostHog/helloworldplugin"}, - ) + # Fails since the plugin already exists self.assertEqual(response.status_code, 400) - self.assertEqual(Plugin.objects.count(), 2) + self.assertEqual(Plugin.objects.count(), 1) def test_cannot_access_others_orgs_plugins(self, mock_get, mock_reload): other_org = Organization.objects.create( diff --git a/posthog/management/commands/test/test_create_batch_export_from_app.py b/posthog/management/commands/test/test_create_batch_export_from_app.py index a5c8fffc5f4d4..9357920f909a5 100644 --- a/posthog/management/commands/test/test_create_batch_export_from_app.py +++ b/posthog/management/commands/test/test_create_batch_export_from_app.py @@ -3,6 +3,7 @@ import datetime as dt import json import typing +import uuid import pytest import temporalio.client @@ -36,11 +37,17 @@ def team(organization): team.delete() +# Used to randomize plugin URLs, to prevent tests stepping on each other, since +# plugin urls are constrained to be unique. +def append_random(url: str) -> str: + return f"{url}?random={uuid.uuid4()}" + + @pytest.fixture def snowflake_plugin(organization) -> typing.Generator[Plugin, None, None]: plugin = Plugin.objects.create( name="Snowflake Export", - url="https://github.com/PostHog/snowflake-export-plugin", + url=append_random("https://github.com/PostHog/snowflake-export-plugin"), plugin_type="custom", organization=organization, ) @@ -52,7 +59,7 @@ def snowflake_plugin(organization) -> typing.Generator[Plugin, None, None]: def s3_plugin(organization) -> typing.Generator[Plugin, None, None]: plugin = Plugin.objects.create( name="S3 Export Plugin", - url="https://github.com/PostHog/s3-export-plugin", + url=append_random("https://github.com/PostHog/s3-export-plugin"), plugin_type="custom", organization=organization, ) @@ -64,7 +71,7 @@ def s3_plugin(organization) -> typing.Generator[Plugin, None, None]: def bigquery_plugin(organization) -> typing.Generator[Plugin, None, None]: plugin = Plugin.objects.create( name="BigQuery Export", - url="https://github.com/PostHog/bigquery-plugin", + url=append_random("https://github.com/PostHog/bigquery-plugin"), plugin_type="custom", organization=organization, ) @@ -76,7 +83,7 @@ def bigquery_plugin(organization) -> typing.Generator[Plugin, None, None]: def postgres_plugin(organization) -> typing.Generator[Plugin, None, None]: plugin = Plugin.objects.create( name="PostgreSQL Export Plugin", - url="https://github.com/PostHog/postgres-plugin", + url=append_random("https://github.com/PostHog/postgres-plugin"), plugin_type="custom", organization=organization, ) @@ -88,7 +95,7 @@ def postgres_plugin(organization) -> typing.Generator[Plugin, None, None]: def redshift_plugin(organization) -> typing.Generator[Plugin, None, None]: plugin = Plugin.objects.create( name="Redshift Export Plugin", - url="https://github.com/PostHog/postgres-plugin", + url=append_random("https://github.com/PostHog/postgres-plugin"), plugin_type="custom", organization=organization, ) diff --git a/posthog/migrations/0449_alter_plugin_organization_alter_plugin_plugin_type_and_more.py b/posthog/migrations/0449_alter_plugin_organization_alter_plugin_plugin_type_and_more.py new file mode 100644 index 0000000000000..acbeebaac82f1 --- /dev/null +++ b/posthog/migrations/0449_alter_plugin_organization_alter_plugin_plugin_type_and_more.py @@ -0,0 +1,90 @@ +# Generated by Django 4.2.14 on 2024-07-22 08:04 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + atomic = False # Added to support concurrent index creation + dependencies = [ + ("posthog", "0448_add_mysql_externaldatasource_source_type"), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.AlterField( + model_name="plugin", + name="organization", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="plugins", + related_query_name="plugin", + to="posthog.organization", + ), + ), + ], + database_operations=[ + migrations.RunSQL( + """ + SET CONSTRAINTS "posthog_plugin_organization_id_d040b9a9_fk_posthog_o" IMMEDIATE; -- existing-table-constraint-ignore + ALTER TABLE "posthog_plugin" DROP CONSTRAINT "posthog_plugin_organization_id_d040b9a9_fk_posthog_o"; -- existing-table-constraint-ignore + ALTER TABLE "posthog_plugin" ALTER COLUMN "organization_id" DROP NOT NULL; + ALTER TABLE "posthog_plugin" ADD CONSTRAINT "posthog_plugin_organization_id_d040b9a9_fk_posthog_o" FOREIGN KEY ("organization_id") REFERENCES "posthog_organization" ("id") DEFERRABLE INITIALLY DEFERRED; -- existing-table-constraint-ignore + """, + reverse_sql=""" + SET CONSTRAINTS "posthog_plugin_organization_id_d040b9a9_fk_posthog_o" IMMEDIATE; -- existing-table-constraint-ignore + ALTER TABLE "posthog_plugin" DROP CONSTRAINT "posthog_plugin_organization_id_d040b9a9_fk_posthog_o"; -- existing-table-constraint-ignore + ALTER TABLE "posthog_plugin" ALTER COLUMN "organization_id" SET NOT NULL; + ALTER TABLE "posthog_plugin" ADD CONSTRAINT "posthog_plugin_organization_id_d040b9a9_fk_posthog_o" FOREIGN KEY ("organization_id") REFERENCES "posthog_organization" ("id") DEFERRABLE INITIALLY DEFERRED; -- existing-table-constraint-ignore + """, + ), + ], + ), + migrations.AlterField( + model_name="plugin", + name="plugin_type", + field=models.CharField( + blank=True, + choices=[ + ("local", "local"), + ("custom", "custom"), + ("repository", "repository"), + ("source", "source"), + ("inline", "inline"), + ], + default=None, + max_length=200, + null=True, + ), + ), + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.AlterField( + model_name="plugin", + name="url", + field=models.CharField(blank=True, max_length=800, null=True, unique=True), + ) + ], + database_operations=[ + migrations.RunSQL( + """ + ALTER TABLE "posthog_plugin" ADD CONSTRAINT "posthog_plugin_url_bccac89d_uniq" UNIQUE ("url"); -- existing-table-constraint-ignore + """, + reverse_sql=""" + ALTER TABLE "posthog_plugin" DROP CONSTRAINT IF EXISTS "posthog_plugin_url_bccac89d_uniq"; + """, + ), + # We add the index seperately + migrations.RunSQL( + """ + CREATE INDEX CONCURRENTLY "posthog_plugin_url_bccac89d_like" ON "posthog_plugin" ("url" varchar_pattern_ops); + """, + reverse_sql=""" + DROP INDEX IF EXISTS "posthog_plugin_url_bccac89d_like"; + """, + ), + ], + ), + ] diff --git a/posthog/models/plugin.py b/posthog/models/plugin.py index 26b3cdde676ca..19d07578cf4a5 100644 --- a/posthog/models/plugin.py +++ b/posthog/models/plugin.py @@ -38,15 +38,11 @@ pass -def raise_if_plugin_installed(url: str, organization_id: str): +def raise_if_plugin_installed(url: str): url_without_private_key = url.split("?")[0] - if ( - Plugin.objects.filter( - models.Q(url=url_without_private_key) | models.Q(url__startswith=f"{url_without_private_key}?") - ) - .filter(organization_id=organization_id) - .exists() - ): + if Plugin.objects.filter( + models.Q(url=url_without_private_key) | models.Q(url__startswith=f"{url_without_private_key}?") + ).exists(): raise ValidationError(f'Plugin from URL "{url_without_private_key}" already installed!') @@ -125,7 +121,7 @@ def install(self, **kwargs) -> "Plugin": plugin_json: Optional[dict[str, Any]] = None if kwargs.get("plugin_type", None) != Plugin.PluginType.SOURCE: plugin_json = update_validated_data_from_url(kwargs, kwargs["url"]) - raise_if_plugin_installed(kwargs["url"], kwargs["organization_id"]) + raise_if_plugin_installed(kwargs["url"]) plugin = Plugin.objects.create(**kwargs) if plugin_json: PluginSourceFile.objects.sync_from_plugin_archive(plugin, plugin_json) @@ -149,12 +145,18 @@ class PluginType(models.TextChoices): "source", "source", ) # coded inside the browser (versioned via plugin_source_version) + INLINE = ( + "inline", + "inline", + ) # Code checked into plugin_server, url starts with "inline:" + # DEPRECATED: plugin-server will own all plugin code, org relations don't make sense organization: models.ForeignKey = models.ForeignKey( "posthog.Organization", on_delete=models.CASCADE, related_name="plugins", related_query_name="plugin", + null=True, ) plugin_type: models.CharField = models.CharField( max_length=200, null=True, blank=True, choices=PluginType.choices, default=None @@ -167,7 +169,7 @@ class PluginType(models.TextChoices): name: models.CharField = models.CharField(max_length=200, null=True, blank=True) description: models.TextField = models.TextField(null=True, blank=True) - url: models.CharField = models.CharField(max_length=800, null=True, blank=True) + url: models.CharField = models.CharField(max_length=800, null=True, blank=True, unique=True) icon: models.CharField = models.CharField(max_length=800, null=True, blank=True) # Describe the fields to ask in the interface; store answers in PluginConfig->config # - config_schema = { [fieldKey]: { name: 'api key', type: 'string', default: '', required: true } }