From e51852a4a30bdea9d54c7f829d93e07f01803991 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 7 Nov 2023 22:03:39 +0000 Subject: [PATCH 1/2] feat(plugin-server): remove frontend transpilation --- plugin-server/src/types.ts | 6 -- plugin-server/src/utils/db/db.ts | 35 ---------- .../src/worker/frontend/transpile.ts | 35 ---------- .../src/worker/plugins/loadPlugin.ts | 67 +++---------------- 4 files changed, 8 insertions(+), 135 deletions(-) delete mode 100644 plugin-server/src/worker/frontend/transpile.ts diff --git a/plugin-server/src/types.ts b/plugin-server/src/types.ts index e295f5414e05a..ffede0c424f65 100644 --- a/plugin-server/src/types.ts +++ b/plugin-server/src/types.ts @@ -457,12 +457,6 @@ export interface PluginLogEntry { instance_id: string } -export enum PluginSourceFileStatus { - Transpiled = 'TRANSPILED', - Locked = 'LOCKED', - Error = 'ERROR', -} - export enum PluginTaskType { Job = 'job', Schedule = 'schedule', diff --git a/plugin-server/src/utils/db/db.ts b/plugin-server/src/utils/db/db.ts index e456335afcc8c..cf64f86b600a2 100644 --- a/plugin-server/src/utils/db/db.ts +++ b/plugin-server/src/utils/db/db.ts @@ -36,7 +36,6 @@ import { PluginLogEntrySource, PluginLogEntryType, PluginLogLevel, - PluginSourceFileStatus, PropertiesLastOperation, PropertiesLastUpdatedAt, PropertyDefinitionType, @@ -1532,38 +1531,4 @@ export class DB { ) return rows[0]?.source ?? null } - - public async setPluginTranspiled(pluginId: Plugin['id'], filename: string, transpiled: string): Promise { - await this.postgres.query( - PostgresUse.COMMON_WRITE, - `INSERT INTO posthog_pluginsourcefile (id, plugin_id, filename, status, transpiled, updated_at) VALUES($1, $2, $3, $4, $5, NOW()) - ON CONFLICT ON CONSTRAINT unique_filename_for_plugin - DO UPDATE SET status = $4, transpiled = $5, error = NULL, updated_at = NOW()`, - [new UUIDT().toString(), pluginId, filename, PluginSourceFileStatus.Transpiled, transpiled], - 'setPluginTranspiled' - ) - } - - public async setPluginTranspiledError(pluginId: Plugin['id'], filename: string, error: string): Promise { - await this.postgres.query( - PostgresUse.COMMON_WRITE, - `INSERT INTO posthog_pluginsourcefile (id, plugin_id, filename, status, error, updated_at) VALUES($1, $2, $3, $4, $5, NOW()) - ON CONFLICT ON CONSTRAINT unique_filename_for_plugin - DO UPDATE SET status = $4, error = $5, transpiled = NULL, updated_at = NOW()`, - [new UUIDT().toString(), pluginId, filename, PluginSourceFileStatus.Error, error], - 'setPluginTranspiledError' - ) - } - - public async getPluginTranspilationLock(pluginId: Plugin['id'], filename: string): Promise { - const response = await this.postgres.query( - PostgresUse.COMMON_WRITE, - `INSERT INTO posthog_pluginsourcefile (id, plugin_id, filename, status, transpiled, updated_at) VALUES($1, $2, $3, $4, NULL, NOW()) - ON CONFLICT ON CONSTRAINT unique_filename_for_plugin - DO UPDATE SET status = $4, updated_at = NOW() WHERE (posthog_pluginsourcefile.status IS NULL OR posthog_pluginsourcefile.status = $5) RETURNING status`, - [new UUIDT().toString(), pluginId, filename, PluginSourceFileStatus.Locked, ''], - 'getPluginTranspilationLock' - ) - return response.rowCount > 0 - } } diff --git a/plugin-server/src/worker/frontend/transpile.ts b/plugin-server/src/worker/frontend/transpile.ts deleted file mode 100644 index 96372df98dfd5..0000000000000 --- a/plugin-server/src/worker/frontend/transpile.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { transform } from '@babel/standalone' - -export function transpileFrontend(rawCode: string): string { - const { code } = transform(rawCode, { - envName: 'production', - code: true, - babelrc: false, - configFile: false, - filename: 'frontend.tsx', - plugins: ['transform-react-jsx'], - presets: [ - ['typescript', { isTSX: true, allExtensions: true }], - ['env', { targets: { esmodules: false } }], - ], - }) - if (!code) { - throw new Error('Could not transpile frontend code') - } - return `"use strict";\nexport function getFrontendApp (require) { let exports = {}; ${code}; return exports; }` -} - -export function transpileSite(rawCode: string): string { - const { code } = transform(rawCode, { - envName: 'production', - code: true, - babelrc: false, - configFile: false, - filename: 'site.ts', - presets: [['typescript', { isTSX: false, allExtensions: true }], 'env'], - }) - if (!code) { - throw new Error('Could not transpile web code') - } - return `(function () {let exports={};${code};return exports;})` -} diff --git a/plugin-server/src/worker/plugins/loadPlugin.ts b/plugin-server/src/worker/plugins/loadPlugin.ts index b5581ef545d42..26a7d45f97e62 100644 --- a/plugin-server/src/worker/plugins/loadPlugin.ts +++ b/plugin-server/src/worker/plugins/loadPlugin.ts @@ -3,9 +3,7 @@ import * as path from 'path' import { Hub, Plugin, PluginConfig, PluginJsonConfig } from '../../types' import { processError } from '../../utils/db/error' -import { status } from '../../utils/status' import { pluginDigest } from '../../utils/utils' -import { transpileFrontend, transpileSite } from '../frontend/transpile' function readFileIfExists(baseDir: string, plugin: Plugin, file: string): string | null { const fullPath = path.resolve(baseDir, plugin.url!.substring(5), file) @@ -40,62 +38,6 @@ export async function loadPlugin(hub: Hub, pluginConfig: PluginConfig): Promise< } } - const transpileIfNeeded = async ({ - type, - filename, - pluginKey, - transpile, - }: { - type: 'frontend' | 'site' - filename: 'frontend.tsx' | 'site.ts' - pluginKey: 'source__frontend_tsx' | 'source__site_ts' - transpile: (source: string) => string - }): Promise => { - const source = isLocalPlugin ? readFileIfExists(hub.BASE_DIR, plugin, filename) : plugin[pluginKey] - if (source && hub.capabilities.transpileFrontendApps) { - if (await hub.db.getPluginTranspilationLock(plugin.id, filename)) { - status.info('🔌', `Transpiling ${pluginDigest(plugin)}`) - const transpilationStartTimer = new Date() - try { - const transpiled = transpile(source) - await hub.db.setPluginTranspiled(plugin.id, filename, transpiled) - } catch (error: any) { - await processError(hub, pluginConfig, error) - await hub.db.setPluginTranspiledError( - plugin.id, - filename, - typeof error === 'string' - ? error - : [error.message, error.stack].filter((a) => !!a).join('\n') - ) - hub.statsd?.increment(`transpile_${type}.ERROR`, { - plugin: plugin.name ?? '?', - pluginId: `${plugin.id ?? '?'}`, - }) - } - hub.statsd?.timing(`transpile_${type}`, transpilationStartTimer, { - plugin: plugin.name ?? '?', - pluginId: `${plugin.id ?? '?'}`, - }) - } - } - return !!source - } - - const hasFrontend = await transpileIfNeeded({ - type: 'frontend', - filename: 'frontend.tsx', - pluginKey: 'source__frontend_tsx', - transpile: transpileFrontend, - }) - - const hasSite = await transpileIfNeeded({ - type: 'site', - filename: 'site.ts', - pluginKey: 'source__site_ts', - transpile: transpileSite, - }) - // setup "backend" app const pluginSource = isLocalPlugin ? config['main'] @@ -110,7 +52,14 @@ export async function loadPlugin(hub: Hub, pluginConfig: PluginConfig): Promise< // always call this if no backend app present, will signal that the VM is done pluginConfig.vm?.failInitialization!() - // if we transpiled a frontend app, don't save an error if no backend app + // if there is a frontend or site app, don't save an error if no backend app + const hasFrontend = isLocalPlugin + ? readFileIfExists(hub.BASE_DIR, plugin, 'frontend.tsx') + : plugin['source__frontend_tsx'] + const hasSite = isLocalPlugin + ? readFileIfExists(hub.BASE_DIR, plugin, 'site.ts') + : plugin['source__site_ts'] + if (!hasFrontend && !hasSite) { await processError( hub, From d26934927f1d2b52ef20e3c25a5172a25c416cb6 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 7 Nov 2023 22:06:35 +0000 Subject: [PATCH 2/2] remove tests --- plugin-server/tests/worker/plugins.test.ts | 86 ---------------------- 1 file changed, 86 deletions(-) diff --git a/plugin-server/tests/worker/plugins.test.ts b/plugin-server/tests/worker/plugins.test.ts index 1b5bbd48efc28..a8d08aaffadcd 100644 --- a/plugin-server/tests/worker/plugins.test.ts +++ b/plugin-server/tests/worker/plugins.test.ts @@ -3,7 +3,6 @@ import { PluginEvent } from '@posthog/plugin-scaffold/src/types' import { Hub, LogLevel } from '../../src/types' import { processError } from '../../src/utils/db/error' import { createHub } from '../../src/utils/db/hub' -import { PostgresUse } from '../../src/utils/db/postgres' import { delay, IllegalOperationError } from '../../src/utils/utils' import { loadPlugin } from '../../src/worker/plugins/loadPlugin' import { loadSchedule } from '../../src/worker/plugins/loadSchedule' @@ -628,91 +627,6 @@ describe('plugins', () => { expect(pluginConfig.plugin!.capabilities!.scheduled_tasks).toEqual([]) }) - test('plugin with frontend source transpiles it', async () => { - getPluginRows.mockReturnValueOnce([ - { ...mockPluginSourceCode(), source__frontend_tsx: `export const scene = {}` }, - ]) - getPluginConfigRows.mockReturnValueOnce([pluginConfig39]) - getPluginAttachmentRows.mockReturnValueOnce([pluginAttachment1]) - await setupPlugins(hub) - const { - rows: [{ transpiled }], - } = await hub.db.postgres.query( - PostgresUse.COMMON_WRITE, - `SELECT transpiled FROM posthog_pluginsourcefile WHERE plugin_id = $1 AND filename = $2`, - [60, 'frontend.tsx'], - '' - ) - expect(transpiled).toEqual(`"use strict"; -export function getFrontendApp (require) { let exports = {}; "use strict"; - -Object.defineProperty(exports, "__esModule", { - value: true -}); -exports.scene = void 0; -var scene = {}; -exports.scene = scene;; return exports; }`) - }) - - test('plugin with frontend source with error', async () => { - getPluginRows.mockReturnValueOnce([ - { ...mockPluginSourceCode(), source__frontend_tsx: `export const scene = {}/` }, - ]) - getPluginConfigRows.mockReturnValueOnce([pluginConfig39]) - getPluginAttachmentRows.mockReturnValueOnce([pluginAttachment1]) - await setupPlugins(hub) - const { - rows: [plugin], - } = await hub.db.postgres.query( - PostgresUse.COMMON_WRITE, - `SELECT * FROM posthog_pluginsourcefile WHERE plugin_id = $1 AND filename = $2`, - [60, 'frontend.tsx'], - '' - ) - expect(plugin.transpiled).toEqual(null) - expect(plugin.status).toEqual('ERROR') - expect(plugin.error).toContain(`SyntaxError: /frontend.tsx: Unexpected token (1:24)`) - expect(plugin.error).toContain(`export const scene = {}/`) - }) - - test('getTranspilationLock returns just once', async () => { - getPluginRows.mockReturnValueOnce([ - { - ...mockPluginSourceCode(), - source__index_ts: `function processEvent (event, meta) { event.properties={"x": 1}; return event }`, - source__frontend_tsx: `export const scene = {}`, - }, - ]) - getPluginConfigRows.mockReturnValueOnce([pluginConfig39]) - getPluginAttachmentRows.mockReturnValueOnce([pluginAttachment1]) - - await setupPlugins(hub) - const getStatus = async () => - ( - await hub.db.postgres.query( - PostgresUse.COMMON_WRITE, - `SELECT status FROM posthog_pluginsourcefile WHERE plugin_id = $1 AND filename = $2`, - [60, 'frontend.tsx'], - '' - ) - )?.rows?.[0]?.status || null - - expect(await getStatus()).toEqual('TRANSPILED') - expect(await hub.db.getPluginTranspilationLock(60, 'frontend.tsx')).toEqual(false) - expect(await hub.db.getPluginTranspilationLock(60, 'frontend.tsx')).toEqual(false) - - await hub.db.postgres.query( - PostgresUse.COMMON_WRITE, - 'UPDATE posthog_pluginsourcefile SET transpiled = NULL, status = NULL WHERE filename = $1', - ['frontend.tsx'], - '' - ) - - expect(await hub.db.getPluginTranspilationLock(60, 'frontend.tsx')).toEqual(true) - expect(await hub.db.getPluginTranspilationLock(60, 'frontend.tsx')).toEqual(false) - expect(await getStatus()).toEqual('LOCKED') - }) - test('reloading plugins after config changes', async () => { const makePlugin = (id: number, updated_at = '2020-11-02'): any => ({ ...plugin60,