Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(plugin-server): remove frontend transpilation #18466

Merged
merged 2 commits into from
Nov 8, 2023
Merged

feat(plugin-server): remove frontend transpilation #18466

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e51852a
feat(plugin-server): remove frontend transpilation
mariusandra Nov 7, 2023
d269349
remove tests
mariusandra Nov 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions plugin-server/src/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -457,12 +457,6 @@ export interface PluginLogEntry {
instance_id: string
}

export enum PluginSourceFileStatus {
Transpiled = 'TRANSPILED',
Locked = 'LOCKED',
Error = 'ERROR',
}

export enum PluginTaskType {
Job = 'job',
Schedule = 'schedule',
Expand Down
35 changes: 0 additions & 35 deletions plugin-server/src/utils/db/db.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ import {
PluginLogEntrySource,
PluginLogEntryType,
PluginLogLevel,
PluginSourceFileStatus,
PropertiesLastOperation,
PropertiesLastUpdatedAt,
PropertyDefinitionType,
Expand Down Expand Up @@ -1532,38 +1531,4 @@ export class DB {
)
return rows[0]?.source ?? null
}

public async setPluginTranspiled(pluginId: Plugin['id'], filename: string, transpiled: string): Promise<void> {
await this.postgres.query(
PostgresUse.COMMON_WRITE,
`INSERT INTO posthog_pluginsourcefile (id, plugin_id, filename, status, transpiled, updated_at) VALUES($1, $2, $3, $4, $5, NOW())
ON CONFLICT ON CONSTRAINT unique_filename_for_plugin
DO UPDATE SET status = $4, transpiled = $5, error = NULL, updated_at = NOW()`,
[new UUIDT().toString(), pluginId, filename, PluginSourceFileStatus.Transpiled, transpiled],
'setPluginTranspiled'
)
}

public async setPluginTranspiledError(pluginId: Plugin['id'], filename: string, error: string): Promise<void> {
await this.postgres.query(
PostgresUse.COMMON_WRITE,
`INSERT INTO posthog_pluginsourcefile (id, plugin_id, filename, status, error, updated_at) VALUES($1, $2, $3, $4, $5, NOW())
ON CONFLICT ON CONSTRAINT unique_filename_for_plugin
DO UPDATE SET status = $4, error = $5, transpiled = NULL, updated_at = NOW()`,
[new UUIDT().toString(), pluginId, filename, PluginSourceFileStatus.Error, error],
'setPluginTranspiledError'
)
}

public async getPluginTranspilationLock(pluginId: Plugin['id'], filename: string): Promise<boolean> {
const response = await this.postgres.query(
PostgresUse.COMMON_WRITE,
`INSERT INTO posthog_pluginsourcefile (id, plugin_id, filename, status, transpiled, updated_at) VALUES($1, $2, $3, $4, NULL, NOW())
ON CONFLICT ON CONSTRAINT unique_filename_for_plugin
DO UPDATE SET status = $4, updated_at = NOW() WHERE (posthog_pluginsourcefile.status IS NULL OR posthog_pluginsourcefile.status = $5) RETURNING status`,
[new UUIDT().toString(), pluginId, filename, PluginSourceFileStatus.Locked, ''],
'getPluginTranspilationLock'
)
return response.rowCount > 0
}
}
35 changes: 0 additions & 35 deletions plugin-server/src/worker/frontend/transpile.ts
View file
Open in desktop

This file was deleted.

67 changes: 8 additions & 59 deletions plugin-server/src/worker/plugins/loadPlugin.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@ import * as path from 'path'

import { Hub, Plugin, PluginConfig, PluginJsonConfig } from '../../types'
import { processError } from '../../utils/db/error'
import { status } from '../../utils/status'
import { pluginDigest } from '../../utils/utils'
import { transpileFrontend, transpileSite } from '../frontend/transpile'

function readFileIfExists(baseDir: string, plugin: Plugin, file: string): string | null {
const fullPath = path.resolve(baseDir, plugin.url!.substring(5), file)
Expand Down Expand Up @@ -40,62 +38,6 @@ export async function loadPlugin(hub: Hub, pluginConfig: PluginConfig): Promise<
}
}

const transpileIfNeeded = async ({
type,
filename,
pluginKey,
transpile,
}: {
type: 'frontend' | 'site'
filename: 'frontend.tsx' | 'site.ts'
pluginKey: 'source__frontend_tsx' | 'source__site_ts'
transpile: (source: string) => string
}): Promise<boolean> => {
const source = isLocalPlugin ? readFileIfExists(hub.BASE_DIR, plugin, filename) : plugin[pluginKey]
if (source && hub.capabilities.transpileFrontendApps) {
if (await hub.db.getPluginTranspilationLock(plugin.id, filename)) {
status.info('🔌', `Transpiling ${pluginDigest(plugin)}`)
const transpilationStartTimer = new Date()
try {
const transpiled = transpile(source)
await hub.db.setPluginTranspiled(plugin.id, filename, transpiled)
} catch (error: any) {
await processError(hub, pluginConfig, error)
await hub.db.setPluginTranspiledError(
plugin.id,
filename,
typeof error === 'string'
? error
: [error.message, error.stack].filter((a) => !!a).join('\n')
)
hub.statsd?.increment(`transpile_${type}.ERROR`, {
plugin: plugin.name ?? '?',
pluginId: `${plugin.id ?? '?'}`,
})
}
hub.statsd?.timing(`transpile_${type}`, transpilationStartTimer, {
plugin: plugin.name ?? '?',
pluginId: `${plugin.id ?? '?'}`,
})
}
}
return !!source
}

const hasFrontend = await transpileIfNeeded({
type: 'frontend',
filename: 'frontend.tsx',
pluginKey: 'source__frontend_tsx',
transpile: transpileFrontend,
})

const hasSite = await transpileIfNeeded({
type: 'site',
filename: 'site.ts',
pluginKey: 'source__site_ts',
transpile: transpileSite,
})

// setup "backend" app
const pluginSource = isLocalPlugin
? config['main']
Expand All @@ -110,7 +52,14 @@ export async function loadPlugin(hub: Hub, pluginConfig: PluginConfig): Promise<
// always call this if no backend app present, will signal that the VM is done
pluginConfig.vm?.failInitialization!()

// if we transpiled a frontend app, don't save an error if no backend app
// if there is a frontend or site app, don't save an error if no backend app
const hasFrontend = isLocalPlugin
? readFileIfExists(hub.BASE_DIR, plugin, 'frontend.tsx')
: plugin['source__frontend_tsx']
const hasSite = isLocalPlugin
? readFileIfExists(hub.BASE_DIR, plugin, 'site.ts')
: plugin['source__site_ts']

if (!hasFrontend && !hasSite) {
await processError(
hub,
Expand Down
86 changes: 0 additions & 86 deletions plugin-server/tests/worker/plugins.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import { PluginEvent } from '@posthog/plugin-scaffold/src/types'
import { Hub, LogLevel } from '../../src/types'
import { processError } from '../../src/utils/db/error'
import { createHub } from '../../src/utils/db/hub'
import { PostgresUse } from '../../src/utils/db/postgres'
import { delay, IllegalOperationError } from '../../src/utils/utils'
import { loadPlugin } from '../../src/worker/plugins/loadPlugin'
import { loadSchedule } from '../../src/worker/plugins/loadSchedule'
Expand Down Expand Up @@ -628,91 +627,6 @@ describe('plugins', () => {
expect(pluginConfig.plugin!.capabilities!.scheduled_tasks).toEqual([])
})

test('plugin with frontend source transpiles it', async () => {
getPluginRows.mockReturnValueOnce([
{ ...mockPluginSourceCode(), source__frontend_tsx: `export const scene = {}` },
])
getPluginConfigRows.mockReturnValueOnce([pluginConfig39])
getPluginAttachmentRows.mockReturnValueOnce([pluginAttachment1])
await setupPlugins(hub)
const {
rows: [{ transpiled }],
} = await hub.db.postgres.query(
PostgresUse.COMMON_WRITE,
`SELECT transpiled FROM posthog_pluginsourcefile WHERE plugin_id = $1 AND filename = $2`,
[60, 'frontend.tsx'],
''
)
expect(transpiled).toEqual(`"use strict";
export function getFrontendApp (require) { let exports = {}; "use strict";

Object.defineProperty(exports, "__esModule", {
value: true
});
exports.scene = void 0;
var scene = {};
exports.scene = scene;; return exports; }`)
})

test('plugin with frontend source with error', async () => {
getPluginRows.mockReturnValueOnce([
{ ...mockPluginSourceCode(), source__frontend_tsx: `export const scene = {}/` },
])
getPluginConfigRows.mockReturnValueOnce([pluginConfig39])
getPluginAttachmentRows.mockReturnValueOnce([pluginAttachment1])
await setupPlugins(hub)
const {
rows: [plugin],
} = await hub.db.postgres.query(
PostgresUse.COMMON_WRITE,
`SELECT * FROM posthog_pluginsourcefile WHERE plugin_id = $1 AND filename = $2`,
[60, 'frontend.tsx'],
''
)
expect(plugin.transpiled).toEqual(null)
expect(plugin.status).toEqual('ERROR')
expect(plugin.error).toContain(`SyntaxError: /frontend.tsx: Unexpected token (1:24)`)
expect(plugin.error).toContain(`export const scene = {}/`)
})

test('getTranspilationLock returns just once', async () => {
getPluginRows.mockReturnValueOnce([
{
...mockPluginSourceCode(),
source__index_ts: `function processEvent (event, meta) { event.properties={"x": 1}; return event }`,
source__frontend_tsx: `export const scene = {}`,
},
])
getPluginConfigRows.mockReturnValueOnce([pluginConfig39])
getPluginAttachmentRows.mockReturnValueOnce([pluginAttachment1])

await setupPlugins(hub)
const getStatus = async () =>
(
await hub.db.postgres.query(
PostgresUse.COMMON_WRITE,
`SELECT status FROM posthog_pluginsourcefile WHERE plugin_id = $1 AND filename = $2`,
[60, 'frontend.tsx'],
''
)
)?.rows?.[0]?.status || null

expect(await getStatus()).toEqual('TRANSPILED')
expect(await hub.db.getPluginTranspilationLock(60, 'frontend.tsx')).toEqual(false)
expect(await hub.db.getPluginTranspilationLock(60, 'frontend.tsx')).toEqual(false)

await hub.db.postgres.query(
PostgresUse.COMMON_WRITE,
'UPDATE posthog_pluginsourcefile SET transpiled = NULL, status = NULL WHERE filename = $1',
['frontend.tsx'],
''
)

expect(await hub.db.getPluginTranspilationLock(60, 'frontend.tsx')).toEqual(true)
expect(await hub.db.getPluginTranspilationLock(60, 'frontend.tsx')).toEqual(false)
expect(await getStatus()).toEqual('LOCKED')
})

test('reloading plugins after config changes', async () => {
const makePlugin = (id: number, updated_at = '2020-11-02'): any => ({
...plugin60,
Expand Down
Loading