diff --git a/.vscode/launch.json b/.vscode/launch.json index a52ed008b6ea1..5cf3f0f5d2921 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -34,6 +34,7 @@ "BILLING_SERVICE_URL": "https://billing.dev.posthog.dev", "CLOUD_DEPLOYMENT": "dev" }, + "envFile": "${workspaceFolder}/.env", "console": "integratedTerminal", "python": "${workspaceFolder}/env/bin/python", "cwd": "${workspaceFolder}", diff --git a/frontend/__snapshots__/scenes-app-insights--trends-line-edit--dark.png b/frontend/__snapshots__/scenes-app-insights--trends-line-edit--dark.png index cb0028fe4f8ad..b8f2ab3e5f571 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-line-edit--dark.png and b/frontend/__snapshots__/scenes-app-insights--trends-line-edit--dark.png differ diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 73e64be958e59..515da6a68d6c2 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -44,6 +44,8 @@ import { FeatureFlagType, Group, GroupListParams, + HogFunctionIconResponse, + HogFunctionTemplateType, HogFunctionType, InsightModel, IntegrationType, @@ -330,6 +332,14 @@ class ApiRequest { return this.hogFunctions(teamId).addPathComponent(id) } + public hogFunctionTemplates(teamId?: TeamType['id']): ApiRequest { + return this.projectsDetail(teamId).addPathComponent('hog_function_templates') + } + + public hogFunctionTemplate(id: HogFunctionTemplateType['id'], teamId?: TeamType['id']): ApiRequest { + return this.hogFunctionTemplates(teamId).addPathComponent(id) + } + // # Actions public actions(teamId?: TeamType['id']): ApiRequest { return this.projectsDetail(teamId).addPathComponent('actions') @@ -1645,9 +1655,6 @@ const api = { }, hogFunctions: { - async listTemplates(): Promise> { - return await new ApiRequest().hogFunctions().get() - }, async list(): Promise> { return await new ApiRequest().hogFunctions().get() }, @@ -1666,6 +1673,17 @@ const api = { ): Promise> { return await new ApiRequest().hogFunction(id).withAction('logs').withQueryString(params).get() }, + + async listTemplates(): Promise> { + return await new ApiRequest().hogFunctionTemplates().get() + }, + async getTemplate(id: HogFunctionTemplateType['id']): Promise { + return await new ApiRequest().hogFunctionTemplate(id).get() + }, + + async listIcons(params: { query?: string } = {}): Promise { + return await new ApiRequest().hogFunctions().withAction('icons').withQueryString(params).get() + }, }, annotations: { diff --git a/frontend/src/scenes/pipeline/Destinations.tsx b/frontend/src/scenes/pipeline/Destinations.tsx index 500111bdc5d1b..026f7fc2061fc 100644 --- a/frontend/src/scenes/pipeline/Destinations.tsx +++ b/frontend/src/scenes/pipeline/Destinations.tsx @@ -12,6 +12,7 @@ import { AvailableFeature, PipelineNodeTab, PipelineStage, ProductKey } from '~/ import { AppMetricSparkLine } from './AppMetricSparkLine' import { pipelineDestinationsLogic } from './destinationsLogic' +import { HogFunctionIcon } from './hogfunctions/HogFunctionIcon' import { NewButton } from './NewButton' import { pipelineAccessLogic } from './pipelineAccessLogic' import { Destination } from './types' @@ -60,6 +61,8 @@ export function DestinationsTable({ inOverview = false }: { inOverview?: boolean switch (destination.backend) { case 'plugin': return + case 'hog_function': + return case 'batch_export': return default: diff --git a/frontend/src/scenes/pipeline/PipelineNodeNew.tsx b/frontend/src/scenes/pipeline/PipelineNodeNew.tsx index 76eea7518a6b0..e24cca9060531 100644 --- a/frontend/src/scenes/pipeline/PipelineNodeNew.tsx +++ b/frontend/src/scenes/pipeline/PipelineNodeNew.tsx @@ -14,6 +14,7 @@ import { AvailableFeature, BatchExportService, HogFunctionTemplateType, Pipeline import { pipelineDestinationsLogic } from './destinationsLogic' import { frontendAppsLogic } from './frontendAppsLogic' +import { HogFunctionIcon } from './hogfunctions/HogFunctionIcon' import { PipelineHogFunctionConfiguration } from './hogfunctions/PipelineHogFunctionConfiguration' import { PipelineBatchExportConfiguration } from './PipelineBatchExportConfiguration' import { PIPELINE_TAB_TO_NODE_STAGE } from './PipelineNode' @@ -86,7 +87,7 @@ function convertHogFunctionToTableEntry(hogFunction: HogFunctionTemplateType): T id: `hog-${hogFunction.id}`, // TODO: This weird identifier thing isn't great name: hogFunction.name, description: hogFunction.description, - icon: 🦔, + icon: , } } diff --git a/frontend/src/scenes/pipeline/destinationsLogic.tsx b/frontend/src/scenes/pipeline/destinationsLogic.tsx index 05d4e2ff1d1e6..b920265e87760 100644 --- a/frontend/src/scenes/pipeline/destinationsLogic.tsx +++ b/frontend/src/scenes/pipeline/destinationsLogic.tsx @@ -18,7 +18,6 @@ import { } from '~/types' import type { pipelineDestinationsLogicType } from './destinationsLogicType' -import { HOG_FUNCTION_TEMPLATES } from './hogfunctions/templates/hog-templates' import { pipelineAccessLogic } from './pipelineAccessLogic' import { BatchExportDestination, convertToPipelineNode, Destination, PipelineBackend } from './types' import { captureBatchExportEvent, capturePluginEvent, loadPluginsFromUrl } from './utils' @@ -124,7 +123,8 @@ export const pipelineDestinationsLogic = kea([ {} as Record, { loadHogFunctionTemplates: async () => { - return HOG_FUNCTION_TEMPLATES.reduce((acc, template) => { + const templates = await api.hogFunctions.listTemplates() + return templates.results.reduce((acc, template) => { acc[template.id] = template return acc }, {} as Record) diff --git a/frontend/src/scenes/pipeline/hogfunctions/HogFunctionIcon.tsx b/frontend/src/scenes/pipeline/hogfunctions/HogFunctionIcon.tsx new file mode 100644 index 0000000000000..f99ee30ef52bb --- /dev/null +++ b/frontend/src/scenes/pipeline/hogfunctions/HogFunctionIcon.tsx @@ -0,0 +1,156 @@ +import { LemonButton, LemonFileInput, LemonInput, LemonSkeleton, lemonToast, Popover, Spinner } from '@posthog/lemon-ui' +import clsx from 'clsx' +import { useActions, useValues } from 'kea' +import { IconUploadFile } from 'lib/lemon-ui/icons' + +import { hogFunctionIconLogic, HogFunctionIconLogicProps } from './hogFunctionIconLogic' + +const fileToBase64 = (file?: File): Promise => { + return new Promise((resolve) => { + if (!file) { + return + } + + const reader = new FileReader() + + reader.onload = (e) => { + const img = new Image() + img.onload = () => { + const canvas = document.createElement('canvas') + const ctx = canvas.getContext('2d') + + // Set the dimensions at the wanted size. + const wantedWidth = 128 + const wantedHeight = 128 + canvas.width = wantedWidth + canvas.height = wantedHeight + + // Resize the image with the canvas method drawImage(); + ctx!.drawImage(img, 0, 0, wantedWidth, wantedHeight) + + const dataURI = canvas.toDataURL() + + resolve(dataURI) + } + img.src = e.target?.result as string + } + + reader.readAsDataURL(file) + }) +} + +export function HogFunctionIconEditable({ + size = 'medium', + ...props +}: HogFunctionIconLogicProps & { size?: 'small' | 'medium' | 'large' }): JSX.Element { + const { possibleIconsLoading, showPopover, possibleIcons, searchTerm } = useValues(hogFunctionIconLogic(props)) + const { setShowPopover, setSearchTerm } = useActions(hogFunctionIconLogic(props)) + + const content = ( + setShowPopover(!showPopover)} + > + {possibleIconsLoading ? : null} + + + ) + + return props.onChange ? ( + setShowPopover(false)} + overlay={ +
+
+

Choose an icon

+ + { + void fileToBase64(files[0]) + .then((dataURI) => { + props.onChange?.(dataURI) + }) + .catch(() => { + lemonToast.error('Error uploading image') + }) + }} + callToAction={ + }> + Upload image + + } + /> +
+ + : undefined} + /> + +
+ {possibleIcons?.map((icon) => ( + { + const nonTempUrl = icon.url.replace('&temp=true', '') + props.onChange?.(nonTempUrl) + setShowPopover(false) + }} + > + + + )) ?? + (possibleIconsLoading ? ( + + ) : ( + 'No icons found' + ))} +
+
+ } + > + {content} + + ) : ( + content + ) +} + +export function HogFunctionIcon({ + src, + size = 'medium', +}: { + src?: string + size?: 'small' | 'medium' | 'large' +}): JSX.Element { + return ( + + {src ? : 🦔} + + ) +} diff --git a/frontend/src/scenes/pipeline/hogfunctions/PipelineHogFunctionConfiguration.tsx b/frontend/src/scenes/pipeline/hogfunctions/PipelineHogFunctionConfiguration.tsx index caeda41d63eef..ed4ee04287fff 100644 --- a/frontend/src/scenes/pipeline/hogfunctions/PipelineHogFunctionConfiguration.tsx +++ b/frontend/src/scenes/pipeline/hogfunctions/PipelineHogFunctionConfiguration.tsx @@ -1,4 +1,13 @@ -import { LemonButton, LemonInput, LemonSwitch, LemonTextArea, SpinnerOverlay } from '@posthog/lemon-ui' +import { IconInfo } from '@posthog/icons' +import { + LemonButton, + LemonDropdown, + LemonInput, + LemonSwitch, + LemonTextArea, + Link, + SpinnerOverlay, +} from '@posthog/lemon-ui' import { useActions, useValues } from 'kea' import { Form } from 'kea-forms' import { NotFound } from 'lib/components/NotFound' @@ -15,6 +24,7 @@ import { groupsModel } from '~/models/groupsModel' import { NodeKind } from '~/queries/schema' import { EntityTypes } from '~/types' +import { HogFunctionIconEditable } from './HogFunctionIcon' import { HogFunctionInput } from './HogFunctionInputs' import { HogFunctionInputsEditor } from './HogFunctionInputsEditor' import { pipelineHogFunctionConfigurationLogic } from './pipelineHogFunctionConfigurationLogic' @@ -28,9 +38,10 @@ export function PipelineHogFunctionConfiguration({ }): JSX.Element { const logicProps = { templateId, id } const logic = pipelineHogFunctionConfigurationLogic(logicProps) - const { isConfigurationSubmitting, configurationChanged, showSource, configuration, loading, loaded } = + const { isConfigurationSubmitting, configurationChanged, showSource, configuration, loading, loaded, hogFunction } = useValues(logic) - const { submitConfiguration, resetForm, setShowSource } = useActions(logic) + const { submitConfiguration, resetForm, setShowSource, duplicate, resetToTemplate, duplicateFromTemplate } = + useActions(logic) const hogFunctionsEnabled = !!useFeatureFlag('HOG_FUNCTIONS') const { groupsTaxonomicTypes } = useValues(groupsModel) @@ -53,7 +64,18 @@ export function PipelineHogFunctionConfiguration({ ) } - const buttons = ( + + const headerButtons = ( + <> + {!templateId && ( + duplicate()}> + Duplicate + + )} + + ) + + const saveButtons = ( <> - + + {headerButtons} + {saveButtons} + + } + /> +
- 🦔 + + {({ value, onChange }) => ( + onChange(val)} + /> + )} + +
- Hog Function + {configuration.name}
- {({ value, onChange }) => ( + + {hogFunction?.template ? ( +

+ Built from template:{' '} + +

+ This function was built from the template{' '} + {hogFunction.template.name}. If the template is updated, this + function is not affected unless you choose to update it. +

+ +
+
+ Close +
+ resetToTemplate()}> + Reset to template + + + duplicateFromTemplate()} + > + New function from template + +
+
+ } + > + + {hogFunction?.template.name} + + +

+ ) : null}
@@ -216,6 +293,7 @@ export function PipelineHogFunctionConfiguration({ name={`inputs.${schema.key}`} label={schema.label || schema.key} showOptional={!schema.required} + help={schema.description} > {({ value, onChange }) => { return ( @@ -238,7 +316,7 @@ export function PipelineHogFunctionConfiguration({
)} -
{buttons}
+
{saveButtons}
diff --git a/frontend/src/scenes/pipeline/hogfunctions/hogFunctionIconLogic.ts b/frontend/src/scenes/pipeline/hogfunctions/hogFunctionIconLogic.ts new file mode 100644 index 0000000000000..2f92423b57a00 --- /dev/null +++ b/frontend/src/scenes/pipeline/hogfunctions/hogFunctionIconLogic.ts @@ -0,0 +1,97 @@ +import { actions, kea, key, listeners, path, props, propsChanged, reducers } from 'kea' +import { loaders } from 'kea-loaders' +import api from 'lib/api' + +import { HogFunctionIconResponse } from '~/types' + +import type { hogFunctionIconLogicType } from './hogFunctionIconLogicType' + +export interface HogFunctionIconLogicProps { + logicKey: string + search: string + src?: string + onChange?: (src: string) => void +} + +export const hogFunctionIconLogic = kea([ + props({} as HogFunctionIconLogicProps), + key((props) => props.logicKey ?? 'default'), + path((key) => ['scenes', 'pipeline', 'hogfunctions', 'hogFunctionIconLogic', key]), + + actions({ + loadPossibleIcons: true, + setShowPopover: (show: boolean) => ({ show }), + setSearchTerm: (search: string) => ({ search }), + }), + + reducers({ + showPopover: [ + false, + { + setShowPopover: (_, { show }) => show, + }, + ], + + searchTerm: [ + null as string | null, + { + setSearchTerm: (_, { search }) => search, + setShowPopover: () => null, + }, + ], + }), + + loaders(({ props, values }) => ({ + possibleIcons: [ + null as HogFunctionIconResponse[] | null, + { + loadPossibleIcons: async (_, breakpoint) => { + const search = values.searchTerm ?? props.search + + if (!search) { + return [] + } + + await breakpoint(1000) + const res = await api.hogFunctions.listIcons({ query: search }) + return res.map((icon) => ({ + ...icon, + url: icon.url + '&temp=true', + })) + }, + }, + ], + })), + + listeners(({ actions, values, props }) => ({ + loadPossibleIconsSuccess: async () => { + const autoChange = props.onChange && (!props.src || props.src.includes('temp=true')) + if (!autoChange) { + return + } + const firstValue = values.possibleIcons?.[0] + if (firstValue) { + props.onChange?.(firstValue.url) + } + }, + + setShowPopover: ({ show }) => { + if (show) { + actions.loadPossibleIcons() + } + }, + + setSearchTerm: () => { + actions.loadPossibleIcons() + }, + })), + + propsChanged(({ props, actions }, oldProps) => { + if (!props.onChange) { + return + } + if (!props.src || (props.search !== oldProps.search && props.src.includes('temp=true'))) { + actions.loadPossibleIcons() + } + }), +]) diff --git a/frontend/src/scenes/pipeline/hogfunctions/pipelineHogFunctionConfigurationLogic.tsx b/frontend/src/scenes/pipeline/hogfunctions/pipelineHogFunctionConfigurationLogic.tsx index 8c90cb6e93334..5937c80dc4eb7 100644 --- a/frontend/src/scenes/pipeline/hogfunctions/pipelineHogFunctionConfigurationLogic.tsx +++ b/frontend/src/scenes/pipeline/hogfunctions/pipelineHogFunctionConfigurationLogic.tsx @@ -1,3 +1,4 @@ +import { lemonToast } from '@posthog/lemon-ui' import { actions, afterMount, kea, key, listeners, path, props, reducers, selectors } from 'kea' import { forms } from 'kea-forms' import { loaders } from 'kea-loaders' @@ -17,13 +18,23 @@ import { } from '~/types' import type { pipelineHogFunctionConfigurationLogicType } from './pipelineHogFunctionConfigurationLogicType' -import { HOG_FUNCTION_TEMPLATES } from './templates/hog-templates' export interface PipelineHogFunctionConfigurationLogicProps { templateId?: string id?: string } +export type HogFunctionConfigurationType = Omit + +const NEW_FUNCTION_TEMPLATE: HogFunctionTemplateType = { + id: 'new', + name: '', + description: '', + inputs_schema: [], + hog: "print('Hello, world!');", + status: 'stable', +} + function sanitizeFilters(filters?: FilterType): PluginConfigTypeNew['filters'] { if (!filters) { return null @@ -66,7 +77,10 @@ export const pipelineHogFunctionConfigurationLogic = kea ['scenes', 'pipeline', 'pipelineHogFunctionConfigurationLogic', id]), actions({ setShowSource: (showSource: boolean) => ({ showSource }), - resetForm: true, + resetForm: (configuration?: HogFunctionConfigurationType) => ({ configuration }), + duplicate: true, + duplicateFromTemplate: true, + resetToTemplate: true, }), reducers({ showSource: [ @@ -84,7 +98,14 @@ export const pipelineHogFunctionConfigurationLogic = kea template.id === props.templateId) + + if (props.templateId === 'new') { + return { + ...NEW_FUNCTION_TEMPLATE, + } + } + + const res = await api.hogFunctions.getTemplate(props.templateId) if (!res) { throw new Error('Template not found') @@ -109,40 +130,48 @@ export const pipelineHogFunctionConfigurationLogic = kea ({ configuration: { - defaults: {} as HogFunctionType, + defaults: {} as HogFunctionConfigurationType, alwaysShowErrors: true, errors: (data) => { return { - name: !data.name ? 'Name is required' : null, + name: !data.name ? 'Name is required' : undefined, ...values.inputFormErrors, } }, submit: async (data) => { - const sanitizedInputs = {} + try { + const sanitizedInputs = {} - data.inputs_schema?.forEach((input) => { - if (input.type === 'json' && typeof data.inputs[input.key].value === 'string') { - try { + data.inputs_schema?.forEach((input) => { + const value = data.inputs?.[input.key]?.value + + if (input.type === 'json' && typeof value === 'string') { + try { + sanitizedInputs[input.key] = { + value: JSON.parse(value), + } + } catch (e) { + // Ignore + } + } else { sanitizedInputs[input.key] = { - value: JSON.parse(data.inputs[input.key].value), + value: value, } - } catch (e) { - // Ignore - } - } else { - sanitizedInputs[input.key] = { - value: data.inputs[input.key].value, } + }) + + const payload: HogFunctionConfigurationType = { + ...data, + filters: data.filters ? sanitizeFilters(data.filters) : null, + inputs: sanitizedInputs, + icon_url: data.icon_url?.replace('&temp=true', ''), // Remove temp=true so it doesn't try and suggest new options next time } - }) - const payload = { - ...data, - filters: data.filters ? sanitizeFilters(data.filters) : null, - inputs: sanitizedInputs, - } + if (props.templateId) { + // Only sent on create + ;(payload as any).template_id = props.templateId + } - try { if (!props.id) { return await api.hogFunctions.create(payload) } @@ -161,7 +190,11 @@ export const pipelineHogFunctionConfigurationLogic = kea ({ - loadTemplateSuccess: () => actions.resetForm(), - loadHogFunctionSuccess: () => actions.resetForm(), - resetForm: () => { - const savedValue = values.hogFunction ?? values.template + loadTemplateSuccess: ({ template }) => { + // Fill defaults from template + const inputs = {} + + template!.inputs_schema?.forEach((schema) => { + if (schema.default) { + inputs[schema.key] = { value: schema.default } + } + }) + + actions.resetForm({ + ...template!, + inputs, + enabled: false, + }) + }, + loadHogFunctionSuccess: ({ hogFunction }) => actions.resetForm(hogFunction!), + + resetForm: ({ configuration }) => { + const savedValue = configuration actions.resetConfiguration({ ...savedValue, - inputs: (savedValue as any)?.inputs ?? {}, + inputs: savedValue?.inputs ?? {}, ...(cache.configFromUrl || {}), }) }, @@ -226,6 +275,44 @@ export const pipelineHogFunctionConfigurationLogic = kea { + if (values.hogFunction) { + const newConfig = { + ...values.configuration, + name: `${values.configuration.name} (copy)`, + } + router.actions.push( + urls.pipelineNodeNew(PipelineStage.Destination, `hog-template-helloworld`), + undefined, + { + configuration: newConfig, + } + ) + } + }, + duplicateFromTemplate: async () => { + if (values.hogFunction?.template) { + const newConfig = { + ...values.hogFunction.template, + } + router.actions.push( + urls.pipelineNodeNew(PipelineStage.Destination, `hog-${values.hogFunction.template.id}`), + undefined, + { + configuration: newConfig, + } + ) + } + }, + resetToTemplate: async () => { + if (values.hogFunction?.template) { + actions.resetForm({ + ...values.hogFunction.template, + enabled: false, + }) + } + }, })), afterMount(({ props, actions, cache }) => { if (props.templateId) { diff --git a/frontend/src/scenes/pipeline/hogfunctions/templates/hog-templates.tsx b/frontend/src/scenes/pipeline/hogfunctions/templates/hog-templates.tsx deleted file mode 100644 index 294159998ab2b..0000000000000 --- a/frontend/src/scenes/pipeline/hogfunctions/templates/hog-templates.tsx +++ /dev/null @@ -1,58 +0,0 @@ -import { HogFunctionTemplateType } from '~/types' - -export const HOG_FUNCTION_TEMPLATES: HogFunctionTemplateType[] = [ - { - id: 'template-webhook', - name: 'HogHook', - description: 'Sends a webhook templated by the incoming event data', - hog: "fetch(inputs.url, {\n 'headers': inputs.headers,\n 'body': inputs.payload,\n 'method': inputs.method\n});", - inputs_schema: [ - { - key: 'url', - type: 'string', - label: 'Webhook URL', - secret: false, - required: true, - }, - { - key: 'method', - type: 'choice', - label: 'Method', - secret: false, - choices: [ - { - label: 'POST', - value: 'POST', - }, - { - label: 'PUT', - value: 'PUT', - }, - { - label: 'GET', - value: 'GET', - }, - { - label: 'DELETE', - value: 'DELETE', - }, - ], - required: false, - }, - { - key: 'payload', - type: 'json', - label: 'JSON Payload', - secret: false, - required: false, - }, - { - key: 'headers', - type: 'dictionary', - label: 'Headers', - secret: false, - required: false, - }, - ], - }, -] diff --git a/frontend/src/scenes/pipeline/types.ts b/frontend/src/scenes/pipeline/types.ts index f958ebb887ca6..e10801808ae24 100644 --- a/frontend/src/scenes/pipeline/types.ts +++ b/frontend/src/scenes/pipeline/types.ts @@ -44,6 +44,7 @@ export interface BatchExportBasedNode extends PipelineNodeBase { export interface HogFunctionBasedNode extends PipelineNodeBase { backend: PipelineBackend.HogFunction id: string + hog_function: HogFunctionType } // Stage: Transformations @@ -121,6 +122,7 @@ export function convertToPipelineNode( enabled: candidate.enabled, created_at: candidate.created_at, updated_at: candidate.created_at, + hog_function: candidate, } } else if (isPluginConfig(candidate)) { const almostNode: Omit< diff --git a/frontend/src/types.ts b/frontend/src/types.ts index b633882617fdb..bcda304202dc2 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -4169,6 +4169,7 @@ export type HogFunctionInputSchemaType = { export type HogFunctionType = { id: string + icon_url?: string name: string description: string created_by: UserBasicType | null @@ -4177,8 +4178,8 @@ export type HogFunctionType = { enabled: boolean hog: string - inputs_schema: HogFunctionInputSchemaType[] - inputs: Record< + inputs_schema?: HogFunctionInputSchemaType[] + inputs?: Record< string, { value: any @@ -4191,8 +4192,16 @@ export type HogFunctionType = { export type HogFunctionTemplateType = Pick< HogFunctionType, - 'id' | 'name' | 'description' | 'hog' | 'inputs_schema' | 'filters' -> + 'id' | 'name' | 'description' | 'hog' | 'inputs_schema' | 'filters' | 'icon_url' +> & { + status: 'alpha' | 'beta' | 'stable' +} + +export type HogFunctionIconResponse = { + id: string + name: string + url: string +} export interface AnomalyCondition { absoluteThreshold: { diff --git a/latest_migrations.manifest b/latest_migrations.manifest index 905aeb627006b..eb4d108600853 100644 --- a/latest_migrations.manifest +++ b/latest_migrations.manifest @@ -5,7 +5,7 @@ contenttypes: 0002_remove_content_type_name ee: 0016_rolemembership_organization_member otp_static: 0002_throttling otp_totp: 0002_auto_20190420_0723 -posthog: 0426_externaldatasource_sync_frequency +posthog: 0427_hogfunction_icon_url_hogfunction_template_id sessions: 0001_initial social_django: 0010_uid_db_index two_factor: 0007_auto_20201201_1019 diff --git a/plugin-server/src/cdp/async-function-executor.ts b/plugin-server/src/cdp/async-function-executor.ts index 8b29a94edaa4e..2d7309f9e739c 100644 --- a/plugin-server/src/cdp/async-function-executor.ts +++ b/plugin-server/src/cdp/async-function-executor.ts @@ -79,11 +79,16 @@ export class AsyncFunctionExecutor { timeout: this.serverConfig.EXTERNAL_REQUEST_TIMEOUT_MS, }) - const maybeJson = await fetchResponse.json().catch(() => null) + let body = await fetchResponse.text() + try { + body = JSON.parse(body) + } catch (err) { + body + } response.vmResponse = { status: fetchResponse.status, - body: maybeJson ? maybeJson : await fetchResponse.text(), + body: body, } } catch (err) { status.error('🦔', `[HogExecutor] Error during fetch`, { ...request, error: String(err) }) diff --git a/plugin-server/src/cdp/cdp-processed-events-consumer.ts b/plugin-server/src/cdp/cdp-processed-events-consumer.ts index 51914f9bd969d..0ba2f3e246ada 100644 --- a/plugin-server/src/cdp/cdp-processed-events-consumer.ts +++ b/plugin-server/src/cdp/cdp-processed-events-consumer.ts @@ -283,7 +283,7 @@ export class CdpProcessedEventsConsumer extends CdpConsumerBase { export class CdpFunctionCallbackConsumer extends CdpConsumerBase { protected name = 'CdpFunctionCallbackConsumer' protected topic = KAFKA_CDP_FUNCTION_CALLBACKS - protected consumerGroupId = 'cdp-processed-events-consumer' + protected consumerGroupId = 'cdp-function-callback-consumer' public async handleEachBatch(messages: Message[], heartbeat: () => void): Promise { await runInstrumentedFunction({ diff --git a/plugin-server/src/cdp/hog-executor.ts b/plugin-server/src/cdp/hog-executor.ts index 7ed9d598f22e9..bda63f316bfba 100644 --- a/plugin-server/src/cdp/hog-executor.ts +++ b/plugin-server/src/cdp/hog-executor.ts @@ -246,7 +246,7 @@ export class HogExecutor { log('warn', `Function was not finished but also had no async function to execute.`) } } else { - log('debug', `Function completed (${hogFunction.id}) (${hogFunction.name})!`) + log('debug', `Function completed`) } } catch (err) { error = err diff --git a/plugin-server/src/cdp/types.ts b/plugin-server/src/cdp/types.ts index ec9cc7f8be6c3..438dc88d9fe8f 100644 --- a/plugin-server/src/cdp/types.ts +++ b/plugin-server/src/cdp/types.ts @@ -54,8 +54,9 @@ export type HogFunctionInvocationGlobals = { } person?: { uuid: string - properties: Record + name: string url: string + properties: Record } groups?: Record< string, diff --git a/plugin-server/src/cdp/utils.ts b/plugin-server/src/cdp/utils.ts index 82f2739944dc8..f8fe9c6dc075b 100644 --- a/plugin-server/src/cdp/utils.ts +++ b/plugin-server/src/cdp/utils.ts @@ -4,6 +4,27 @@ import { GroupTypeToColumnIndex, RawClickHouseEvent, Team } from '../types' import { clickHouseTimestampToISO } from '../utils/utils' import { HogFunctionFilterGlobals, HogFunctionInvocationGlobals } from './types' +export const PERSON_DEFAULT_DISPLAY_NAME_PROPERTIES = [ + 'email', + 'Email', + 'name', + 'Name', + 'username', + 'Username', + 'UserName', +] + +const getPersonDisplayName = (team: Team, distinctId: string, properties: Record): string => { + const personDisplayNameProperties = team.person_display_name_properties ?? PERSON_DEFAULT_DISPLAY_NAME_PROPERTIES + const customPropertyKey = personDisplayNameProperties.find((x) => properties?.[x]) + const propertyIdentifier = customPropertyKey ? properties[customPropertyKey] : undefined + + const customIdentifier: string = + typeof propertyIdentifier !== 'string' ? JSON.stringify(propertyIdentifier) : propertyIdentifier + + return (customIdentifier || distinctId)?.trim() +} + // that we can keep to as a contract export function convertToHogFunctionInvocationGlobals( event: RawClickHouseEvent, @@ -18,6 +39,20 @@ export function convertToHogFunctionInvocationGlobals( properties['$elements_chain'] = event.elements_chain } + let person: HogFunctionInvocationGlobals['person'] + + if (event.person_id) { + const personProperties = event.person_properties ? JSON.parse(event.person_properties) : {} + const personDisplayName = getPersonDisplayName(team, event.distinct_id, personProperties) + + person = { + uuid: event.person_id, + name: personDisplayName, + properties: personProperties, + url: `${projectUrl}/person/${encodeURIComponent(event.distinct_id)}`, + } + } + let groups: HogFunctionInvocationGlobals['groups'] = undefined if (groupTypes) { @@ -59,14 +94,7 @@ export function convertToHogFunctionInvocationGlobals( clickHouseTimestampToISO(event.timestamp) )}`, }, - person: event.person_id - ? { - uuid: event.person_id, - properties: event.person_properties ? JSON.parse(event.person_properties) : {}, - // TODO: IS this distinct_id or person_id? - url: `${projectUrl}/person/${encodeURIComponent(event.distinct_id)}`, - } - : undefined, + person, groups, } diff --git a/posthog/api/__init__.py b/posthog/api/__init__.py index dbe3d1bd17515..7968f3d1f2171 100644 --- a/posthog/api/__init__.py +++ b/posthog/api/__init__.py @@ -21,6 +21,7 @@ exports, feature_flag, hog_function, + hog_function_template, ingestion_warnings, instance_settings, instance_status, @@ -416,6 +417,13 @@ def api_not_found(request): ["team_id"], ) +projects_router.register( + r"hog_function_templates", + hog_function_template.HogFunctionTemplateViewSet, + "project_hog_function_templates", + ["team_id"], +) + projects_router.register( r"alerts", alert.AlertViewSet, diff --git a/posthog/api/hog_function.py b/posthog/api/hog_function.py index 77cb4c88e8a7b..17ee1bb2b0995 100644 --- a/posthog/api/hog_function.py +++ b/posthog/api/hog_function.py @@ -2,81 +2,26 @@ from django_filters.rest_framework import DjangoFilterBackend from rest_framework import serializers, viewsets from rest_framework.serializers import BaseSerializer +from rest_framework.decorators import action +from rest_framework.request import Request +from rest_framework.response import Response + from posthog.api.forbid_destroy_model import ForbidDestroyModel +from posthog.api.hog_function_template import HogFunctionTemplateSerializer from posthog.api.log_entries import LogEntryMixin from posthog.api.routing import TeamAndOrgViewSetMixin from posthog.api.shared import UserBasicSerializer -from posthog.hogql.bytecode import create_bytecode -from posthog.hogql.parser import parse_program + +from posthog.cdp.services.icons import CDPIconsService +from posthog.cdp.validation import compile_hog, validate_inputs, validate_inputs_schema from posthog.models.hog_functions.hog_function import HogFunction -from posthog.models.hog_functions.utils import generate_template_bytecode from posthog.permissions import PostHogFeatureFlagPermission logger = structlog.get_logger(__name__) -class InputsSchemaItemSerializer(serializers.Serializer): - type = serializers.ChoiceField(choices=["string", "boolean", "dictionary", "choice", "json"]) - key = serializers.CharField() - label = serializers.CharField(required=False) # type: ignore - choices = serializers.ListField(child=serializers.DictField(), required=False) - required = serializers.BooleanField(default=False) # type: ignore - default = serializers.JSONField(required=False) - secret = serializers.BooleanField(default=False) - description = serializers.CharField(required=False) - - # TODO Validate choices if type=choice - - -class AnyInputField(serializers.Field): - def to_internal_value(self, data): - return data - - def to_representation(self, value): - return value - - -class InputsItemSerializer(serializers.Serializer): - value = AnyInputField(required=False) - bytecode = serializers.ListField(required=False, read_only=True) - - def validate(self, attrs): - schema = self.context["schema"] - value = attrs.get("value") - - if schema.get("required") and not value: - raise serializers.ValidationError("This field is required.") - - if not value: - return attrs - - name: str = schema["key"] - item_type = schema["type"] - value = attrs["value"] - - # Validate each type - if item_type == "string": - if not isinstance(value, str): - raise serializers.ValidationError("Value must be a string.") - elif item_type == "boolean": - if not isinstance(value, bool): - raise serializers.ValidationError("Value must be a boolean.") - elif item_type == "dictionary": - if not isinstance(value, dict): - raise serializers.ValidationError("Value must be a dictionary.") - - try: - if value: - if item_type in ["string", "dictionary", "json"]: - attrs["bytecode"] = generate_template_bytecode(value) - except Exception as e: - raise serializers.ValidationError({"inputs": {name: f"Invalid template: {str(e)}"}}) - - return attrs - - class HogFunctionMinimalSerializer(serializers.ModelSerializer): created_by = UserBasicSerializer(read_only=True) @@ -92,11 +37,14 @@ class Meta: "enabled", "hog", "filters", + "icon_url", ] read_only_fields = fields class HogFunctionSerializer(HogFunctionMinimalSerializer): + template = HogFunctionTemplateSerializer(read_only=True) + class Meta: model = HogFunction fields = [ @@ -112,6 +60,9 @@ class Meta: "inputs_schema", "inputs", "filters", + "icon_url", + "template", + "template_id", ] read_only_fields = [ "id", @@ -119,46 +70,22 @@ class Meta: "created_by", "updated_at", "bytecode", + "template", ] + extra_kwargs = { + "template_id": {"write_only": True}, + } def validate_inputs_schema(self, value): - if not isinstance(value, list): - raise serializers.ValidationError("inputs_schema must be a list of objects.") - - serializer = InputsSchemaItemSerializer(data=value, many=True) - - if not serializer.is_valid(): - raise serializers.ValidationError(serializer.errors) - - return serializer.validated_data or [] + return validate_inputs_schema(value) def validate(self, attrs): team = self.context["get_team"]() attrs["team"] = team attrs["inputs_schema"] = attrs.get("inputs_schema", []) - attrs["inputs"] = attrs.get("inputs", {}) attrs["filters"] = attrs.get("filters", {}) - - validated_inputs = {} - - for schema in attrs["inputs_schema"]: - value = attrs["inputs"].get(schema["key"], {}) - serializer = InputsItemSerializer(data=value, context={"schema": schema}) - - if not serializer.is_valid(): - first_error = next(iter(serializer.errors.values()))[0] - raise serializers.ValidationError({"inputs": {schema["key"]: first_error}}) - - validated_inputs[schema["key"]] = serializer.validated_data - - attrs["inputs"] = validated_inputs - - # Attempt to compile the hog - try: - program = parse_program(attrs["hog"]) - attrs["bytecode"] = create_bytecode(program, supported_functions={"fetch"}) - except Exception as e: - raise serializers.ValidationError({"hog": str(e)}) + attrs["inputs"] = validate_inputs(attrs["inputs_schema"], attrs.get("inputs", {})) + attrs["bytecode"] = compile_hog(attrs["hog"]) return attrs @@ -180,3 +107,23 @@ class HogFunctionViewSet(TeamAndOrgViewSetMixin, LogEntryMixin, ForbidDestroyMod def get_serializer_class(self) -> type[BaseSerializer]: return HogFunctionMinimalSerializer if self.action == "list" else HogFunctionSerializer + + @action(detail=False, methods=["GET"]) + def icons(self, request: Request, *args, **kwargs): + query = request.GET.get("query") + if not query: + return Response([]) + + icons = CDPIconsService().list_icons(query, icon_url_base="/api/projects/@current/hog_functions/icon/?id=") + + return Response(icons) + + @action(detail=False, methods=["GET"]) + def icon(self, request: Request, *args, **kwargs): + id = request.GET.get("id") + if not id: + raise serializers.ValidationError("id is required") + + icon_service = CDPIconsService() + + return icon_service.get_icon_http_response(id) diff --git a/posthog/api/hog_function_template.py b/posthog/api/hog_function_template.py new file mode 100644 index 0000000000000..6d04fa69ced6c --- /dev/null +++ b/posthog/api/hog_function_template.py @@ -0,0 +1,53 @@ +import structlog +from django_filters.rest_framework import DjangoFilterBackend +from rest_framework import viewsets +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.exceptions import NotFound + +from posthog.api.routing import TeamAndOrgViewSetMixin +from posthog.cdp.templates import HOG_FUNCTION_TEMPLATES +from posthog.cdp.templates.hog_function_template import HogFunctionTemplate +from posthog.models.hog_functions.hog_function import HogFunction +from posthog.permissions import PostHogFeatureFlagPermission +from rest_framework_dataclasses.serializers import DataclassSerializer + + +logger = structlog.get_logger(__name__) + + +class HogFunctionTemplateSerializer(DataclassSerializer): + class Meta: + dataclass = HogFunctionTemplate + + +class HogFunctionTemplateViewSet(TeamAndOrgViewSetMixin, viewsets.GenericViewSet): + scope_object = "INTERNAL" # Keep internal until we are happy to release this GA + queryset = HogFunction.objects.none() + filter_backends = [DjangoFilterBackend] + filterset_fields = ["id", "team", "created_by", "enabled"] + + permission_classes = [PostHogFeatureFlagPermission] + posthog_feature_flag = {"hog-functions": ["create", "partial_update", "update"]} + + serializer_class = HogFunctionTemplateSerializer + + def _get_templates(self): + # TODO: Filtering for status? + data = HOG_FUNCTION_TEMPLATES + return data + + def list(self, request: Request, *args, **kwargs): + page = self.paginate_queryset(self._get_templates()) + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + + def retrieve(self, request: Request, *args, **kwargs): + data = self._get_templates() + item = next((item for item in data if item.id == kwargs["pk"]), None) + + if not item: + raise NotFound(f"Template with id {kwargs['pk']} not found.") + + serializer = self.get_serializer(item) + return Response(serializer.data) diff --git a/posthog/api/test/__snapshots__/test_decide.ambr b/posthog/api/test/__snapshots__/test_decide.ambr index d03538893572b..e3fed7b7e261d 100644 --- a/posthog/api/test/__snapshots__/test_decide.ambr +++ b/posthog/api/test/__snapshots__/test_decide.ambr @@ -338,11 +338,13 @@ "posthog_hogfunction"."deleted", "posthog_hogfunction"."updated_at", "posthog_hogfunction"."enabled", + "posthog_hogfunction"."icon_url", "posthog_hogfunction"."hog", "posthog_hogfunction"."bytecode", "posthog_hogfunction"."inputs_schema", "posthog_hogfunction"."inputs", "posthog_hogfunction"."filters", + "posthog_hogfunction"."template_id", "posthog_team"."id", "posthog_team"."uuid", "posthog_team"."organization_id", @@ -455,11 +457,13 @@ "posthog_hogfunction"."deleted", "posthog_hogfunction"."updated_at", "posthog_hogfunction"."enabled", + "posthog_hogfunction"."icon_url", "posthog_hogfunction"."hog", "posthog_hogfunction"."bytecode", "posthog_hogfunction"."inputs_schema", "posthog_hogfunction"."inputs", "posthog_hogfunction"."filters", + "posthog_hogfunction"."template_id", "posthog_team"."id", "posthog_team"."uuid", "posthog_team"."organization_id", @@ -697,11 +701,13 @@ "posthog_hogfunction"."deleted", "posthog_hogfunction"."updated_at", "posthog_hogfunction"."enabled", + "posthog_hogfunction"."icon_url", "posthog_hogfunction"."hog", "posthog_hogfunction"."bytecode", "posthog_hogfunction"."inputs_schema", "posthog_hogfunction"."inputs", "posthog_hogfunction"."filters", + "posthog_hogfunction"."template_id", "posthog_team"."id", "posthog_team"."uuid", "posthog_team"."organization_id", @@ -1057,11 +1063,13 @@ "posthog_hogfunction"."deleted", "posthog_hogfunction"."updated_at", "posthog_hogfunction"."enabled", + "posthog_hogfunction"."icon_url", "posthog_hogfunction"."hog", "posthog_hogfunction"."bytecode", "posthog_hogfunction"."inputs_schema", "posthog_hogfunction"."inputs", "posthog_hogfunction"."filters", + "posthog_hogfunction"."template_id", "posthog_team"."id", "posthog_team"."uuid", "posthog_team"."organization_id", diff --git a/posthog/api/test/test_hog_function.py b/posthog/api/test/test_hog_function.py index 63b6fbc22ec93..94c3b5b1c4d14 100644 --- a/posthog/api/test/test_hog_function.py +++ b/posthog/api/test/test_hog_function.py @@ -5,6 +5,7 @@ from posthog.models.action.action import Action from posthog.test.base import APIBaseTest, ClickhouseTestMixin, QueryMatchingTest +from posthog.cdp.templates.webhook.template_webhook import template as template_webhook EXAMPLE_FULL = { @@ -96,6 +97,31 @@ def test_create_hog_function(self, *args): "inputs_schema": [], "inputs": {}, "filters": {"bytecode": ["_h", 29]}, + "icon_url": None, + "template": None, + } + + @patch("posthog.permissions.posthoganalytics.feature_enabled", return_value=True) + def test_creates_with_template_id(self, *args): + response = self.client.post( + f"/api/projects/{self.team.id}/hog_functions/", + data={ + "name": "Fetch URL", + "description": "Test description", + "hog": "fetch(inputs.url);", + "template_id": template_webhook.id, + }, + ) + assert response.status_code == status.HTTP_201_CREATED, response.json() + assert response.json()["template"] == { + "name": template_webhook.name, + "description": template_webhook.description, + "id": template_webhook.id, + "status": template_webhook.status, + "icon_url": template_webhook.icon_url, + "inputs_schema": template_webhook.inputs_schema, + "hog": template_webhook.hog, + "filters": None, } @patch("posthog.permissions.posthoganalytics.feature_enabled", return_value=True) diff --git a/posthog/cdp/services/icons.py b/posthog/cdp/services/icons.py new file mode 100644 index 0000000000000..f4cd744d5acb9 --- /dev/null +++ b/posthog/cdp/services/icons.py @@ -0,0 +1,48 @@ +from django.conf import settings +from django.http import HttpResponse +import requests + +from rest_framework.exceptions import NotFound + + +class CDPIconsService: + @property + def supported(self): + return bool(settings.LOGO_DEV_TOKEN) + + def list_icons(self, query: str, icon_url_base: str): + if not self.supported: + return [] + + res = requests.get( + f"https://search.logo.dev/api/icons", + params={ + "token": settings.LOGO_DEV_TOKEN, + "query": query, + }, + ) + data = res.json() + + parsed = [ + { + "id": item["domain"], + "name": item["name"], + "url": f"{icon_url_base}{item['domain']}", + } + for item in data + ] + + return parsed + + def get_icon_http_response(self, id: str): + if not self.supported: + raise NotFound() + + res = requests.get( + f"https://img.logo.dev/{id}", + { + "token": settings.LOGO_DEV_TOKEN, + }, + ) + + return HttpResponse(res.content, content_type=res.headers["Content-Type"]) diff --git a/posthog/cdp/templates/README.md b/posthog/cdp/templates/README.md new file mode 100644 index 0000000000000..40ae30f222dc7 --- /dev/null +++ b/posthog/cdp/templates/README.md @@ -0,0 +1,19 @@ +# CDP Hog Function Templates + +Hog function templates are the main way that people will get started with the CDP V2. Templates are ephemeral configurations of HogFunctions. They are purposefully ephemeral for a few reasons: + +1. Easier to manage as they are just code in the `posthog` repo - we don't need to worry about keeping a database in sync +2. Update conflicts are left to the user to manage - changing the template doesn't change their function. It only will indicate in the UI that it is out of sync and they can choose to pull in the updated template changes, resolving any issues at that point. +3. Sharing templates becomes very simple - it can all be done via a URL + +## Notes on building good templates + +Templates should be as generic as possible. The underlying Hog code should only do what is required for communicating with the destination or triggering the workflow. + +All input data should be controlled via the `inputs_schema` wherever possible as this leaves ultimate flexibility in the hands of the user of the template as they can inject data from the source (whether it is an Event, an ActivityLog or anything else) using Hog templating. If you hit the limits of the input templating, considering working with #team-cdp to extend these before writing a Hog function that is too tightly coupled to + +## Filtering + +Filtering of the incoming source should almost always **not** be done in the Hog code itself. PostHog provides a filtering UI when setting up the source that is powerful and generic to the source to ensure the function is only run when it needs to. + +This isn't a hard rule of course, you can also do filtering in Hog just be aware that it limits the re-usability of your function. diff --git a/posthog/cdp/templates/__init__.py b/posthog/cdp/templates/__init__.py new file mode 100644 index 0000000000000..3e29e2a21d68f --- /dev/null +++ b/posthog/cdp/templates/__init__.py @@ -0,0 +1,9 @@ +from .webhook.template_webhook import template as webhook +from .helloworld.template_helloworld import template as hello_world +from .slack.template_slack import template as slack + + +HOG_FUNCTION_TEMPLATES = [webhook, hello_world, slack] +HOG_FUNCTION_TEMPLATES_BY_ID = {template.id: template for template in HOG_FUNCTION_TEMPLATES} + +__all__ = ["HOG_FUNCTION_TEMPLATES", "HOG_FUNCTION_TEMPLATES_BY_ID"] diff --git a/posthog/cdp/templates/helloworld/template_helloworld.py b/posthog/cdp/templates/helloworld/template_helloworld.py new file mode 100644 index 0000000000000..84a8349476564 --- /dev/null +++ b/posthog/cdp/templates/helloworld/template_helloworld.py @@ -0,0 +1,22 @@ +from posthog.cdp.templates.hog_function_template import HogFunctionTemplate + + +template: HogFunctionTemplate = HogFunctionTemplate( + status="alpha", + id="template-hello-workd", + name="Hello world", + description="Prints your message or hello world!", + icon_url="/api/projects/@current/hog_functions/icon/?id=posthog.com&temp=true", + hog=""" +print(inputs.message ?? 'hello world!'); +""".strip(), + inputs_schema=[ + { + "key": "message", + "type": "string", + "label": "Message to print", + "secret": False, + "required": False, + } + ], +) diff --git a/posthog/cdp/templates/helpers.py b/posthog/cdp/templates/helpers.py new file mode 100644 index 0000000000000..b5b3c2081decd --- /dev/null +++ b/posthog/cdp/templates/helpers.py @@ -0,0 +1,36 @@ +from typing import Any +from unittest.mock import MagicMock +from posthog.cdp.templates.hog_function_template import HogFunctionTemplate +from posthog.cdp.validation import compile_hog +from posthog.test.base import BaseTest +from hogvm.python.execute import execute_bytecode + + +class BaseHogFunctionTemplateTest(BaseTest): + template: HogFunctionTemplate + compiled_hog: Any + + mock_fetch = MagicMock() + + def setUp(self): + super().setUp() + self.compiled_hog = compile_hog(self.template.hog) + + def createHogGlobals(self, globals=None) -> dict: + # Return an object simulating the + return {} + + def run_function(self, inputs: dict, globals=None): + # Create the globals object + globals = self.createHogGlobals(globals) + globals["inputs"] = inputs + + # Run the function + + return execute_bytecode( + self.compiled_hog, + globals, + functions={ + "fetch": self.mock_fetch, + }, + ) diff --git a/posthog/cdp/templates/hog_function_template.py b/posthog/cdp/templates/hog_function_template.py new file mode 100644 index 0000000000000..ac4f36e58f27c --- /dev/null +++ b/posthog/cdp/templates/hog_function_template.py @@ -0,0 +1,14 @@ +import dataclasses +from typing import Literal, Optional + + +@dataclasses.dataclass(frozen=True) +class HogFunctionTemplate: + status: Literal["alpha", "beta", "stable"] + id: str + name: str + description: str + hog: str + inputs_schema: list[dict] + filters: Optional[dict] = None + icon_url: Optional[str] = None diff --git a/posthog/cdp/templates/slack/template_slack.py b/posthog/cdp/templates/slack/template_slack.py new file mode 100644 index 0000000000000..0d8c9b100951a --- /dev/null +++ b/posthog/cdp/templates/slack/template_slack.py @@ -0,0 +1,62 @@ +from posthog.cdp.templates.hog_function_template import HogFunctionTemplate + + +template: HogFunctionTemplate = HogFunctionTemplate( + status="alpha", + id="template-slack", + name="Slack webhook", + description="Sends a webhook templated by the incoming event data", + icon_url="/api/projects/@current/hog_functions/icon/?id=slack.com", + hog=""" +fetch(inputs.url, { + 'headers': inputs.headers, + 'body': inputs.body, + 'method': inputs.method +}); +""".strip(), + inputs_schema=[ + { + "key": "url", + "type": "string", + "label": "Slack webhook URL", + "description": "Create a slack webhook URL in your (see https://api.slack.com/messaging/webhooks)", + "placeholder": "https://hooks.slack.com/services/XXX/YYY", + "secret": False, + "required": True, + }, + { + "key": "body", + "type": "json", + "label": "Message", + "description": "Message to send to Slack (see https://api.slack.com/block-kit/building)", + "default": { + "blocks": [ + { + "text": { + "text": "*{person.name}* triggered event: '{event.name}'", + "type": "mrkdwn", + }, + "type": "section", + }, + { + "type": "actions", + "elements": [ + { + "url": "{person.url}", + "text": {"text": "View Person in PostHog", "type": "plain_text"}, + "type": "button", + }, + { + "url": "{source.url}", + "text": {"text": "Message source", "type": "plain_text"}, + "type": "button", + }, + ], + }, + ] + }, + "secret": False, + "required": False, + }, + ], +) diff --git a/posthog/cdp/templates/slack/test_template_slack.py b/posthog/cdp/templates/slack/test_template_slack.py new file mode 100644 index 0000000000000..50144106615ed --- /dev/null +++ b/posthog/cdp/templates/slack/test_template_slack.py @@ -0,0 +1,29 @@ +import json +from unittest.mock import call +from posthog.cdp.templates.helpers import BaseHogFunctionTemplateTest +from posthog.cdp.templates.slack.template_slack import template as template_slack + + +class TestTemplateSlack(BaseHogFunctionTemplateTest): + template = template_slack + + def test_function_works(self): + res = self.run_function( + inputs={ + "url": "https://posthog.com", + "method": "GET", + "headers": {}, + "body": json.dumps({"hello": "world"}), + } + ) + + assert res.result is None + + assert self.mock_fetch.mock_calls[0] == call( + "https://posthog.com", + { + "headers": {}, + "body": '{"hello": "world"}', + "method": "GET", + }, + ) diff --git a/posthog/cdp/templates/test_cdp_templates.py b/posthog/cdp/templates/test_cdp_templates.py new file mode 100644 index 0000000000000..21d5d841b3a9d --- /dev/null +++ b/posthog/cdp/templates/test_cdp_templates.py @@ -0,0 +1,14 @@ +from posthog.cdp.templates import HOG_FUNCTION_TEMPLATES +from posthog.cdp.validation import compile_hog, validate_inputs_schema +from posthog.test.base import BaseTest + + +class TestTemplatesGeneral(BaseTest): + def setUp(self): + super().setUp() + + def test_templates_are_valid(self): + for template in HOG_FUNCTION_TEMPLATES: + bytecode = compile_hog(template.hog) + assert bytecode[0] == "_h" + assert validate_inputs_schema(template.inputs_schema) diff --git a/posthog/cdp/templates/webhook/template_webhook.py b/posthog/cdp/templates/webhook/template_webhook.py new file mode 100644 index 0000000000000..68c84fbcc7807 --- /dev/null +++ b/posthog/cdp/templates/webhook/template_webhook.py @@ -0,0 +1,65 @@ +from posthog.cdp.templates.hog_function_template import HogFunctionTemplate + + +template: HogFunctionTemplate = HogFunctionTemplate( + status="alpha", + id="template-webhook", + name="HTTP Webhook", + description="Sends a webhook templated by the incoming event data", + icon_url="/api/projects/@current/hog_functions/icon/?id=posthog.com&temp=true", + hog=""" +fetch(inputs.url, { + 'headers': inputs.headers, + 'body': inputs.body, + 'method': inputs.method +}); +""".strip(), + inputs_schema=[ + { + "key": "url", + "type": "string", + "label": "Webhook URL", + "secret": False, + "required": True, + }, + { + "key": "method", + "type": "choice", + "label": "Method", + "secret": False, + "choices": [ + { + "label": "POST", + "value": "POST", + }, + { + "label": "PUT", + "value": "PUT", + }, + { + "label": "GET", + "value": "GET", + }, + { + "label": "DELETE", + "value": "DELETE", + }, + ], + "required": False, + }, + { + "key": "body", + "type": "json", + "label": "JSON Body", + "secret": False, + "required": False, + }, + { + "key": "headers", + "type": "dictionary", + "label": "Headers", + "secret": False, + "required": False, + }, + ], +) diff --git a/posthog/cdp/templates/webhook/test_template_webhook.py b/posthog/cdp/templates/webhook/test_template_webhook.py new file mode 100644 index 0000000000000..0e29f66632210 --- /dev/null +++ b/posthog/cdp/templates/webhook/test_template_webhook.py @@ -0,0 +1,29 @@ +import json +from unittest.mock import call +from posthog.cdp.templates.helpers import BaseHogFunctionTemplateTest +from posthog.cdp.templates.webhook.template_webhook import template as template_webhook + + +class TestTemplateWebhook(BaseHogFunctionTemplateTest): + template = template_webhook + + def test_function_works(self): + res = self.run_function( + inputs={ + "url": "https://posthog.com", + "method": "GET", + "headers": {}, + "body": json.dumps({"hello": "world"}), + } + ) + + assert res.result is None + + assert self.mock_fetch.mock_calls[0] == call( + "https://posthog.com", + { + "headers": {}, + "body": '{"hello": "world"}', + "method": "GET", + }, + ) diff --git a/posthog/cdp/validation.py b/posthog/cdp/validation.py new file mode 100644 index 0000000000000..ea467ef54ac47 --- /dev/null +++ b/posthog/cdp/validation.py @@ -0,0 +1,103 @@ +from typing import Any +from rest_framework import serializers + +from posthog.hogql.bytecode import create_bytecode +from posthog.hogql.parser import parse_program +from posthog.models.hog_functions.utils import generate_template_bytecode + + +class InputsSchemaItemSerializer(serializers.Serializer): + type = serializers.ChoiceField(choices=["string", "boolean", "dictionary", "choice", "json"]) + key = serializers.CharField() + label = serializers.CharField(required=False) # type: ignore + choices = serializers.ListField(child=serializers.DictField(), required=False) + required = serializers.BooleanField(default=False) # type: ignore + default = serializers.JSONField(required=False) + secret = serializers.BooleanField(default=False) + description = serializers.CharField(required=False) + + # TODO Validate choices if type=choice + + +class AnyInputField(serializers.Field): + def to_internal_value(self, data): + return data + + def to_representation(self, value): + return value + + +class InputsItemSerializer(serializers.Serializer): + value = AnyInputField(required=False) + bytecode = serializers.ListField(required=False, read_only=True) + + def validate(self, attrs): + schema = self.context["schema"] + value = attrs.get("value") + + if schema.get("required") and not value: + raise serializers.ValidationError("This field is required.") + + if not value: + return attrs + + name: str = schema["key"] + item_type = schema["type"] + value = attrs["value"] + + # Validate each type + if item_type == "string": + if not isinstance(value, str): + raise serializers.ValidationError("Value must be a string.") + elif item_type == "boolean": + if not isinstance(value, bool): + raise serializers.ValidationError("Value must be a boolean.") + elif item_type == "dictionary": + if not isinstance(value, dict): + raise serializers.ValidationError("Value must be a dictionary.") + + try: + if value: + if item_type in ["string", "dictionary", "json"]: + attrs["bytecode"] = generate_template_bytecode(value) + except Exception as e: + raise serializers.ValidationError({"inputs": {name: f"Invalid template: {str(e)}"}}) + + return attrs + + +def validate_inputs_schema(value: list) -> list: + if not isinstance(value, list): + raise serializers.ValidationError("inputs_schema must be a list of objects.") + + serializer = InputsSchemaItemSerializer(data=value, many=True) + + if not serializer.is_valid(): + raise serializers.ValidationError(serializer.errors) + + return serializer.validated_data or [] + + +def validate_inputs(inputs_schema: list, inputs: dict) -> dict: + validated_inputs = {} + + for schema in inputs_schema: + value = inputs.get(schema["key"], {}) + serializer = InputsItemSerializer(data=value, context={"schema": schema}) + + if not serializer.is_valid(): + first_error = next(iter(serializer.errors.values()))[0] + raise serializers.ValidationError({"inputs": {schema["key"]: first_error}}) + + validated_inputs[schema["key"]] = serializer.validated_data + + return validated_inputs + + +def compile_hog(hog: str) -> list[Any]: + # Attempt to compile the hog + try: + program = parse_program(hog) + return create_bytecode(program, supported_functions={"fetch"}) + except Exception: + raise serializers.ValidationError({"hog": "Hog code has errors."}) diff --git a/posthog/migrations/0427_hogfunction_icon_url_hogfunction_template_id.py b/posthog/migrations/0427_hogfunction_icon_url_hogfunction_template_id.py new file mode 100644 index 0000000000000..54d93ce758184 --- /dev/null +++ b/posthog/migrations/0427_hogfunction_icon_url_hogfunction_template_id.py @@ -0,0 +1,22 @@ +# Generated by Django 4.2.11 on 2024-06-13 07:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("posthog", "0426_externaldatasource_sync_frequency"), + ] + + operations = [ + migrations.AddField( + model_name="hogfunction", + name="icon_url", + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name="hogfunction", + name="template_id", + field=models.CharField(blank=True, max_length=400, null=True), + ), + ] diff --git a/posthog/models/hog_functions/hog_function.py b/posthog/models/hog_functions/hog_function.py index c61c5a9533247..0307178e2bd33 100644 --- a/posthog/models/hog_functions/hog_function.py +++ b/posthog/models/hog_functions/hog_function.py @@ -4,6 +4,7 @@ from django.db.models.signals import post_save from django.dispatch.dispatcher import receiver +from posthog.cdp.templates.hog_function_template import HogFunctionTemplate from posthog.models.action.action import Action from posthog.models.team.team import Team from posthog.models.utils import UUIDModel @@ -20,11 +21,19 @@ class HogFunction(UUIDModel): updated_at: models.DateTimeField = models.DateTimeField(auto_now=True) enabled: models.BooleanField = models.BooleanField(default=False) + icon_url: models.TextField = models.TextField(null=True, blank=True) hog: models.TextField = models.TextField() bytecode: models.JSONField = models.JSONField(null=True, blank=True) inputs_schema: models.JSONField = models.JSONField(null=True) inputs: models.JSONField = models.JSONField(null=True) filters: models.JSONField = models.JSONField(null=True, blank=True) + template_id: models.CharField = models.CharField(max_length=400, null=True, blank=True) + + @property + def template(self) -> Optional[HogFunctionTemplate]: + from posthog.cdp.templates import HOG_FUNCTION_TEMPLATES_BY_ID + + return HOG_FUNCTION_TEMPLATES_BY_ID.get(self.template_id, None) @property def filter_action_ids(self) -> list[int]: diff --git a/posthog/settings/web.py b/posthog/settings/web.py index b031ef3df3633..50f4069559db4 100644 --- a/posthog/settings/web.py +++ b/posthog/settings/web.py @@ -378,3 +378,5 @@ def add_recorder_js_headers(headers, path, url): PROXY_PROVISIONER_ADDR = get_from_env("PROXY_PROVISIONER_ADDR", "") PROXY_TARGET_CNAME = get_from_env("PROXY_TARGET_CNAME", "") PROXY_BASE_CNAME = get_from_env("PROXY_BASE_CNAME", "") + +LOGO_DEV_TOKEN = get_from_env("LOGO_DEV_TOKEN", "")