feat(pipeline-ui): url linking and v1 towards batch exports into dest (…

…#19682)

Co-authored-by: Michael Matloka <[email protected]>

frontend/src/scenes/pipeline/NewButton.tsx

@@ -1,7 +1,7 @@
 import { LemonButton } from 'lib/lemon-ui/LemonButton'
 import { urls } from 'scenes/urls'
 
-import { PipelineTabs } from '~/types'
+import { PipelineAppTabs, PipelineTabs } from '~/types'
 
 import { singularName } from './pipelineLogic'
 
@@ -12,7 +12,11 @@ type NewButtonProps = {
 export function NewButton({ tab }: NewButtonProps): JSX.Element {
     const singular = singularName(tab)
     return (
-        <LemonButton data-attr={`new-${singular}`} to={urls.pipelineNew(tab)} type="primary">
+        <LemonButton
+            data-attr={`new-${singular}`}
+            to={urls.pipelineApp(tab, 'new', PipelineAppTabs.Configuration)}
+            type="primary"
+        >
             New {singular}
         </LemonButton>
     )

frontend/src/scenes/pipeline/Pipeline.stories.tsx

@@ -86,7 +86,7 @@ export function PipelineDestinationsPage(): JSX.Element {
 
 export function PipelineAppConfiguration(): JSX.Element {
     useEffect(() => {
-        router.actions.push(urls.pipelineApp(1, PipelineAppTabs.Configuration))
+        router.actions.push(urls.pipelineApp(PipelineTabs.Destinations, 1, PipelineAppTabs.Configuration))
     }, [])
     return <App />
 }
@@ -99,7 +99,7 @@ export function PipelineAppMetrics(): JSX.Element {
         },
     })
     useEffect(() => {
-        router.actions.push(urls.pipelineApp(4, PipelineAppTabs.Metrics))
+        router.actions.push(urls.pipelineApp(PipelineTabs.Destinations, 4, PipelineAppTabs.Metrics))
         appMetricsLogic({ pluginConfigId: 4 }).mount()
     }, [])
     return <App />
@@ -113,7 +113,7 @@ export function PipelineAppMetricsErrorModal(): JSX.Element {
         },
     })
     useEffect(() => {
-        router.actions.push(urls.pipelineApp(4, PipelineAppTabs.Metrics))
+        router.actions.push(urls.pipelineApp(PipelineTabs.Destinations, 4, PipelineAppTabs.Metrics))
         const logic = appMetricsLogic({ pluginConfigId: 4 })
         logic.mount()
         logic.actions.openErrorDetailsModal('Error')
@@ -128,7 +128,7 @@ export function PipelineAppLogs(): JSX.Element {
         },
     })
     useEffect(() => {
-        router.actions.push(urls.pipelineApp(1, PipelineAppTabs.Logs))
+        router.actions.push(urls.pipelineApp(PipelineTabs.Destinations, 1, PipelineAppTabs.Logs))
     }, [])
     return <App />
 }

frontend/src/scenes/pipeline/PipelineApp.tsx

@@ -1,6 +1,7 @@
 import { Spinner } from '@posthog/lemon-ui'
 import { useValues } from 'kea'
 import { router } from 'kea-router'
+import { NotFound } from 'lib/components/NotFound'
 import { PageHeader } from 'lib/components/PageHeader'
 import { FEATURE_FLAGS } from 'lib/constants'
 import { LemonTabs } from 'lib/lemon-ui/LemonTabs/LemonTabs'
@@ -10,21 +11,27 @@ import { PluginLogs } from 'scenes/plugins/plugin/PluginLogs'
 import { SceneExport } from 'scenes/sceneTypes'
 import { urls } from 'scenes/urls'
 
-import { PipelineAppTabs } from '~/types'
+import { PipelineAppTabs, PipelineTabs } from '~/types'
 
 import { AppMetrics } from './AppMetrics'
 import { pipelineAppLogic } from './pipelineAppLogic'
 
 export const scene: SceneExport = {
     component: PipelineApp,
     logic: pipelineAppLogic,
-    paramsToProps: ({ params: { id } }: { params: { id?: string } }) => ({ id: id ? parseInt(id) : 'new' }),
+    paramsToProps: ({ params: { kind, id } }: { params: { kind?: string; id?: string } }) => ({
+        kind: kind,
+        id: id ? parseInt(id) : 'new',
+    }),
 }
 
-export function PipelineApp({ id }: { id?: string } = {}): JSX.Element {
+export function PipelineApp({ kind, id }: { kind?: string; id?: string } = {}): JSX.Element {
     const { featureFlags } = useValues(featureFlagLogic)
     if (!featureFlags[FEATURE_FLAGS.PIPELINE_UI]) {
-        return <></>
+        return <p>Pipeline 3000 not available yet</p>
+    }
+    if (!Object.values(PipelineTabs).includes(kind as PipelineTabs)) {
+        return <NotFound object="pipeline app" />
     }
     const { currentTab } = useValues(pipelineAppLogic)
 
@@ -45,7 +52,9 @@ export function PipelineApp({ id }: { id?: string } = {}): JSX.Element {
             <PageHeader title={`Pipeline App`} />
             <LemonTabs
                 activeKey={currentTab}
-                onChange={(tab) => router.actions.push(urls.pipelineApp(confId, tab as PipelineAppTabs))}
+                onChange={(tab) =>
+                    router.actions.push(urls.pipelineApp(kind as PipelineTabs, confId, tab as PipelineAppTabs))
+                }
                 tabs={Object.values(PipelineAppTabs).map((tab) => ({
                     label: capitalizeFirstLetter(tab),
                     key: tab,

frontend/src/scenes/pipeline/Transformations.tsx

@@ -33,7 +33,7 @@ import { RenderApp } from './utils'
 export function Transformations(): JSX.Element {
     const { featureFlags } = useValues(featureFlagLogic)
     if (!featureFlags[FEATURE_FLAGS.PIPELINE_UI]) {
-        return <></>
+        return <p>Pipeline 3000 not available yet</p>
     }
     const {
         loading,
@@ -109,6 +109,7 @@ export function Transformations(): JSX.Element {
                                             <Tooltip title={'Click to update configuration, view metrics, and more'}>
                                                 <Link
                                                     to={urls.pipelineApp(
+                                                        PipelineTabs.Transformations,
                                                         pluginConfig.id,
                                                         PipelineAppTabs.Configuration
                                                     )}
@@ -190,6 +191,7 @@ export function Transformations(): JSX.Element {
                                                     )}
                                                     <LemonButton
                                                         to={urls.pipelineApp(
+                                                            PipelineTabs.Transformations,
                                                             pluginConfig.id,
                                                             PipelineAppTabs.Configuration
                                                         )}
@@ -199,14 +201,22 @@ export function Transformations(): JSX.Element {
                                                         {canConfigurePlugins ? 'Edit' : 'View'} app configuration
                                                     </LemonButton>
                                                     <LemonButton
-                                                        to={urls.pipelineApp(pluginConfig.id, PipelineAppTabs.Metrics)}
+                                                        to={urls.pipelineApp(
+                                                            PipelineTabs.Transformations,
+                                                            pluginConfig.id,
+                                                            PipelineAppTabs.Metrics
+                                                        )}
                                                         id={`app-${pluginConfig.id}-metrics`}
                                                         fullWidth
                                                     >
                                                         View app metrics
                                                     </LemonButton>
                                                     <LemonButton
-                                                        to={urls.pipelineApp(pluginConfig.id, PipelineAppTabs.Logs)}
+                                                        to={urls.pipelineApp(
+                                                            PipelineTabs.Transformations,
+                                                            pluginConfig.id,
+                                                            PipelineAppTabs.Logs
+                                                        )}
                                                         id={`app-${pluginConfig.id}-logs`}
                                                         fullWidth
                                                     >
@@ -234,7 +244,7 @@ export function Transformations(): JSX.Element {
                                                                 callback: loadPluginConfigs,
                                                             })
                                                         }}
-                                                        id={`app-reorder`}
+                                                        id={`app-delete`}
                                                         disabledReason={
                                                             canConfigurePlugins
                                                                 ? undefined

frontend/src/scenes/pipeline/destinationsLogic.tsx

@@ -3,13 +3,55 @@ import { loaders } from 'kea-loaders'
 import api from 'lib/api'
 import { canConfigurePlugins } from 'scenes/plugins/access'
 import { teamLogic } from 'scenes/teamLogic'
+import { urls } from 'scenes/urls'
 import { userLogic } from 'scenes/userLogic'
 
-import { PluginConfigTypeNew, PluginConfigWithPluginInfoNew, PluginType, ProductKey } from '~/types'
+import {
+    PipelineAppTabs,
+    PipelineTabs,
+    PluginConfigTypeNew,
+    PluginConfigWithPluginInfoNew,
+    PluginType,
+    ProductKey,
+} from '~/types'
 
 import type { pipelineDestinationsLogicType } from './destinationsLogicType'
 import { capturePluginEvent } from './utils'
 
+interface WebhookSuccessRate {
+    '24h': number | null
+    '7d': number | null
+}
+interface BatchExportSuccessRate {
+    '24h': [successes: number, failures: number]
+    '7d': [successes: number, failures: number]
+}
+
+interface DestinationTypeBase {
+    name: string
+    description?: string
+    enabled: boolean
+    config_url: string
+    metrics_url: string
+    logs_url: string
+    updated_at: string
+    frequency: 'realtime' | 'hourly' | 'daily'
+}
+export interface BatchExportDestination extends DestinationTypeBase {
+    type: 'batch_export'
+    id: string
+    success_rates: BatchExportSuccessRate
+    app_source_code_url?: never
+}
+export interface WebhookDestination extends DestinationTypeBase {
+    type: 'webhook'
+    id: number
+    plugin: PluginType
+    app_source_code_url?: string
+    success_rates: WebhookSuccessRate
+}
+export type DestinationType = BatchExportDestination | WebhookDestination
+
 export const pipelineDestinationsLogic = kea<pipelineDestinationsLogicType>([
     path(['scenes', 'pipeline', 'destinationsLogic']),
     connect({
@@ -96,6 +138,35 @@ export const pipelineDestinationsLogic = kea<pipelineDestinationsLogicType>([
                 return withPluginInfo
             },
         ],
+        destinations: [
+            (s) => [s.pluginConfigs, s.plugins],
+            (pluginConfigs, plugins): DestinationType[] => {
+                const dests = Object.values(pluginConfigs).map<DestinationType>((pluginConfig) => ({
+                    type: 'webhook',
+                    frequency: 'realtime',
+                    id: pluginConfig.id,
+                    name: pluginConfig.name,
+                    description: pluginConfig.description,
+                    enabled: pluginConfig.enabled,
+                    config_url: urls.pipelineApp(
+                        PipelineTabs.Destinations,
+                        pluginConfig.id,
+                        PipelineAppTabs.Configuration
+                    ),
+                    metrics_url: urls.pipelineApp(PipelineTabs.Destinations, pluginConfig.id, PipelineAppTabs.Metrics),
+                    logs_url: urls.pipelineApp(PipelineTabs.Destinations, pluginConfig.id, PipelineAppTabs.Logs),
+                    app_source_code_url: '',
+                    plugin: plugins[pluginConfig.plugin],
+                    success_rates: {
+                        '24h': pluginConfig.delivery_rate_24h === undefined ? null : pluginConfig.delivery_rate_24h,
+                        '7d': null, // TODO: start populating real data for this
+                    },
+                    updated_at: pluginConfig.updated_at,
+                }))
+                const enabledFirst = Object.values(dests).sort((a, b) => Number(b.enabled) - Number(a.enabled))
+                return enabledFirst
+            },
+        ],
         // This is currently an organization level setting but might in the future be user level
         // it's better to add the permission checks everywhere now
         canConfigurePlugins: [(s) => [s.user], (user) => canConfigurePlugins(user?.organization)],

frontend/src/scenes/pipeline/pipelineAppLogic.tsx

@@ -1,14 +1,16 @@
 import { actions, kea, key, path, props, reducers, selectors } from 'kea'
 import { actionToUrl, urlToAction } from 'kea-router'
+import { capitalizeFirstLetter } from 'lib/utils'
 import { Scene } from 'scenes/sceneTypes'
 import { urls } from 'scenes/urls'
 
-import { Breadcrumb, PipelineAppTabs } from '~/types'
+import { Breadcrumb, PipelineAppTabs, PipelineTabs } from '~/types'
 
 import type { pipelineAppLogicType } from './pipelineAppLogicType'
 
 export interface PipelineAppLogicProps {
     id: number
+    kind: PipelineTabs
 }
 
 export const pipelineAppLogic = kea<pipelineAppLogicType>([
@@ -17,6 +19,7 @@ export const pipelineAppLogic = kea<pipelineAppLogicType>([
     path((id) => ['scenes', 'pipeline', 'pipelineAppLogic', id]),
     actions({
         setCurrentTab: (tab: PipelineAppTabs = PipelineAppTabs.Configuration) => ({ tab }),
+        setKind: (kind: PipelineTabs) => ({ kind }),
     }),
     reducers({
         currentTab: [
@@ -28,13 +31,17 @@ export const pipelineAppLogic = kea<pipelineAppLogicType>([
     }),
     selectors({
         breadcrumbs: [
-            () => [],
-            (): Breadcrumb[] => [
+            (_, p) => [p.kind],
+            (kind): Breadcrumb[] => [
                 {
                     key: Scene.Pipeline,
-                    name: 'Pipeline',
+                    name: 'Data pipeline',
                     path: urls.pipeline(),
                 },
+                {
+                    key: 'Kind',
+                    name: capitalizeFirstLetter(kind),
+                },
                 {
                     key: 'todo',
                     name: 'App name',
@@ -44,12 +51,12 @@ export const pipelineAppLogic = kea<pipelineAppLogicType>([
     }),
     actionToUrl(({ values, props }) => {
         return {
-            setCurrentTab: () => [urls.pipelineApp(props.id, values.currentTab)],
+            setCurrentTab: () => [urls.pipelineApp(props.kind, props.id, values.currentTab)],
         }
     }),
     urlToAction(({ actions, values }) => ({
-        '/pipeline/:id/:tab': ({ tab }) => {
-            if (tab !== values.currentTab) {
+        '/pipeline/:kind/:id/:tab': ({ tab }) => {
+            if (tab !== values.currentTab && Object.values(PipelineAppTabs).includes(tab as PipelineAppTabs)) {
                 actions.setCurrentTab(tab as PipelineAppTabs)
             }
         },

frontend/src/scenes/scenes.ts

@@ -9,7 +9,7 @@ import { Error404 as Error404Component } from '~/layout/Error404'
 import { ErrorNetwork as ErrorNetworkComponent } from '~/layout/ErrorNetwork'
 import { ErrorProjectUnavailable as ErrorProjectUnavailableComponent } from '~/layout/ErrorProjectUnavailable'
 import { EventsQuery } from '~/queries/schema'
-import { ActivityScope, InsightShortId, PipelineAppTabs, PipelineTabs, PropertyFilterType, ReplayTabs } from '~/types'
+import { ActivityScope, InsightShortId, PropertyFilterType, ReplayTabs } from '~/types'
 
 export const emptySceneParams = { params: {}, searchParams: {}, hashParams: {} }
 
@@ -476,16 +476,8 @@ export const routes: Record<string, Scene> = {
     [urls.personByDistinctId('*', false)]: Scene.Person,
     [urls.personByUUID('*', false)]: Scene.Person,
     [urls.persons()]: Scene.PersonsManagement,
-    [urls.pipeline()]: Scene.Pipeline,
-    // One entry for every available tab
-    ...(Object.fromEntries(Object.values(PipelineTabs).map((tab) => [urls.pipeline(tab), Scene.Pipeline])) as Record<
-        string,
-        Scene
-    >),
-    // One entry for each available tab (key by app config id)
-    ...(Object.fromEntries(
-        Object.values(PipelineAppTabs).map((tab) => [urls.pipelineApp(':id', tab), Scene.PipelineApp])
-    ) as Record<string, Scene>),
+    [urls.pipeline(':tab')]: Scene.Pipeline,
+    [urls.pipelineApp(':kind', ':id', ':tab')]: Scene.PipelineApp,
     [urls.groups(':groupTypeIndex')]: Scene.PersonsManagement,
     [urls.group(':groupTypeIndex', ':groupKey', false)]: Scene.Group,
     [urls.group(':groupTypeIndex', ':groupKey', false, ':groupTab')]: Scene.Group,

frontend/src/scenes/urls.ts

@@ -101,10 +101,10 @@ export const urls = {
         encode ? `/persons/${encodeURIComponent(uuid)}` : `/persons/${uuid}`,
     persons: (): string => '/persons',
     // TODO: Default to the landing page, once it's ready
-    pipeline: (tab?: PipelineTabs): string => `/pipeline/${tab ? tab : PipelineTabs.Destinations}`,
-    pipelineApp: (id: string | number, tab?: PipelineAppTabs): string =>
-        `/pipeline/${id}/${tab ? tab : PipelineAppTabs.Configuration}`,
-    pipelineNew: (tab?: PipelineTabs): string => `/pipeline/${tab ? tab : PipelineTabs.Destinations}/new`,
+    pipeline: (tab?: PipelineTabs | ':tab'): string => `/pipeline/${tab ? tab : PipelineTabs.Destinations}`,
+    /** @param id 'new' for new, uuid for batch exports and numbers for plugins */
+    pipelineApp: (kind: PipelineTabs | ':kind', id: string | number, tab?: PipelineAppTabs | ':tab'): string =>
+        `/pipeline/${kind}/${id}/${tab ? tab : PipelineAppTabs.Configuration}`,
     groups: (groupTypeIndex: string | number): string => `/groups/${groupTypeIndex}`,
     // :TRICKY: Note that groupKey is provided by user. We need to override urlPatternOptions for kea-router.
     group: (groupTypeIndex: string | number, groupKey: string, encode: boolean = true, tab?: string | null): string =>