feat(data-warehouse): external data resource (#17554)

* create airbyte source api

* comment test

* rename and add connection

* comment out test

* frontend updates and additional API calls on airbyte

* some more UI and retrieve endpoint

* restore lock file

* connecting the dots

* add destination logic

* make destinatino parquet

* ui updates

* rename data

* more renaming

* migration

* remove test

* rename

* Update UI snapshots for `chromium` (2)

* missing file

* typing

* remove unneeded field

* add rollback deletions if one fo the related resources fails

* add feature flag

* fix error handling

* adjust warning messages

* fix external link listener

* Update UI snapshots for `chromium` (2)

* format

* remove unnecessary env var

* adjustments

* add more env vars

* fix import

---------

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

frontend/src/lib/api.ts

@@ -50,6 +50,8 @@ import {
     BatchExportRun,
     UserBasicType,
     NotebookNodeResource,
+    ExternalDataStripeSourceCreatePayload,
+    ExternalDataStripeSource,
 } from '~/types'
 import { getCurrentOrganizationId, getCurrentTeamId } from './utils/logics'
 import { CheckboxValueType } from 'antd/lib/checkbox/Group'
@@ -566,6 +568,11 @@ class ApiRequest {
         return this.batchExportRun(id, runId, teamId).addPathComponent('logs')
     }
 
+    // External Data Source
+    public externalDataSources(teamId?: TeamType['id']): ApiRequest {
+        return this.projectsDetail(teamId).addPathComponent('external_data_sources')
+    }
+
     // Request finalization
     public async get(options?: ApiMethodOptions): Promise<any> {
         return await api.get(this.assembleFullUrl(), options)
@@ -1571,6 +1578,17 @@ const api = {
         },
     },
 
+    externalDataSources: {
+        async list(): Promise<PaginatedResponse<ExternalDataStripeSource>> {
+            return await new ApiRequest().externalDataSources().get()
+        },
+        async create(
+            data: Partial<ExternalDataStripeSourceCreatePayload>
+        ): Promise<ExternalDataStripeSourceCreatePayload> {
+            return await new ApiRequest().externalDataSources().create({ data })
+        },
+    },
+
     dataWarehouseViewLinks: {
         async list(): Promise<PaginatedResponse<DataWarehouseViewLink>> {
             return await new ApiRequest().dataWarehouseViewLinks().get()

frontend/src/lib/constants.tsx

@@ -151,6 +151,7 @@ export const FEATURE_FLAGS = {
     EXCEPTION_AUTOCAPTURE: 'exception-autocapture',
     DATA_WAREHOUSE: 'data-warehouse', // owner: @EDsCODE
     DATA_WAREHOUSE_VIEWS: 'data-warehouse-views', // owner: @EDsCODE
+    DATA_WAREHOUSE_EXTERNAL_LINK: 'data-warehouse-external-link', // owner: @EDsCODE
     FF_DASHBOARD_TEMPLATES: 'ff-dashboard-templates', // owner: @EDsCODE
     SHOW_PRODUCT_INTRO_EXISTING_PRODUCTS: 'show-product-intro-existing-products', // owner: @raquelmsmith
     ARTIFICIAL_HOG: 'artificial-hog', // owner: @Twixes

frontend/src/scenes/appScenes.ts

@@ -45,7 +45,7 @@ export const appScenes: Record<Scene, () => any> = {
     [Scene.DataWarehousePosthog]: () => import('./data-warehouse/posthog/DataWarehousePosthogScene'),
     [Scene.DataWarehouseExternal]: () => import('./data-warehouse/external/DataWarehouseExternalScene'),
     [Scene.DataWarehouseSavedQueries]: () => import('./data-warehouse/saved_queries/DataWarehouseSavedQueriesScene'),
-    [Scene.DataWarehouseTable]: () => import('./data-warehouse/DataWarehouseTable'),
+    [Scene.DataWarehouseSettings]: () => import('./data-warehouse/settings/DataWarehouseSettingsScene'),
     [Scene.OrganizationSettings]: () => import('./organization/Settings'),
     [Scene.OrganizationCreateFirst]: () => import('./organization/Create'),
     [Scene.OrganizationCreationConfirm]: () => import('./organization/ConfirmOrganization/ConfirmOrganization'),

frontend/src/scenes/data-warehouse/DataWarehouseTable.tsx → frontend/src/scenes/data-warehouse/DataWarehouseTableForm.tsx

@@ -1,76 +1,15 @@
-import { SceneExport } from 'scenes/sceneTypes'
 import { dataWarehouseTableLogic } from './dataWarehouseTableLogic'
-import { useActions, useValues } from 'kea'
 import { Form } from 'kea-forms'
-import { PageHeader } from 'lib/components/PageHeader'
-import { LemonSkeleton } from 'lib/lemon-ui/LemonSkeleton'
-import { LemonButton, LemonDivider, LemonInput, LemonSelect, Link } from '@posthog/lemon-ui'
-import { router } from 'kea-router'
-import { urls } from 'scenes/urls'
+import { LemonInput, LemonSelect } from '@posthog/lemon-ui'
 import { Field } from 'lib/forms/Field'
 
-export const scene: SceneExport = {
-    component: DataWarehousetTable,
-    logic: dataWarehouseTableLogic,
-    paramsToProps: ({ params: { id } }): (typeof dataWarehouseTableLogic)['props'] => ({
-        id: id,
-    }),
+interface DataWarehouseTableFormProps {
+    footer?: JSX.Element
 }
 
-export function DataWarehousetTable({ id }: { id?: string } = {}): JSX.Element {
-    const { isEditingTable } = useValues(dataWarehouseTableLogic)
-    const showTableForm = id === 'new' || isEditingTable
-    return <div>{!id ? <LemonSkeleton /> : <>{showTableForm ? <TableForm id={id} /> : <></>}</>}</div>
-}
-
-export function TableForm({ id }: { id: string }): JSX.Element {
-    const { table, tableLoading, isEditingTable } = useValues(dataWarehouseTableLogic)
-    const { loadTable, editingTable } = useActions(dataWarehouseTableLogic)
-
+export function DatawarehouseTableForm({ footer }: DataWarehouseTableFormProps): JSX.Element {
     return (
         <Form formKey="table" logic={dataWarehouseTableLogic} className="space-y-4" enableFormOnSubmit>
-            <PageHeader
-                title={id === 'new' ? 'New table' : table.name}
-                buttons={
-                    <div className="flex items-center gap-2">
-                        <LemonButton
-                            data-attr="cancel-table"
-                            type="secondary"
-                            loading={tableLoading}
-                            onClick={() => {
-                                if (isEditingTable) {
-                                    editingTable(false)
-                                    loadTable()
-                                } else {
-                                    router.actions.push(urls.dataWarehouse())
-                                }
-                            }}
-                        >
-                            Cancel
-                        </LemonButton>
-                        <LemonButton
-                            type="primary"
-                            data-attr="save-feature-flag"
-                            htmlType="submit"
-                            loading={tableLoading}
-                        >
-                            Save
-                        </LemonButton>
-                    </div>
-                }
-                caption={
-                    <div>
-                        External tables are supported through object storage systems like S3.{' '}
-                        <Link
-                            to="https://posthog.com/docs/data/data-warehouse#step-1-creating-a-bucket-in-s3"
-                            target="_blank"
-                        >
-                            Learn how to set up your data
-                        </Link>
-                    </div>
-                }
-            />
-            <LemonDivider />
             <div className="flex flex-col gap-2 max-w-160">
                 <Field name="name" label="Table Name">
                     <LemonInput
@@ -136,6 +75,7 @@ export function TableForm({ id }: { id: string }): JSX.Element {
                     />
                 </Field>
             </div>
+            {footer}
         </Form>
     )
 }

frontend/src/scenes/data-warehouse/dataWarehouseTableLogic.tsx

@@ -1,14 +1,15 @@
 import { lemonToast } from '@posthog/lemon-ui'
-import { kea, path, props, key, listeners, afterMount, reducers, actions, selectors, connect } from 'kea'
+import { kea, path, props, listeners, reducers, actions, selectors, connect } from 'kea'
 import { forms } from 'kea-forms'
 import { loaders } from 'kea-loaders'
-import { router, urlToAction } from 'kea-router'
+import { router } from 'kea-router'
 import api from 'lib/api'
 import { urls } from 'scenes/urls'
 import { AnyPropertyFilter, Breadcrumb, DataWarehouseTable } from '~/types'
 import { DataTableNode } from '~/queries/schema'
 import { databaseSceneLogic } from 'scenes/data-management/database/databaseSceneLogic'
 import type { dataWarehouseTableLogicType } from './dataWarehouseTableLogicType'
+import { dataWarehouseSceneLogic } from './external/dataWarehouseSceneLogic'
 
 export interface TableLogicProps {
     id: string | 'new'
@@ -29,9 +30,13 @@ const NEW_WAREHOUSE_TABLE: DataWarehouseTable = {
 export const dataWarehouseTableLogic = kea<dataWarehouseTableLogicType>([
     path(['scenes', 'data-warehouse', 'tableLogic']),
     props({} as TableLogicProps),
-    key(({ id }) => id),
     connect(() => ({
-        actions: [databaseSceneLogic, ['loadDatabase']],
+        actions: [
+            databaseSceneLogic,
+            ['loadDatabase'],
+            dataWarehouseSceneLogic,
+            ['loadDataWarehouse', 'toggleSourceModal'],
+        ],
     })),
     actions({
         editingTable: (editing: boolean) => ({ editing }),
@@ -65,6 +70,8 @@ export const dataWarehouseTableLogic = kea<dataWarehouseTableLogicType>([
         createTableSuccess: async ({ table }) => {
             lemonToast.success(<>Table {table.name} created</>)
             actions.loadDatabase()
+            actions.loadDataWarehouse()
+            actions.toggleSourceModal()
             router.actions.replace(urls.dataWarehouse())
         },
         updateTableSuccess: async ({ table }) => {
@@ -122,25 +129,4 @@ export const dataWarehouseTableLogic = kea<dataWarehouseTableLogicType>([
             },
         },
     })),
-    urlToAction(({ actions, props }) => ({
-        [urls.dataWarehouseTable(props.id ?? 'new')]: (_, __, ___, { method }) => {
-            // If the URL was pushed (user clicked on a link), reset the scene's data.
-            // This avoids resetting form fields if you click back/forward.
-            if (method === 'PUSH') {
-                if (props.id) {
-                    actions.loadTable()
-                } else {
-                    actions.resetTable()
-                }
-            }
-        },
-    })),
-    afterMount(async ({ props, actions }) => {
-        // if (props.id !== 'new') {
-        //     await actions.loadTable()
-        // }
-        if (props.id === 'new') {
-            actions.resetTable()
-        }
-    }),
 ])

frontend/src/scenes/data-warehouse/external/DataWarehouseExternalScene.tsx

@@ -1,22 +1,29 @@
-import { LemonButton, LemonTag, Link } from '@posthog/lemon-ui'
+import { LemonTag, Link, LemonButtonWithSideAction } from '@posthog/lemon-ui'
 import { PageHeader } from 'lib/components/PageHeader'
 import { SceneExport } from 'scenes/sceneTypes'
 import { urls } from 'scenes/urls'
-import { useValues } from 'kea'
+import { useActions, useValues } from 'kea'
 import { router } from 'kea-router'
 import { ProductIntroduction } from 'lib/components/ProductIntroduction/ProductIntroduction'
 import { ProductKey } from '~/types'
 import { DataWarehouseTablesContainer } from './DataWarehouseTables'
 import { dataWarehouseSceneLogic } from './dataWarehouseSceneLogic'
 import { DataWarehousePageTabs, DataWarehouseTab } from '../DataWarehousePageTabs'
+import SourceModal from './SourceModal'
+import { IconSettings } from 'lib/lemon-ui/icons'
+import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
+import { FEATURE_FLAGS } from 'lib/constants'
 
 export const scene: SceneExport = {
     component: DataWarehouseExternalScene,
     logic: dataWarehouseSceneLogic,
 }
 
 export function DataWarehouseExternalScene(): JSX.Element {
-    const { shouldShowEmptyState, shouldShowProductIntroduction } = useValues(dataWarehouseSceneLogic)
+    const { shouldShowEmptyState, shouldShowProductIntroduction, isSourceModalOpen } =
+        useValues(dataWarehouseSceneLogic)
+    const { toggleSourceModal } = useActions(dataWarehouseSceneLogic)
+    const { featureFlags } = useValues(featureFlagLogic)
 
     return (
         <div>
@@ -30,14 +37,20 @@ export function DataWarehouseExternalScene(): JSX.Element {
                     </div>
                 }
                 buttons={
-                    !shouldShowProductIntroduction ? (
-                        <LemonButton
+                    featureFlags[FEATURE_FLAGS.DATA_WAREHOUSE_EXTERNAL_LINK] ? (
+                        <LemonButtonWithSideAction
                             type="primary"
-                            to={urls.dataWarehouseTable('new')}
-                            data-attr="new-data-warehouse-table"
+                            sideAction={{
+                                icon: <IconSettings />,
+                                onClick: () => router.actions.push(urls.dataWarehouseSettings()),
+                                'data-attr': 'saved-insights-new-insight-dropdown',
+                            }}
+                            data-attr="new-data-warehouse-easy-link"
+                            key={'new-data-warehouse-easy-link'}
+                            onClick={toggleSourceModal}
                         >
-                            New Table
-                        </LemonButton>
+                            Link Source
+                        </LemonButtonWithSideAction>
                     ) : undefined
                 }
                 caption={
@@ -59,13 +72,14 @@ export function DataWarehouseExternalScene(): JSX.Element {
                     description={
                         'Bring your production database, revenue data, CRM contacts or any other data into PostHog.'
                     }
-                    action={() => router.actions.push(urls.dataWarehouseTable('new'))}
+                    action={() => toggleSourceModal()}
                     isEmpty={shouldShowEmptyState}
                     docsURL="https://posthog.com/docs/data/data-warehouse"
                     productKey={ProductKey.DATA_WAREHOUSE}
                 />
             )}
             {!shouldShowEmptyState && <DataWarehouseTablesContainer />}
+            <SourceModal isOpen={isSourceModalOpen} onClose={toggleSourceModal} />
         </div>
     )
 }