chore(data-warehouse): postgres integration (#19797)

* basics

* add db schema endpoint

* frontend progress

* form working

* working

* schema selection

* refactor frontend

* add team

* format api.ts

* typing

* add internal network filter

* typing

* fix codescan

* typing

* fix test

* add schemas

* update feature flag from hubspot to postgres

* remove sslmode

* cleanup

* fix test

* fix test

* add logo

frontend/src/lib/api.ts

@@ -31,6 +31,7 @@ import {
     EventType,
     Experiment,
     ExportedAssetType,
+    ExternalDataPostgresSchema,
     ExternalDataSourceCreatePayload,
     ExternalDataSourceSchema,
     ExternalDataStripeSource,
@@ -1826,6 +1827,22 @@ const api = {
         async reload(sourceId: ExternalDataStripeSource['id']): Promise<void> {
             await new ApiRequest().externalDataSource(sourceId).withAction('reload').create()
         },
+        async database_schema(
+            host: string,
+            port: string,
+            dbname: string,
+            user: string,
+            password: string,
+            schema: string
+        ): Promise<ExternalDataPostgresSchema[]> {
+            const queryParams = toParams({ host, port, dbname, user, password, schema })
+
+            return await new ApiRequest()
+                .externalDataSources()
+                .withAction('database_schema')
+                .withQueryString(queryParams)
+                .get()
+        },
     },
 
     externalDataSchemas: {

frontend/src/lib/constants.tsx

@@ -159,7 +159,7 @@ export const FEATURE_FLAGS = {
     EXCEPTION_AUTOCAPTURE: 'exception-autocapture',
     DATA_WAREHOUSE: 'data-warehouse', // owner: @EDsCODE
     DATA_WAREHOUSE_VIEWS: 'data-warehouse-views', // owner: @EDsCODE
-    DATA_WAREHOUSE_HUBSPOT_IMPORT: 'data-warehouse-hubspot-import', // owner: @EDsCODE
+    DATA_WAREHOUSE_POSTGRES_IMPORT: 'data-warehouse-postgres-import', // owner: @EDsCODE
     FF_DASHBOARD_TEMPLATES: 'ff-dashboard-templates', // owner: @EDsCODE
     SHOW_PRODUCT_INTRO_EXISTING_PRODUCTS: 'show-product-intro-existing-products', // owner: @raquelmsmith
     ARTIFICIAL_HOG: 'artificial-hog', // owner: @Twixes

frontend/src/scenes/data-warehouse/external/SourceModal.tsx

@@ -1,45 +1,106 @@
-import { LemonButton, LemonDivider, LemonInput, LemonModal, LemonModalProps, Link } from '@posthog/lemon-ui'
+import { LemonButton, LemonModal, LemonModalProps } from '@posthog/lemon-ui'
 import { useActions, useValues } from 'kea'
-import { Form } from 'kea-forms'
 import { FEATURE_FLAGS } from 'lib/constants'
-import { Field } from 'lib/forms/Field'
 import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
 import hubspotLogo from 'public/hubspot-logo.svg'
+import postgresLogo from 'public/postgres-logo.svg'
 import stripeLogo from 'public/stripe-logo.svg'
 
-import { ExternalDataSourceType } from '~/types'
-
 import { DatawarehouseTableForm } from '../new_table/DataWarehouseTableForm'
-import { SOURCE_DETAILS, SourceConfig, sourceFormLogic } from './sourceFormLogic'
+import PostgresSchemaForm from './forms/PostgresSchemaForm'
+import SourceForm from './forms/SourceForm'
+import { SourceConfig } from './sourceModalLogic'
 import { sourceModalLogic } from './sourceModalLogic'
 
 interface SourceModalProps extends LemonModalProps {}
 
 export default function SourceModal(props: SourceModalProps): JSX.Element {
-    const { tableLoading, selectedConnector, isManualLinkFormVisible, connectors, addToHubspotButtonUrl } =
-        useValues(sourceModalLogic)
-    const { selectConnector, toggleManualLinkFormVisible, onClear } = useActions(sourceModalLogic)
+    const { modalTitle, modalCaption } = useValues(sourceModalLogic)
+    const { onClear, onBack, onSubmit } = useActions(sourceModalLogic)
+    const { currentStep } = useValues(sourceModalLogic)
+
+    const footer = (): JSX.Element | null => {
+        if (currentStep === 1) {
+            return null
+        }
+
+        return (
+            <div className="mt-2 flex flex-row justify-end gap-2">
+                <LemonButton type="secondary" center data-attr="source-modal-back-button" onClick={onBack}>
+                    Back
+                </LemonButton>
+                <LemonButton type="primary" center onClick={() => onSubmit()} data-attr="source-link">
+                    Link
+                </LemonButton>
+            </div>
+        )
+    }
+
+    return (
+        <LemonModal
+            {...props}
+            width={600}
+            onAfterClose={() => onClear()}
+            title={modalTitle}
+            description={modalCaption}
+            footer={footer()}
+        >
+            <FirstStep />
+            <SecondStep />
+            <ThirdStep />
+        </LemonModal>
+    )
+}
+
+interface ModalPageProps {
+    page: number
+    children?: React.ReactNode
+}
+
+function ModalPage({ children, page }: ModalPageProps): JSX.Element {
+    const { currentStep } = useValues(sourceModalLogic)
+
+    if (currentStep !== page) {
+        return <></>
+    }
+
+    return <div>{children}</div>
+}
+
+function FirstStep(): JSX.Element {
+    const { connectors, addToHubspotButtonUrl } = useValues(sourceModalLogic)
+    const { selectConnector, toggleManualLinkFormVisible, onNext } = useActions(sourceModalLogic)
     const { featureFlags } = useValues(featureFlagLogic)
 
     const MenuButton = (config: SourceConfig): JSX.Element => {
         const onClick = (): void => {
             selectConnector(config)
+            onNext()
         }
 
         if (config.name === 'Stripe') {
             return (
-                <LemonButton onClick={onClick} className="w-100" center type="secondary">
+                <LemonButton onClick={onClick} fullWidth center type="secondary">
                     <img src={stripeLogo} alt="stripe logo" height={50} />
                 </LemonButton>
             )
         }
-        if (config.name === 'Hubspot' && featureFlags[FEATURE_FLAGS.DATA_WAREHOUSE_HUBSPOT_IMPORT]) {
+        if (config.name === 'Hubspot') {
             return (
-                <Link to={addToHubspotButtonUrl() || ''}>
-                    <LemonButton className="w-100" center type="secondary">
-                        <img src={hubspotLogo} alt="hubspot logo" height={45} />
-                    </LemonButton>
-                </Link>
+                <LemonButton fullWidth center type="secondary" to={addToHubspotButtonUrl() || ''}>
+                    <img src={hubspotLogo} alt="hubspot logo" height={45} />
+                </LemonButton>
+            )
+        }
+
+        if (config.name === 'Postgres' && featureFlags[FEATURE_FLAGS.DATA_WAREHOUSE_POSTGRES_IMPORT]) {
+            return (
+                <LemonButton onClick={onClick} fullWidth center type="secondary">
+                    <div className="flex flex-row gap-2 justify-center items-center">
+                        <img src={postgresLogo} alt="postgres logo" height={45} />
+                        <div className="text-base">Postgres</div>
+                    </div>
+                </LemonButton>
             )
         }
 
@@ -48,110 +109,37 @@ export default function SourceModal(props: SourceModalProps): JSX.Element {
 
     const onManualLinkClick = (): void => {
         toggleManualLinkFormVisible(true)
+        onNext()
     }
 
-    const formToShow = (): JSX.Element => {
-        if (selectedConnector) {
-            return <SourceForm sourceType={selectedConnector.name} />
-        }
-
-        if (isManualLinkFormVisible) {
-            return (
-                <div>
-                    <DatawarehouseTableForm
-                        footer={
-                            <>
-                                <LemonDivider className="mt-4" />
-                                <div className="mt-2 flex flex-row justify-end gap-2">
-                                    <LemonButton
-                                        type="secondary"
-                                        center
-                                        data-attr="source-modal-back-button"
-                                        onClick={onClear}
-                                    >
-                                        Back
-                                    </LemonButton>
-                                    <LemonButton
-                                        type="primary"
-                                        center
-                                        htmlType="submit"
-                                        data-attr="source-link"
-                                        loading={tableLoading}
-                                    >
-                                        Link
-                                    </LemonButton>
-                                </div>
-                            </>
-                        }
-                    />
-                </div>
-            )
-        }
-
-        return (
-            <div className="flex flex-col gap-2">
+    return (
+        <ModalPage page={1}>
+            <div className="flex flex-col gap-2 items-center">
                 {connectors.map((config, index) => (
                     <MenuButton key={config.name + '_' + index} {...config} />
                 ))}
-                <LemonButton onClick={onManualLinkClick} className="w-100" center type="secondary">
+                <LemonButton onClick={onManualLinkClick} className="w-full" center type="secondary">
                     Manual Link
                 </LemonButton>
             </div>
-        )
-    }
-
-    return (
-        <LemonModal
-            {...props}
-            onAfterClose={() => onClear()}
-            title={selectedConnector ? 'Link ' + selectedConnector.name : 'Select source to link'}
-            description={selectedConnector ? selectedConnector.caption : null}
-        >
-            {formToShow()}
-        </LemonModal>
+        </ModalPage>
     )
 }
 
-interface SourceFormProps {
-    sourceType: ExternalDataSourceType
-}
+function SecondStep(): JSX.Element {
+    const { selectedConnector } = useValues(sourceModalLogic)
 
-function SourceForm({ sourceType }: SourceFormProps): JSX.Element {
-    const logic = sourceFormLogic({ sourceType })
-    const { isExternalDataSourceSubmitting } = useValues(logic)
-    const { onBack } = useActions(logic)
+    return (
+        <ModalPage page={2}>
+            {selectedConnector ? <SourceForm sourceType={selectedConnector.name} /> : <DatawarehouseTableForm />}
+        </ModalPage>
+    )
+}
 
+function ThirdStep(): JSX.Element {
     return (
-        <Form
-            logic={sourceFormLogic}
-            props={{ sourceType }}
-            formKey="externalDataSource"
-            className="space-y-4"
-            enableFormOnSubmit
-        >
-            {SOURCE_DETAILS[sourceType].fields.map((field) => (
-                <Field key={field.name} name={['payload', field.name]} label={field.label}>
-                    <LemonInput className="ph-ignore-input" data-attr={field.name} />
-                </Field>
-            ))}
-            <Field name="prefix" label="Table Prefix (optional)">
-                <LemonInput className="ph-ignore-input" data-attr="prefix" placeholder="internal_" />
-            </Field>
-            <LemonDivider className="mt-4" />
-            <div className="mt-2 flex flex-row justify-end gap-2">
-                <LemonButton type="secondary" center data-attr="source-modal-back-button" onClick={onBack}>
-                    Back
-                </LemonButton>
-                <LemonButton
-                    type="primary"
-                    center
-                    htmlType="submit"
-                    data-attr="source-link"
-                    loading={isExternalDataSourceSubmitting}
-                >
-                    Link
-                </LemonButton>
-            </div>
-        </Form>
+        <ModalPage page={3}>
+            <PostgresSchemaForm />
+        </ModalPage>
     )
 }

frontend/src/scenes/data-warehouse/external/forms/PostgresSchemaForm.tsx

@@ -0,0 +1,45 @@
+import { LemonSwitch, LemonTable } from '@posthog/lemon-ui'
+import { useActions, useMountedLogic, useValues } from 'kea'
+
+import { sourceModalLogic } from '../sourceModalLogic'
+import { sourceFormLogic } from './sourceFormLogic'
+
+export default function PostgresSchemaForm(): JSX.Element {
+    useMountedLogic(sourceFormLogic({ sourceType: 'Postgres' }))
+    const { selectSchema } = useActions(sourceModalLogic)
+    const { databaseSchema } = useValues(sourceModalLogic)
+
+    return (
+        <div className="flex flex-col gap-2">
+            <div>
+                <LemonTable
+                    emptyState="No schemas found"
+                    dataSource={databaseSchema}
+                    columns={[
+                        {
+                            title: 'Table',
+                            key: 'table',
+                            render: function RenderTable(_, schema) {
+                                return schema.table
+                            },
+                        },
+                        {
+                            title: 'Sync',
+                            key: 'should_sync',
+                            render: function RenderShouldSync(_, schema) {
+                                return (
+                                    <LemonSwitch
+                                        checked={schema.should_sync}
+                                        onChange={() => {
+                                            selectSchema(schema)
+                                        }}
+                                    />
+                                )
+                            },
+                        },
+                    ]}
+                />
+            </div>
+        </div>
+    )
+}

frontend/src/scenes/data-warehouse/external/forms/SourceForm.tsx

@@ -0,0 +1,33 @@
+import { LemonInput } from '@posthog/lemon-ui'
+import { Form } from 'kea-forms'
+import { Field } from 'lib/forms/Field'
+
+import { ExternalDataSourceType } from '~/types'
+
+import { SOURCE_DETAILS } from '../sourceModalLogic'
+import { sourceFormLogic } from './sourceFormLogic'
+
+interface SourceFormProps {
+    sourceType: ExternalDataSourceType
+}
+
+export default function SourceForm({ sourceType }: SourceFormProps): JSX.Element {
+    return (
+        <Form
+            logic={sourceFormLogic}
+            props={{ sourceType }}
+            formKey={sourceType == 'Postgres' ? 'databaseSchemaForm' : 'externalDataSource'}
+            className="space-y-4"
+            enableFormOnSubmit
+        >
+            {SOURCE_DETAILS[sourceType].fields.map((field) => (
+                <Field key={field.name} name={['payload', field.name]} label={field.label}>
+                    <LemonInput className="ph-ignore-input" data-attr={field.name} />
+                </Field>
+            ))}
+            <Field name="prefix" label="Table Prefix (optional)">
+                <LemonInput className="ph-ignore-input" data-attr="prefix" placeholder="internal_" />
+            </Field>
+        </Form>
+    )
+}