feat: Move data warehouse sources into pipeline (#24072)

Co-authored-by: eric <[email protected]>
Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Eric Duong <[email protected]>
Co-authored-by: Michael Matloka <[email protected]>

frontend/src/layout/navigation-3000/navigationLogic.tsx

@@ -486,6 +486,14 @@ export const navigation3000Logic = kea<navigation3000LogicType>([
                             icon: <IconChat />,
                             to: urls.surveys(),
                         },
+                        featureFlags[FEATURE_FLAGS.PRODUCT_INTRO_PAGES] !== 'test' || hasOnboardedFeatureFlags
+                            ? {
+                                  identifier: Scene.EarlyAccessFeatures,
+                                  label: 'Early access features',
+                                  icon: <IconRocket />,
+                                  to: urls.earlyAccessFeatures(),
+                              }
+                            : null,
                         {
                             identifier: Scene.DataWarehouse,
                             label: 'Data warehouse',
@@ -500,14 +508,6 @@ export const navigation3000Logic = kea<navigation3000LogicType>([
                                   to: urls.pipeline(),
                               }
                             : null,
-                        featureFlags[FEATURE_FLAGS.PRODUCT_INTRO_PAGES] !== 'test' || hasOnboardedFeatureFlags
-                            ? {
-                                  identifier: Scene.EarlyAccessFeatures,
-                                  label: 'Early access features',
-                                  icon: <IconRocket />,
-                                  to: urls.earlyAccessFeatures(),
-                              }
-                            : null,
                     ].filter(isNotNil),
                 ]
             },

frontend/src/lib/components/TaxonomicFilter/taxonomicFilterLogic.tsx

@@ -16,7 +16,7 @@ import { CORE_FILTER_DEFINITIONS_BY_GROUP } from 'lib/taxonomy'
 import { capitalizeFirstLetter, pluralize, toParams } from 'lib/utils'
 import { getEventDefinitionIcon, getPropertyDefinitionIcon } from 'scenes/data-management/events/DefinitionHeader'
 import { dataWarehouseJoinsLogic } from 'scenes/data-warehouse/external/dataWarehouseJoinsLogic'
-import { dataWarehouseSceneLogic } from 'scenes/data-warehouse/external/dataWarehouseSceneLogic'
+import { dataWarehouseSceneLogic } from 'scenes/data-warehouse/settings/dataWarehouseSceneLogic'
 import { experimentsLogic } from 'scenes/experiments/experimentsLogic'
 import { featureFlagsLogic } from 'scenes/feature-flags/featureFlagsLogic'
 import { groupDisplayId } from 'scenes/persons/GroupActorDisplay'

frontend/src/queries/nodes/HogQLQuery/HogQLQueryEditor.tsx

@@ -11,7 +11,7 @@ import { CodeEditor } from 'lib/monaco/CodeEditor'
 import { codeEditorLogic } from 'lib/monaco/codeEditorLogic'
 import type { editor as importedEditor, IDisposable } from 'monaco-editor'
 import { useEffect, useRef, useState } from 'react'
-import { dataWarehouseSceneLogic } from 'scenes/data-warehouse/external/dataWarehouseSceneLogic'
+import { dataWarehouseSceneLogic } from 'scenes/data-warehouse/settings/dataWarehouseSceneLogic'
 
 import { HogQLQuery } from '~/queries/schema'
 

frontend/src/queries/nodes/HogQLQuery/hogQLQueryEditorLogic.tsx

@@ -12,8 +12,8 @@ import { LemonField } from 'lib/lemon-ui/LemonField'
 // esbuild doesn't support manual chunks as of 2023, so we can't just put Monaco in its own chunk, which would prevent
 // re-importing. As for @monaco-editor/react, it does some lazy loading and doesn't have this problem.
 import type { editor } from 'monaco-editor'
-import { dataWarehouseSceneLogic } from 'scenes/data-warehouse/external/dataWarehouseSceneLogic'
 import { dataWarehouseViewsLogic } from 'scenes/data-warehouse/saved_queries/dataWarehouseViewsLogic'
+import { dataWarehouseSceneLogic } from 'scenes/data-warehouse/settings/dataWarehouseSceneLogic'
 import { preflightLogic } from 'scenes/PreflightCheck/preflightLogic'
 
 import { DataNode, HogQLQuery, NodeKind } from '~/queries/schema'

frontend/src/scenes/appScenes.ts

@@ -22,7 +22,6 @@ export const appScenes: Record<Scene, () => any> = {
     [Scene.ReplayFilePlayback]: () => import('./session-recordings/file-playback/SessionRecordingFilePlaybackScene'),
     [Scene.PersonsManagement]: () => import('./persons-management/PersonsManagementScene'),
     [Scene.Person]: () => import('./persons/PersonScene'),
-    [Scene.pipelineNodeDataWarehouseNew]: () => import('./data-warehouse/new/NewSourceWizard'),
     [Scene.PipelineNodeNew]: () => import('./pipeline/PipelineNodeNew'),
     [Scene.Pipeline]: () => import('./pipeline/Pipeline'),
     [Scene.PipelineNode]: () => import('./pipeline/PipelineNode'),
@@ -42,10 +41,7 @@ export const appScenes: Record<Scene, () => any> = {
     [Scene.DataWarehouse]: () => import('./data-warehouse/external/DataWarehouseExternalScene'),
     [Scene.DataWarehouseTable]: () => import('./data-warehouse/new/NewSourceWizard'),
     [Scene.DataWarehouseExternal]: () => import('./data-warehouse/external/DataWarehouseExternalScene'),
-    [Scene.DataWarehouseSettings]: () => import('./data-warehouse/external/DataWarehouseExternalScene'),
     [Scene.DataWarehouseRedirect]: () => import('./data-warehouse/redirect/DataWarehouseRedirectScene'),
-    [Scene.dataWarehouseSourceSettings]: () =>
-        import('./data-warehouse/settings/source/DataWarehouseSourceSettingsScene'),
     [Scene.OrganizationCreateFirst]: () => import('./organization/Create'),
     [Scene.OrganizationCreationConfirm]: () => import('./organization/ConfirmOrganization/ConfirmOrganization'),
     [Scene.ProjectHomepage]: () => import('./project-homepage/ProjectHomepage'),

frontend/src/scenes/data-management/DataManagementScene.stories.tsx

@@ -248,12 +248,6 @@ const meta: Meta = {
     ],
 }
 export default meta
-export function Database(): JSX.Element {
-    useEffect(() => {
-        router.actions.push(urls.database())
-    }, [])
-    return <App />
-}
 
 export function IngestionWarnings(): JSX.Element {
     setFeatureFlags([FEATURE_FLAGS.INGESTION_WARNINGS_ENABLED])

frontend/src/scenes/data-management/DataManagementScene.tsx

@@ -6,7 +6,6 @@ import { PageHeader } from 'lib/components/PageHeader'
 import { TitleWithIcon } from 'lib/components/TitleWithIcon'
 import { FEATURE_FLAGS } from 'lib/constants'
 import { LemonTab, LemonTabs } from 'lib/lemon-ui/LemonTabs'
-import { LemonTag } from 'lib/lemon-ui/LemonTag/LemonTag'
 import { Tooltip } from 'lib/lemon-ui/Tooltip'
 import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
 import { capitalizeFirstLetter } from 'lib/utils'
@@ -20,7 +19,6 @@ import { urls } from 'scenes/urls'
 import { ActivityScope, Breadcrumb } from '~/types'
 
 import { ActionsTable } from './actions/ActionsTable'
-import { DatabaseTableList } from './database/DatabaseTableList'
 import type { dataManagementSceneLogicType } from './DataManagementSceneType'
 import { EventDefinitionsTable } from './events/EventDefinitionsTable'
 import { IngestionWarningsView } from './ingestion-warnings/IngestionWarningsView'
@@ -33,7 +31,6 @@ export enum DataManagementTab {
     Annotations = 'annotations',
     History = 'history',
     IngestionWarnings = 'warnings',
-    Database = 'database',
 }
 
 const tabs: Record<
@@ -97,18 +94,6 @@ const tabs: Record<
         label: 'Ingestion warnings',
         content: <IngestionWarningsView />,
     },
-    [DataManagementTab.Database]: {
-        url: urls.database(),
-        label: (
-            <>
-                Database
-                <LemonTag type="warning" className="uppercase ml-2">
-                    Beta
-                </LemonTag>
-            </>
-        ),
-        content: <DatabaseTableList />,
-    },
 }
 
 const dataManagementSceneLogic = kea<dataManagementSceneLogicType>([

frontend/src/scenes/data-management/database/DatabaseTable.tsx

@@ -7,7 +7,7 @@ import { Link } from 'lib/lemon-ui/Link'
 import { deleteWithUndo } from 'lib/utils/deleteWithUndo'
 import { useCallback } from 'react'
 import { dataWarehouseJoinsLogic } from 'scenes/data-warehouse/external/dataWarehouseJoinsLogic'
-import { dataWarehouseSceneLogic } from 'scenes/data-warehouse/external/dataWarehouseSceneLogic'
+import { dataWarehouseSceneLogic } from 'scenes/data-warehouse/settings/dataWarehouseSceneLogic'
 import { viewLinkLogic } from 'scenes/data-warehouse/viewLinkLogic'
 import { teamLogic } from 'scenes/teamLogic'
 import { urls } from 'scenes/urls'

frontend/src/scenes/data-warehouse/external/DataWarehouseExternalScene.tsx

@@ -1,45 +1,23 @@
-import { LemonButton, LemonTabs, Link } from '@posthog/lemon-ui'
+import { LemonButton, Link } from '@posthog/lemon-ui'
 import { BindLogic, useActions, useValues } from 'kea'
-import { router } from 'kea-router'
 import { PageHeader } from 'lib/components/PageHeader'
 import { insightLogic } from 'scenes/insights/insightLogic'
 import { insightSceneLogic } from 'scenes/insights/insightSceneLogic'
 import { SceneExport } from 'scenes/sceneTypes'
 import { urls } from 'scenes/urls'
 
-import { DataWarehouseTab } from '~/types'
+import { PipelineTab } from '~/types'
 
 import { DataWarehouseInitialBillingLimitNotice } from '../DataWarehouseInitialBillingLimitNotice'
-import { DataWarehouseManagedSourcesTable } from '../settings/DataWarehouseManagedSourcesTable'
-import { DataWarehouseSelfManagedSourcesTable } from '../settings/DataWarehouseSelfManagedSourcesTable'
-import { dataWarehouseSceneLogic } from './dataWarehouseSceneLogic'
+import { dataWarehouseExternalSceneLogic } from './dataWarehouseExternalSceneLogic'
 import { DataWarehouseTables } from './DataWarehouseTables'
 
 export const scene: SceneExport = {
     component: DataWarehouseExternalScene,
-    logic: dataWarehouseSceneLogic,
-}
-
-const tabToContent: Partial<Record<DataWarehouseTab, JSX.Element>> = {
-    [DataWarehouseTab.Explore]: <Explore />,
-    [DataWarehouseTab.ManagedSources]: <DataWarehouseManagedSourcesTable />,
-    [DataWarehouseTab.SelfManagedSources]: <DataWarehouseSelfManagedSourcesTable />,
-}
-
-export const humanFriendlyDataWarehouseTabName = (tab: DataWarehouseTab): string => {
-    switch (tab) {
-        case DataWarehouseTab.Explore:
-            return 'Explore'
-        case DataWarehouseTab.ManagedSources:
-            return 'Managed sources'
-        case DataWarehouseTab.SelfManagedSources:
-            return 'Self-Managed sources'
-    }
+    logic: dataWarehouseExternalSceneLogic,
 }
 
 export function DataWarehouseExternalScene(): JSX.Element {
-    const { currentTab } = useValues(dataWarehouseSceneLogic)
-
     const { insightSaving } = useValues(
         insightLogic({
             dashboardItemId: 'new-dataWarehouse',
@@ -58,24 +36,16 @@ export function DataWarehouseExternalScene(): JSX.Element {
             <PageHeader
                 buttons={
                     <>
-                        {currentTab === DataWarehouseTab.Explore && (
-                            <LemonButton
-                                type="primary"
-                                data-attr="save-exploration"
-                                onClick={() => saveAs(true)}
-                                loading={insightSaving}
-                            >
-                                Save as insight
-                            </LemonButton>
-                        )}
-
                         <LemonButton
                             type="primary"
-                            data-attr="new-data-warehouse-easy-link"
-                            key="new-data-warehouse-easy-link"
-                            to={urls.dataWarehouseTable()}
+                            data-attr="save-exploration"
+                            onClick={() => saveAs(true)}
+                            loading={insightSaving}
                         >
-                            Link source
+                            Save as insight
+                        </LemonButton>
+                        <LemonButton type="secondary" to={urls.pipeline(PipelineTab.Sources)}>
+                            Manage sources
                         </LemonButton>
                     </>
                 }
@@ -91,27 +61,9 @@ export function DataWarehouseExternalScene(): JSX.Element {
                 }
             />
             <DataWarehouseInitialBillingLimitNotice />
-            <LemonTabs
-                activeKey={currentTab}
-                onChange={(tab) => router.actions.push(urls.dataWarehouse(tab as DataWarehouseTab))}
-                tabs={Object.entries(tabToContent).map(([tab, content]) => ({
-                    label: (
-                        <span className="flex items-center justify-between gap-1">
-                            {humanFriendlyDataWarehouseTabName(tab as DataWarehouseTab)}{' '}
-                        </span>
-                    ),
-                    key: tab,
-                    content: content,
-                }))}
-            />
+            <BindLogic logic={insightSceneLogic} props={{}}>
+                <DataWarehouseTables />
+            </BindLogic>
         </div>
     )
 }
-
-function Explore(): JSX.Element {
-    return (
-        <BindLogic logic={insightSceneLogic} props={{}}>
-            <DataWarehouseTables />
-        </BindLogic>
-    )
-}

frontend/src/scenes/data-warehouse/external/DataWarehouseTables.tsx

@@ -13,9 +13,9 @@ import { urls } from 'scenes/urls'
 import { Query } from '~/queries/Query/Query'
 import { DatabaseSchemaTable, NodeKind } from '~/queries/schema'
 
+import { dataWarehouseSceneLogic } from '../settings/dataWarehouseSceneLogic'
 import { viewLinkLogic } from '../viewLinkLogic'
 import { ViewLinkModal } from '../ViewLinkModal'
-import { dataWarehouseSceneLogic } from './dataWarehouseSceneLogic'
 import { DeleteTableModal, TableData } from './TableData'
 
 export const DataWarehouseTables = (): JSX.Element => {

frontend/src/scenes/data-warehouse/external/TableData.tsx

@@ -8,7 +8,7 @@ import { DatabaseTable } from 'scenes/data-management/database/DatabaseTable'
 
 import { DatabaseSchemaTable } from '~/queries/schema'
 
-import { dataWarehouseSceneLogic } from './dataWarehouseSceneLogic'
+import { dataWarehouseSceneLogic } from '../settings/dataWarehouseSceneLogic'
 
 export function TableData(): JSX.Element {
     const {

frontend/src/scenes/data-warehouse/external/dataWarehouseExternalSceneLogic.ts

@@ -0,0 +1,23 @@
+import { kea, path, selectors } from 'kea'
+import { Scene } from 'scenes/sceneTypes'
+import { urls } from 'scenes/urls'
+
+import { Breadcrumb } from '~/types'
+
+import type { dataWarehouseExternalSceneLogicType } from './dataWarehouseExternalSceneLogicType'
+
+export const dataWarehouseExternalSceneLogic = kea<dataWarehouseExternalSceneLogicType>([
+    path(() => ['scenes', 'data-warehouse', 'external', 'dataWarehouseExternalSceneLogic']),
+    selectors(() => ({
+        breadcrumbs: [
+            () => [],
+            (): Breadcrumb[] => [
+                {
+                    key: Scene.DataWarehouse,
+                    name: 'Explore',
+                    path: urls.dataWarehouse(),
+                },
+            ],
+        ],
+    })),
+])