Skip to content

Commit

Permalink
feat(data-warehouse): edit SQL based import configs (#25685)
Browse files Browse the repository at this point in the history
Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
  • Loading branch information
EDsCODE and github-actions[bot] authored Oct 18, 2024
1 parent a585c33 commit dbfefae
Show file tree
Hide file tree
Showing 9 changed files with 171 additions and 52 deletions.
20 changes: 10 additions & 10 deletions frontend/src/lib/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,11 @@ import {
Experiment,
ExportedAssetType,
ExternalDataJob,
ExternalDataSource,
ExternalDataSourceCreatePayload,
ExternalDataSourceSchema,
ExternalDataSourceSyncSchema,
ExternalDataSourceType,
ExternalDataStripeSource,
FeatureFlagAssociatedRoleType,
FeatureFlagType,
Group,
Expand Down Expand Up @@ -854,7 +854,7 @@ class ApiRequest {
return this.projectsDetail(teamId).addPathComponent('external_data_sources')
}

public externalDataSource(sourceId: ExternalDataStripeSource['id'], teamId?: TeamType['id']): ApiRequest {
public externalDataSource(sourceId: ExternalDataSource['id'], teamId?: TeamType['id']): ApiRequest {
return this.externalDataSources(teamId).addPathComponent(sourceId)
}

Expand Down Expand Up @@ -2199,25 +2199,25 @@ const api = {
},
},
externalDataSources: {
async list(options?: ApiMethodOptions | undefined): Promise<PaginatedResponse<ExternalDataStripeSource>> {
async list(options?: ApiMethodOptions | undefined): Promise<PaginatedResponse<ExternalDataSource>> {
return await new ApiRequest().externalDataSources().get(options)
},
async get(sourceId: ExternalDataStripeSource['id']): Promise<ExternalDataStripeSource> {
async get(sourceId: ExternalDataSource['id']): Promise<ExternalDataSource> {
return await new ApiRequest().externalDataSource(sourceId).get()
},
async create(data: Partial<ExternalDataSourceCreatePayload>): Promise<{ id: string }> {
return await new ApiRequest().externalDataSources().create({ data })
},
async delete(sourceId: ExternalDataStripeSource['id']): Promise<void> {
async delete(sourceId: ExternalDataSource['id']): Promise<void> {
await new ApiRequest().externalDataSource(sourceId).delete()
},
async reload(sourceId: ExternalDataStripeSource['id']): Promise<void> {
async reload(sourceId: ExternalDataSource['id']): Promise<void> {
await new ApiRequest().externalDataSource(sourceId).withAction('reload').create()
},
async update(
sourceId: ExternalDataStripeSource['id'],
data: Partial<ExternalDataStripeSource>
): Promise<ExternalDataStripeSource> {
sourceId: ExternalDataSource['id'],
data: Partial<ExternalDataSource>
): Promise<ExternalDataSource> {
return await new ApiRequest().externalDataSource(sourceId).update({ data })
},
async database_schema(
Expand All @@ -2239,7 +2239,7 @@ const api = {
.create({ data: { source_type, prefix } })
},
async jobs(
sourceId: ExternalDataStripeSource['id'],
sourceId: ExternalDataSource['id'],
before: string | null,
after: string | null
): Promise<ExternalDataJob[]> {
Expand Down
62 changes: 39 additions & 23 deletions frontend/src/scenes/data-warehouse/external/forms/SourceForm.tsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import { LemonFileInput, LemonInput, LemonSelect, LemonSwitch, LemonTextArea } from '@posthog/lemon-ui'
import { useValues } from 'kea'
import { Form, Group } from 'kea-forms'
import { LemonField } from 'lib/lemon-ui/LemonField'

Expand All @@ -8,22 +7,27 @@ import { SourceConfig, SourceFieldConfig } from '~/types'
import { SOURCE_DETAILS, sourceWizardLogic } from '../../new/sourceWizardLogic'
import { DataWarehouseIntegrationChoice } from './DataWarehouseIntegrationChoice'

interface SourceFormProps {
export interface SourceFormProps {
sourceConfig: SourceConfig
showPrefix?: boolean
showSourceFields?: boolean
jobInputs?: Record<string, any>
}

const sourceFieldToElement = (field: SourceFieldConfig, sourceConfig: SourceConfig): JSX.Element => {
const sourceFieldToElement = (field: SourceFieldConfig, sourceConfig: SourceConfig, lastValue?: any): JSX.Element => {
if (field.type === 'switch-group') {
return (
<LemonField key={field.name} name={[field.name, 'enabled']} label={field.label}>
{({ value, onChange }) => (
<>
<LemonSwitch checked={value} onChange={onChange} />
<LemonSwitch
checked={value === undefined || value === null ? lastValue?.['enabled'] : value}
onChange={onChange}
/>
{value && (
<Group name={field.name}>
{field.fields.map((field) => sourceFieldToElement(field, sourceConfig))}
{field.fields.map((field) =>
sourceFieldToElement(field, sourceConfig, lastValue?.[field.name])
)}
</Group>
)}
</>
Expand All @@ -43,11 +47,21 @@ const sourceFieldToElement = (field: SourceFieldConfig, sourceConfig: SourceConf
>
{({ value, onChange }) => (
<>
<LemonSelect options={field.options} value={value ?? field.defaultValue} onChange={onChange} />
<LemonSelect
options={field.options}
value={
value === undefined || value === null
? lastValue?.[field.name]
: value || field.defaultValue
}
onChange={onChange}
/>
<Group name={field.name}>
{field.options
.find((n) => n.value === (value ?? field.defaultValue))
?.fields?.map((field) => sourceFieldToElement(field, sourceConfig))}
?.fields?.map((field) =>
sourceFieldToElement(field, sourceConfig, lastValue?.[field.name])
)}
</Group>
</>
)}
Expand All @@ -63,6 +77,7 @@ const sourceFieldToElement = (field: SourceFieldConfig, sourceConfig: SourceConf
data-attr={field.name}
placeholder={field.placeholder}
minRows={4}
defaultValue={lastValue}
/>
</LemonField>
)
Expand Down Expand Up @@ -102,32 +117,33 @@ const sourceFieldToElement = (field: SourceFieldConfig, sourceConfig: SourceConf
data-attr={field.name}
placeholder={field.placeholder}
type={field.type}
defaultValue={lastValue}
/>
</LemonField>
)
}

export default function SourceForm({ sourceConfig }: SourceFormProps): JSX.Element {
const { source } = useValues(sourceWizardLogic)
const showSourceFields = SOURCE_DETAILS[sourceConfig.name].showSourceForm
? SOURCE_DETAILS[sourceConfig.name].showSourceForm?.(source.payload)
: true
const showPrefix = SOURCE_DETAILS[sourceConfig.name].showPrefix
? SOURCE_DETAILS[sourceConfig.name].showPrefix?.(source.payload)
: true
export default function SourceFormContainer(props: SourceFormProps): JSX.Element {
return (
<Form logic={sourceWizardLogic} formKey="sourceConnectionDetails" enableFormOnSubmit>
<SourceFormComponent {...props} />
</Form>
)
}

export function SourceFormComponent({ sourceConfig, showPrefix = true, jobInputs }: SourceFormProps): JSX.Element {
return (
<Form logic={sourceWizardLogic} formKey="sourceConnectionDetails" className="space-y-4" enableFormOnSubmit>
{showSourceFields && (
<Group name="payload">
{SOURCE_DETAILS[sourceConfig.name].fields.map((field) => sourceFieldToElement(field, sourceConfig))}
</Group>
)}
<div className="space-y-4">
<Group name="payload">
{SOURCE_DETAILS[sourceConfig.name].fields.map((field) =>
sourceFieldToElement(field, sourceConfig, jobInputs?.[field.name])
)}
</Group>
{showPrefix && (
<LemonField name="prefix" label="Table Prefix (optional)">
<LemonInput className="ph-ignore-input" data-attr="prefix" placeholder="internal_" />
</LemonField>
)}
</Form>
</div>
)
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import posthog from 'posthog-js'
import { databaseTableListLogic } from 'scenes/data-management/database/databaseTableListLogic'

import { DatabaseSchemaDataWarehouseTable } from '~/queries/schema'
import { DataWarehouseSettingsTab, ExternalDataSourceSchema, ExternalDataStripeSource } from '~/types'
import { DataWarehouseSettingsTab, ExternalDataSource, ExternalDataSourceSchema } from '~/types'

import type { dataWarehouseSettingsLogicType } from './dataWarehouseSettingsLogicType'

Expand All @@ -31,17 +31,17 @@ export const dataWarehouseSettingsLogic = kea<dataWarehouseSettingsLogicType>([
actions: [databaseTableListLogic, ['loadDatabase']],
})),
actions({
deleteSource: (source: ExternalDataStripeSource) => ({ source }),
reloadSource: (source: ExternalDataStripeSource) => ({ source }),
sourceLoadingFinished: (source: ExternalDataStripeSource) => ({ source }),
deleteSource: (source: ExternalDataSource) => ({ source }),
reloadSource: (source: ExternalDataSource) => ({ source }),
sourceLoadingFinished: (source: ExternalDataSource) => ({ source }),
schemaLoadingFinished: (schema: ExternalDataSourceSchema) => ({ schema }),
abortAnyRunningQuery: true,
deleteSelfManagedTable: (tableId: string) => ({ tableId }),
refreshSelfManagedTableSchema: (tableId: string) => ({ tableId }),
}),
loaders(({ cache, actions, values }) => ({
dataWarehouseSources: [
null as PaginatedResponse<ExternalDataStripeSource> | null,
null as PaginatedResponse<ExternalDataSource> | null,
{
loadSources: async (_, breakpoint) => {
await breakpoint(300)
Expand All @@ -59,7 +59,7 @@ export const dataWarehouseSettingsLogic = kea<dataWarehouseSettingsLogicType>([

return res
},
updateSource: async (source: ExternalDataStripeSource) => {
updateSource: async (source: ExternalDataSource) => {
const updatedSource = await api.externalDataSources.update(source.id, source)
return {
...values.dataWarehouseSources,
Expand All @@ -77,7 +77,7 @@ export const dataWarehouseSettingsLogic = kea<dataWarehouseSettingsLogicType>([
// Optimistic UI updates before sending updates to the backend
const clonedSources = JSON.parse(
JSON.stringify(values.dataWarehouseSources?.results ?? [])
) as ExternalDataStripeSource[]
) as ExternalDataSource[]
const sourceIndex = clonedSources.findIndex((n) => n.schemas.find((m) => m.id === schema.id))
const schemaIndex = clonedSources[sourceIndex].schemas.findIndex((n) => n.id === schema.id)
clonedSources[sourceIndex].schemas[schemaIndex] = schema
Expand Down Expand Up @@ -166,7 +166,7 @@ export const dataWarehouseSettingsLogic = kea<dataWarehouseSettingsLogicType>([
// Optimistic UI updates before sending updates to the backend
const clonedSources = JSON.parse(
JSON.stringify(values.dataWarehouseSources?.results ?? [])
) as ExternalDataStripeSource[]
) as ExternalDataSource[]
const sourceIndex = clonedSources.findIndex((n) => n.id === source.id)
clonedSources[sourceIndex].status = 'Running'
clonedSources[sourceIndex].schemas = clonedSources[sourceIndex].schemas.map((n) => {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import { LemonBanner, LemonButton, LemonSkeleton } from '@posthog/lemon-ui'
import { BindLogic, useValues } from 'kea'
import { Form } from 'kea-forms'
import { SourceFormComponent, SourceFormProps } from 'scenes/data-warehouse/external/forms/SourceForm'

import { dataWarehouseSourceSettingsLogic } from './dataWarehouseSourceSettingsLogic'

interface SourceConfigurationProps {
id: string
}

export const SourceConfiguration = ({ id }: SourceConfigurationProps): JSX.Element => {
const { sourceFieldConfig } = useValues(dataWarehouseSourceSettingsLogic({ id }))
return (
<BindLogic logic={dataWarehouseSourceSettingsLogic} props={{ id }}>
{sourceFieldConfig ? (
<UpdateSourceConnectionFormContainer id={id} sourceConfig={sourceFieldConfig} showPrefix={false} />
) : (
<LemonSkeleton />
)}
</BindLogic>
)
}

interface UpdateSourceConnectionFormContainerProps extends SourceFormProps {
id: string
}

function UpdateSourceConnectionFormContainer(props: UpdateSourceConnectionFormContainerProps): JSX.Element {
const { source, sourceLoading } = useValues(dataWarehouseSourceSettingsLogic({ id: props.id }))

if (source?.source_type !== 'MSSQL' && source?.source_type !== 'MySQL' && source?.source_type !== 'Postgres') {
return (
<LemonBanner type="warning" className="mt-2">
<p>
Only Postgres, MSSQL, and MySQL are configurable. Please delete and recreate your source if you need
to connect to a new source of the same type.
</p>
</LemonBanner>
)
}
return (
<Form logic={dataWarehouseSourceSettingsLogic} formKey="sourceConfig" enableFormOnSubmit>
<SourceFormComponent {...props} jobInputs={source?.job_inputs} />
<div className="mt-4 flex flex-row justify-end gap-2">
<LemonButton
loading={sourceLoading && !source}
type="primary"
center
htmlType="submit"
data-attr="source-update"
>
Save
</LemonButton>
</div>
</Form>
)
}
Loading

0 comments on commit dbfefae

Please sign in to comment.