-
Notifications
You must be signed in to change notification settings - Fork 1.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(data-warehouse): Added incremental syncs for postgres (#23145)
* Added incremental syncs for postgres * Tests * Fixed mypy * Added extra protection on the sync form * Set the correct write disposition for SQL * PR fixes * Fixed tests * Fixed tests * Fixed tests
- Loading branch information
Showing
34 changed files
with
2,042 additions
and
347 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
187 changes: 104 additions & 83 deletions
187
frontend/src/scenes/data-warehouse/external/forms/PostgresSchemaForm.tsx
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,98 +1,119 @@ | ||
| import { LemonSelect, LemonSelectOptionLeaf, LemonSwitch, LemonTable, Link } from '@posthog/lemon-ui' | ||
| import { LemonButton, LemonCheckbox, LemonModal, LemonTable } from '@posthog/lemon-ui' | ||
| import { useActions, useValues } from 'kea' | ||
| import { useState } from 'react' | ||
|
|
||
| import { ExternalDataSourceSyncSchema } from '~/types' | ||
|
|
||
| import { sourceWizardLogic } from '../../new/sourceWizardLogic' | ||
|
|
||
| const syncTypesToOptions = ( | ||
| schema: ExternalDataSourceSyncSchema | ||
| ): LemonSelectOptionLeaf<ExternalDataSourceSyncSchema['sync_type']>[] => { | ||
| const options: LemonSelectOptionLeaf<ExternalDataSourceSyncSchema['sync_type']>[] = [] | ||
|
|
||
| if (schema.sync_types.full_refresh) { | ||
| options.push({ value: 'full_refresh', label: 'Full refresh' }) | ||
| } | ||
|
|
||
| if (schema.sync_types.incremental) { | ||
| options.push({ value: 'incremental', label: 'Incremental' }) | ||
| } | ||
|
|
||
| return options | ||
| } | ||
| import { SyncMethodForm } from './SyncMethodForm' | ||
|
|
||
| export default function PostgresSchemaForm(): JSX.Element { | ||
| const { toggleSchemaShouldSync, updateSchemaSyncType } = useActions(sourceWizardLogic) | ||
| const { toggleSchemaShouldSync, openSyncMethodModal } = useActions(sourceWizardLogic) | ||
| const { databaseSchema } = useValues(sourceWizardLogic) | ||
| const [toggleAllState, setToggleAllState] = useState(false) | ||
|
|
||
| const toggleAllSwitches = (): void => { | ||
| databaseSchema.forEach((schema) => { | ||
| toggleSchemaShouldSync(schema, toggleAllState) | ||
| }) | ||
|
|
||
| setToggleAllState(!toggleAllState) | ||
| } | ||
|
|
||
| return ( | ||
| <div className="flex flex-col gap-2"> | ||
| <div> | ||
| <LemonTable | ||
| emptyState="No schemas found" | ||
| dataSource={databaseSchema} | ||
| columns={[ | ||
| { | ||
| title: 'Table', | ||
| key: 'table', | ||
| render: function RenderTable(_, schema) { | ||
| return schema.table | ||
| <> | ||
| <div className="flex flex-col gap-2"> | ||
| <div> | ||
| <LemonTable | ||
| emptyState="No schemas found" | ||
| dataSource={databaseSchema} | ||
| columns={[ | ||
| { | ||
| width: 0, | ||
| key: 'enabled', | ||
| render: (_, schema) => { | ||
| return ( | ||
| <LemonCheckbox | ||
| checked={schema.should_sync} | ||
| onChange={(checked) => { | ||
| toggleSchemaShouldSync(schema, checked) | ||
| }} | ||
| /> | ||
| ) | ||
| }, | ||
| }, | ||
| }, | ||
| { | ||
| title: ( | ||
| <> | ||
| <span>Sync</span> | ||
| <Link | ||
| className="ml-2 w-[60px] overflow-visible" | ||
| onClick={() => toggleAllSwitches()} | ||
| > | ||
| {toggleAllState ? 'Enable' : 'Disable'} all | ||
| </Link> | ||
| </> | ||
| ), | ||
| key: 'should_sync', | ||
| render: function RenderShouldSync(_, schema) { | ||
| return ( | ||
| <LemonSwitch | ||
| checked={schema.should_sync} | ||
| onChange={(checked) => { | ||
| toggleSchemaShouldSync(schema, checked) | ||
| }} | ||
| /> | ||
| ) | ||
| { | ||
| title: 'Table', | ||
| key: 'table', | ||
| render: function RenderTable(_, schema) { | ||
| return schema.table | ||
| }, | ||
| }, | ||
| }, | ||
| { | ||
| key: 'sync_type', | ||
| title: 'Sync type', | ||
| tooltip: | ||
| 'Full refresh will refresh the full table on every sync, whereas incremental will only sync new and updated rows since the last sync', | ||
| render: (_, schema) => { | ||
| const options = syncTypesToOptions(schema) | ||
| { | ||
| key: 'sync_type', | ||
| title: 'Sync method', | ||
| align: 'right', | ||
| tooltip: | ||
| 'Full refresh will refresh the full table on every sync, whereas incremental will only sync new and updated rows since the last sync', | ||
| render: (_, schema) => { | ||
| if (!schema.sync_type) { | ||
| return ( | ||
| <div className="justify-end flex"> | ||
| <LemonButton | ||
| className="my-1" | ||
| type="primary" | ||
| onClick={() => openSyncMethodModal(schema)} | ||
| > | ||
| Set up | ||
| </LemonButton> | ||
| </div> | ||
| ) | ||
| } | ||
|
|
||
| return ( | ||
| <LemonSelect | ||
| options={options} | ||
| value={schema.sync_type} | ||
| onChange={(newValue) => updateSchemaSyncType(schema, newValue)} | ||
| /> | ||
| ) | ||
| return ( | ||
| <div className="justify-end flex"> | ||
| <LemonButton | ||
| className="my-1" | ||
| size="small" | ||
| type="secondary" | ||
| onClick={() => openSyncMethodModal(schema)} | ||
| > | ||
| {schema.sync_type === 'full_refresh' ? 'Full refresh' : 'Incremental'} | ||
| </LemonButton> | ||
| </div> | ||
| ) | ||
| }, | ||
| }, | ||
| }, | ||
| ]} | ||
| /> | ||
| ]} | ||
| /> | ||
| </div> | ||
| </div> | ||
| </div> | ||
| <SyncMethodModal /> | ||
| </> | ||
| ) | ||
| } | ||
|
|
||
| const SyncMethodModal = (): JSX.Element => { | ||
| const { cancelSyncMethodModal, updateSchemaSyncType, toggleSchemaShouldSync } = useActions(sourceWizardLogic) | ||
| const { syncMethodModalOpen, currentSyncMethodModalSchema } = useValues(sourceWizardLogic) | ||
|
|
||
| if (!currentSyncMethodModalSchema) { | ||
| return <></> | ||
| } | ||
|
|
||
| return ( | ||
| <LemonModal | ||
| title={`Sync method for ${currentSyncMethodModalSchema.table}`} | ||
| isOpen={syncMethodModalOpen} | ||
| onClose={cancelSyncMethodModal} | ||
| > | ||
| <SyncMethodForm | ||
| schema={currentSyncMethodModalSchema} | ||
| onClose={cancelSyncMethodModal} | ||
| onSave={(syncType, incrementalField, incrementalFieldType) => { | ||
| if (syncType === 'incremental') { | ||
| updateSchemaSyncType( | ||
| currentSyncMethodModalSchema, | ||
| syncType, | ||
| incrementalField, | ||
| incrementalFieldType | ||
| ) | ||
| } else { | ||
| updateSchemaSyncType(currentSyncMethodModalSchema, syncType ?? null, null, null) | ||
| } | ||
|
|
||
| toggleSchemaShouldSync(currentSyncMethodModalSchema, true) | ||
| cancelSyncMethodModal() | ||
| }} | ||
| /> | ||
| </LemonModal> | ||
| ) | ||
| } |
Oops, something went wrong.