diff --git a/.vscode/launch.json b/.vscode/launch.json index 23c945f6cfbb8..1624d13d056dc 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -75,12 +75,33 @@ "console": "integratedTerminal", "python": "${workspaceFolder}/env/bin/python", "cwd": "${workspaceFolder}" + }, + { + "name": "Temporal Worker", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/manage.py", + "args": ["start_temporal_worker"], + "django": true, + "env": { + "PYTHONUNBUFFERED": "1", + "DJANGO_SETTINGS_MODULE": "posthog.settings", + "DEBUG": "1", + "CLICKHOUSE_SECURE": "False", + "KAFKA_HOSTS": "localhost", + "DATABASE_URL": "postgres://posthog:posthog@localhost:5432/posthog", + "SKIP_SERVICE_VERSION_REQUIREMENTS": "1", + "PRINT_SQL": "1" + }, + "console": "integratedTerminal", + "python": "${workspaceFolder}/env/bin/python", + "cwd": "${workspaceFolder}" } ], "compounds": [ { "name": "PostHog", - "configurations": ["Backend", "Celery", "Frontend", "Plugin Server"], + "configurations": ["Backend", "Celery", "Frontend", "Plugin Server", "Temporal Worker"], "stopAll": true } ] diff --git a/ee/api/test/__snapshots__/test_organization_resource_access.ambr b/ee/api/test/__snapshots__/test_organization_resource_access.ambr index 65894ce59ee76..75b0190684e35 100644 --- a/ee/api/test/__snapshots__/test_organization_resource_access.ambr +++ b/ee/api/test/__snapshots__/test_organization_resource_access.ambr @@ -117,6 +117,20 @@ LIMIT 100 /*controller='organization_resource_access-list',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/resource_access/%3F%24'*/ ' --- +# name: TestOrganizationResourceAccessAPI.test_list_organization_resource_access_is_not_nplus1.15 + ' + SELECT "ee_organizationresourceaccess"."id", + "ee_organizationresourceaccess"."resource", + "ee_organizationresourceaccess"."access_level", + "ee_organizationresourceaccess"."organization_id", + "ee_organizationresourceaccess"."created_by_id", + "ee_organizationresourceaccess"."created_at", + "ee_organizationresourceaccess"."updated_at" + FROM "ee_organizationresourceaccess" + WHERE "ee_organizationresourceaccess"."organization_id" = '00000000-0000-0000-0000-000000000000'::uuid + LIMIT 100 /*controller='organization_resource_access-list',route='api/organizations/%28%3FP%3Cparent_lookup_organization_id%3E%5B%5E/.%5D%2B%29/resource_access/%3F%24'*/ + ' +--- # name: TestOrganizationResourceAccessAPI.test_list_organization_resource_access_is_not_nplus1.2 ' SELECT "posthog_organization"."id", diff --git a/frontend/__snapshots__/scenes-app-batchexports--create-export.png b/frontend/__snapshots__/scenes-app-batchexports--create-export.png new file mode 100644 index 0000000000000..9adb989486a9c Binary files /dev/null and b/frontend/__snapshots__/scenes-app-batchexports--create-export.png differ diff --git a/frontend/__snapshots__/scenes-app-batchexports--exports.png b/frontend/__snapshots__/scenes-app-batchexports--exports.png new file mode 100644 index 0000000000000..3916251c3c058 Binary files /dev/null and b/frontend/__snapshots__/scenes-app-batchexports--exports.png differ diff --git a/frontend/__snapshots__/scenes-app-batchexports--view-export.png b/frontend/__snapshots__/scenes-app-batchexports--view-export.png new file mode 100644 index 0000000000000..d3e19cc05db8f Binary files /dev/null and b/frontend/__snapshots__/scenes-app-batchexports--view-export.png differ diff --git a/frontend/__snapshots__/scenes-app-exports--create-export.png b/frontend/__snapshots__/scenes-app-exports--create-export.png index 53b64b09cbffd..430b3eaef704c 100644 Binary files a/frontend/__snapshots__/scenes-app-exports--create-export.png and b/frontend/__snapshots__/scenes-app-exports--create-export.png differ diff --git a/frontend/__snapshots__/scenes-app-exports--exports.png b/frontend/__snapshots__/scenes-app-exports--exports.png deleted file mode 100644 index 357b54126e8a8..0000000000000 Binary files a/frontend/__snapshots__/scenes-app-exports--exports.png and /dev/null differ diff --git a/frontend/__snapshots__/scenes-app-insights--trends-table-breakdown--webkit.png b/frontend/__snapshots__/scenes-app-insights--trends-table-breakdown--webkit.png index ffc35d8080d7c..25ac744a61e77 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-table-breakdown--webkit.png and b/frontend/__snapshots__/scenes-app-insights--trends-table-breakdown--webkit.png differ diff --git a/frontend/__snapshots__/scenes-app-insights--trends-table-breakdown.png b/frontend/__snapshots__/scenes-app-insights--trends-table-breakdown.png index 642ac3e312b6e..27429218f4d83 100644 Binary files a/frontend/__snapshots__/scenes-app-insights--trends-table-breakdown.png and b/frontend/__snapshots__/scenes-app-insights--trends-table-breakdown.png differ diff --git a/frontend/__snapshots__/scenes-app-notebooks--text-only-notebook.png b/frontend/__snapshots__/scenes-app-notebooks--text-only-notebook.png index d7f0b0c199d71..6c6df40fe306c 100644 Binary files a/frontend/__snapshots__/scenes-app-notebooks--text-only-notebook.png and b/frontend/__snapshots__/scenes-app-notebooks--text-only-notebook.png differ diff --git a/frontend/__snapshots__/scenes-app-recordings--recordings-list.png b/frontend/__snapshots__/scenes-app-recordings--recordings-list.png index b8502c455d9fb..4be8221040d3f 100644 Binary files a/frontend/__snapshots__/scenes-app-recordings--recordings-list.png and b/frontend/__snapshots__/scenes-app-recordings--recordings-list.png differ diff --git a/frontend/__snapshots__/scenes-app-recordings--recordings-play-list-no-pinned-recordings.png b/frontend/__snapshots__/scenes-app-recordings--recordings-play-list-no-pinned-recordings.png index cb7fa0b0a0125..2d00330023073 100644 Binary files a/frontend/__snapshots__/scenes-app-recordings--recordings-play-list-no-pinned-recordings.png and b/frontend/__snapshots__/scenes-app-recordings--recordings-play-list-no-pinned-recordings.png differ diff --git a/frontend/__snapshots__/scenes-app-recordings--recordings-play-list-with-pinned-recordings.png b/frontend/__snapshots__/scenes-app-recordings--recordings-play-list-with-pinned-recordings.png index 4dbbac886b34d..101ad4954e80e 100644 Binary files a/frontend/__snapshots__/scenes-app-recordings--recordings-play-list-with-pinned-recordings.png and b/frontend/__snapshots__/scenes-app-recordings--recordings-play-list-with-pinned-recordings.png differ diff --git a/frontend/src/layout/navigation/SideBar/SideBar.tsx b/frontend/src/layout/navigation/SideBar/SideBar.tsx index ecbc507ee546e..20397e56595fe 100644 --- a/frontend/src/layout/navigation/SideBar/SideBar.tsx +++ b/frontend/src/layout/navigation/SideBar/SideBar.tsx @@ -233,6 +233,7 @@ function Pages(): JSX.Element { to={urls.projectApps()} /> )} + {Object.keys(frontendApps).length > 0 && } ) : null} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index d75126d45289b..58ea85dfa2dc4 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -44,6 +44,8 @@ import { Survey, TeamType, UserType, + BatchExportConfiguration, + BatchExportRun, NotebookNodeType, } from '~/types' import { getCurrentOrganizationId, getCurrentTeamId } from './utils/logics' @@ -513,8 +515,28 @@ class ApiRequest { return this.notebooks(teamId).addPathComponent(id) } - // Request finalization + // Batch Exports + public batchExports(teamId?: TeamType['id']): ApiRequest { + return this.projectsDetail(teamId).addPathComponent('batch_exports') + } + + public batchExport(id: BatchExportConfiguration['id'], teamId?: TeamType['id']): ApiRequest { + return this.batchExports(teamId).addPathComponent(id) + } + + public batchExportRuns(id: BatchExportConfiguration['id'], teamId?: TeamType['id']): ApiRequest { + return this.batchExports(teamId).addPathComponent(id).addPathComponent('runs') + } + + public batchExportRun( + id: BatchExportConfiguration['id'], + runId: BatchExportRun['id'], + teamId?: TeamType['id'] + ): ApiRequest { + return this.batchExportRuns(id, teamId).addPathComponent(runId) + } + // Request finalization public async get(options?: ApiMethodOptions): Promise { return await api.get(this.assembleFullUrl(), options) } @@ -1282,6 +1304,49 @@ const api = { }, }, + batchExports: { + async list(params: Record = {}): Promise> { + return await new ApiRequest().batchExports().withQueryString(toParams(params)).get() + }, + async get(id: BatchExportConfiguration['id']): Promise { + return await new ApiRequest().batchExport(id).get() + }, + async update( + id: BatchExportConfiguration['id'], + data: Partial + ): Promise { + return await new ApiRequest().batchExport(id).update({ data }) + }, + + async create(data?: Partial): Promise { + return await new ApiRequest().batchExports().create({ data }) + }, + async delete(id: BatchExportConfiguration['id']): Promise { + return await new ApiRequest().batchExport(id).delete() + }, + + async pause(id: BatchExportConfiguration['id']): Promise { + return await new ApiRequest().batchExport(id).withAction('pause').create() + }, + + async unpause(id: BatchExportConfiguration['id']): Promise { + return await new ApiRequest().batchExport(id).withAction('unpause').create() + }, + + async listRuns( + id: BatchExportConfiguration['id'], + params: Record = {} + ): Promise> { + return await new ApiRequest().batchExportRuns(id).withQueryString(toParams(params)).get() + }, + async createBackfill( + id: BatchExportConfiguration['id'], + data: Pick + ): Promise { + return await new ApiRequest().batchExport(id).withAction('backfill').create({ data }) + }, + }, + earlyAccessFeatures: { async get(featureId: EarlyAccessFeatureType['id']): Promise { return await new ApiRequest().earlyAccessFeature(featureId).get() @@ -1452,7 +1517,7 @@ const api = { }, /** Fetch data from specified URL. The result already is JSON-parsed. */ - async get(url: string, options?: ApiMethodOptions): Promise { + async get(url: string, options?: ApiMethodOptions): Promise { const res = await api.getResponse(url, options) return await getJSONOrThrow(res) }, diff --git a/frontend/src/lib/components/UUIDShortener.tsx b/frontend/src/lib/components/UUIDShortener.tsx new file mode 100644 index 0000000000000..c943725ba270b --- /dev/null +++ b/frontend/src/lib/components/UUIDShortener.tsx @@ -0,0 +1,32 @@ +import { Tooltip } from 'lib/lemon-ui/Tooltip' +import { copyToClipboard } from 'lib/utils' + +export function truncateUuid(uuid: string): string { + // Simple function to truncate a UUID. Useful for more simple displaying but should always be made clear it is truncated. + return uuid + .split('-') + .map((x) => x.slice(0, 2)) + .join('') +} + +export function UUIDShortener({ uuid, clickToCopy = false }: { uuid: string; clickToCopy?: boolean }): JSX.Element { + return ( + + {uuid} + {clickToCopy && ( + <> +
+ Double click to copy + + )} + + } + > + copyToClipboard(uuid) : undefined} title={uuid}> + {truncateUuid(uuid)}... + + + ) +} diff --git a/frontend/src/lib/lemon-ui/LemonCalendar/LemonCalendarSelect.tsx b/frontend/src/lib/lemon-ui/LemonCalendar/LemonCalendarSelect.tsx index 97f2c45e7c054..d06f263ab8dc0 100644 --- a/frontend/src/lib/lemon-ui/LemonCalendar/LemonCalendarSelect.tsx +++ b/frontend/src/lib/lemon-ui/LemonCalendar/LemonCalendarSelect.tsx @@ -1,8 +1,9 @@ import { LemonCalendar } from 'lib/lemon-ui/LemonCalendar/LemonCalendar' import { useState } from 'react' import { dayjs } from 'lib/dayjs' -import { LemonButton } from 'lib/lemon-ui/LemonButton' +import { LemonButton, LemonButtonProps, LemonButtonWithSideAction, SideAction } from 'lib/lemon-ui/LemonButton' import { IconClose } from 'lib/lemon-ui/icons' +import { Popover } from '../Popover' export interface LemonCalendarSelectProps { value?: dayjs.Dayjs | null @@ -58,3 +59,58 @@ export function LemonCalendarSelect({ value, onChange, months, onClose }: LemonC ) } + +export function LemonCalendarSelectInput( + props: LemonCalendarSelectProps & { + onChange: (date: dayjs.Dayjs | null) => void + buttonProps?: LemonButtonProps + placeholder?: string + clearable?: boolean + } +): JSX.Element { + const { buttonProps, placeholder, clearable, ...calendarProps } = props + const [visible, setVisible] = useState(false) + + const showClear = props.value && clearable + + const ButtonComponent = showClear ? LemonButtonWithSideAction : LemonButton + + return ( + setVisible(false)} + visible={visible} + overlay={ + { + props.onChange(value) + setVisible(false) + }} + onClose={() => { + setVisible(false) + props.onClose?.() + }} + /> + } + > + setVisible(true)} + type="secondary" + status="stealth" + fullWidth + sideAction={ + showClear + ? { + icon: , + onClick: () => props.onChange(null), + } + : (undefined as unknown as SideAction) // We know it will be a normal button if not clearable + } + {...props.buttonProps} + > + {props.value?.format('MMMM D, YYYY') ?? placeholder ?? 'Select date'} + + + ) +} diff --git a/frontend/src/scenes/appScenes.ts b/frontend/src/scenes/appScenes.ts index a393ad40dbadb..63877fe7dcd0e 100644 --- a/frontend/src/scenes/appScenes.ts +++ b/frontend/src/scenes/appScenes.ts @@ -12,9 +12,9 @@ export const appScenes: Record any> = { [Scene.Cohort]: () => import('./cohorts/Cohort'), [Scene.DataManagement]: () => import('./data-management/events/EventDefinitionsTable'), [Scene.Events]: () => import('./events/Events'), - [Scene.Exports]: () => import('./exports/ExportsList'), - [Scene.CreateExport]: () => import('./exports/CreateExport'), - [Scene.ViewExport]: () => import('./exports/ViewExport'), + [Scene.BatchExports]: () => import('./batch_exports/BatchExportsListScene'), + [Scene.BatchExportEdit]: () => import('./batch_exports/BatchExportEditScene'), + [Scene.BatchExport]: () => import('./batch_exports/BatchExportScene'), [Scene.Actions]: () => import('./actions/ActionsTable'), [Scene.EventDefinitions]: () => import('./data-management/events/EventDefinitionsTable'), [Scene.EventDefinition]: () => import('./data-management/definition/DefinitionView'), diff --git a/frontend/src/scenes/apps/appMetricsSceneLogic.ts b/frontend/src/scenes/apps/appMetricsSceneLogic.ts index 2f7e9ec180346..082945687bfa2 100644 --- a/frontend/src/scenes/apps/appMetricsSceneLogic.ts +++ b/frontend/src/scenes/apps/appMetricsSceneLogic.ts @@ -4,7 +4,7 @@ import { loaders } from 'kea-loaders' import type { appMetricsSceneLogicType } from './appMetricsSceneLogicType' import { urls } from 'scenes/urls' import { Breadcrumb, PluginConfigWithPluginInfo, UserBasicType } from '~/types' -import api from 'lib/api' +import api, { PaginatedResponse } from 'lib/api' import { teamLogic } from 'scenes/teamLogic' import { actionToUrl, urlToAction } from 'kea-router' import { toParams } from 'lib/utils' @@ -142,7 +142,7 @@ export const appMetricsSceneLogic = kea([ null as PluginConfigWithPluginInfo | null, { loadPluginConfig: async () => { - return await api.get( + return await api.get( `api/projects/${teamLogic.values.currentTeamId}/plugin_configs/${props.pluginConfigId}` ) }, @@ -152,12 +152,14 @@ export const appMetricsSceneLogic = kea([ null as AppMetricsResponse | null, { loadMetrics: async () => { - if (values.activeTab && values.dateFrom) { - const params = toParams({ category: values.activeTab, date_from: values.dateFrom }) - return await api.get( - `api/projects/${teamLogic.values.currentTeamId}/app_metrics/${props.pluginConfigId}?${params}` - ) + if (!values.activeTab || !values.dateFrom) { + return null } + + const params = toParams({ category: values.activeTab, date_from: values.dateFrom }) + return await api.get( + `api/projects/${teamLogic.values.currentTeamId}/app_metrics/${props.pluginConfigId}?${params}` + ) }, }, ], @@ -165,10 +167,10 @@ export const appMetricsSceneLogic = kea([ [] as Array, { loadHistoricalExports: async () => { - const { results } = await api.get( + const { results } = await api.get>( `api/projects/${teamLogic.values.currentTeamId}/app_metrics/${props.pluginConfigId}/historical_exports` ) - return results as Array + return results }, }, ], diff --git a/frontend/src/scenes/batch_exports/BatchExportBackfillModal.tsx b/frontend/src/scenes/batch_exports/BatchExportBackfillModal.tsx new file mode 100644 index 0000000000000..5616f80e9d333 --- /dev/null +++ b/frontend/src/scenes/batch_exports/BatchExportBackfillModal.tsx @@ -0,0 +1,74 @@ +import { useActions, useValues } from 'kea' + +import { LemonButton } from 'lib/lemon-ui/LemonButton' + +import { LemonModal } from 'lib/lemon-ui/LemonModal' +import { Form } from 'kea-forms' +import { Field } from 'lib/forms/Field' +import { batchExportLogic } from './batchExportLogic' +import { LemonCalendarSelectInput } from 'lib/lemon-ui/LemonCalendar/LemonCalendarSelect' + +export function BatchExportBackfillModal(): JSX.Element { + const { batchExportConfig, isBackfillModalOpen, isBackfillFormSubmitting } = useValues(batchExportLogic) + const { closeBackfillModal } = useActions(batchExportLogic) + + return ( + + + Cancel + + + Schedule runs + + + } + > +

+ Triggering a historic export will create multiple runs, one after another for the range specified below. + The runs will export data in {batchExportConfig?.interval} intervals, from the start date until + the end date is reached. +

+ +
+ + {({ value, onChange }) => ( + + )} + + + + {({ value, onChange }) => ( + + )} + +
+ + ) +} diff --git a/frontend/src/scenes/batch_exports/BatchExportEditForm.tsx b/frontend/src/scenes/batch_exports/BatchExportEditForm.tsx new file mode 100644 index 0000000000000..f6d190bf6d89f --- /dev/null +++ b/frontend/src/scenes/batch_exports/BatchExportEditForm.tsx @@ -0,0 +1,262 @@ +import { LemonInput, LemonSelect, LemonCheckbox, LemonDivider, LemonButton } from '@posthog/lemon-ui' +import { useValues, useActions } from 'kea' +import { Form } from 'kea-forms' +import { LemonBanner } from 'lib/lemon-ui/LemonBanner' +import { LemonCalendarSelectInput } from 'lib/lemon-ui/LemonCalendar/LemonCalendarSelect' +import { LemonSkeleton } from 'lib/lemon-ui/LemonSkeleton' +import { IconInfo } from 'lib/lemon-ui/icons' +import { BatchExportsEditLogicProps, batchExportsEditLogic } from './batchExportEditLogic' +import { Field } from 'lib/forms/Field' +import { Tooltip } from 'lib/lemon-ui/Tooltip' + +export function BatchExportsEditForm(props: BatchExportsEditLogicProps): JSX.Element { + const logic = batchExportsEditLogic(props) + const { isNew, batchExportConfigForm, isBatchExportConfigFormSubmitting, batchExportConfigLoading } = + useValues(logic) + const { submitBatchExportConfigForm, cancelEditing } = useActions(logic) + + return ( + <> + {batchExportConfigLoading ? ( + <> + + + + + + ) : ( + <> +
+
+ + + + +
+ + The intervals of data exports. For example, if you select hourly, every hour + a run will be created to export that hours data. + + } + > + + + {/* + The date from which data is to be exported. Leaving it unset implies that + data exports start from the next period as given by the frequency. + + } + > + {({ value, onChange }) => ( + + )} + */} + + + The date up to which data is to be exported. Leaving it unset implies that + data exports will continue forever until this export is paused or deleted. + + } + > + {({ value, onChange }) => ( + + )} + +
+ + + This batch exporter will schedule regular batch exports at your indicated interval until + the end date. Once you have configured your exporter, you can trigger a manual export + for historic data. + + + {isNew ? ( + + + Create in paused state + + + + + } + /> + + ) : null} +
+ +
+ + + + + + {!batchExportConfigForm.destination ? ( +

Select a destination to continue configuring

+ ) : batchExportConfigForm.destination === 'S3' ? ( + <> +
+ + + + + + +
+ + + +
+ + + + + + +
+ + ) : batchExportConfigForm.destination === 'Snowflake' ? ( + <> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ) : null} +
+ +
+ cancelEditing()} + disabledReason={isBatchExportConfigFormSubmitting ? 'Currently being saved' : undefined} + > + Cancel + + + {isNew ? 'Create' : 'Save'} + +
+
+ + )} + + ) +} diff --git a/frontend/src/scenes/batch_exports/BatchExportEditScene.tsx b/frontend/src/scenes/batch_exports/BatchExportEditScene.tsx new file mode 100644 index 0000000000000..7f5a66fd4798d --- /dev/null +++ b/frontend/src/scenes/batch_exports/BatchExportEditScene.tsx @@ -0,0 +1,29 @@ +import { SceneExport } from 'scenes/sceneTypes' +import { PageHeader } from 'lib/components/PageHeader' +import { useValues } from 'kea' +import { BatchExportsEditLogicProps, batchExportsEditLogic } from './batchExportEditLogic' +import { batchExportsEditSceneLogic } from './batchExportEditSceneLogic' +import { BatchExportsEditForm } from './BatchExportEditForm' + +export const scene: SceneExport = { + component: BatchExportsEditScene, + logic: batchExportsEditSceneLogic, + paramsToProps: ({ params: { id } }: { params: { id?: string } }): BatchExportsEditLogicProps => ({ + id: id ?? 'new', + }), +} + +export function BatchExportsEditScene(): JSX.Element { + const { id } = useValues(batchExportsEditSceneLogic) + const { isNew } = useValues(batchExportsEditLogic({ id })) + + return ( + <> + + +
+ + + + ) +} diff --git a/frontend/src/scenes/batch_exports/BatchExportScene.tsx b/frontend/src/scenes/batch_exports/BatchExportScene.tsx new file mode 100644 index 0000000000000..72f1ef344ad03 --- /dev/null +++ b/frontend/src/scenes/batch_exports/BatchExportScene.tsx @@ -0,0 +1,365 @@ +import { SceneExport } from 'scenes/sceneTypes' +import { PageHeader } from 'lib/components/PageHeader' +import { LemonButton, LemonDivider, LemonTable, LemonTag } from '@posthog/lemon-ui' +import { urls } from 'scenes/urls' +import { useActions, useValues } from 'kea' +import { useEffect, useState } from 'react' +import { BatchExportLogicProps, batchExportLogic } from './batchExportLogic' +import { BatchExportRunIcon, BatchExportTag } from './components' +import { LemonSkeleton } from 'lib/lemon-ui/LemonSkeleton' +import { IconEllipsis, IconRefresh } from 'lib/lemon-ui/icons' +import { capitalizeFirstLetter, identifierToHuman } from 'lib/utils' +import { BatchExportBackfillModal } from './BatchExportBackfillModal' +import { humanizeDestination, intervalToFrequency, isRunInProgress } from './utils' +import { TZLabel } from '@posthog/apps-common' +import { Popover } from 'lib/lemon-ui/Popover' +import { LemonCalendarRange } from 'lib/lemon-ui/LemonCalendarRange/LemonCalendarRange' +import { NotFound } from 'lib/components/NotFound' +import { LemonMenu } from 'lib/lemon-ui/LemonMenu' +import { LemonDialog } from 'lib/lemon-ui/LemonDialog' +import { Tooltip } from 'lib/lemon-ui/Tooltip' + +export const scene: SceneExport = { + component: BatchExportScene, + logic: batchExportLogic, + paramsToProps: ({ params: { id } }: { params: { id?: string } }): BatchExportLogicProps => ({ + id: id ?? 'missing', + }), +} + +export function BatchExportScene(): JSX.Element { + const { + batchExportRunsResponse, + batchExportConfig, + batchExportConfigLoading, + groupedRuns, + batchExportRunsResponseLoading, + runsDateRange, + } = useValues(batchExportLogic) + const { + loadBatchExportConfig, + loadBatchExportRuns, + loadNextBatchExportRuns, + openBackfillModal, + setRunsDateRange, + retryRun, + pause, + unpause, + archive, + } = useActions(batchExportLogic) + + const [dateRangeVisible, setDateRangeVisible] = useState(false) + + useEffect(() => { + loadBatchExportConfig() + loadBatchExportRuns() + }, []) + + if (!batchExportConfig && !batchExportConfigLoading) { + return + } + + return ( + <> + + {batchExportConfig?.name ?? (batchExportConfigLoading ? 'Loading...' : 'Missing')} + + } + buttons={ + batchExportConfig ? ( + <> + { + batchExportConfig.paused ? unpause() : pause() + }, + disabledReason: batchExportConfigLoading ? 'Loading...' : undefined, + }, + { + label: 'Archive', + status: 'danger', + onClick: () => + LemonDialog.open({ + title: 'Archive Batch Export?', + description: + 'Are you sure you want to archive this Batch Export? This will stop all future runs', + + primaryButton: { + children: 'Archive', + status: 'danger', + onClick: () => archive(), + }, + secondaryButton: { + children: 'Cancel', + }, + }), + disabledReason: batchExportConfigLoading ? 'Loading...' : undefined, + }, + ]} + > + } status="stealth" size="small" /> + + + openBackfillModal()}> + Create historic export + + + + Edit + + + ) : undefined + } + /> + +
+ {batchExportConfig ? ( + <> +
+ + + {capitalizeFirstLetter(intervalToFrequency(batchExportConfig.interval))} + + + + {batchExportConfig.end_at ? ( + <> + + Ends + + + ) : ( + 'Indefinite' + )} + + + +
    +
  • + Destination: + + {batchExportConfig.destination.type} + +
    • + + {Object.keys(batchExportConfig.destination.config).map((x) => ( +
  • + {identifierToHuman(x)}: + + {batchExportConfig.destination.config[x]} + +
    • + ))} +
+ + } + > + {humanizeDestination(batchExportConfig.destination)} + +
+ + ) : ( + + )} +
+ + {batchExportConfig ? ( +
+
+
+

Latest Runs

+ { + setRunsDateRange({ from: start.startOf('day'), to: end.endOf('day') }) + setDateRangeVisible(false) + }} + onClose={function noRefCheck() { + setDateRangeVisible(false) + }} + /> + } + > + + {runsDateRange.from.format('MMMM D, YYYY')} -{' '} + {runsDateRange.to.format('MMMM D, YYYY')} + + +
+ + + Load more button in the footer! + +
+ ) + } + expandable={{ + noIndent: true, + expandedRowRender: (groupedRuns) => { + return ( + , + }, + { + title: 'ID', + key: 'runId', + render: (_, run) => run.id, + }, + { + title: 'Run start', + key: 'runStart', + tooltip: 'Date and time when this BatchExport run started', + render: (_, run) => , + }, + ]} + /> + ) + }, + }} + columns={[ + { + key: 'icon', + width: 0, + render: (_, groupedRun) => { + return + }, + }, + + { + title: 'Data interval start', + key: 'dataIntervalStart', + tooltip: 'Start of the time range to export', + render: (_, run) => { + return ( + + ) + }, + }, + { + title: 'Data interval end', + key: 'dataIntervalEnd', + tooltip: 'End of the time range to export', + render: (_, run) => { + return ( + + ) + }, + }, + { + title: 'Latest run start', + key: 'runStart', + tooltip: 'Date and time when this BatchExport run started', + render: (_, groupedRun) => { + return + }, + }, + { + // title: 'Actions', + key: 'actions', + width: 0, + render: function RenderName(_, groupedRun) { + return ( + + {!isRunInProgress(groupedRun.runs[0]) && ( + } + onClick={() => + LemonDialog.open({ + title: 'Retry export?', + description: ( + <> +

+ This will schedule a new run for the same + interval. Any changes to the configuration + will be applied to the new run. +

+

+ Please note - there may be a slight + delay before the new run appears. +

+ + ), + width: '20rem', + primaryButton: { + children: 'Retry', + onClick: () => retryRun(groupedRun.runs[0]), + }, + secondaryButton: { + children: 'Cancel', + }, + }) + } + /> + )} + + ) + }, + }, + ]} + emptyState={ + <> + No runs yet. Your exporter runs every {batchExportConfig.interval}. +
+ openBackfillModal()}> + Create historic export + + + } + /> +
+
+ ) : null} + + + + ) +} diff --git a/frontend/src/scenes/batch_exports/BatchExports.scss b/frontend/src/scenes/batch_exports/BatchExports.scss new file mode 100644 index 0000000000000..da0646417ab23 --- /dev/null +++ b/frontend/src/scenes/batch_exports/BatchExports.scss @@ -0,0 +1,20 @@ +.BatchExportRunIcon--pulse { + outline: 2px solid transparent; + outline-offset: 0; + animation: pulse 2s infinite ease-out; +} + +@keyframes pulse { + 0% { + outline-offset: 0; + outline-color: var(--primary-light); + } + 80% { + outline-offset: 20px; + outline-color: transparent; + } + 100% { + outline-offset: 20px; + outline-color: transparent; + } +} diff --git a/frontend/src/scenes/batch_exports/BatchExports.stories.tsx b/frontend/src/scenes/batch_exports/BatchExports.stories.tsx new file mode 100644 index 0000000000000..533ed8b51e200 --- /dev/null +++ b/frontend/src/scenes/batch_exports/BatchExports.stories.tsx @@ -0,0 +1,96 @@ +import { Meta, Story } from '@storybook/react' +import { App } from 'scenes/App' +import { useEffect } from 'react' +import { router } from 'kea-router' +import { mswDecorator } from '~/mocks/browser' +import { urls } from 'scenes/urls' +import { createExportServiceHandlers } from './__mocks__/api-mocks' + +export default { + title: 'Scenes-App/BatchExports', + parameters: { + layout: 'fullscreen', + options: { showPanel: false }, + testOptions: { + excludeNavigationFromSnapshot: true, + waitForLoadersToDisappear: true, + }, + viewMode: 'story', + }, + decorators: [ + mswDecorator( + createExportServiceHandlers({ + 1: { + id: '1', + name: 'My S3 Exporter', + destination: { + type: 'S3', + config: { + bucket_name: 'my-bucket', + region: 'us-east-1', + prefix: 'my-prefix', + aws_access_key_id: 'my-access-key-id', + aws_secret_access_key: '', + }, + }, + start_at: null, + end_at: null, + interval: 'hour', + paused: false, + created_at: '2021-09-01T00:00:00.000000Z', + latest_runs: [ + { + id: '4', + status: 'Running', + created_at: '2023-01-01T12:00:00Z' as any, + data_interval_start: '2023-01-01T05:00:00Z' as any, + data_interval_end: '2023-01-01T06:00:00Z' as any, + }, + { + id: '3', + status: 'Failed', + created_at: '2023-01-01T12:00:00Z' as any, + data_interval_start: '2023-01-01T03:00:00Z' as any, + data_interval_end: '2023-01-01T04:00:00Z' as any, + }, + { + id: '2', + status: 'Completed', + created_at: '2023-01-01T12:00:00Z' as any, + data_interval_start: '2023-01-01T01:00:00Z' as any, + data_interval_end: '2023-01-01T02:00:00Z' as any, + }, + { + id: '1', + status: 'Completed', + created_at: '2023-01-01T12:00:00Z' as any, + data_interval_start: '2023-01-01T00:00:00Z' as any, + data_interval_end: '2023-01-01T01:00:00Z' as any, + }, + ], + }, + }).handlers + ), + ], +} as Meta + +export const Exports: Story = () => { + useEffect(() => { + router.actions.push(urls.batchExports()) + }) + return +} + +export const CreateExport: Story = () => { + useEffect(() => { + router.actions.push(urls.batchExportNew()) + }) + return +} + +export const ViewExport: Story = () => { + useEffect(() => { + router.actions.push(urls.batchExport('1')) + }) + return +} diff --git a/frontend/src/scenes/batch_exports/BatchExportsListScene.tsx b/frontend/src/scenes/batch_exports/BatchExportsListScene.tsx new file mode 100644 index 0000000000000..c982f3538c2f6 --- /dev/null +++ b/frontend/src/scenes/batch_exports/BatchExportsListScene.tsx @@ -0,0 +1,123 @@ +import { SceneExport } from 'scenes/sceneTypes' +import { PageHeader } from 'lib/components/PageHeader' +import { LemonButton, LemonTable, Link } from '@posthog/lemon-ui' +import { urls } from 'scenes/urls' +import { useValues } from 'kea' +import { batchExportsListLogic } from './batchExportsListLogic' +import { LemonMenu, LemonMenuItems } from 'lib/lemon-ui/LemonMenu' +import { IconEllipsis } from 'lib/lemon-ui/icons' +import { BatchExportRunIcon, BatchExportTag } from './components' + +export const scene: SceneExport = { + component: BatchExportsListScene, +} + +export function BatchExportsListScene(): JSX.Element { + const { batchExportConfigs, batchExportConfigsLoading, pagination } = useValues(batchExportsListLogic) + + return ( + <> + + + Create export workflow + + + } + /> +

Batch exports allow you to export your data to a destination of your choice.

+ + + {batchExport.name} + + ) + }, + }, + { + title: 'Latest runs', + key: 'runs', + render: function RenderStatus(_, batchExport) { + return ( +
+ {[...(batchExport.latest_runs || [])].reverse()?.map((run) => ( + // TODO: Link to run details + + + + ))} +
+ ) + }, + }, + + { + title: 'Destination', + key: 'destination', + render: function RenderType(_, batchExport) { + return <>{batchExport.destination.type} + }, + }, + + { + title: 'Frequency', + key: 'frequency', + dataIndex: 'interval', + }, + { + title: 'Status', + key: 'status', + width: 0, + render: function RenderStatus(_, batchExport) { + return + }, + }, + + { + width: 0, + render: function Render(_, batchExport) { + const menuItems: LemonMenuItems = [ + { + label: 'View', + to: urls.batchExport(batchExport.id), + }, + { + label: 'Edit', + to: urls.batchExportEdit(batchExport.id), + }, + { + label: batchExport.paused ? 'Resume' : 'Pause', + status: batchExport.paused ? 'primary' : 'danger', + onClick: () => {}, + }, + ] + return ( + + } /> + + ) + }, + }, + ]} + /> + + ) +} diff --git a/frontend/src/scenes/exports/api-mocks.ts b/frontend/src/scenes/batch_exports/__mocks__/api-mocks.ts similarity index 77% rename from frontend/src/scenes/exports/api-mocks.ts rename to frontend/src/scenes/batch_exports/__mocks__/api-mocks.ts index 9748da31c1e3c..d8d37f7cad618 100644 --- a/frontend/src/scenes/exports/api-mocks.ts +++ b/frontend/src/scenes/batch_exports/__mocks__/api-mocks.ts @@ -1,8 +1,9 @@ -import { BatchExport, BatchExportData, BatchExportsResponse } from './api' +import { CountedPaginatedResponse } from 'lib/api' +import { BatchExportConfiguration } from '~/types' export const createExportServiceHandlers = ( - exports: { [id: number]: BatchExport } = {} -): { exports: { [id: number]: BatchExport }; handlers: any } => { + exports: { [id: number]: BatchExportConfiguration } = {} +): { exports: { [id: number]: BatchExportConfiguration }; handlers: any } => { const handlers = { get: { '/api/projects/:team_id/batch_exports/': (_req: any, res: any, ctx: any) => { @@ -10,7 +11,7 @@ export const createExportServiceHandlers = ( ctx.delay(1000), ctx.json({ results: Object.values(exports), - } as BatchExportsResponse) + } as CountedPaginatedResponse) ) }, '/api/projects/:team_id/batch_exports/:export_id': (req: any, res: any, ctx: any) => { @@ -22,22 +23,14 @@ export const createExportServiceHandlers = ( return res( ctx.delay(1000), ctx.json({ - results: [ - { - export_id: id, - run_id: 1, - status: 'RUNNING', - created_at: '2021-09-01T00:00:00.000000Z', - last_updated_at: '2021-09-01T00:00:00.000000Z', - }, - ], + results: exports[id]?.latest_runs ?? [], }) ) }, }, post: { '/api/projects/:team_id/batch_exports/': (req: any, res: any, ctx: any) => { - const body = req.body as BatchExportData + const body = req.body as BatchExportConfiguration const id = (Object.keys(exports).length + 1).toString() exports[id] = { ...body, diff --git a/frontend/src/scenes/batch_exports/batchExportEditLogic.ts b/frontend/src/scenes/batch_exports/batchExportEditLogic.ts new file mode 100644 index 0000000000000..32e095e053033 --- /dev/null +++ b/frontend/src/scenes/batch_exports/batchExportEditLogic.ts @@ -0,0 +1,208 @@ +import { actions, afterMount, connect, kea, key, listeners, path, props, selectors } from 'kea' + +import { + BatchExportConfiguration, + BatchExportDestination, + BatchExportDestinationS3, + BatchExportDestinationSnowflake, + Breadcrumb, +} from '~/types' + +import api from 'lib/api' +import { forms } from 'kea-forms' +import { urls } from 'scenes/urls' +import { beforeUnload, router } from 'kea-router' + +import type { batchExportsEditLogicType } from './batchExportEditLogicType' +import { dayjs, Dayjs } from 'lib/dayjs' +import { batchExportLogic } from './batchExportLogic' + +export type BatchExportsEditLogicProps = { + id: string | 'new' +} + +export type BatchExportConfigurationForm = Omit< + BatchExportConfiguration, + 'id' | 'destination' | 'start_at' | 'end_at' +> & + Partial & + Partial & { + destination: 'S3' | 'Snowflake' + start_at: Dayjs | null + end_at: Dayjs | null + } + +const formFields = ( + props: BatchExportsEditLogicProps, + { name, destination, interval, start_at, end_at, paused, ...config }: BatchExportConfigurationForm +): Record => { + // Important! All fields that are required must be checked here as it is used also to sanitise the existing + const isNew = props.id === 'new' + + return { + name: !name ? 'Please enter a name' : '', + destination: !destination ? 'Please select a destination' : '', + interval: !interval ? 'Please select a frequency' : '', + paused: '', + start_at: '', + end_at: '', + ...(destination === 'S3' + ? { + bucket_name: !config.bucket_name ? 'This field is required' : '', + region: !config.region ? 'This field is required' : '', + prefix: !config.prefix ? 'This field is required' : '', + aws_access_key_id: isNew ? (!config.aws_access_key_id ? 'This field is required' : '') : '', + aws_secret_access_key: isNew ? (!config.aws_secret_access_key ? 'This field is required' : '') : '', + } + : destination === 'Snowflake' + ? { + account: !config.account ? 'This field is required' : '', + database: !config.database ? 'This field is required' : '', + warehouse: !config.warehouse ? 'This field is required' : '', + user: isNew ? (!config.user ? 'This field is required' : '') : '', + password: isNew ? (!config.password ? 'This field is required' : '') : '', + schema: !config.schema ? 'This field is required' : '', + table_name: !config.table_name ? 'This field is required' : '', + role: '', + } + : {}), + } +} + +export const batchExportsEditLogic = kea([ + props({} as BatchExportsEditLogicProps), + key(({ id }) => id), + path((key) => ['scenes', 'batch_exports', 'batchExportsEditLogic', key]), + connect((props: BatchExportsEditLogicProps) => ({ + values: [batchExportLogic(props), ['batchExportConfig', 'batchExportConfigLoading']], + actions: [batchExportLogic(props), ['loadBatchExportConfig', 'loadBatchExportConfigSuccess']], + })), + + actions({ + cancelEditing: true, + }), + + forms(({ props, actions }) => ({ + batchExportConfigForm: { + defaults: { + name: '', + } as BatchExportConfigurationForm, + errors: (form) => formFields(props, form), + submit: async ({ name, destination, interval, start_at, end_at, paused, ...config }) => { + const destinationObject: BatchExportDestination = + destination === 'S3' + ? ({ + type: 'S3', + config: config, + } as unknown as BatchExportDestinationS3) + : ({ + type: 'Snowflake', + config: config, + } as unknown as BatchExportDestinationSnowflake) + + const data: Omit = { + paused, + name, + interval, + start_at: start_at?.toISOString() ?? null, + end_at: end_at?.toISOString() ?? null, + destination: destinationObject, + } + + const result = + props.id === 'new' + ? await api.batchExports.create(data) + : await api.batchExports.update(props.id, data) + + await new Promise((resolve) => setTimeout(resolve, 1000)) + + actions.resetBatchExportConfigForm() + router.actions.replace(urls.batchExport(result.id)) + + return + }, + }, + })), + + listeners(({ values, props, actions }) => ({ + cancelEditing: () => { + if (values.isNew) { + router.actions.push(urls.batchExports()) + } else { + router.actions.push(urls.batchExport(props.id)) + } + }, + + loadBatchExportConfigSuccess: ({ batchExportConfig }) => { + if (!batchExportConfig) { + return + } + + const destination = batchExportConfig.destination.type + + const transformedConfig: BatchExportConfigurationForm = { + ...batchExportConfig, + destination, + start_at: batchExportConfig.start_at ? dayjs(batchExportConfig.start_at) : null, + end_at: batchExportConfig.end_at ? dayjs(batchExportConfig.end_at) : null, + ...batchExportConfig.destination.config, + } + + // Filter out any values that aren't part of our from + + const validFormFields = Object.keys(formFields(props, transformedConfig)) + + Object.keys(transformedConfig).forEach((key) => { + if (!validFormFields.includes(key)) { + delete transformedConfig[key] + } + }) + + actions.resetBatchExportConfigForm(transformedConfig) + }, + })), + + selectors({ + isNew: [() => [(_, props) => props], (props): boolean => props.id === 'new'], + breadcrumbs: [ + (s) => [s.batchExportConfig, s.isNew], + (config, isNew): Breadcrumb[] => [ + { + name: 'Batch Exports', + path: urls.batchExports(), + }, + ...(isNew + ? [ + { + name: 'New', + }, + ] + : [ + { + name: config?.name ?? 'Loading', + path: config?.id ? urls.batchExport(config.id) : undefined, + }, + + { + name: 'Edit', + }, + ]), + ], + ], + }), + + afterMount(({ values, actions }) => { + if (!values.isNew) { + if (values.batchExportConfig) { + actions.loadBatchExportConfigSuccess(values.batchExportConfig) + } else { + actions.loadBatchExportConfig() + } + } + }), + + beforeUnload(({ values }) => ({ + enabled: () => values.batchExportConfigFormChanged, + message: `Leave?\nChanges you made will be discarded.`, + })), +]) diff --git a/frontend/src/scenes/batch_exports/batchExportEditSceneLogic.ts b/frontend/src/scenes/batch_exports/batchExportEditSceneLogic.ts new file mode 100644 index 0000000000000..b944ad32f546a --- /dev/null +++ b/frontend/src/scenes/batch_exports/batchExportEditSceneLogic.ts @@ -0,0 +1,48 @@ +import { connect, kea, key, path, props, selectors } from 'kea' + +import { Breadcrumb } from '~/types' + +import { urls } from 'scenes/urls' + +import { batchExportLogic } from './batchExportLogic' +import { BatchExportsEditLogicProps } from './batchExportEditLogic' + +import type { batchExportsEditSceneLogicType } from './batchExportEditSceneLogicType' + +export const batchExportsEditSceneLogic = kea([ + props({} as BatchExportsEditLogicProps), + key(({ id }) => id), + path((key) => ['scenes', 'batch_exports', 'batchExportsEditSceneLogic', key]), + connect((props: BatchExportsEditLogicProps) => ({ + values: [batchExportLogic(props), ['batchExportConfig']], + })), + + selectors({ + id: [() => [(_, props) => props], (props): string => props.id], + breadcrumbs: [ + (s) => [s.batchExportConfig, s.id], + (config, id): Breadcrumb[] => [ + { + name: 'Batch Exports', + path: urls.batchExports(), + }, + ...(id === 'new' + ? [ + { + name: 'New', + }, + ] + : [ + { + name: config?.name ?? 'Loading', + path: config?.id ? urls.batchExport(config.id) : undefined, + }, + + { + name: 'Edit', + }, + ]), + ], + ], + }), +]) diff --git a/frontend/src/scenes/batch_exports/batchExportLogic.ts b/frontend/src/scenes/batch_exports/batchExportLogic.ts new file mode 100644 index 0000000000000..782b41f78f869 --- /dev/null +++ b/frontend/src/scenes/batch_exports/batchExportLogic.ts @@ -0,0 +1,266 @@ +import { actions, beforeUnmount, kea, key, listeners, path, props, reducers, selectors } from 'kea' + +import { loaders } from 'kea-loaders' +import { BatchExportConfiguration, BatchExportRun, Breadcrumb, GroupedBatchExportRuns } from '~/types' + +import api, { PaginatedResponse } from 'lib/api' + +import { lemonToast } from '@posthog/lemon-ui' +import { forms } from 'kea-forms' +import { dayjs, Dayjs } from 'lib/dayjs' +import { urls } from 'scenes/urls' +import type { batchExportLogicType } from './batchExportLogicType' +import { router } from 'kea-router' + +export type BatchExportLogicProps = { + id: string +} + +// TODO: Fix this +const RUNS_REFRESH_INTERVAL = 5000 + +const convert = (run: BatchExportRun): BatchExportRun => { + return { + ...run, + data_interval_start: dayjs(run.data_interval_start), + data_interval_end: dayjs(run.data_interval_end), + created_at: dayjs(run.created_at), + last_updated_at: run.last_updated_at ? dayjs(run.last_updated_at) : undefined, + } +} + +const mergeRuns = (oldRuns: BatchExportRun[], newRuns: BatchExportRun[]): BatchExportRun[] => { + const runs = [...oldRuns] + + newRuns.forEach((rawRun) => { + const newRun = convert(rawRun) + const index = runs.findIndex((run) => run.id === newRun.id) + + if (index > -1) { + runs[index] = newRun + } else { + runs.push(newRun) + } + }) + + return runs +} + +export const batchExportLogic = kea([ + props({} as BatchExportLogicProps), + key(({ id }) => id), + path((key) => ['scenes', 'batch_exports', 'batchExportLogic', key]), + + actions({ + loadBatchExportRuns: true, + loadNextBatchExportRuns: true, + openBackfillModal: true, + closeBackfillModal: true, + retryRun: (run: BatchExportRun) => ({ run }), + setRunsDateRange: (data: { from: Dayjs; to: Dayjs }) => data, + }), + + reducers({ + runsDateRange: [ + { from: dayjs().subtract(7, 'day').startOf('day'), to: dayjs().endOf('day') } as { from: Dayjs; to: Dayjs }, + { + setRunsDateRange: (_, { from, to }) => ({ from, to }), + }, + ], + isBackfillModalOpen: [ + false, + { + openBackfillModal: () => true, + closeBackfillModal: () => false, + }, + ], + }), + + loaders(({ props, values }) => ({ + batchExportConfig: [ + null as BatchExportConfiguration | null, + { + loadBatchExportConfig: async () => { + const res = await api.batchExports.get(props.id) + return res + }, + + pause: async () => { + if (!values.batchExportConfig) { + return null + } + await api.batchExports.pause(props.id) + lemonToast.success('Batch export paused. No future runs will be scheduled') + return { + ...values.batchExportConfig, + paused: true, + } + }, + unpause: async () => { + if (!values.batchExportConfig) { + return null + } + await api.batchExports.unpause(props.id) + lemonToast.success('Batch export unpaused. Future runs will be scheduled') + return { + ...values.batchExportConfig, + paused: false, + } + }, + archive: async () => { + if (!values.batchExportConfig) { + return null + } + await api.batchExports.delete(props.id) + + router.actions.replace(urls.batchExports()) + return null + }, + }, + ], + + batchExportRunsResponse: [ + null as PaginatedResponse | null, + { + loadBatchExportRuns: async () => { + const res = await api.batchExports.listRuns(props.id, { + after: values.runsDateRange.from, + before: values.runsDateRange.to.add(1, 'day'), + }) + + res.results = mergeRuns(values.batchExportRunsResponse?.results ?? [], res.results) + + return res + }, + loadNextBatchExportRuns: async () => { + const nextUrl = values.batchExportRunsResponse?.next + + if (!nextUrl) { + return values.batchExportRunsResponse + } + + const res = await api.get>(nextUrl) + + res.results = mergeRuns(values.batchExportRunsResponse?.results ?? [], res.results) + + return res + }, + }, + ], + })), + + forms(({ props, actions }) => ({ + backfillForm: { + defaults: { end_at: dayjs() } as { + start_at?: Dayjs + end_at?: Dayjs + }, + errors: ({ start_at, end_at }) => ({ + start_at: !start_at ? 'Start date is required' : undefined, + end_at: !end_at ? 'End date is required' : undefined, + }), + submit: async ({ start_at, end_at }) => { + await new Promise((resolve) => setTimeout(resolve, 1000)) + await api.batchExports + .createBackfill(props.id, { + start_at: start_at?.toISOString() ?? null, + end_at: end_at?.toISOString() ?? null, + }) + .catch((e) => { + if (e.detail) { + actions.setBackfillFormManualErrors({ + [e.attr ?? 'start_at']: e.detail, + }) + } else { + lemonToast.error('Unknown error occurred') + } + + throw e + }) + + actions.closeBackfillModal() + actions.loadBatchExportRuns() + + return + }, + }, + })), + + selectors(({}) => ({ + groupedRuns: [ + (s) => [s.batchExportRuns], + (runs): GroupedBatchExportRuns[] => { + // Runs are grouped by the date range they cover + const groupedRuns: Record = {} + + runs.forEach((run) => { + const key = `${run.data_interval_start}-${run.data_interval_end}` + if (!groupedRuns[key]) { + groupedRuns[key] = { + data_interval_start: run.data_interval_start, + data_interval_end: run.data_interval_end, + runs: [], + last_run_at: run.created_at, + } + } + + groupedRuns[key].runs.push(run) + groupedRuns[key].runs.sort((a, b) => b.created_at.diff(a.created_at)) + groupedRuns[key].last_run_at = groupedRuns[key].runs[0].created_at + }) + + return Object.values(groupedRuns).sort((a, b) => b.data_interval_end.diff(a.data_interval_end)) + }, + ], + breadcrumbs: [ + (s) => [s.batchExportConfig], + (config): Breadcrumb[] => [ + { + name: 'Batch Exports', + path: urls.batchExports(), + }, + { + name: config?.name ?? 'Loading', + }, + ], + ], + + batchExportRuns: [ + (s) => [s.batchExportRunsResponse], + (batchExportRunsResponse): BatchExportRun[] => batchExportRunsResponse?.results ?? [], + ], + })), + + listeners(({ actions, cache, props }) => ({ + setRunsDateRange: () => { + actions.loadBatchExportRuns() + }, + loadBatchExportRunsSuccess: () => { + clearTimeout(cache.refreshTimeout) + + // NOTE: This isn't perfect - it assumes that the first page will contain the currently running run. + // In practice the in progress runs are almost always in the first page + cache.refreshTimeout = setTimeout(() => { + actions.loadBatchExportRuns() + }, RUNS_REFRESH_INTERVAL) + }, + + retryRun: async ({ run }) => { + await api.batchExports.createBackfill(props.id, { + start_at: run.data_interval_start.toISOString(), + end_at: run.data_interval_end.toISOString(), + }) + + lemonToast.success('Retry has been scheduled.') + + clearTimeout(cache.refreshTimeout) + cache.refreshTimeout = setTimeout(() => { + actions.loadBatchExportRuns() + }, 2000) + }, + })), + + beforeUnmount(({ cache }) => { + clearTimeout(cache.refreshTimeout) + }), +]) diff --git a/frontend/src/scenes/batch_exports/batchExportsListLogic.ts b/frontend/src/scenes/batch_exports/batchExportsListLogic.ts new file mode 100644 index 0000000000000..7e003a2e3934a --- /dev/null +++ b/frontend/src/scenes/batch_exports/batchExportsListLogic.ts @@ -0,0 +1,80 @@ +import { actions, afterMount, beforeUnmount, kea, listeners, path, reducers, selectors } from 'kea' + +import { loaders } from 'kea-loaders' +import { BatchExportConfiguration } from '~/types' + +import api, { CountedPaginatedResponse } from 'lib/api' + +import type { batchExportsListLogicType } from './batchExportsListLogicType' +import { PaginationManual } from 'lib/lemon-ui/PaginationControl' + +const PAGE_SIZE = 10 +// Refresh the current page of exports periodically to see whats up. +const REFRESH_INTERVAL = 10000 + +export const batchExportsListLogic = kea([ + path(['scenes', 'batch_exports', 'batchExportsListLogic']), + actions({ + loadBatchExports: (offset?: number) => ({ offset }), + }), + + reducers({ + offset: [ + 0, + { + loadBatchExports: (_, { offset }) => offset || 0, + }, + ], + }), + + loaders(({}) => ({ + batchExportConfigs: [ + null as null | CountedPaginatedResponse, + { + loadBatchExports: async ({ offset }, breakpoint) => { + // TODO: Support pagination + await breakpoint(100) + const res = await api.batchExports.list({ + offset: offset || 0, + limit: PAGE_SIZE, + }) + return res + }, + }, + ], + })), + + listeners(({ actions, values, cache }) => ({ + loadBatchExportsSuccess: () => { + clearTimeout(cache.refreshTimeout) + + cache.refreshTimeout = setTimeout(() => { + actions.loadBatchExports(values.offset) + }, REFRESH_INTERVAL) + }, + })), + + beforeUnmount(({ cache }) => { + clearTimeout(cache.refreshTimeout) + }), + + afterMount(({ actions }) => { + actions.loadBatchExports() + }), + + selectors(({ actions }) => ({ + pagination: [ + (s) => [s.offset, s.batchExportConfigs], + (offset, configs): PaginationManual => { + return { + controlled: true, + pageSize: PAGE_SIZE, + currentPage: Math.floor(offset / PAGE_SIZE) + 1, + entryCount: configs?.count ?? 0, + onBackward: () => actions.loadBatchExports(offset - PAGE_SIZE), + onForward: () => actions.loadBatchExports(offset + PAGE_SIZE), + } + }, + ], + })), +]) diff --git a/frontend/src/scenes/batch_exports/components.tsx b/frontend/src/scenes/batch_exports/components.tsx new file mode 100644 index 0000000000000..1970b4ca01d14 --- /dev/null +++ b/frontend/src/scenes/batch_exports/components.tsx @@ -0,0 +1,84 @@ +import { LemonTag } from '@posthog/lemon-ui' +import clsx from 'clsx' +import { Tooltip } from 'lib/lemon-ui/Tooltip' +import { BatchExportConfiguration, BatchExportRun } from '~/types' + +import './BatchExports.scss' + +export function BatchExportTag({ batchExportConfig }: { batchExportConfig: BatchExportConfiguration }): JSX.Element { + return ( + + {batchExportConfig.paused + ? 'This export is paused - no future export runs will be scheduled ' + : 'This export is active - new runs will be triggered at the configured interval.'} + + } + > + + {batchExportConfig.paused ? 'Paused' : 'Active'} + + + ) +} + +export const colorForStatus = ( + status: BatchExportRun['status'] +): 'success' | 'primary' | 'warning' | 'danger' | 'default' => { + switch (status) { + case 'Completed': + return 'success' + case 'ContinuedAsNew': + case 'Running': + case 'Starting': + return 'primary' + case 'Cancelled': + case 'Terminated': + case 'TimedOut': + return 'warning' + case 'Failed': + return 'danger' + default: + return 'default' + } +} + +export function BatchExportRunIcon({ + runs, + showLabel = false, +}: { + runs: BatchExportRun[] + showLabel?: boolean +}): JSX.Element { + // We assume these are pre-sorted + const latestRun = runs[0] + + const color = colorForStatus(latestRun.status) + + return ( + + Run status: {latestRun.status} + {runs.length > 1 && ( + <> +
+ Attempts: {runs.length} + + )} + + } + > + + {showLabel ? {latestRun.status} : runs.length} + + + ) +} diff --git a/frontend/src/scenes/batch_exports/utils.ts b/frontend/src/scenes/batch_exports/utils.ts new file mode 100644 index 0000000000000..377107a58d0c1 --- /dev/null +++ b/frontend/src/scenes/batch_exports/utils.ts @@ -0,0 +1,24 @@ +import { BatchExportConfiguration, BatchExportDestination, BatchExportRun } from '~/types' + +export function intervalToFrequency(interval: BatchExportConfiguration['interval']): string { + return { + day: 'daily', + hour: 'hourly', + }[interval] +} + +export function isRunInProgress(run: BatchExportRun): boolean { + return ['Running', 'Starting'].includes(run.status) +} + +export function humanizeDestination(destination: BatchExportDestination): string { + if (destination.type === 'S3') { + return `s3://${destination.config.bucket_name}/${destination.config.prefix}` + } + + if (destination.type === 'Snowflake') { + return `snowflake:${destination.config.account}:${destination.config.database}:${destination.config.table_name}` + } + + return 'Unknown' +} diff --git a/frontend/src/scenes/exports/CreateExport.spec.tsx b/frontend/src/scenes/exports/CreateExport.spec.tsx deleted file mode 100644 index ba3b0f0d9f12a..0000000000000 --- a/frontend/src/scenes/exports/CreateExport.spec.tsx +++ /dev/null @@ -1,108 +0,0 @@ -// Here we test the S3 and Snowflake export creation forms. We use MSW to mock -// out the batch export API calls, and we use the userEvent library to simulate -// user interactions with the form. -// -// We use the screen object from the testing-library/react library to render the -// form and get references to the form elements. We use the waitFor function to -// wait for the form to be rendered before we start interacting with it. -// -// We use the waitFor function again to wait for the form to be submitted before -// we start asserting on the results. -// -// We use the expect function from the jest-dom library to assert on the form -// elements. We use the toBeInTheDocument matcher to assert that the form is -// rendered, and we use the toHaveTextContent matcher to assert that the form -// contains the expected text. -// -// We use ARIA roles, semantics, and labels to make our forms accessible. We use -// these for selection of elements within tests, and we use them to make our -// forms accessible to users with disabilities. - -import { getByLabelText, getByRole, render, screen, waitFor } from '@testing-library/react' -import '@testing-library/jest-dom' - -import { CreateExport } from './CreateExport' -import { initKeaTests } from '../../test/init' -import { createExportServiceHandlers } from './api-mocks' -import { useMocks } from '../../mocks/jest' -import userEvent from '@testing-library/user-event' - -// Required as LemonSelect uses this when we click on the select button. -global.ResizeObserver = require('resize-observer-polyfill') - -afterEach(() => { - jest.useRealTimers() -}) - -jest.setTimeout(5000) - -describe('CreateExport', () => { - it('renders an S3 export form by default and allows submission', async () => { - const { exports, handlers } = createExportServiceHandlers() - useMocks(handlers) - initKeaTests() - - render() - - // Wait for the form with aria label "Create Export" to be rendered - const form = await waitFor(() => { - const form = screen.getByRole('form') - expect(form).toBeInTheDocument() - return form - }) - - // Should be able to input values into the form inputs - // Generate a random name to avoid conflicts with other tests - const name = `test-export-${Math.random().toString(36).substring(7)}` - userEvent.type(getByLabelText(form, 'Name'), name) - userEvent.type(getByLabelText(form, 'Bucket'), 'test-bucket') - userEvent.type(getByLabelText(form, 'Key prefix'), 'test-export') - userEvent.type(getByLabelText(form, 'AWS Access Key ID'), 'test-access-key-id') - userEvent.type(getByLabelText(form, 'AWS Secret Access Key'), 'test-secret-access-key') - - // Should be able to select values from the form selects. LemonSelect - // components are not html select elements, so we need to 1. click on - // the component element to open the dropdown, and 2. click on the - // dropdown option to select it. - const frequencyComponent = getByLabelText(form, 'Frequency') - userEvent.click(frequencyComponent) - userEvent.click(await screen.findByText('Daily')) - - userEvent.click(getByLabelText(form, 'Region')) - userEvent.click(await screen.findByText('Canada (Central)')) - - // Should be able to submit the form - - jest.useFakeTimers({ advanceTimers: true }) - userEvent.click(getByRole(form, 'button', { name: 'Create Export' })) - - // Wait e.g. for the create request to complete - jest.advanceTimersByTime(5000) - - // Validate that the export was added to the list of exports, with the - // correct values. - await waitFor(() => { - // Filter the exports object values to find an export with the name - // we specified in the form. - const [export_] = Object.values(exports).filter((export_: any) => export_.name === name) - - // Validate that the export has the correct values - expect(export_).toEqual( - expect.objectContaining({ - name, - interval: 'day', - destination: { - type: 'S3', - config: { - bucket_name: 'test-bucket', - prefix: 'test-export', - region: 'ca-central-1', - aws_access_key_id: 'test-access-key-id', - aws_secret_access_key: 'test-secret-access-key', - }, - }, - }) - ) - }) - }) -}) diff --git a/frontend/src/scenes/exports/CreateExport.tsx b/frontend/src/scenes/exports/CreateExport.tsx deleted file mode 100644 index fbe8ff1edbf6a..0000000000000 --- a/frontend/src/scenes/exports/CreateExport.tsx +++ /dev/null @@ -1,408 +0,0 @@ -import { dayjs } from 'lib/dayjs' -import { LemonButton } from 'lib/lemon-ui/LemonButton' -import { LemonCalendarSelect } from 'lib/lemon-ui/LemonCalendar/LemonCalendarSelect' -import { LemonInput } from '../../lib/lemon-ui/LemonInput/LemonInput' -import { LemonSelect } from '../../lib/lemon-ui/LemonSelect' -import { Popover } from 'lib/lemon-ui/Popover/Popover' -import { PureField } from '../../lib/forms/Field' -import { router } from 'kea-router' -import { SceneExport } from '../sceneTypes' -import { useCallback, useRef, useState } from 'react' -import { useCreateExport, useCurrentTeamId } from './api' -import { urls } from '../urls' - -// TODO: rewrite this to not use explicit refs for the form fields. Possibly use -// kea-forms instead. - -// TODO: this file could end up getting pretty large. We should split when it -// before too much. - -// TODO: if we want to enable e.g. adding a new export destination type without -// having to change this codebase we might want to consider defining either some -// configuration description of the export types, or having the config component -// be injected somehow from elsewhere. We're early days so I don't think we need -// to worry about this right now. - -export const scene: SceneExport = { - component: CreateExport, -} - -export function CreateExport(): JSX.Element { - // At the top level we offer a select to choose the export type, and then - // render the appropriate component for that export type. - const [exportType, setExportType] = useState<'S3' | 'Snowflake'>('S3') - const [exportStartAt, setExportStartAt] = useState(null) - const [exportEndAt, setExportEndAt] = useState(null) - const [startAtSelectVisible, setStartAtSelectVisible] = useState(false) - const [endAtSelectVisible, setEndAtSelectVisible] = useState(false) - - return ( - // a form for inputting the config for an export, using aria labels to - // make it accessible. -
-

Create Export

- - setExportType(value as any)} - /> - - - { - setExportStartAt(value) - setStartAtSelectVisible(false) - }} - onClose={function noRefCheck() { - setStartAtSelectVisible(false) - }} - /> - } - > - - {exportStartAt ? exportStartAt.format('MMMM D, YYYY') : 'Select start date (optional)'} - - - - - { - setExportEndAt(value) - setEndAtSelectVisible(false) - }} - onClose={function noRefCheck() { - setEndAtSelectVisible(false) - }} - /> - } - > - - {exportEndAt ? exportEndAt.format('MMMM D, YYYY') : 'Select end date (optional)'} - - - - {exportType === 'S3' && } - {exportType === 'Snowflake' && } - - ) -} - -export interface ExportCommonProps { - startAt: dayjs.Dayjs | null - endAt: dayjs.Dayjs | null -} - -export function CreateS3Export({ startAt, endAt }: ExportCommonProps): JSX.Element { - const { currentTeamId } = useCurrentTeamId() - - // We use references to elements rather than maintaining state for each - // field. This is a bit more verbose but it means we avoids risks of - // re-rendering the component. - // TODO: use kea-forms instead. - const nameRef = useRef(null) - const bucketRef = useRef(null) - const prefixRef = useRef(null) - const regionRef = useRef(null) - const accessKeyIdRef = useRef(null) - const secretAccessKeyRef = useRef(null) - const intervalRef = useRef<'hour' | 'day' | null>(null) - - const { createExport, loading, error } = useCreateExport() - - const handleCreateExport = useCallback(() => { - if ( - !nameRef.current || - !bucketRef.current || - !prefixRef.current || - !regionRef.current || - !accessKeyIdRef.current || - !secretAccessKeyRef.current || - !intervalRef.current - ) { - console.warn('Missing ref') - } - - // Get the values from the form fields. - const name = nameRef.current?.value ?? '' - const bucket = bucketRef.current?.value ?? '' - const prefix = prefixRef.current?.value ?? '' - const region = regionRef.current ?? '' - const accessKeyId = accessKeyIdRef.current?.value ?? '' - const secretAccessKey = secretAccessKeyRef.current?.value ?? '' - const interval = intervalRef.current ?? '' - - const exportData = { - name, - destination: { - type: 'S3', - config: { - bucket_name: bucket, - region: region, - prefix: prefix, - aws_access_key_id: accessKeyId, - aws_secret_access_key: secretAccessKey, - }, - }, - interval: interval || 'hour', - start_at: startAt ? startAt.format('YYYY-MM-DDTHH:mm:ss.SSSZ') : null, - end_at: endAt ? endAt.format('YYYY-MM-DDTHH:mm:ss.SSSZ') : null, - } as const - - // Create the export. - createExport(currentTeamId, exportData).then(() => { - // Navigate back to the exports list. - router.actions.push(urls.exports()) - }) - }, [startAt, endAt]) - - return ( -
- - - - - - - - - - { - regionRef.current = value - }} - options={[ - { value: 'us-east-1', label: 'US East (N. Virginia)' }, - { value: 'us-east-2', label: 'US East (Ohio)' }, - { value: 'us-west-1', label: 'US West (N. California)' }, - { value: 'us-west-2', label: 'US West (Oregon)' }, - { value: 'af-south-1', label: 'Africa (Cape Town)' }, - { value: 'ap-east-1', label: 'Asia Pacific (Hong Kong)' }, - { value: 'ap-south-1', label: 'Asia Pacific (Mumbai)' }, - { value: 'ap-northeast-3', label: 'Asia Pacific (Osaka-Local)' }, - { value: 'ap-northeast-2', label: 'Asia Pacific (Seoul)' }, - { value: 'ap-southeast-1', label: 'Asia Pacific (Singapore)' }, - { value: 'ap-southeast-2', label: 'Asia Pacific (Sydney)' }, - { value: 'ap-northeast-1', label: 'Asia Pacific (Tokyo)' }, - { value: 'ca-central-1', label: 'Canada (Central)' }, - { value: 'cn-north-1', label: 'China (Beijing)' }, - { value: 'cn-northwest-1', label: 'China (Ningxia)' }, - { value: 'eu-central-1', label: 'Europe (Frankfurt)' }, - { value: 'eu-west-1', label: 'Europe (Ireland)' }, - { value: 'eu-west-2', label: 'Europe (London)' }, - { value: 'eu-south-1', label: 'Europe (Milan)' }, - { value: 'eu-west-3', label: 'Europe (Paris)' }, - { value: 'eu-north-1', label: 'Europe (Stockholm)' }, - { value: 'me-south-1', label: 'Middle East (Bahrain)' }, - { value: 'sa-east-1', label: 'South America (São Paulo)' }, - ]} - /> - - - - - - - - - - - - - - - - { - intervalRef.current = value - }} - options={[ - { value: 'hour', label: 'Hourly' }, - { value: 'day', label: 'Daily' }, - ]} - /> - - - Create Export - - {loading &&
Saving...
} - {error &&
Error: {error?.toString()}
} -
- ) -} - -export function CreateSnowflakeExport({ startAt, endAt }: ExportCommonProps): JSX.Element { - const { currentTeamId } = useCurrentTeamId() - - // Matches up with the backend config schema: - // - // user: str - // password: str - // account: str - // database: str - // warehouse: str - // schema: str - // table_name: str = "events" - // - - const nameRef = useRef(null) - const userRef = useRef(null) - const passwordRef = useRef(null) - const accountRef = useRef(null) - const databaseRef = useRef(null) - const warehouseRef = useRef(null) - const schemaRef = useRef(null) - const tableNameRef = useRef(null) - const intervalRef = useRef<'hour' | 'day' | null>(null) - const roleRef = useRef(null) - - const { createExport, loading, error } = useCreateExport() - - const handleCreateExport = useCallback(() => { - if ( - !nameRef.current || - !roleRef.current || - !userRef.current || - !passwordRef.current || - !accountRef.current || - !databaseRef.current || - !warehouseRef.current || - !schemaRef.current || - !tableNameRef.current - ) { - console.warn('Missing ref') - } - - // Get the values from the form fields. - const name = nameRef.current?.value ?? '' - const user = userRef.current?.value ?? '' - const password = passwordRef.current?.value ?? '' - const account = accountRef.current?.value ?? '' - const database = databaseRef.current?.value ?? '' - const warehouse = warehouseRef.current?.value ?? '' - const schema = schemaRef.current?.value ?? '' - const tableName = tableNameRef.current?.value ?? '' - const interval = intervalRef.current - const role = roleRef.current?.value ?? '' - - const exportData = { - name, - destination: { - type: 'Snowflake', - config: { - user, - password, - account, - database, - warehouse, - schema, - role: role === '' ? null : role, - table_name: tableName, - }, - }, - interval: interval || 'hour', - start_at: startAt ? startAt.format('YYYY-MM-DDTHH:mm:ss.SSSZ') : null, - end_at: endAt ? endAt.format('YYYY-MM-DDTHH:mm:ss.SSSZ') : null, - } as const - - // Create the export. - createExport(currentTeamId, exportData).then(() => { - // Navigate back to the exports list. - router.actions.push(urls.exports()) - }) - }, [startAt, endAt]) - - return ( -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - { - intervalRef.current = value - }} - options={[ - { value: 'hour', label: 'Hourly' }, - { value: 'day', label: 'Daily' }, - ]} - /> - - - - - - - Create Export - - {loading &&
Saving...
} - {error &&
Error: {error?.toString()}
} -
- ) -} diff --git a/frontend/src/scenes/exports/ExportsList.spec.tsx b/frontend/src/scenes/exports/ExportsList.spec.tsx deleted file mode 100644 index 9d86b233b2db6..0000000000000 --- a/frontend/src/scenes/exports/ExportsList.spec.tsx +++ /dev/null @@ -1,298 +0,0 @@ -import { render, screen, waitFor } from '@testing-library/react' -import userEvent from '@testing-library/user-event' -import '@testing-library/jest-dom' - -import { BatchExport } from './api' -import { createExportServiceHandlers } from './api-mocks' -import { NestedExportActionButtons, Exports } from './ExportsList' -import { initKeaTests } from '../../test/init' -import { useMocks } from '../../mocks/jest' - -jest.setTimeout(20000) - -describe('ExportActionButtons', () => { - it('renders a pause button that can be clicked to pause an export', async () => { - const exportId = 123 - const name = `test-export-${Math.random().toString(36).substring(7)}` - - const testExports: { [id: number]: BatchExport } = { - 123: { - id: exportId.toString(), - name: name, - team_id: 1, - status: 'RUNNING', - paused: false, - created_at: new Date().toISOString(), - last_updated_at: new Date().toISOString(), - interval: 'hour' as const, - start_at: null, - end_at: null, - destination: { - type: 'S3', - config: { - bucket_name: 'my-bucket', - region: 'us-east-1', - prefix: 'posthog-events', - aws_access_key_id: 'accessKeyId', - aws_secret_access_key: 'secretAccessKey', - }, - }, - }, - } - const { exports, handlers } = createExportServiceHandlers(testExports) - useMocks(handlers) - initKeaTests() - - render( - {}} - /> - ) - const dropdownButton = await waitFor(() => { - const dropdownButton = screen.getByRole('button', { name: 'more' }) - expect(dropdownButton).toBeInTheDocument() - return dropdownButton - }) - - userEvent.click(dropdownButton) - - const pauseButton = await waitFor(() => { - const pauseButton = screen.getByRole('button', { name: 'Pause this active BatchExport' }) - expect(pauseButton).toBeInTheDocument() - return pauseButton - }) - - userEvent.click(pauseButton) - - await waitFor(() => { - const [export_] = Object.values(exports).filter((export_: any) => export_.name === name) - expect(export_).toEqual( - expect.objectContaining({ - name, - paused: true, - }) - ) - }) - }) - - it('renders a resume button that can be clicked to resume an export', async () => { - const exportId = 456 - const name = `test-export-${Math.random().toString(36).substring(7)}` - - const testExports: { [id: number]: BatchExport } = { - 456: { - id: exportId.toString(), - name: name, - team_id: 1, - status: 'RUNNING', - paused: true, - created_at: new Date().toISOString(), - last_updated_at: new Date().toISOString(), - interval: 'hour' as const, - start_at: null, - end_at: null, - destination: { - type: 'S3', - config: { - bucket_name: 'my-bucket', - region: 'us-east-1', - prefix: 'posthog-events', - aws_access_key_id: 'accessKeyId', - aws_secret_access_key: 'secretAccessKey', - }, - }, - }, - } - const { exports, handlers } = createExportServiceHandlers(testExports) - useMocks(handlers) - initKeaTests() - - render( - {}} - /> - ) - const dropdownButton = await waitFor(() => { - const dropdownButton = screen.getByRole('button', { name: 'more' }) - expect(dropdownButton).toBeInTheDocument() - return dropdownButton - }) - - userEvent.click(dropdownButton) - - const resumeButton = await waitFor(() => { - const resumeButton = screen.getByRole('button', { name: 'Resume this paused BatchExport' }) - expect(resumeButton).toBeInTheDocument() - return resumeButton - }) - - userEvent.click(resumeButton) - - await waitFor(() => { - const [export_] = Object.values(exports).filter((export_: any) => export_.name === name) - expect(export_).toEqual( - expect.objectContaining({ - name, - paused: false, - }) - ) - }) - }) - - it('renders a delete button that can be clicked to delete an export', async () => { - const exportId = 789 - const name = `test-export-${Math.random().toString(36).substring(7)}` - - const testExports: { [id: number]: BatchExport } = { - 789: { - id: exportId.toString(), - name: name, - team_id: 1, - status: 'RUNNING', - paused: false, - created_at: new Date().toISOString(), - last_updated_at: new Date().toISOString(), - interval: 'hour' as const, - start_at: null, - end_at: null, - destination: { - type: 'S3', - config: { - bucket_name: 'my-bucket', - region: 'us-east-1', - prefix: 'posthog-events', - aws_access_key_id: 'accessKeyId', - aws_secret_access_key: 'secretAccessKey', - }, - }, - }, - } - const { exports, handlers } = createExportServiceHandlers(testExports) - useMocks(handlers) - initKeaTests() - - render( - {}} - /> - ) - const dropdownButton = await waitFor(() => { - const dropdownButton = screen.getByRole('button', { name: 'more' }) - expect(dropdownButton).toBeInTheDocument() - return dropdownButton - }) - - userEvent.click(dropdownButton) - - const deleteButton = await waitFor(() => { - const deleteButton = screen.getByRole('button', { name: 'Permanently delete this BatchExport' }) - expect(deleteButton).toBeInTheDocument() - return deleteButton - }) - - userEvent.click(deleteButton) - - await waitFor(() => { - expect(Object.keys(exports).length).toEqual(0) - }) - }) -}) - -describe('Exports', () => { - it('renders a table with 3 exports', async () => { - const testExports: { [id: number]: BatchExport } = { - 1: { - id: '1', - name: 'test-export-1', - team_id: 1, - status: 'RUNNING' as const, - paused: false, - created_at: new Date().toISOString(), - last_updated_at: new Date().toISOString(), - interval: 'hour' as const, - start_at: null, - end_at: null, - destination: { - type: 'S3', - config: { - bucket_name: 'my-bucket', - region: 'us-east-1', - prefix: 'posthog-events', - aws_access_key_id: 'accessKeyId', - aws_secret_access_key: 'secretAccessKey', - }, - }, - }, - 2: { - id: '2', - name: 'test-export-2', - team_id: 1, - status: 'RUNNING' as const, - paused: false, - created_at: new Date().toISOString(), - last_updated_at: new Date().toISOString(), - interval: 'hour' as const, - start_at: null, - end_at: null, - destination: { - type: 'S3', - config: { - bucket_name: 'my-bucket', - region: 'us-east-1', - prefix: 'posthog-events', - aws_access_key_id: 'accessKeyId', - aws_secret_access_key: 'secretAccessKey', - }, - }, - }, - 3: { - id: '3', - name: 'test-export-3', - team_id: 1, - status: 'RUNNING' as const, - paused: false, - created_at: new Date().toISOString(), - last_updated_at: new Date().toISOString(), - interval: 'hour' as const, - start_at: null, - end_at: null, - destination: { - type: 'S3', - config: { - bucket_name: 'my-bucket', - region: 'us-east-1', - prefix: 'posthog-events', - aws_access_key_id: 'accessKeyId', - aws_secret_access_key: 'secretAccessKey', - }, - }, - }, - } - - const { handlers } = createExportServiceHandlers(testExports) - useMocks(handlers) - initKeaTests() - - render() - - await waitFor( - () => { - const exportsTable = screen.getByRole('table') - expect(exportsTable).toBeInTheDocument() - - const rows = screen.getAllByRole('row') - expect(rows).toHaveLength(4) - }, - { timeout: 5000 } - ) - }) -}) diff --git a/frontend/src/scenes/exports/ExportsList.tsx b/frontend/src/scenes/exports/ExportsList.tsx deleted file mode 100644 index 01ba62635fab0..0000000000000 --- a/frontend/src/scenes/exports/ExportsList.tsx +++ /dev/null @@ -1,331 +0,0 @@ -import { SceneExport } from 'scenes/sceneTypes' -import { urls } from '../urls' -import { LemonButton } from '../../lib/lemon-ui/LemonButton' -import { More } from '../../lib/lemon-ui/LemonButton/More' -import { LemonDivider } from '../../lib/lemon-ui/LemonDivider' -import { LemonTag } from '../../lib/lemon-ui/LemonTag/LemonTag' -import { lemonToast } from 'lib/lemon-ui/lemonToast' -import { useCurrentTeamId, useExports, useExportAction, useDeleteExport, BatchExport } from './api' -import { LemonTable } from '../../lib/lemon-ui/LemonTable' -import { Link } from 'lib/lemon-ui/Link' -import clsx from 'clsx' -import { Spinner } from 'lib/lemon-ui/Spinner' - -export const scene: SceneExport = { - component: Exports, -} - -export interface ExportActionButtonsProps { - currentTeamId: number - export_: BatchExport - loading: boolean - buttonFullWidth: boolean - buttonType: 'primary' | 'secondary' | 'tertiary' - dividerVertical: boolean - updateCallback: (signal: AbortSignal | undefined) => void -} - -export function ExportActionButtons({ - currentTeamId, - export_, - loading, - buttonFullWidth, - buttonType, - dividerVertical, - updateCallback, -}: ExportActionButtonsProps): JSX.Element { - const { executeExportAction: pauseExport, error: pauseError } = useExportAction(currentTeamId, export_.id, 'pause') - const { executeExportAction: resumeExport, error: resumeError } = useExportAction( - currentTeamId, - export_.id, - 'unpause' - ) - - const { deleteExport, error: deleteError } = useDeleteExport(currentTeamId, export_.id) - - return ( - <> - { - export_.paused - ? resumeExport(undefined) - .then(() => { - updateCallback(undefined) - lemonToast['success']( - <> - {export_.name} has been resumed - , - { - toastId: `resume-export-success-${export_.id}`, - } - ) - }) - .catch(() => { - lemonToast['error']( - <> - {export_.name} could not be resumed: {resumeError} - , - { - toastId: `resume-export-error-${export_.id}`, - } - ) - }) - : pauseExport(undefined) - .then(() => { - updateCallback(undefined) - lemonToast['info']( - <> - {export_.name} has been paused - , - { - toastId: `pause-export-info-${export_.id}`, - } - ) - }) - .catch(() => { - lemonToast['error']( - <> - {export_.name} could not be resumed: {pauseError} - , - { - toastId: `pause-export-error-${export_.id}`, - } - ) - }) - }} - tooltip={export_.paused ? 'Resume this paused BatchExport' : 'Pause this active BatchExport'} - disabled={loading} - loading={loading} - fullWidth={buttonFullWidth} - > - {export_.paused ? 'Resume' : 'Pause'} - - {export_.paused ? ( - { - resumeExport({ backfill: true }) - .then(() => { - updateCallback(undefined) - lemonToast['success']( - <> - {export_.name} has been resumed - , - { - toastId: `resume-export-success-${export_.id}`, - } - ) - }) - .catch(() => { - lemonToast['error']( - <> - {export_.name} could not be resumed: {resumeError} - , - { - toastId: `resume-export-error-${export_.id}`, - } - ) - }) - }} - tooltip={'Resume this paused BatchExport and trigger backfilling for any runs missed while paused'} - disabled={loading} - loading={loading} - fullWidth={buttonFullWidth} - > - Resume and backfill - - ) : undefined} - - { - deleteExport() - .then(() => { - updateCallback(undefined) - lemonToast['success']( - <> - {export_.name} has been deleted - , - { - toastId: `delete-export-success-${export_.id}`, - } - ) - }) - .catch(() => { - lemonToast['error']( - <> - {export_.name} could not be deleted: {deleteError} - , - { - toastId: `delete-export-error-${export_.id}`, - } - ) - }) - }} - tooltip="Permanently delete this BatchExport" - disabled={loading} - loading={loading} - fullWidth={buttonFullWidth} - > - Delete - - - ) -} - -export interface SubExportActionButtonsProps { - currentTeamId: number - export_: BatchExport - loading: boolean - updateCallback: (signal: AbortSignal | undefined) => void -} - -export function InlineExportActionButtons({ - currentTeamId, - export_, - loading, - updateCallback, -}: SubExportActionButtonsProps): JSX.Element { - return ( -
- -
- ) -} - -export function NestedExportActionButtons({ - currentTeamId, - export_, - loading, - updateCallback, -}: SubExportActionButtonsProps): JSX.Element { - return ( - - } - /> - ) -} - -export function Exports(): JSX.Element { - // Displays a list of exports for the current project. We use the - // useCurrentTeamId hook to get the current team ID, and then use the - // useExports hook to fetch the list of exports for that team. - const { currentTeamId } = useCurrentTeamId() - const { exportsState, updateCallback } = useExports(currentTeamId) - const { loading, error, exports } = exportsState - - // If exports hasn't been set yet, we display a placeholder and a loading - // spinner. - if (exports === undefined) { - return ( -
-

Exports

-

- Fetching exports... -

-
- ) - } - - // If we have an error, we display the error message. - if (error) { - return ( -
-

Exports

-

Error fetching exports: {error}

-
- ) - } - - // If we have exports we display them in a table, showing: - // - The export type e.g. S3, Snowflake, etc. - // - The export frequency e.g. hourly, daily, etc. - // - The export status e.g. running, failed, etc. - // - The export last run time. - return ( - <> -

Exports

- {export_.name} - }, - }, - { - title: 'Type', - key: 'type', - render: function RenderType(_, export_) { - return <>{export_.destination.type} - }, - }, - { - title: 'Status', - key: 'status', - render: function RenderStatus(_, export_) { - return ( - <> - {export_.paused === true ? ( - - Paused - - ) : ( - - Running - - )} - - ) - }, - }, - { - title: 'Frequency', - key: 'frequency', - dataIndex: 'interval', - }, - { - title: 'Actions', - render: function Render(_, export_) { - return ( - - ) - }, - }, - ]} - /> - Create export - {/* If we are loading, we overlay a spinner */} - {loading &&
Loading...
} - - ) -} diff --git a/frontend/src/scenes/exports/ExportsScene.stories.tsx b/frontend/src/scenes/exports/ExportsScene.stories.tsx deleted file mode 100644 index 9385a8242c485..0000000000000 --- a/frontend/src/scenes/exports/ExportsScene.stories.tsx +++ /dev/null @@ -1,62 +0,0 @@ -import { Meta, Story } from '@storybook/react' -import { App } from 'scenes/App' -import { useEffect } from 'react' -import { router } from 'kea-router' -import { mswDecorator } from '~/mocks/browser' -import { urls } from 'scenes/urls' -import { createExportServiceHandlers } from './api-mocks' - -export default { - title: 'Scenes-App/Exports', - parameters: { - layout: 'fullscreen', - options: { showPanel: false }, - testOptions: { - excludeNavigationFromSnapshot: true, - waitForLoadersToDisappear: true, - }, - viewMode: 'story', - }, - decorators: [ - mswDecorator( - createExportServiceHandlers({ - 1: { - id: '1', - team_id: 1, - name: 'S3', - destination: { - type: 'S3', - config: { - bucket_name: 'my-bucket', - region: 'us-east-1', - prefix: 'my-prefix', - aws_access_key_id: 'my-access-key-id', - aws_secret_access_key: '', - }, - }, - start_at: null, - end_at: null, - interval: 'hour', - status: 'RUNNING', - paused: false, - created_at: '2021-09-01T00:00:00.000000Z', - last_updated_at: '2021-09-01T00:00:00.000000Z', - }, - }).handlers - ), - ], -} as Meta - -export const Exports: Story = () => { - useEffect(() => { - router.actions.push(urls.exports()) - }) - return -} - -export const CreateExport: Story = () => { - useEffect(() => { - router.actions.push(urls.createExport()) - }) - return -} diff --git a/frontend/src/scenes/exports/ViewExport.tsx b/frontend/src/scenes/exports/ViewExport.tsx deleted file mode 100644 index f2e520ebc4e51..0000000000000 --- a/frontend/src/scenes/exports/ViewExport.tsx +++ /dev/null @@ -1,452 +0,0 @@ -import { dayjs } from 'lib/dayjs' -import { useValues } from 'kea' -import { useCurrentTeamId, useExport, useExportRuns, BatchExport, BatchExportRun, useExportRunAction } from './api' -import { PageHeader } from 'lib/components/PageHeader' -import { Popover } from 'lib/lemon-ui/Popover/Popover' -import { LemonTag } from 'lib/lemon-ui/LemonTag/LemonTag' -import { LemonInput } from 'lib/lemon-ui/LemonInput/LemonInput' -import { lemonToast } from 'lib/lemon-ui/lemonToast' -import { LemonButton } from 'lib/lemon-ui/LemonButton' -import { IconReplay } from 'lib/lemon-ui/icons' -import { IconRefresh } from 'lib/lemon-ui/icons' -import { CopyToClipboardInline } from 'lib/components/CopyToClipboard' -import { LemonCalendarRange } from 'lib/lemon-ui/LemonCalendarRange/LemonCalendarRange' -import { InlineExportActionButtons } from './ExportsList' -import { LemonTable } from '../../lib/lemon-ui/LemonTable' -import { router } from 'kea-router' -import { useState } from 'react' -import clsx from 'clsx' -import { Spinner } from 'lib/lemon-ui/Spinner' - -export const Export = (): JSX.Element => { - // Displays a single export. We use the useCurrentTeamId hook to get the - // current team ID, and then use the useExport hook to fetch the export - // details for that team. We pull out the export_id from the URL. - const { currentLocation } = useValues(router) - const exportId = currentLocation.pathname.split('/').pop() - - if (exportId === undefined) { - throw Error('exportId is undefined') - } - - const { currentTeamId } = useCurrentTeamId() - const { loading, export_, error, updateCallback } = useExport(currentTeamId, exportId) - - // If the export is still undefined and we're loading, show a loading - // message and placeholder. - if (export_ === undefined) { - return ( -
-

Export

-

- Fetching export... -

-
- ) - } - - // If we have an error, show the error message. - if (error) { - return ( -
-

Export

-

Error fetching export: {error}

-
- ) - } - - // If we have an export, show the export details. - return ( - <> - - - {loading ?

Loading...

: null} - - - - ) -} - -export interface ExportHeaderProps { - currentTeamId: number - export_: BatchExport - loading: boolean - updateCallback: (signal: AbortSignal | undefined) => void -} - -function ExportHeader({ currentTeamId, export_, loading, updateCallback }: ExportHeaderProps): JSX.Element { - return ( - <> - - {export_.name} - -
- {export_.paused ? ( - - Paused - - ) : ( - - Running - - )} - {export_.destination.type} - Frequency: {export_.interval} -
- - } - buttons={ - - } - /> - - ) -} - -const ExportRunStatus = ({ exportRun }: { exportRun: BatchExportRun }): JSX.Element => { - if (exportRun.status === 'Running') { - return ( - - Running - - ) - } else if (exportRun.status === 'Completed') { - return ( - - Completed - - ) - } else if (exportRun.status === 'Starting') { - return ( - - Starting - - ) - } else { - return ( - - Error - - ) - } -} - -type ExportRunKey = { - workflow_id: string -} - -function endOfDay(d: dayjs.Dayjs): dayjs.Dayjs { - return d.hour(23).second(59).minute(59) -} - -function startOfDay(d: dayjs.Dayjs): dayjs.Dayjs { - return d.hour(0).second(0).minute(0) -} - -const ExportRuns = ({ exportId }: { exportId: string }): JSX.Element => { - // Displays a list of export runs for the given export ID. We use the - // useCurrentTeamId hook to get the current team ID, and then use the - // useExportRuns hook to fetch the export runs for that team and export ID. - const defaultDateRange: [dayjs.Dayjs, dayjs.Dayjs] = [startOfDay(dayjs().subtract(1, 'day')), endOfDay(dayjs())] - const [dateRange, setDateRange] = useState<[dayjs.Dayjs, dayjs.Dayjs]>(defaultDateRange) - const [dateRangeVisible, setDateRangeVisible] = useState(false) - - const defaultNumberOfRuns = 25 - const [numberOfRuns, setNumberOfRuns] = useState(defaultNumberOfRuns) - const { currentTeamId } = useCurrentTeamId() - const { loading, exportRuns, error, updateCallback } = useExportRuns( - currentTeamId, - exportId, - defaultNumberOfRuns, - dateRange - ) - // If the export runs are still undefined and we're loading, show a loading - // message and placeholder. - if (exportRuns === undefined) { - return ( -
-

Export Runs

-

Fetching export runs...

-
- ) - } - - // If we have an error, show the error message. - if (error) { - return ( -
-

Export Runs

-

Error fetching export runs: {error}

-
- ) - } - - // I originally tried using only a Map here, but that was /too/ simple for the type checker. - // Feel free to change this. - const exportRunKeys = new Array() - const exportRunsMap = new Map() - exportRuns.forEach((run) => { - const key = run.batch_export_id + dayjs(run.data_interval_end).format('YYYY-MM-DDTHH:MM:SSZ') - const arr = exportRunsMap.get(key) - - if (!arr) { - exportRunKeys.push({ workflow_id: key }) - exportRunsMap.set(key, [run]) - } else { - arr.push(run) - } - }) - - // If we have export runs, show the export runs in a table, showing: - // - The export run status e.g. running, failed, etc. - // - The export run start time. - // - The export run end time. - // - The export run duration. - // - The export run size. - return ( - <> -

Export Runs

-
- { - setDateRange([startOfDay(range[0]), endOfDay(range[1])]) - setDateRangeVisible(false) - }} - onClose={function noRefCheck() { - setDateRangeVisible(false) - }} - /> - } - > - - {dateRange[0].format('MMMM D, YYYY')} - {dateRange[1].format('MMMM D, YYYY')} - - - { - setNumberOfRuns(newValue ? newValue : numberOfRuns) - }} - /> - } - disabled={loading} - onClick={() => { - updateCallback(undefined, numberOfRuns, dateRange).then(() => { - if (error === undefined) { - lemonToast['info'](<>Refreshed Export Runs, { - toastId: `refreshed-export-runs-info`, - }) - } else { - lemonToast['error'](<>Export Runs could not be refreshed: {error}, { - toastId: `refreshed-export-runs-error`, - }) - } - }) - }} - > - Refresh - -
- -

Error fetching export runs

- - ) - } - - return ( - - }, - }, - { - title: 'ID', - key: 'runId', - render: function RenderStatus(_, run) { - return <>{run.id} - }, - }, - { - title: 'Run start', - key: 'runStart', - tooltip: 'Date and time when this BatchExport run started', - render: function RenderName(_, run) { - return <>{dayjs(run.created_at).format('YYYY-MM-DD HH:mm:ss z')} - }, - }, - ]} - /> - ) - }, - }} - columns={[ - { - title: 'Last status', - key: 'lastStatus', - render: function RenderStatus(_, exportRunKey) { - const runs = exportRunsMap.get(exportRunKey.workflow_id) - if (runs === undefined || runs.length === 0) { - // Each array will have at least one run (the original). - // So, we should never land here; I am only pleasing the type checker. - return <>{null} - } - const exportRun = runs[0] - - return - }, - }, - { - title: 'Last run start', - key: 'lastRunStart', - tooltip: 'Date and time when the last run for this batch started', - render: function RenderName(_, exportRunKey) { - const runs = exportRunsMap.get(exportRunKey.workflow_id) - if (runs === undefined || runs.length === 0) { - // Each array will have at least one run (the original). - // So, we should never land here; I am only pleasing the type checker. - return <>{null} - } - const exportRun = runs[0] - - return <>{dayjs(exportRun.created_at).format('YYYY-MM-DD HH:mm:ss z')} - }, - }, - { - title: 'Data interval start', - key: 'dataIntervalStart', - tooltip: 'Start of the time range to export', - render: function RenderName(_, exportRunKey) { - const runs = exportRunsMap.get(exportRunKey.workflow_id) - if (runs === undefined || runs.length === 0) { - // Each array will have at least one run (the original). - // So, we should never land here; I am only pleasing the type checker. - return <>{null} - } - const exportRun = runs[0] - - return <>{dayjs(exportRun.data_interval_start).format('YYYY-MM-DD HH:mm:ss z')} - }, - }, - { - title: 'Data interval end', - key: 'dataIntervalEnd', - tooltip: 'End of the time range to export', - render: function RenderName(_, exportRunKey) { - const runs = exportRunsMap.get(exportRunKey.workflow_id) - if (runs === undefined || runs.length === 0) { - // Each array will have at least one run (the original). - // So, we should never land here; I am only pleasing the type checker. - return <>{null} - } - const exportRun = runs[0] - - return <>{dayjs(exportRun.data_interval_end).format('YYYY-MM-DD HH:mm:ss z')} - }, - }, - { - title: 'Actions', - render: function RenderName(_, exportRunKey) { - const runs = exportRunsMap.get(exportRunKey.workflow_id) - if (runs === undefined || runs.length === 0) { - // Each array will have at least one run (the original). - // So, we should never land here; I am only pleasing the type checker. - return <>{null} - } - const exportRun = runs.slice(-1)[0] - - const { - executeExportRunAction: resetExportRun, - loading: restarting, - error: resetError, - } = useExportRunAction(currentTeamId, exportId, exportRun.id, 'reset') - - return ( -
- { - resetExportRun() - .then(() => { - updateCallback(undefined, numberOfRuns, dateRange) - lemonToast['success']( - <> - {exportRun.id} has been restarted - , - { - toastId: `restart-export-run-success-${exportRun.id}`, - } - ) - }) - .catch(() => { - lemonToast['error']( - <> - {exportRun.id} could not be restarted: {resetError} - , - { - toastId: `restart-export-run-error-${exportRun.id}`, - } - ) - }) - }} - tooltip={'Restart this Batch Export run'} - disabled={loading || restarting} - icon={} - /> -
- ) - }, - }, - ]} - /> - - ) -} diff --git a/frontend/src/scenes/exports/api.ts b/frontend/src/scenes/exports/api.ts deleted file mode 100644 index 63ca36203055c..0000000000000 --- a/frontend/src/scenes/exports/api.ts +++ /dev/null @@ -1,404 +0,0 @@ -import { dayjs } from 'lib/dayjs' -import { useValues } from 'kea' -import { teamLogic } from '../teamLogic' -import { useCallback, useEffect, useState } from 'react' -import api from '../../lib/api' - -export const useCurrentTeamId = (): { currentTeamId: number } => { - // Returns the current team ID from the team logic. We assume that in all - // contexts we will have a current team ID, and assert that here such that - // we can ensure the return value type and avoid having to check for - // undefined in the caller. - const { currentTeamId } = useValues(teamLogic) - - if (currentTeamId == null) { - throw Error('currentTeamId should not be undefined') - } - - return { currentTeamId } -} - -type UseExportsReturnType = { - loading: boolean - error?: Error - exports?: BatchExport[] -} - -export const useExports = ( - teamId: number -): { - exportsState: UseExportsReturnType - updateCallback: (signal: AbortSignal | undefined) => void -} => { - // Returns a list of exports for the given team. While we are fetching the - // list, we return a loading: true as part of the state. On component - // unmount we ensure that we clean up the fetch request by use of the - // AbortController. - // - // If we get an error, we return this to the caller. - const [state, setExports] = useState({ - loading: true, - exports: undefined, - error: undefined, - }) - - const updateCallback = useCallback( - (signal: AbortSignal | undefined) => { - fetch(`/api/projects/${teamId}/batch_exports/`, { signal }) - .then((response) => response.json() as Promise) - .then((data) => { - setExports({ loading: false, exports: data.results, error: undefined }) - }) - .catch((error) => { - setExports({ loading: false, exports: undefined, error }) - }) - }, - [teamId] - ) - - // Make the actual fetch request as a side effect. - useEffect(() => { - const controller = new AbortController() - const signal = controller.signal - - updateCallback(signal) - - return () => controller.abort() - }, [teamId]) - - return { exportsState: state, updateCallback } -} - -type S3Destination = { - // At the moment we just support S3, but we include this nesting to - // allow for future expansion easily without needing to change the - // interface. - type: 'S3' - config: { - bucket_name: string - region: string - prefix: string - aws_access_key_id: string - aws_secret_access_key: string - } -} - -type SnowflakeDestination = { - type: 'Snowflake' - config: { - account: string - database: string - warehouse: string - user: string - password: string - schema: string - table_name: string - role: string | null - } -} - -export type Destination = S3Destination | SnowflakeDestination - -export type BatchExportData = { - // User provided data for the export. This is the data that the user - // provides when creating the export. - name: string - destination: Destination - interval: 'hour' | 'day' - start_at: string | null - end_at: string | null -} - -export type BatchExport = { - id: string - team_id: number - status: 'RUNNING' | 'FAILED' | 'COMPLETED' | 'PAUSED' - created_at: string - last_updated_at: string - paused: boolean -} & BatchExportData - -export type BatchExportsResponse = { - results: BatchExport[] -} - -export const useCreateExport = (): { - loading: boolean - error: Error | null - createExport: (teamId: number, exportData: BatchExportData) => Promise -} => { - // Return a callback that can be used to create an export. We also include - // the loading state and error. - const [state, setState] = useState<{ loading: boolean; error: Error | null }>({ loading: false, error: null }) - - const createExport = useCallback((teamId: number, exportData: BatchExportData) => { - setState({ loading: true, error: null }) - return api.createResponse(`/api/projects/${teamId}/batch_exports/`, exportData).then((response) => { - if (response.ok) { - setState({ loading: false, error: null }) - } else { - // TODO: parse the error response. - const error = new Error(response.statusText) - setState({ loading: false, error: error }) - throw error - } - }) - }, []) - - return { createExport, ...state } -} - -export const useDeleteExport = ( - teamId: number, - exportId: string -): { - deleteExport: () => Promise - deleting: boolean - error: Error | null -} => { - // Return a callback that can be used to delete an export. We also include - // the deleting state and error. We take a callback to update any state after delete. - const [state, setState] = useState<{ deleting: boolean; error: Error | null }>({ - deleting: false, - error: null, - }) - - const deleteExport = useCallback(() => { - setState({ deleting: true, error: null }) - return api.delete(`/api/projects/${teamId}/batch_exports/${exportId}`).then((response) => { - if (response.ok) { - setState({ deleting: false, error: null }) - } else { - // TODO: parse the error response. - const error = new Error(response.statusText) - setState({ deleting: false, error: error }) - throw error - } - }) - }, [teamId, exportId]) - - return { deleteExport, ...state } -} - -export const useUpdateExport = (): { - loading: boolean - error: Error | null - updateExport: (teamId: number, exportId: string, exportData: BatchExportData) => Promise -} => { - const [state, setState] = useState<{ loading: boolean; error: Error | null }>({ loading: false, error: null }) - - const updateExport = useCallback((teamId: number, exportId: string, exportData: BatchExportData) => { - setState({ loading: true, error: null }) - return api.createResponse(`/api/projects/${teamId}/batch_exports/${exportId}`, exportData).then((response) => { - if (response.ok) { - setState({ loading: false, error: null }) - } else { - // TODO: parse the error response. - const error = new Error(response.statusText) - setState({ loading: false, error: error }) - throw error - } - }) - }, []) - - return { updateExport, ...state } -} - -export const useExportAction = ( - teamId: number, - exportId: string, - action: 'pause' | 'unpause' -): { - executeExportAction: (data: any) => Promise - loading: boolean - error: Error | null -} => { - // Returns a callback to execute an action for the given team and export ID. - const [state, setState] = useState<{ loading: boolean; error: Error | null }>({ loading: false, error: null }) - - const executeExportAction = useCallback( - (data) => { - setState({ loading: true, error: null }) - return api - .createResponse(`/api/projects/${teamId}/batch_exports/${exportId}/${action}`, data ? data : {}) - .then((response) => { - if (response.ok) { - setState({ loading: false, error: null }) - } else { - // TODO: parse the error response. - const error = new Error(response.statusText) - setState({ loading: false, error: error }) - throw error - } - }) - }, - [teamId, exportId, action] - ) - - return { executeExportAction, ...state } -} - -export const useExport = ( - teamId: number, - exportId: string -): { - loading: boolean - export_: BatchExport | undefined - error: Error | undefined - updateCallback: (signal: AbortSignal | undefined) => void -} => { - // Fetches the export details for the given team and export ID. - const [loading, setLoading] = useState(true) - const [export_, setExport] = useState() - const [error, setError] = useState() - - const updateCallback = useCallback( - (signal: AbortSignal | undefined) => { - fetch(`/api/projects/${teamId}/batch_exports/${exportId}`, { signal }) - .then((res) => res.json()) - .then((data) => { - setExport(data) - setLoading(false) - }) - .catch((error) => { - setError(error) - setLoading(false) - }) - }, - [teamId, exportId] - ) - - useEffect(() => { - const controller = new AbortController() - const signal = controller.signal - - setLoading(true) - setError(undefined) - - updateCallback(signal) - - return () => controller.abort() - }, [teamId, exportId]) - - return { loading, export_, error, updateCallback } -} - -export const useExportRuns = ( - teamId: number, - exportId: string, - limit: number | null, - dateRange: [dayjs.Dayjs, dayjs.Dayjs] -): { - loading: boolean - exportRuns: BatchExportRun[] | undefined - error: Error | undefined - updateCallback: ( - signal: AbortSignal | undefined, - numberOfRows: number | null, - dateRange: [dayjs.Dayjs, dayjs.Dayjs] - ) => Promise -} => { - // Fetches the export runs for the given team and export ID. - const [loading, setLoading] = useState(true) - const [exportRuns, setExportRuns] = useState() - const [error, setError] = useState() - - const updateCallback = useCallback( - (signal: AbortSignal | undefined, numberOfRows: number | null, dateRange: [dayjs.Dayjs, dayjs.Dayjs]) => { - setLoading(true) - setError(undefined) - - const url = numberOfRows - ? `/api/projects/${teamId}/batch_exports/${exportId}/runs?limit=${encodeURIComponent( - numberOfRows - )}&after=${encodeURIComponent(dateRange[0].toISOString())}&before=${encodeURIComponent( - dateRange[1].toISOString() - )}` - : `/api/projects/${teamId}/batch_exports/${exportId}/runs?after=${encodeURIComponent( - dateRange[0].toISOString() - )}&before=${encodeURIComponent(dateRange[1].toISOString())}` - - return fetch(url, { signal }) - .then((res) => res.json() as Promise) - .then((data) => { - setExportRuns(data.results) - setLoading(false) - }) - .catch((error) => { - setError(error) - setLoading(false) - }) - }, - [teamId, exportId] - ) - - useEffect(() => { - const controller = new AbortController() - const signal = controller.signal - - updateCallback(signal, limit, dateRange) - - return () => controller.abort() - }, [teamId, exportId, limit, dateRange]) - - return { loading, exportRuns, error, updateCallback } -} - -export const useExportRunAction = ( - teamId: number, - exportId: string, - exportRunId: string, - action: 'reset' -): { - executeExportRunAction: () => Promise - loading: boolean - error: Error | null -} => { - // Returns a callback to execute an action for the given team, export ID and export run ID. - const [state, setState] = useState<{ loading: boolean; error: Error | null }>({ loading: false, error: null }) - - const executeExportRunAction = useCallback(() => { - setState({ loading: true, error: null }) - return api - .createResponse(`/api/projects/${teamId}/batch_exports/${exportId}/runs/${exportRunId}/${action}`, {}) - .then((response) => { - if (response.ok) { - setState({ loading: false, error: null }) - } else { - // TODO: parse the error response. - const error = new Error(response.statusText) - setState({ loading: false, error: error }) - throw error - } - }) - }, [teamId, exportId, action]) - - return { executeExportRunAction, ...state } -} - -export type BatchExportRunStatus = - | 'Cancelled' - | 'Completed' - | 'ContinuedAsNew' - | 'Failed' - | 'Terminated' - | 'TimedOut' - | 'Running' - | 'Starting' - -export type BatchExportRun = { - id: string - team_id: number - batch_export_id: string - status: BatchExportRunStatus - opened_at: string - closed_at: string - data_interval_start: string - data_interval_end: string - created_at: string - last_updated_at: string -} - -type BatchExportRunsResponse = { - results: BatchExportRun[] -} diff --git a/frontend/src/scenes/groups/groupsListLogic.ts b/frontend/src/scenes/groups/groupsListLogic.ts index 36256f47342c8..d57520f9c0966 100644 --- a/frontend/src/scenes/groups/groupsListLogic.ts +++ b/frontend/src/scenes/groups/groupsListLogic.ts @@ -41,14 +41,15 @@ export const groupsListLogic = kea({ { next: null, previous: null, results: [] } as GroupsPaginatedResponse, { loadGroups: async ({ url }) => { - if (values.groupsEnabled) { - url = - url || - `api/projects/${values.currentTeamId}/groups/?group_type_index=${props.groupTypeIndex}${ - values.search ? '&search=' + encodeURIComponent(values.search) : '' - }` - return await api.get(url) + if (!values.groupsEnabled) { + return values.groups } + url = + url || + `api/projects/${values.currentTeamId}/groups/?group_type_index=${props.groupTypeIndex}${ + values.search ? '&search=' + encodeURIComponent(values.search) : '' + }` + return await api.get(url) }, }, ], diff --git a/frontend/src/scenes/notebooks/notebookSceneLogic.ts b/frontend/src/scenes/notebooks/notebookSceneLogic.ts index a3b3ffa7a1aa2..a3f34d7cf9bab 100644 --- a/frontend/src/scenes/notebooks/notebookSceneLogic.ts +++ b/frontend/src/scenes/notebooks/notebookSceneLogic.ts @@ -10,7 +10,6 @@ export type NotebookSceneLogicProps = { shortId: string } export const notebookSceneLogic = kea([ - path(['scenes', 'notebooks', 'notebookSceneLogic']), path((key) => ['scenes', 'notebooks', 'notebookSceneLogic', key]), props({} as NotebookSceneLogicProps), key(({ shortId }) => shortId), diff --git a/frontend/src/scenes/sceneTypes.ts b/frontend/src/scenes/sceneTypes.ts index 589e42354248a..3eea02454390c 100644 --- a/frontend/src/scenes/sceneTypes.ts +++ b/frontend/src/scenes/sceneTypes.ts @@ -31,9 +31,9 @@ export enum Scene { Actions = 'ActionsTable', Experiments = 'Experiments', Experiment = 'Experiment', - Exports = 'Exports', - CreateExport = 'CreateExport', - ViewExport = 'ViewExport', + BatchExports = 'BatchExports', + BatchExport = 'BatchExport', + BatchExportEdit = 'BatchExportEdit', FeatureFlags = 'FeatureFlags', FeatureFlag = 'FeatureFlag', EarlyAccessFeatures = 'EarlyAccessFeatures', diff --git a/frontend/src/scenes/scenes.ts b/frontend/src/scenes/scenes.ts index bbf2fcadcf1a5..c37bec50b55f4 100644 --- a/frontend/src/scenes/scenes.ts +++ b/frontend/src/scenes/scenes.ts @@ -55,17 +55,17 @@ export const sceneConfigurations: Partial> = { projectBased: true, name: 'Event Explorer', }, - [Scene.Exports]: { + [Scene.BatchExports]: { projectBased: true, - name: 'Exports', + name: 'Batch Exports', }, - [Scene.CreateExport]: { + [Scene.BatchExportEdit]: { projectBased: true, - name: 'Create Export', + name: 'Edit Batch Export', }, - [Scene.ViewExport]: { + [Scene.BatchExport]: { projectBased: true, - name: 'View Export', + name: 'Batch Export', }, [Scene.DataManagement]: { projectBased: true, @@ -361,6 +361,7 @@ export const redirects: Record< return urls.replay() }, '/replay': urls.replay(), + '/exports': urls.batchExports(), } export const routes: Record = { @@ -384,9 +385,10 @@ export const routes: Record = { [urls.actions()]: Scene.Actions, // TODO: remove when "simplify-actions" FF is released [urls.eventDefinitions()]: Scene.EventDefinitions, [urls.eventDefinition(':id')]: Scene.EventDefinition, - [urls.exports()]: Scene.Exports, - [urls.createExport()]: Scene.CreateExport, - [urls.viewExport(':id')]: Scene.ViewExport, + [urls.batchExports()]: Scene.BatchExports, + [urls.batchExportNew()]: Scene.BatchExportEdit, + [urls.batchExport(':id')]: Scene.BatchExport, + [urls.batchExportEdit(':id')]: Scene.BatchExportEdit, [urls.propertyDefinitions()]: Scene.PropertyDefinitions, [urls.propertyDefinition(':id')]: Scene.PropertyDefinition, [urls.dataManagementHistory()]: Scene.DataManagementHistory, diff --git a/frontend/src/scenes/urls.ts b/frontend/src/scenes/urls.ts index 370a2cf04c507..ecd409e2817a4 100644 --- a/frontend/src/scenes/urls.ts +++ b/frontend/src/scenes/urls.ts @@ -11,6 +11,7 @@ import { combineUrl } from 'kea-router' import { ExportOptions } from '~/exporter/types' import { AppMetricsUrlParams } from './apps/appMetricsSceneLogic' import { PluginTab } from './plugins/types' +import { toParams } from 'lib/utils' /** * To add a new URL to the front end: @@ -51,9 +52,11 @@ export const urls = { events: (): string => '/events', event: (id: string, timestamp: string): string => `/events/${encodeURIComponent(id)}/${encodeURIComponent(timestamp)}`, - exports: (): string => '/exports', - createExport: (): string => `/exports/new`, - viewExport: (id: string | number): string => `/exports/${id}`, + batchExports: (): string => '/batch_exports', + batchExportNew: (): string => `/batch_exports/new`, + batchExport: (id: string, params?: { runId?: string }): string => + `/batch_exports/${id}` + (params ? `?${toParams(params)}` : ''), + batchExportEdit: (id: string): string => `/batch_exports/${id}/edit`, ingestionWarnings: (): string => '/data-management/ingestion-warnings', insightNew: (filters?: AnyPartialFilterType, dashboardId?: DashboardType['id'] | null, query?: string): string => combineUrl('/insights/new', dashboardId ? { dashboard: dashboardId } : {}, { diff --git a/frontend/src/types.ts b/frontend/src/types.ts index b63fdd7a94e6f..e1b1996ad3d91 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -3066,3 +3066,60 @@ export interface DataWarehouseSavedQuery { query: HogQLQuery columns: DatabaseSchemaQueryResponseField[] } + +export type BatchExportDestinationS3 = { + type: 'S3' + config: { + bucket_name: string + region: string + prefix: string + aws_access_key_id: string + aws_secret_access_key: string + } +} + +export type BatchExportDestinationSnowflake = { + type: 'Snowflake' + config: { + account: string + database: string + warehouse: string + user: string + password: string + schema: string + table_name: string + role: string | null + } +} + +export type BatchExportDestination = BatchExportDestinationS3 | BatchExportDestinationSnowflake + +export type BatchExportConfiguration = { + // User provided data for the export. This is the data that the user + // provides when creating the export. + id: string + name: string + destination: BatchExportDestination + interval: 'hour' | 'day' + created_at: string + start_at: string | null + end_at: string | null + paused: boolean + latest_runs?: BatchExportRun[] +} + +export type BatchExportRun = { + id: string + status: 'Cancelled' | 'Completed' | 'ContinuedAsNew' | 'Failed' | 'Terminated' | 'TimedOut' | 'Running' | 'Starting' + created_at: Dayjs + data_interval_start: Dayjs + data_interval_end: Dayjs + last_updated_at?: Dayjs +} + +export type GroupedBatchExportRuns = { + last_run_at: Dayjs + data_interval_start: Dayjs + data_interval_end: Dayjs + runs: BatchExportRun[] +} diff --git a/jest.setup.ts b/jest.setup.ts index 78f64a1f94ba3..847624fd0224f 100644 --- a/jest.setup.ts +++ b/jest.setup.ts @@ -2,4 +2,6 @@ import 'whatwg-fetch' import 'jest-canvas-mock' window.scrollTo = jest.fn() -window.matchMedia = jest.fn(() => ({ matches: false } as MediaQueryList)) +window.matchMedia = jest.fn( + () => ({ matches: false, addListener: jest.fn(), removeListener: jest.fn() } as MediaQueryList) +) diff --git a/posthog/api/test/batch_exports/operations.py b/posthog/api/test/batch_exports/operations.py index 4ee9ffe3f30b7..2000099a9385e 100644 --- a/posthog/api/test/batch_exports/operations.py +++ b/posthog/api/test/batch_exports/operations.py @@ -104,13 +104,3 @@ def patch_batch_export(client, team_id, batch_export_id, new_batch_export_data): new_batch_export_data, content_type="application/json", ) - - -def reset_batch_export_run(client: TestClient, team_id: int, batch_export_id: str, batch_export_run_id: str): - return client.post(f"/api/projects/{team_id}/batch_exports/{batch_export_id}/runs/{batch_export_run_id}/reset") - - -def reset_batch_export_run_ok(client: TestClient, team_id: int, batch_export_id: str, batch_export_run_id: str): - response = reset_batch_export_run(client, team_id, batch_export_id, batch_export_run_id) - assert response.status_code == status.HTTP_200_OK, response.json() - return response.json() diff --git a/posthog/api/test/batch_exports/test_reset.py b/posthog/api/test/batch_exports/test_reset.py deleted file mode 100644 index 19e1752759843..0000000000000 --- a/posthog/api/test/batch_exports/test_reset.py +++ /dev/null @@ -1,92 +0,0 @@ -import datetime as dt -import time - -import pytest -from django.test.client import Client as HttpClient - -from posthog.api.test.batch_exports.conftest import start_test_worker -from posthog.api.test.batch_exports.operations import ( - create_batch_export_ok, - get_batch_export_runs_ok, - reset_batch_export_run_ok, -) -from posthog.api.test.test_organization import create_organization -from posthog.api.test.test_team import create_team -from posthog.api.test.test_user import create_user -from posthog.temporal.client import sync_connect - - -def wait_for_runs(client, team_id, batch_export_id, timeout=10, number_of_runs=1): - """Wait for BatchExportRuns to be created. - - As these rows are created by Temporal, and the worker is running in a separate thread, we allow it - to take a few seconds. - - Raises: - TimeoutError: If there are less than number_of_runs BatchExportRuns after around timeout seconds. - - Returns: - The BatchExportRuns response. - """ - start = dt.datetime.utcnow() - batch_export_runs = get_batch_export_runs_ok(client, team_id, batch_export_id) - - while batch_export_runs["count"] < number_of_runs: - batch_export_runs = get_batch_export_runs_ok(client, team_id, batch_export_id) - time.sleep(1) - if (dt.datetime.utcnow() - start).seconds > timeout: - raise TimeoutError("BatchExportRuns never created") - - return batch_export_runs - - -@pytest.mark.django_db(transaction=True) -def test_can_reset_export_run(client: HttpClient): - """Test calling the reset endpoint to reset a BatchExportRun a couple of times.""" - temporal = sync_connect() - - destination_data = { - "type": "S3", - "config": { - "bucket_name": "my-production-s3-bucket", - "region": "us-east-1", - "prefix": "posthog-events/", - "aws_access_key_id": "abc123", - "aws_secret_access_key": "secret", - }, - } - - batch_export_data = { - "name": "my-production-s3-bucket-destination", - "destination": destination_data, - "interval": "hour", - "trigger_immediately": True, - } - - organization = create_organization("Test Org") - team = create_team(organization) - user = create_user("reset.test@user.com", "Reset test user", organization) - client.force_login(user) - - with start_test_worker(temporal): - batch_export = create_batch_export_ok( - client, - team.pk, - batch_export_data, - ) - - batch_export_runs = wait_for_runs(client, team.pk, batch_export["id"]) - assert batch_export_runs["count"] == 1 - - first_batch_export_run = batch_export_runs["results"][0] - reset_batch_export_run_ok(client, team.pk, batch_export["id"], first_batch_export_run["id"]) - - batch_export_runs = wait_for_runs(client, team.pk, batch_export["id"], number_of_runs=2) - assert batch_export_runs["count"] == 2 - assert batch_export_runs["results"][1] == first_batch_export_run - - reset_batch_export_run_ok(client, team.pk, batch_export["id"], first_batch_export_run["id"]) - - batch_export_runs = wait_for_runs(client, team.pk, batch_export["id"], number_of_runs=3) - assert batch_export_runs["count"] == 3 - assert batch_export_runs["results"][2] == first_batch_export_run diff --git a/posthog/batch_exports/http.py b/posthog/batch_exports/http.py index 719f5242d321c..eebc57787759d 100644 --- a/posthog/batch_exports/http.py +++ b/posthog/batch_exports/http.py @@ -1,35 +1,31 @@ import datetime as dt from typing import Any +from django.utils.timezone import now from rest_framework import request, response, serializers, viewsets from rest_framework.decorators import action from rest_framework.exceptions import NotAuthenticated, NotFound, ValidationError +from rest_framework.pagination import CursorPagination from rest_framework.permissions import IsAuthenticated from posthog.api.routing import StructuredViewSetMixin +from posthog.batch_exports.models import BATCH_EXPORT_INTERVALS from posthog.batch_exports.service import ( BatchExportIdError, BatchExportServiceError, BatchExportServiceRPCError, backfill_export, - create_batch_export, delete_schedule, pause_batch_export, - reset_batch_export_run, + sync_batch_export, unpause_batch_export, - update_batch_export, -) -from posthog.models import ( - BatchExport, - BatchExportDestination, - BatchExportRun, - User, -) -from posthog.permissions import ( - ProjectMembershipNecessaryPermissions, - TeamMemberAccessPermission, ) +from django.db import transaction + +from posthog.models import BatchExport, BatchExportDestination, BatchExportRun, User +from posthog.permissions import ProjectMembershipNecessaryPermissions, TeamMemberAccessPermission from posthog.temporal.client import sync_connect +from posthog.utils import relative_date_parse def validate_date_input(date_input: Any) -> dt.datetime: @@ -61,13 +57,20 @@ class BatchExportRunSerializer(serializers.ModelSerializer): class Meta: model = BatchExportRun fields = "__all__" + # TODO: Why aren't all these read only? read_only_fields = ["batch_export"] -class BatchExportRunViewSet(StructuredViewSetMixin, viewsets.ModelViewSet): +class RunsCursorPagination(CursorPagination): + ordering = "-created_at" + page_size = 100 + + +class BatchExportRunViewSet(StructuredViewSetMixin, viewsets.ReadOnlyModelViewSet): queryset = BatchExportRun.objects.all() permission_classes = [IsAuthenticated, ProjectMembershipNecessaryPermissions, TeamMemberAccessPermission] serializer_class = BatchExportRunSerializer + pagination_class = RunsCursorPagination def get_queryset(self, date_range: tuple[dt.datetime, dt.datetime] | None = None): if not isinstance(self.request.user, User) or self.request.user.current_team is None: @@ -87,45 +90,15 @@ def list(self, request: request.Request, *args, **kwargs) -> response.Response: if not isinstance(request.user, User) or request.user.current_team is None: raise NotAuthenticated() - after = self.request.query_params.get("after", None) + after = self.request.query_params.get("after", "-7d") before = self.request.query_params.get("before", None) - date_range = None - if after is not None and before is not None: - after_datetime = validate_date_input(after) - before_datetime = validate_date_input(before) - date_range = (after_datetime, before_datetime) - - runs = self.get_queryset(date_range=date_range) - limit = self.request.query_params.get("limit", None) - if limit is not None: - try: - limit = int(limit) - except (TypeError, ValueError): - raise ValidationError(f"Invalid value for 'limit' parameter: '{limit}'") - - runs = runs[:limit] - - page = self.paginate_queryset(runs) - if page is not None: - serializer = self.get_serializer(page, many=True) - return self.get_paginated_response(serializer.data) - - serializer = self.get_serializer(runs, many=True) - return response.Response(serializer.data) + after_datetime = relative_date_parse(after) + before_datetime = relative_date_parse(before) if before else now() + date_range = (after_datetime, before_datetime) - @action(methods=["POST"], detail=True) - def reset(self, request: request.Request, *args, **kwargs) -> response.Response: - """Reset a BatchExportRun by resetting its associated Temporal Workflow.""" - if not isinstance(request.user, User) or request.user.current_team is None: - raise NotAuthenticated() - - batch_export_run = self.get_object() - temporal = sync_connect() - - scheduled_id = f"{batch_export_run.batch_export.id}-{batch_export_run.data_interval_end:%Y-%m-%dT%H:%M:%SZ}" - new_run_id = reset_batch_export_run(temporal, batch_export_id=scheduled_id) - - return response.Response({"new_run_id": new_run_id}) + page = self.paginate_queryset(self.get_queryset(date_range=date_range)) + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) class BatchExportDestinationSerializer(serializers.ModelSerializer): @@ -152,7 +125,8 @@ class BatchExportSerializer(serializers.ModelSerializer): """Serializer for a BatchExport model.""" destination = BatchExportDestinationSerializer() - trigger_immediately = serializers.BooleanField(default=False) + latest_runs = BatchExportRunSerializer(many=True, read_only=True) + interval = serializers.ChoiceField(choices=BATCH_EXPORT_INTERVALS) class Meta: model = BatchExport @@ -167,51 +141,44 @@ class Meta: "last_paused_at", "start_at", "end_at", - "trigger_immediately", - ] - read_only_fields = [ - "id", - "paused", - "created_at", - "last_updated_at", + "latest_runs", ] + read_only_fields = ["id", "created_at", "last_updated_at", "latest_runs"] def create(self, validated_data: dict) -> BatchExport: """Create a BatchExport.""" destination_data = validated_data.pop("destination") team_id = self.context["team_id"] - interval = validated_data.pop("interval") - name = validated_data.pop("name") - start_at = validated_data.get("start_at", None) - end_at = validated_data.get("end_at", None) - trigger_immediately = validated_data.get("trigger_immediately", False) - - return create_batch_export( - team_id=team_id, - interval=interval, - name=name, - destination_data=destination_data, - start_at=start_at, - end_at=end_at, - trigger_immediately=trigger_immediately, - ) - def update(self, instance: BatchExport, validated_data: dict) -> BatchExport: + destination = BatchExportDestination(**destination_data) + batch_export = BatchExport(team_id=team_id, destination=destination, **validated_data) + + sync_batch_export(batch_export, created=True) + + with transaction.atomic(): + destination.save() + batch_export.save() + + return batch_export + + def update(self, batch_export: BatchExport, validated_data: dict) -> BatchExport: """Update a BatchExport.""" destination_data = validated_data.pop("destination", None) - interval = validated_data.get("interval", None) - name = validated_data.get("name", None) - start_at = validated_data.get("start_at", None) - end_at = validated_data.get("end_at", None) - - return update_batch_export( - batch_export=instance, - interval=interval, - name=name, - destination_data=destination_data, - start_at=start_at, - end_at=end_at, - ) + + with transaction.atomic(): + if destination_data: + batch_export.destination.type = destination_data.get("type", batch_export.destination.type) + batch_export.destination.config = { + **batch_export.destination.config, + **destination_data.get("config", {}), + } + + batch_export.destination.save() + batch_export = super().update(batch_export, validated_data) + + sync_batch_export(batch_export, created=False) + + return batch_export class BatchExportViewSet(StructuredViewSetMixin, viewsets.ModelViewSet): @@ -223,7 +190,12 @@ def get_queryset(self): if not isinstance(self.request.user, User) or self.request.user.current_team is None: raise NotAuthenticated() - return self.queryset.filter(team_id=self.team_id).exclude(deleted=True).prefetch_related("destination") + return ( + self.queryset.filter(team_id=self.team_id) + .exclude(deleted=True) + .order_by("-created_at") + .prefetch_related("destination") + ) @action(methods=["POST"], detail=True) def backfill(self, request: request.Request, *args, **kwargs) -> response.Response: @@ -234,6 +206,9 @@ def backfill(self, request: request.Request, *args, **kwargs) -> response.Respon start_at_input = request.data.get("start_at", None) end_at_input = request.data.get("end_at", None) + if start_at_input is None or end_at_input is None: + raise ValidationError("Both 'start_at' and 'end_at' must be specified") + start_at = validate_date_input(start_at_input) end_at = validate_date_input(end_at_input) diff --git a/posthog/batch_exports/models.py b/posthog/batch_exports/models.py index a81c9bf889617..55ab202ec4fb4 100644 --- a/posthog/batch_exports/models.py +++ b/posthog/batch_exports/models.py @@ -1,4 +1,5 @@ from django.db import models +from datetime import timedelta from posthog.models.utils import UUIDModel @@ -87,6 +88,9 @@ class Status(models.TextChoices): ) +BATCH_EXPORT_INTERVALS = [("hour", "hour"), ("day", "day"), ("week", "week")] + + class BatchExport(UUIDModel): """ Defines the configuration of PostHog to export data to a destination, @@ -103,7 +107,7 @@ class BatchExport(UUIDModel): interval = models.CharField( max_length=64, null=False, - choices=[("hour", "hour"), ("day", "day"), ("week", "week")], + choices=BATCH_EXPORT_INTERVALS, default="hour", help_text="The interval at which to export data.", ) @@ -124,3 +128,17 @@ class BatchExport(UUIDModel): end_at: models.DateTimeField = models.DateTimeField( null=True, default=None, help_text="Time after which any Batch Export runs won't be triggered." ) + + @property + def latest_runs(self): + return self.batchexportrun_set.all().order_by("-created_at")[:10] + + @property + def interval_time_delta(self) -> timedelta: + if self.interval == "hour": + return timedelta(hours=1) + elif self.interval == "day": + return timedelta(days=1) + elif self.interval == "week": + return timedelta(weeks=1) + raise ValueError("Invalid interval") diff --git a/posthog/batch_exports/service.py b/posthog/batch_exports/service.py index 49e8c20295e83..6db01a5a21530 100644 --- a/posthog/batch_exports/service.py +++ b/posthog/batch_exports/service.py @@ -1,9 +1,8 @@ import datetime as dt from dataclasses import asdict, dataclass -from uuid import UUID, uuid4 +from uuid import UUID -from asgiref.sync import async_to_sync, sync_to_async -from temporalio.api.workflowservice.v1 import ResetWorkflowExecutionRequest +from asgiref.sync import async_to_sync from temporalio.client import ( Client, Schedule, @@ -21,7 +20,6 @@ from posthog import settings from posthog.batch_exports.models import ( BatchExport, - BatchExportDestination, BatchExportRun, ) from posthog.temporal.client import sync_connect @@ -93,16 +91,6 @@ class BatchExportServiceRPCError(BatchExportServiceError): """Exception raised when the underlying Temporal RPC fails.""" -@async_to_sync -async def create_schedule(temporal: Client, id: str, schedule: Schedule, trigger_immediately: bool = False): - """Create a Temporal Schedule.""" - return await temporal.create_schedule( - id=id, - schedule=schedule, - trigger_immediately=trigger_immediately, - ) - - def pause_batch_export(temporal: Client, batch_export_id: str, note: str | None = None) -> None: """Pause this BatchExport. @@ -262,34 +250,7 @@ def update_batch_export_run_status(run_id: UUID, status: str, latest_error: str raise ValueError(f"BatchExportRun with id {run_id} not found.") -def update_batch_export( - batch_export: BatchExport, - interval: str | None, - name: str | None, - destination_data: dict | None = None, - start_at: dt.datetime | None = None, - end_at: dt.datetime | None = None, -): - if destination_data: - batch_export.destination.type = destination_data.get("type", batch_export.destination.type) - batch_export.destination.config = {**batch_export.destination.config, **destination_data.get("config", {})} - - batch_export.name = name or batch_export.name - batch_export.start_at = start_at or batch_export.start_at - batch_export.end_at = end_at or batch_export.end_at - - if interval is None: - interval = batch_export.interval - - if interval == "hour": - time_delta_from_interval = dt.timedelta(hours=1) - elif interval == "day": - time_delta_from_interval = dt.timedelta(days=1) - else: - raise ValueError(f"Unsupported interval '{interval}'") - - batch_export.interval = interval or batch_export.interval - +def sync_batch_export(batch_export: BatchExport, created: bool): workflow, workflow_inputs = DESTINATION_WORKFLOWS[batch_export.destination.type] state = ScheduleState( note=f"Schedule updated for BatchExport {batch_export.id} to Destination {batch_export.destination.id} in Team {batch_export.team.id}.", @@ -297,7 +258,7 @@ def update_batch_export( ) temporal = sync_connect() - new_schedule = Schedule( + schedule = Schedule( action=ScheduleActionStartWorkflow( workflow, asdict( @@ -314,141 +275,38 @@ def update_batch_export( spec=ScheduleSpec( start_at=batch_export.start_at, end_at=batch_export.end_at, - intervals=[ScheduleIntervalSpec(every=time_delta_from_interval)], + intervals=[ScheduleIntervalSpec(every=batch_export.interval_time_delta)], ), state=state, + policy=SchedulePolicy(overlap=ScheduleOverlapPolicy.ALLOW_ALL), ) - update_schedule(temporal, schedule_id=str(batch_export.id), schedule=new_schedule) + if created: + create_schedule(temporal, id=str(batch_export.id), schedule=schedule) + else: + update_schedule(temporal, id=str(batch_export.id), schedule=schedule) - batch_export.save() - batch_export.destination.save() return batch_export @async_to_sync -async def update_schedule(temporal: Client, schedule_id: str, schedule: Schedule) -> None: - """Update a Temporal Schedule.""" - handle = temporal.get_schedule_handle(schedule_id) - - async def updater(_: ScheduleUpdateInput) -> ScheduleUpdate: - return ScheduleUpdate(schedule=schedule) - - return await handle.update( - updater=updater, - ) - - -def create_batch_export( - team_id: int, - interval: str, - name: str, - destination_data: dict, - start_at: dt.datetime | None = None, - end_at: dt.datetime | None = None, - trigger_immediately: bool = False, -): - """Create a BatchExport and its underlying Temporal Schedule. - - Args: - team_id: The team this BatchExport belongs to. - interval: The time interval the Schedule will use. - name: An informative name for the BatchExport. - destination_data: Deserialized data for a BatchExportDestination. - start_at: No runs will be scheduled before the start_at datetime. - end_at: No runs will be scheduled after the end_at datetime. - trigger_immediately: Whether a run should be trigger as soon as the Schedule is created - or when the next Schedule interval begins. - """ - destination = BatchExportDestination.objects.create(**destination_data) - - batch_export = BatchExport.objects.create( - team_id=team_id, name=name, interval=interval, destination=destination, start_at=start_at, end_at=end_at - ) - - workflow, workflow_inputs = DESTINATION_WORKFLOWS[batch_export.destination.type] - - state = ScheduleState( - note=f"Schedule created for BatchExport {batch_export.id} to Destination {batch_export.destination.id} in Team {batch_export.team.id}.", - paused=batch_export.paused, - ) - - temporal = sync_connect() - - time_delta_from_interval = dt.timedelta(hours=1) if interval == "hour" else dt.timedelta(days=1) - - create_schedule( - temporal, - id=str(batch_export.id), - schedule=Schedule( - action=ScheduleActionStartWorkflow( - workflow, - asdict( - workflow_inputs( - team_id=batch_export.team.id, - # We could take the batch_export_id from the Workflow id - # But temporal appends a timestamp at the end we would have to parse out. - batch_export_id=str(batch_export.id), - interval=str(batch_export.interval), - **batch_export.destination.config, - ) - ), - id=str(batch_export.id), - task_queue=settings.TEMPORAL_TASK_QUEUE, - ), - spec=ScheduleSpec( - start_at=start_at, - end_at=end_at, - intervals=[ScheduleIntervalSpec(every=time_delta_from_interval)], - ), - state=state, - policy=SchedulePolicy(overlap=ScheduleOverlapPolicy.ALLOW_ALL), - ), +async def create_schedule(temporal: Client, id: str, schedule: Schedule, trigger_immediately: bool = False): + """Create a Temporal Schedule.""" + return await temporal.create_schedule( + id=id, + schedule=schedule, trigger_immediately=trigger_immediately, ) - return batch_export - - -async def acreate_batch_export(team_id: int, interval: str, name: str, destination_data: dict) -> BatchExport: - """Create a BatchExport and its underlying Schedule.""" - return await sync_to_async(create_batch_export)(team_id, interval, name, destination_data) # type: ignore - - -def fetch_batch_export_runs(batch_export_id: UUID, limit: int = 100) -> list[BatchExportRun]: - """Fetch the BatchExportRuns for a given BatchExport.""" - return list(BatchExportRun.objects.filter(batch_export_id=batch_export_id).order_by("-created_at")[:limit]) - - -async def afetch_batch_export_runs(batch_export_id: UUID, limit: int = 100) -> list[BatchExportRun]: - """Fetch the BatchExportRuns for a given BatchExport.""" - return await sync_to_async(fetch_batch_export_runs)(batch_export_id, limit) # type: ignore - @async_to_sync -async def reset_batch_export_run(temporal, batch_export_id: str | UUID) -> str: - """Reset an individual batch export run corresponding to a given batch export. - - Resetting a workflow is considered an "advanced concept" by Temporal, hence it's not exposed - cleanly via the SDK, and it requries us to make a raw request. +async def update_schedule(temporal: Client, id: str, schedule: Schedule) -> None: + """Update a Temporal Schedule.""" + handle = temporal.get_schedule_handle(id) - Resetting a workflow will create a new run with the same workflow id. The new run will have a - reference to the original run_id that we can use to tie up re-runs with their originals. + async def updater(_: ScheduleUpdateInput) -> ScheduleUpdate: + return ScheduleUpdate(schedule=schedule) - Returns: - The run_id assigned to the new run. - """ - request = ResetWorkflowExecutionRequest( - namespace=settings.TEMPORAL_NAMESPACE, - workflow_execution={ - "workflow_id": str(batch_export_id), - }, - # Any unique identifier for the request would work. - request_id=str(uuid4()), - # Reset can only happen from 'WorkflowTaskStarted' events. The first one always has id = 3. - # In other words, this means "reset from the beginning". - workflow_task_finish_event_id=3, + return await handle.update( + updater=updater, ) - resp = await temporal.workflow_service.reset_workflow_execution(request) - - return resp.run_id diff --git a/posthog/temporal/tests/batch_exports/fixtures.py b/posthog/temporal/tests/batch_exports/fixtures.py new file mode 100644 index 0000000000000..65de3fd4910c3 --- /dev/null +++ b/posthog/temporal/tests/batch_exports/fixtures.py @@ -0,0 +1,34 @@ +from uuid import UUID +from asgiref.sync import sync_to_async + +from posthog.batch_exports.models import BatchExport, BatchExportDestination, BatchExportRun +from posthog.batch_exports.service import sync_batch_export + + +def create_batch_export(team_id: int, interval: str, name: str, destination_data: dict) -> BatchExport: + """Create a BatchExport and its underlying Schedule.""" + + destination = BatchExportDestination(**destination_data) + batch_export = BatchExport(team_id=team_id, destination=destination, interval=interval, name=name) + + sync_batch_export(batch_export, created=True) + + destination.save() + batch_export.save() + + return batch_export + + +async def acreate_batch_export(team_id: int, interval: str, name: str, destination_data: dict) -> BatchExport: + """Create a BatchExport and its underlying Schedule.""" + return await sync_to_async(create_batch_export)(team_id, interval, name, destination_data) # type: ignore + + +def fetch_batch_export_runs(batch_export_id: UUID, limit: int = 100) -> list[BatchExportRun]: + """Fetch the BatchExportRuns for a given BatchExport.""" + return list(BatchExportRun.objects.filter(batch_export_id=batch_export_id).order_by("-created_at")[:limit]) + + +async def afetch_batch_export_runs(batch_export_id: UUID, limit: int = 100) -> list[BatchExportRun]: + """Fetch the BatchExportRuns for a given BatchExport.""" + return await sync_to_async(fetch_batch_export_runs)(batch_export_id, limit) # type: ignore diff --git a/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py b/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py index 8e0af428de2e4..99700ad24e984 100644 --- a/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py +++ b/posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py @@ -19,7 +19,7 @@ from ee.clickhouse.materialized_columns.columns import materialize from posthog.api.test.test_organization import acreate_organization from posthog.api.test.test_team import acreate_team -from posthog.batch_exports.service import acreate_batch_export, afetch_batch_export_runs +from posthog.temporal.tests.batch_exports.fixtures import acreate_batch_export, afetch_batch_export_runs from posthog.temporal.workflows.base import create_export_run, update_export_run_status from posthog.temporal.workflows.batch_exports import get_results_iterator from posthog.temporal.workflows.clickhouse import ClickHouseClient diff --git a/posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py b/posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py index 29ac95bc4e57f..2c1bba56d249b 100644 --- a/posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py +++ b/posthog/temporal/tests/batch_exports/test_snowflake_batch_export_workflow.py @@ -18,7 +18,7 @@ from posthog.api.test.test_organization import acreate_organization from posthog.api.test.test_team import acreate_team -from posthog.batch_exports.service import acreate_batch_export, afetch_batch_export_runs +from posthog.temporal.tests.batch_exports.fixtures import acreate_batch_export, afetch_batch_export_runs from posthog.temporal.workflows.base import create_export_run, update_export_run_status from posthog.temporal.workflows.clickhouse import ClickHouseClient from posthog.temporal.workflows.snowflake_batch_export import (