From ce61ea321afe642a9e0ce7ffc62bc856e620e2bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Far=C3=ADas=20Santana?= Date: Mon, 25 Sep 2023 17:19:54 +0200 Subject: [PATCH] feat: Batch export logs frontend --- frontend/src/lib/api.ts | 72 +++ .../scenes/batch_exports/BatchExportScene.tsx | 551 +++++++++++------- .../scenes/batch_exports/batchExportLogic.ts | 21 + .../batch_exports/batchExportLogsLogic.ts | 179 ++++++ frontend/src/types.ts | 17 + 5 files changed, 637 insertions(+), 203 deletions(-) create mode 100644 frontend/src/scenes/batch_exports/batchExportLogsLogic.ts diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 83b2b232de88ab..07e50dfb3ff27a 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1,6 +1,7 @@ import posthog from 'posthog-js' import { ActionType, + BatchExportLogEntry, CohortType, DashboardCollaboratorType, DashboardTemplateEditorType, @@ -537,6 +538,10 @@ class ApiRequest { return this.batchExports(teamId).addPathComponent(id) } + public batchExportLogs(id: BatchExportConfiguration['id'], teamId?: TeamType['id']): ApiRequest { + return this.batchExport(id, teamId).addPathComponent('logs') + } + public batchExportRuns(id: BatchExportConfiguration['id'], teamId?: TeamType['id']): ApiRequest { return this.batchExports(teamId).addPathComponent(id).addPathComponent('runs') } @@ -549,6 +554,14 @@ class ApiRequest { return this.batchExportRuns(id, teamId).addPathComponent(runId) } + public batchExportRunLogs( + id: BatchExportConfiguration['id'], + runId: BatchExportRun['id'], + teamId?: TeamType['id'] + ): ApiRequest { + return this.batchExportRun(id, runId, teamId).addPathComponent('logs') + } + // Request finalization public async get(options?: ApiMethodOptions): Promise { return await api.get(this.assembleFullUrl(), options) @@ -1186,6 +1199,65 @@ const api = { }, }, + batchExportLogs: { + async search( + batchExportId: string, + currentTeamId: number | null, + searchTerm: string | null = null, + typeFilters: CheckboxValueType[] = [], + trailingEntry: BatchExportLogEntry | null = null, + leadingEntry: BatchExportLogEntry | null = null + ): Promise { + const params = toParams( + { + limit: LOGS_PORTION_LIMIT, + type_filter: typeFilters, + search: searchTerm || undefined, + before: trailingEntry?.timestamp, + after: leadingEntry?.timestamp, + }, + true + ) + + const response = await new ApiRequest() + .batchExportLogs(batchExportId, currentTeamId || undefined) + .withQueryString(params) + .get() + + return response.results + }, + }, + + batchExportRunLogs: { + async search( + batchExportId: string, + batchExportRunId: string, + currentTeamId: number | null, + searchTerm: string | null = null, + typeFilters: CheckboxValueType[] = [], + trailingEntry: BatchExportLogEntry | null = null, + leadingEntry: BatchExportLogEntry | null = null + ): Promise { + const params = toParams( + { + limit: LOGS_PORTION_LIMIT, + type_filter: typeFilters, + search: searchTerm || undefined, + before: trailingEntry?.timestamp, + after: leadingEntry?.timestamp, + }, + true + ) + + const response = await new ApiRequest() + .batchExportRunLogs(batchExportId, batchExportRunId, currentTeamId || undefined) + .withQueryString(params) + .get() + + return response.results + }, + }, + annotations: { async get(annotationId: RawAnnotationType['id']): Promise { return await new ApiRequest().annotation(annotationId).get() diff --git a/frontend/src/scenes/batch_exports/BatchExportScene.tsx b/frontend/src/scenes/batch_exports/BatchExportScene.tsx index 609d1103bec3cc..2f44c776f0aa01 100644 --- a/frontend/src/scenes/batch_exports/BatchExportScene.tsx +++ b/frontend/src/scenes/batch_exports/BatchExportScene.tsx @@ -1,10 +1,12 @@ +import { Checkbox } from 'antd' import { SceneExport } from 'scenes/sceneTypes' import { PageHeader } from 'lib/components/PageHeader' -import { LemonButton, LemonDivider, LemonTable, LemonTag } from '@posthog/lemon-ui' +import { LemonButton, LemonDivider, LemonTable, LemonTag, LemonInput } from '@posthog/lemon-ui' import { urls } from 'scenes/urls' import { useActions, useValues } from 'kea' import { useEffect, useState } from 'react' -import { BatchExportLogicProps, batchExportLogic } from './batchExportLogic' +import { BatchExportLogicProps, batchExportLogic, BatchExportTab } from './batchExportLogic' +import { BatchExportLogsProps, batchExportLogsLogic, LOGS_PORTION_LIMIT } from './batchExportLogsLogic' import { BatchExportRunIcon, BatchExportTag } from './components' import { LemonSkeleton } from 'lib/lemon-ui/LemonSkeleton' import { IconEllipsis, IconRefresh } from 'lib/lemon-ui/icons' @@ -18,6 +20,11 @@ import { NotFound } from 'lib/components/NotFound' import { LemonMenu } from 'lib/lemon-ui/LemonMenu' import { LemonDialog } from 'lib/lemon-ui/LemonDialog' import { Tooltip } from 'lib/lemon-ui/Tooltip' +import { LemonTabs } from 'lib/lemon-ui/LemonTabs' +import { ResizableTable, ResizableColumnType } from 'lib/components/ResizableTable' +import { dayjs } from 'lib/dayjs' +import { BatchExportLogEntryLevel, BatchExportLogEntry } from '~/types' +import { pluralize } from 'lib/utils' export const scene: SceneExport = { component: BatchExportScene, @@ -27,29 +34,338 @@ export const scene: SceneExport = { }), } -export function BatchExportScene(): JSX.Element { +export function RunsTab(): JSX.Element { const { batchExportRunsResponse, batchExportConfig, - batchExportConfigLoading, groupedRuns, batchExportRunsResponseLoading, runsDateRange, + batchExportConfigLoading, } = useValues(batchExportLogic) - const { - loadBatchExportConfig, - loadBatchExportRuns, - loadNextBatchExportRuns, - openBackfillModal, - setRunsDateRange, - retryRun, - pause, - unpause, - archive, - } = useActions(batchExportLogic) + const { loadNextBatchExportRuns, openBackfillModal, setRunsDateRange, retryRun } = useActions(batchExportLogic) const [dateRangeVisible, setDateRangeVisible] = useState(false) + if (!batchExportConfig && !batchExportConfigLoading) { + return + } + + return ( +
+
+
+

Latest Runs

+ { + setRunsDateRange({ from: start.startOf('day'), to: end.endOf('day') }) + setDateRangeVisible(false) + }} + onClose={function noRefCheck() { + setDateRangeVisible(false) + }} + /> + } + > + + {runsDateRange.from.format('MMMM D, YYYY')} - {runsDateRange.to.format('MMMM D, YYYY')} + + +
+ + + Load more button in the footer! + +
+ ) + } + expandable={{ + noIndent: true, + expandedRowRender: (groupedRuns) => { + return ( + , + }, + { + title: 'ID', + key: 'runId', + render: (_, run) => run.id, + }, + { + title: 'Run start', + key: 'runStart', + tooltip: 'Date and time when this BatchExport run started', + render: (_, run) => , + }, + ]} + /> + ) + }, + }} + columns={[ + { + key: 'icon', + width: 0, + render: (_, groupedRun) => { + return + }, + }, + + { + title: 'Data interval start', + key: 'dataIntervalStart', + tooltip: 'Start of the time range to export', + render: (_, run) => { + return ( + + ) + }, + }, + { + title: 'Data interval end', + key: 'dataIntervalEnd', + tooltip: 'End of the time range to export', + render: (_, run) => { + return ( + + ) + }, + }, + { + title: 'Latest run start', + key: 'runStart', + tooltip: 'Date and time when this BatchExport run started', + render: (_, groupedRun) => { + return + }, + }, + { + // title: 'Actions', + key: 'actions', + width: 0, + render: function RenderName(_, groupedRun) { + return ( + + {!isRunInProgress(groupedRun.runs[0]) && ( + } + onClick={() => + LemonDialog.open({ + title: 'Retry export?', + description: ( + <> +

+ This will schedule a new run for the same interval. + Any changes to the configuration will be applied to + the new run. +

+

+ Please note - there may be a slight delay + before the new run appears. +

+ + ), + width: '20rem', + primaryButton: { + children: 'Retry', + onClick: () => retryRun(groupedRun.runs[0]), + }, + secondaryButton: { + children: 'Cancel', + }, + }) + } + /> + )} + + ) + }, + }, + ]} + emptyState={ + <> + No runs yet. Your exporter runs every {batchExportConfig.interval}. +
+ openBackfillModal()}> + Create historic export + + + } + /> +
+ + ) +} + +function BatchExportLogEntryLevelDisplay(type: BatchExportLogEntryLevel): JSX.Element { + let color: string | undefined + switch (type) { + case BatchExportLogEntryLevel.Debug: + color = 'var(--muted)' + break + case BatchExportLogEntryLevel.Log: + color = 'var(--default)' + break + case BatchExportLogEntryLevel.Info: + color = 'var(--blue)' + break + case BatchExportLogEntryLevel.Warning: + color = 'var(--warning)' + break + case BatchExportLogEntryLevel.Error: + color = 'var(--danger)' + break + default: + break + } + return {type} +} + +const columns: ResizableColumnType[] = [ + { + title: 'Timestamp', + key: 'timestamp', + dataIndex: 'timestamp', + span: 3, + render: (timestamp: string) => dayjs(timestamp).format('YYYY-MM-DD HH:mm:ss.SSS UTC'), + }, + { + title: 'Level', + key: 'level', + dataIndex: 'level', + span: 1, + render: BatchExportLogEntryLevelDisplay, + } as ResizableColumnType, + { + title: 'Message', + key: 'message', + dataIndex: 'message', + span: 6, + } as ResizableColumnType, +] + +export function LogsTab({ batchExportId }: BatchExportLogsProps): JSX.Element { + const { activeTab, batchExportConfig, batchExportConfigLoading } = useValues(batchExportLogic) + + if (!batchExportConfig || batchExportConfigLoading || !activeTab) { + return + } + + const logic = batchExportLogsLogic({ batchExportId }) + const { + batchExportLogs, + batchExportLogsLoading, + batchExportLogsBackground, + isThereMoreToLoad, + batchExportLogsTypes, + } = useValues(logic) + const { revealBackground, loadBatchExportLogsMore, setBatchExportLogsTypes, setSearchTerm } = useActions(logic) + + return ( +
+ +
+ Show logs of type:  + +
+ + {batchExportLogsBackground.length + ? `Load ${pluralize(batchExportLogsBackground.length, 'newer entry', 'newer entries')}` + : 'No new entries'} + + + {!!batchExportLogs.length && ( + + {isThereMoreToLoad ? `Load up to ${LOGS_PORTION_LIMIT} older entries` : 'No older entries'} + + )} +
+ ) +} + +export function BatchExportScene(): JSX.Element { + const { batchExportConfig, batchExportConfigLoading, showTab, activeTab } = useValues(batchExportLogic) + const { loadBatchExportConfig, loadBatchExportRuns, openBackfillModal, pause, unpause, archive, setActiveTab } = + useActions(batchExportLogic) + useEffect(() => { loadBatchExportConfig() loadBatchExportRuns() @@ -181,194 +497,23 @@ export function BatchExportScene(): JSX.Element { )} - {batchExportConfig ? ( -
-
-
-

Latest Runs

- { - setRunsDateRange({ from: start.startOf('day'), to: end.endOf('day') }) - setDateRangeVisible(false) - }} - onClose={function noRefCheck() { - setDateRangeVisible(false) - }} - /> - } - > - - {runsDateRange.from.format('MMMM D, YYYY')} -{' '} - {runsDateRange.to.format('MMMM D, YYYY')} - - -
- - - Load more button in the footer! - -
- ) - } - expandable={{ - noIndent: true, - expandedRowRender: (groupedRuns) => { - return ( - , - }, - { - title: 'ID', - key: 'runId', - render: (_, run) => run.id, - }, - { - title: 'Run start', - key: 'runStart', - tooltip: 'Date and time when this BatchExport run started', - render: (_, run) => , - }, - ]} - /> - ) - }, - }} - columns={[ - { - key: 'icon', - width: 0, - render: (_, groupedRun) => { - return - }, - }, - - { - title: 'Data interval start', - key: 'dataIntervalStart', - tooltip: 'Start of the time range to export', - render: (_, run) => { - return ( - - ) - }, - }, - { - title: 'Data interval end', - key: 'dataIntervalEnd', - tooltip: 'End of the time range to export', - render: (_, run) => { - return ( - - ) - }, - }, - { - title: 'Latest run start', - key: 'runStart', - tooltip: 'Date and time when this BatchExport run started', - render: (_, groupedRun) => { - return - }, - }, - { - // title: 'Actions', - key: 'actions', - width: 0, - render: function RenderName(_, groupedRun) { - return ( - - {!isRunInProgress(groupedRun.runs[0]) && ( - } - onClick={() => - LemonDialog.open({ - title: 'Retry export?', - description: ( - <> -

- This will schedule a new run for the same - interval. Any changes to the configuration - will be applied to the new run. -

-

- Please note - there may be a slight - delay before the new run appears. -

- - ), - width: '20rem', - primaryButton: { - children: 'Retry', - onClick: () => retryRun(groupedRun.runs[0]), - }, - secondaryButton: { - children: 'Cancel', - }, - }) - } - /> - )} - - ) - }, - }, - ]} - emptyState={ - <> - No runs yet. Your exporter runs every {batchExportConfig.interval}. -
- openBackfillModal()}> - Create historic export - - - } - /> -
- + {batchExportConfig && activeTab ? ( + setActiveTab(newKey)} + tabs={[ + showTab(BatchExportTab.Runs) && { + key: BatchExportTab.Runs, + label: <>Latest runs, + content: , + }, + showTab(BatchExportTab.Logs) && { + key: BatchExportTab.Logs, + label: <>Logs, + content: , + }, + ]} + /> ) : null} diff --git a/frontend/src/scenes/batch_exports/batchExportLogic.ts b/frontend/src/scenes/batch_exports/batchExportLogic.ts index d8a07ffaacb2ec..24279747cff7b6 100644 --- a/frontend/src/scenes/batch_exports/batchExportLogic.ts +++ b/frontend/src/scenes/batch_exports/batchExportLogic.ts @@ -16,6 +16,11 @@ export type BatchExportLogicProps = { id: string } +export enum BatchExportTab { + Runs = 'runs', + Logs = 'logs', +} + // TODO: Fix this const RUNS_REFRESH_INTERVAL = 5000 @@ -58,9 +63,16 @@ export const batchExportLogic = kea([ closeBackfillModal: true, retryRun: (run: BatchExportRun) => ({ run }), setRunsDateRange: (data: { from: Dayjs; to: Dayjs }) => data, + setActiveTab: (tab: BatchExportTab) => ({ tab }), }), reducers({ + activeTab: [ + 'runs' as BatchExportTab | null, + { + setActiveTab: (_, { tab }) => tab, + }, + ], runsDateRange: [ { from: dayjs().subtract(7, 'day').startOf('day'), to: dayjs().endOf('day') } as { from: Dayjs; to: Dayjs }, { @@ -229,6 +241,15 @@ export const batchExportLogic = kea([ (s) => [s.batchExportRunsResponse], (batchExportRunsResponse): BatchExportRun[] => batchExportRunsResponse?.results ?? [], ], + + defaultTab: [(s) => [s.batchExportConfig], () => BatchExportTab.Runs], + + showTab: [ + () => [() => 1], + () => (): boolean => { + return true + }, + ], })), listeners(({ actions, cache, props }) => ({ diff --git a/frontend/src/scenes/batch_exports/batchExportLogsLogic.ts b/frontend/src/scenes/batch_exports/batchExportLogsLogic.ts new file mode 100644 index 00000000000000..5323175d4c6e3f --- /dev/null +++ b/frontend/src/scenes/batch_exports/batchExportLogsLogic.ts @@ -0,0 +1,179 @@ +import { kea } from 'kea' +import api from '~/lib/api' +import { BatchExportLogEntryLevel, BatchExportLogEntry } from '~/types' +import { CheckboxValueType } from 'antd/lib/checkbox/Group' +import { teamLogic } from 'scenes/teamLogic' + +import type { batchExportLogsLogicType } from './batchExportLogsLogicType' + +export interface BatchExportLogsProps { + batchExportId: string +} + +export const LOGS_PORTION_LIMIT = 50 + +export const batchExportLogsLogic = kea({ + path: (batchExportId) => ['scenes', 'batch_exports', 'batchExportLogsLogic', batchExportId], + props: {} as BatchExportLogsProps, + key: ({ batchExportId }: BatchExportLogsProps) => batchExportId, + + connect: { + values: [teamLogic, ['currentTeamId']], + }, + + actions: { + clearBatchExportLogsBackground: true, + markLogsEnd: true, + setBatchExportLogsTypes: (typeFilters: CheckboxValueType[]) => ({ + typeFilters, + }), + setSearchTerm: (searchTerm: string) => ({ searchTerm }), + }, + + loaders: ({ props: { batchExportId }, values, actions, cache }) => ({ + batchExportLogs: { + __default: [] as BatchExportLogEntry[], + loadBatchExportLogs: async () => { + const results = await api.batchExportLogs.search( + batchExportId, + values.currentTeamId, + values.searchTerm, + values.typeFilters + ) + + if (!cache.pollingInterval) { + cache.pollingInterval = setInterval(actions.loadBatchExportLogsBackgroundPoll, 2000) + } + actions.clearBatchExportLogsBackground() + return results + }, + loadBatchExportLogsMore: async () => { + const results = await api.batchExportLogs.search( + batchExportId, + values.currentTeamId, + values.searchTerm, + values.typeFilters, + values.trailingEntry + ) + + if (results.length < LOGS_PORTION_LIMIT) { + actions.markLogsEnd() + } + return [...values.batchExportLogs, ...results] + }, + revealBackground: () => { + const newArray = [...values.batchExportLogsBackground, ...values.batchExportLogs] + actions.clearBatchExportLogsBackground() + return newArray + }, + }, + batchExportLogsBackground: { + __default: [] as BatchExportLogEntry[], + loadBatchExportLogsBackgroundPoll: async () => { + if (values.batchExportLogsLoading) { + return values.batchExportLogsBackground + } + + const results = await api.batchExportLogs.search( + batchExportId, + values.currentTeamId, + values.searchTerm, + values.typeFilters, + null, + values.leadingEntry + ) + + return [...results, ...values.batchExportLogsBackground] + }, + }, + }), + + listeners: ({ actions }) => ({ + setBatchExportLogsTypes: () => { + actions.loadBatchExportLogs() + }, + setSearchTerm: async ({ searchTerm }, breakpoint) => { + if (searchTerm) { + await breakpoint(1000) + } + actions.loadBatchExportLogs() + }, + }), + + reducers: { + batchExportLogsTypes: [ + Object.values(BatchExportLogEntryLevel).filter((type) => type !== 'DEBUG'), + { + setBatchExportLogsTypes: (_, { typeFilters }) => + typeFilters.map((tf) => tf as BatchExportLogEntryLevel), + }, + ], + batchExportLogsBackground: [ + [] as BatchExportLogEntry[], + { + clearBatchExportLogsBackground: () => [], + }, + ], + searchTerm: [ + '', + { + setSearchTerm: (_, { searchTerm }) => searchTerm, + }, + ], + typeFilters: [ + Object.values(BatchExportLogEntryLevel).filter((type) => type !== 'DEBUG') as CheckboxValueType[], + { + setBatchExportLogsTypes: (_, { typeFilters }) => typeFilters || [], + }, + ], + isThereMoreToLoad: [ + true, + { + loadBatchExportLogsSuccess: (_, { batchExportLogs }) => batchExportLogs.length >= LOGS_PORTION_LIMIT, + markLogsEnd: () => false, + }, + ], + }, + + selectors: ({ selectors }) => ({ + leadingEntry: [ + () => [selectors.batchExportLogs, selectors.batchExportLogsBackground], + ( + batchExportLogs: BatchExportLogEntry[], + batchExportLogsBackground: BatchExportLogEntry[] + ): BatchExportLogEntry | null => { + if (batchExportLogsBackground.length) { + return batchExportLogsBackground[0] + } + if (batchExportLogs.length) { + return batchExportLogs[0] + } + return null + }, + ], + trailingEntry: [ + () => [selectors.batchExportLogs, selectors.batchExportLogsBackground], + ( + batchExportLogs: BatchExportLogEntry[], + batchExportLogsBackground: BatchExportLogEntry[] + ): BatchExportLogEntry | null => { + if (batchExportLogs.length) { + return batchExportLogs[batchExportLogs.length - 1] + } + if (batchExportLogsBackground.length) { + return batchExportLogsBackground[batchExportLogsBackground.length - 1] + } + return null + }, + ], + }), + + events: ({ actions, cache }) => ({ + afterMount: () => { + actions.loadBatchExportLogs() + }, + beforeUnmount: () => { + clearInterval(cache.pollingInterval) + }, + }), +}) diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 0d529adc829cda..f38ba8af34a5c5 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -1522,6 +1522,23 @@ export interface PluginLogEntry { instance_id: string } +export enum BatchExportLogEntryLevel { + Debug = 'DEBUG', + Log = 'LOG', + Info = 'INFO', + Warning = 'WARNING', + Error = 'ERROR', +} + +export interface BatchExportLogEntry { + team_id: number + batch_export_id: number + run_id: number + timestamp: string + level: BatchExportLogEntryLevel + message: string +} + export enum AnnotationScope { Insight = 'dashboard_item', Project = 'project',