diff --git a/frontend/__snapshots__/scenes-app-batchexports--view-export.png b/frontend/__snapshots__/scenes-app-batchexports--view-export.png index c8c9b7b1e73ba..be0cac9238f46 100644 Binary files a/frontend/__snapshots__/scenes-app-batchexports--view-export.png and b/frontend/__snapshots__/scenes-app-batchexports--view-export.png differ diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index e0331a81655af..86f09892d6082 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1,6 +1,7 @@ import posthog from 'posthog-js' import { ActionType, + BatchExportLogEntry, CohortType, DashboardCollaboratorType, DashboardTemplateEditorType, @@ -541,6 +542,10 @@ class ApiRequest { return this.batchExports(teamId).addPathComponent(id) } + public batchExportLogs(id: BatchExportConfiguration['id'], teamId?: TeamType['id']): ApiRequest { + return this.batchExport(id, teamId).addPathComponent('logs') + } + public batchExportRuns(id: BatchExportConfiguration['id'], teamId?: TeamType['id']): ApiRequest { return this.batchExports(teamId).addPathComponent(id).addPathComponent('runs') } @@ -553,6 +558,14 @@ class ApiRequest { return this.batchExportRuns(id, teamId).addPathComponent(runId) } + public batchExportRunLogs( + id: BatchExportConfiguration['id'], + runId: BatchExportRun['id'], + teamId?: TeamType['id'] + ): ApiRequest { + return this.batchExportRun(id, runId, teamId).addPathComponent('logs') + } + // Request finalization public async get(options?: ApiMethodOptions): Promise { return await api.get(this.assembleFullUrl(), options) @@ -1191,6 +1204,65 @@ const api = { }, }, + batchExportLogs: { + async search( + batchExportId: string, + currentTeamId: number | null, + searchTerm: string | null = null, + typeFilters: CheckboxValueType[] = [], + trailingEntry: BatchExportLogEntry | null = null, + leadingEntry: BatchExportLogEntry | null = null + ): Promise { + const params = toParams( + { + limit: LOGS_PORTION_LIMIT, + type_filter: typeFilters, + search: searchTerm || undefined, + before: trailingEntry?.timestamp, + after: leadingEntry?.timestamp, + }, + true + ) + + const response = await new ApiRequest() + .batchExportLogs(batchExportId, currentTeamId || undefined) + .withQueryString(params) + .get() + + return response.results + }, + }, + + batchExportRunLogs: { + async search( + batchExportId: string, + batchExportRunId: string, + currentTeamId: number | null, + searchTerm: string | null = null, + typeFilters: CheckboxValueType[] = [], + trailingEntry: BatchExportLogEntry | null = null, + leadingEntry: BatchExportLogEntry | null = null + ): Promise { + const params = toParams( + { + limit: LOGS_PORTION_LIMIT, + type_filter: typeFilters, + search: searchTerm || undefined, + before: trailingEntry?.timestamp, + after: leadingEntry?.timestamp, + }, + true + ) + + const response = await new ApiRequest() + .batchExportRunLogs(batchExportId, batchExportRunId, currentTeamId || undefined) + .withQueryString(params) + .get() + + return response.results + }, + }, + annotations: { async get(annotationId: RawAnnotationType['id']): Promise { return await new ApiRequest().annotation(annotationId).get() diff --git a/frontend/src/scenes/batch_exports/BatchExportScene.tsx b/frontend/src/scenes/batch_exports/BatchExportScene.tsx index 609d1103bec3c..9de5c77a43a03 100644 --- a/frontend/src/scenes/batch_exports/BatchExportScene.tsx +++ b/frontend/src/scenes/batch_exports/BatchExportScene.tsx @@ -1,10 +1,12 @@ +import { Checkbox } from 'antd' import { SceneExport } from 'scenes/sceneTypes' import { PageHeader } from 'lib/components/PageHeader' -import { LemonButton, LemonDivider, LemonTable, LemonTag } from '@posthog/lemon-ui' +import { LemonButton, LemonDivider, LemonTable, LemonTag, LemonInput, LemonTableColumns } from '@posthog/lemon-ui' import { urls } from 'scenes/urls' import { useActions, useValues } from 'kea' import { useEffect, useState } from 'react' -import { BatchExportLogicProps, batchExportLogic } from './batchExportLogic' +import { BatchExportLogicProps, batchExportLogic, BatchExportTab } from './batchExportLogic' +import { BatchExportLogsProps, batchExportLogsLogic, LOGS_PORTION_LIMIT } from './batchExportLogsLogic' import { BatchExportRunIcon, BatchExportTag } from './components' import { LemonSkeleton } from 'lib/lemon-ui/LemonSkeleton' import { IconEllipsis, IconRefresh } from 'lib/lemon-ui/icons' @@ -18,6 +20,10 @@ import { NotFound } from 'lib/components/NotFound' import { LemonMenu } from 'lib/lemon-ui/LemonMenu' import { LemonDialog } from 'lib/lemon-ui/LemonDialog' import { Tooltip } from 'lib/lemon-ui/Tooltip' +import { LemonTabs } from 'lib/lemon-ui/LemonTabs' +import { dayjs } from 'lib/dayjs' +import { BatchExportLogEntryLevel, BatchExportLogEntry } from '~/types' +import { pluralize } from 'lib/utils' export const scene: SceneExport = { component: BatchExportScene, @@ -27,165 +33,29 @@ export const scene: SceneExport = { }), } -export function BatchExportScene(): JSX.Element { +export function RunsTab(): JSX.Element { const { batchExportRunsResponse, batchExportConfig, - batchExportConfigLoading, groupedRuns, batchExportRunsResponseLoading, runsDateRange, + batchExportConfigLoading, } = useValues(batchExportLogic) - const { - loadBatchExportConfig, - loadBatchExportRuns, - loadNextBatchExportRuns, - openBackfillModal, - setRunsDateRange, - retryRun, - pause, - unpause, - archive, - } = useActions(batchExportLogic) + const { loadNextBatchExportRuns, openBackfillModal, setRunsDateRange, retryRun } = useActions(batchExportLogic) const [dateRangeVisible, setDateRangeVisible] = useState(false) - useEffect(() => { - loadBatchExportConfig() - loadBatchExportRuns() - }, []) - if (!batchExportConfig && !batchExportConfigLoading) { return } return ( <> - - {batchExportConfig?.name ?? (batchExportConfigLoading ? 'Loading...' : 'Missing')} - - } - buttons={ - batchExportConfig ? ( - <> - { - batchExportConfig.paused ? unpause() : pause() - }, - disabledReason: batchExportConfigLoading ? 'Loading...' : undefined, - }, - { - label: 'Archive', - status: 'danger', - onClick: () => - LemonDialog.open({ - title: 'Archive Batch Export?', - description: - 'Are you sure you want to archive this Batch Export? This will stop all future runs', - - primaryButton: { - children: 'Archive', - status: 'danger', - onClick: () => archive(), - }, - secondaryButton: { - children: 'Cancel', - }, - }), - disabledReason: batchExportConfigLoading ? 'Loading...' : undefined, - }, - ]} - > - } status="stealth" size="small" /> - - - openBackfillModal()}> - Create historic export - - - - Edit - - - ) : undefined - } - /> - -
- {batchExportConfig ? ( - <> -
- - - {capitalizeFirstLetter(intervalToFrequency(batchExportConfig.interval))} - - - - {batchExportConfig.end_at ? ( - <> - - Ends - - - ) : ( - 'Indefinite' - )} - - - -
    -
  • - Destination: - - {batchExportConfig.destination.type} - -
    • - - {Object.keys(batchExportConfig.destination.config) - .filter( - (x) => - ![ - 'password', - 'aws_secret_access_key', - 'client_email', - 'token_uri', - 'private_key', - 'private_key_id', - ].includes(x) - ) - .map((x) => ( -
  • - {identifierToHuman(x)}: - - {batchExportConfig.destination.config[x]} - -
    • - ))} -
- - } - > - {humanizeDestination(batchExportConfig.destination)} - -
- - ) : ( - - )} -
- {batchExportConfig ? ( -
+
-
-

Latest Runs

+
) : null} + + ) +} + +function BatchExportLogEntryLevelDisplay(type: BatchExportLogEntryLevel): JSX.Element { + let color: string | undefined + switch (type) { + case BatchExportLogEntryLevel.Debug: + color = 'var(--muted)' + break + case BatchExportLogEntryLevel.Log: + color = 'var(--default)' + break + case BatchExportLogEntryLevel.Info: + color = 'var(--blue)' + break + case BatchExportLogEntryLevel.Warning: + color = 'var(--warning)' + break + case BatchExportLogEntryLevel.Error: + color = 'var(--danger)' + break + default: + break + } + return {type} +} + +const columns: LemonTableColumns = [ + { + title: 'Timestamp', + key: 'timestamp', + dataIndex: 'timestamp', + width: 1, + render: (_, entry: BatchExportLogEntry) => dayjs(entry.timestamp).format('YYYY-MM-DD HH:mm:ss.SSS UTC'), + }, + { + title: 'Level', + key: 'level', + dataIndex: 'level', + width: 1, + render: (_, entry: BatchExportLogEntry) => BatchExportLogEntryLevelDisplay(entry.level), + }, + { + title: 'Run ID', + key: 'run_id', + dataIndex: 'run_id', + width: 1, + render: (_, entry) => entry.run_id, + }, + { + title: 'Message', + key: 'message', + dataIndex: 'message', + width: 6, + }, +] + +export function LogsTab({ batchExportId }: BatchExportLogsProps): JSX.Element { + const { activeTab, batchExportConfig, batchExportConfigLoading } = useValues(batchExportLogic) + + if (!batchExportConfig || batchExportConfigLoading || !activeTab) { + return + } + + const logic = batchExportLogsLogic({ batchExportId }) + const { + batchExportLogs, + batchExportLogsLoading, + batchExportLogsBackground, + isThereMoreToLoad, + batchExportLogsTypes, + } = useValues(logic) + const { revealBackground, loadBatchExportLogsMore, setBatchExportLogsTypes, setSearchTerm } = useActions(logic) + + return ( +
+ +
+ Show logs of type:  + +
+ + {batchExportLogsBackground.length + ? `Load ${pluralize(batchExportLogsBackground.length, 'newer entry', 'newer entries')}` + : 'No new entries'} + + + {!!batchExportLogs.length && ( + + {isThereMoreToLoad ? `Load up to ${LOGS_PORTION_LIMIT} older entries` : 'No older entries'} + + )} +
+ ) +} + +export function BatchExportScene(): JSX.Element { + const { batchExportConfig, batchExportConfigLoading, activeTab } = useValues(batchExportLogic) + const { loadBatchExportConfig, loadBatchExportRuns, openBackfillModal, pause, unpause, archive, setActiveTab } = + useActions(batchExportLogic) + + useEffect(() => { + loadBatchExportConfig() + loadBatchExportRuns() + }, []) + + if (!batchExportConfig && !batchExportConfigLoading) { + return + } + + return ( + <> + + {batchExportConfig?.name ?? (batchExportConfigLoading ? 'Loading...' : 'Missing')} + + } + buttons={ + batchExportConfig ? ( + <> + { + batchExportConfig.paused ? unpause() : pause() + }, + disabledReason: batchExportConfigLoading ? 'Loading...' : undefined, + }, + { + label: 'Archive', + status: 'danger', + onClick: () => + LemonDialog.open({ + title: 'Archive Batch Export?', + description: + 'Are you sure you want to archive this Batch Export? This will stop all future runs', + + primaryButton: { + children: 'Archive', + status: 'danger', + onClick: () => archive(), + }, + secondaryButton: { + children: 'Cancel', + }, + }), + disabledReason: batchExportConfigLoading ? 'Loading...' : undefined, + }, + ]} + > + } status="stealth" size="small" /> + + + openBackfillModal()}> + Create historic export + + + + Edit + + + ) : undefined + } + /> + +
+ {batchExportConfig ? ( + <> +
+ + + {capitalizeFirstLetter(intervalToFrequency(batchExportConfig.interval))} + + + + {batchExportConfig.end_at ? ( + <> + + Ends + + + ) : ( + 'Indefinite' + )} + + + +
    +
  • + Destination: + + {batchExportConfig.destination.type} + +
    • + + {Object.keys(batchExportConfig.destination.config) + .filter( + (x) => + ![ + 'password', + 'aws_secret_access_key', + 'client_email', + 'token_uri', + 'private_key', + 'private_key_id', + ].includes(x) + ) + .map((x) => ( +
  • + {identifierToHuman(x)}: + + {batchExportConfig.destination.config[x]} + +
    • + ))} +
+ + } + > + {humanizeDestination(batchExportConfig.destination)} + +
+ + ) : ( + + )} +
+ + {batchExportConfig && activeTab ? ( + setActiveTab(newKey)} + tabs={[ + { + key: BatchExportTab.Runs, + label: <>Latest runs, + content: , + }, + { + key: BatchExportTab.Logs, + label: <>Logs, + content: , + }, + ]} + /> + ) : null} diff --git a/frontend/src/scenes/batch_exports/batchExportLogic.ts b/frontend/src/scenes/batch_exports/batchExportLogic.ts index d8a07ffaacb2e..66cf9de3e2bfa 100644 --- a/frontend/src/scenes/batch_exports/batchExportLogic.ts +++ b/frontend/src/scenes/batch_exports/batchExportLogic.ts @@ -16,6 +16,11 @@ export type BatchExportLogicProps = { id: string } +export enum BatchExportTab { + Runs = 'runs', + Logs = 'logs', +} + // TODO: Fix this const RUNS_REFRESH_INTERVAL = 5000 @@ -58,9 +63,16 @@ export const batchExportLogic = kea([ closeBackfillModal: true, retryRun: (run: BatchExportRun) => ({ run }), setRunsDateRange: (data: { from: Dayjs; to: Dayjs }) => data, + setActiveTab: (tab: BatchExportTab) => ({ tab }), }), reducers({ + activeTab: [ + 'runs' as BatchExportTab | null, + { + setActiveTab: (_, { tab }) => tab, + }, + ], runsDateRange: [ { from: dayjs().subtract(7, 'day').startOf('day'), to: dayjs().endOf('day') } as { from: Dayjs; to: Dayjs }, { @@ -229,6 +241,8 @@ export const batchExportLogic = kea([ (s) => [s.batchExportRunsResponse], (batchExportRunsResponse): BatchExportRun[] => batchExportRunsResponse?.results ?? [], ], + + defaultTab: [(s) => [s.batchExportConfig], () => BatchExportTab.Runs], })), listeners(({ actions, cache, props }) => ({ diff --git a/frontend/src/scenes/batch_exports/batchExportLogsLogic.ts b/frontend/src/scenes/batch_exports/batchExportLogsLogic.ts new file mode 100644 index 0000000000000..5323175d4c6e3 --- /dev/null +++ b/frontend/src/scenes/batch_exports/batchExportLogsLogic.ts @@ -0,0 +1,179 @@ +import { kea } from 'kea' +import api from '~/lib/api' +import { BatchExportLogEntryLevel, BatchExportLogEntry } from '~/types' +import { CheckboxValueType } from 'antd/lib/checkbox/Group' +import { teamLogic } from 'scenes/teamLogic' + +import type { batchExportLogsLogicType } from './batchExportLogsLogicType' + +export interface BatchExportLogsProps { + batchExportId: string +} + +export const LOGS_PORTION_LIMIT = 50 + +export const batchExportLogsLogic = kea({ + path: (batchExportId) => ['scenes', 'batch_exports', 'batchExportLogsLogic', batchExportId], + props: {} as BatchExportLogsProps, + key: ({ batchExportId }: BatchExportLogsProps) => batchExportId, + + connect: { + values: [teamLogic, ['currentTeamId']], + }, + + actions: { + clearBatchExportLogsBackground: true, + markLogsEnd: true, + setBatchExportLogsTypes: (typeFilters: CheckboxValueType[]) => ({ + typeFilters, + }), + setSearchTerm: (searchTerm: string) => ({ searchTerm }), + }, + + loaders: ({ props: { batchExportId }, values, actions, cache }) => ({ + batchExportLogs: { + __default: [] as BatchExportLogEntry[], + loadBatchExportLogs: async () => { + const results = await api.batchExportLogs.search( + batchExportId, + values.currentTeamId, + values.searchTerm, + values.typeFilters + ) + + if (!cache.pollingInterval) { + cache.pollingInterval = setInterval(actions.loadBatchExportLogsBackgroundPoll, 2000) + } + actions.clearBatchExportLogsBackground() + return results + }, + loadBatchExportLogsMore: async () => { + const results = await api.batchExportLogs.search( + batchExportId, + values.currentTeamId, + values.searchTerm, + values.typeFilters, + values.trailingEntry + ) + + if (results.length < LOGS_PORTION_LIMIT) { + actions.markLogsEnd() + } + return [...values.batchExportLogs, ...results] + }, + revealBackground: () => { + const newArray = [...values.batchExportLogsBackground, ...values.batchExportLogs] + actions.clearBatchExportLogsBackground() + return newArray + }, + }, + batchExportLogsBackground: { + __default: [] as BatchExportLogEntry[], + loadBatchExportLogsBackgroundPoll: async () => { + if (values.batchExportLogsLoading) { + return values.batchExportLogsBackground + } + + const results = await api.batchExportLogs.search( + batchExportId, + values.currentTeamId, + values.searchTerm, + values.typeFilters, + null, + values.leadingEntry + ) + + return [...results, ...values.batchExportLogsBackground] + }, + }, + }), + + listeners: ({ actions }) => ({ + setBatchExportLogsTypes: () => { + actions.loadBatchExportLogs() + }, + setSearchTerm: async ({ searchTerm }, breakpoint) => { + if (searchTerm) { + await breakpoint(1000) + } + actions.loadBatchExportLogs() + }, + }), + + reducers: { + batchExportLogsTypes: [ + Object.values(BatchExportLogEntryLevel).filter((type) => type !== 'DEBUG'), + { + setBatchExportLogsTypes: (_, { typeFilters }) => + typeFilters.map((tf) => tf as BatchExportLogEntryLevel), + }, + ], + batchExportLogsBackground: [ + [] as BatchExportLogEntry[], + { + clearBatchExportLogsBackground: () => [], + }, + ], + searchTerm: [ + '', + { + setSearchTerm: (_, { searchTerm }) => searchTerm, + }, + ], + typeFilters: [ + Object.values(BatchExportLogEntryLevel).filter((type) => type !== 'DEBUG') as CheckboxValueType[], + { + setBatchExportLogsTypes: (_, { typeFilters }) => typeFilters || [], + }, + ], + isThereMoreToLoad: [ + true, + { + loadBatchExportLogsSuccess: (_, { batchExportLogs }) => batchExportLogs.length >= LOGS_PORTION_LIMIT, + markLogsEnd: () => false, + }, + ], + }, + + selectors: ({ selectors }) => ({ + leadingEntry: [ + () => [selectors.batchExportLogs, selectors.batchExportLogsBackground], + ( + batchExportLogs: BatchExportLogEntry[], + batchExportLogsBackground: BatchExportLogEntry[] + ): BatchExportLogEntry | null => { + if (batchExportLogsBackground.length) { + return batchExportLogsBackground[0] + } + if (batchExportLogs.length) { + return batchExportLogs[0] + } + return null + }, + ], + trailingEntry: [ + () => [selectors.batchExportLogs, selectors.batchExportLogsBackground], + ( + batchExportLogs: BatchExportLogEntry[], + batchExportLogsBackground: BatchExportLogEntry[] + ): BatchExportLogEntry | null => { + if (batchExportLogs.length) { + return batchExportLogs[batchExportLogs.length - 1] + } + if (batchExportLogsBackground.length) { + return batchExportLogsBackground[batchExportLogsBackground.length - 1] + } + return null + }, + ], + }), + + events: ({ actions, cache }) => ({ + afterMount: () => { + actions.loadBatchExportLogs() + }, + beforeUnmount: () => { + clearInterval(cache.pollingInterval) + }, + }), +}) diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 8a038927dc90e..c8d5298492603 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -1522,6 +1522,23 @@ export interface PluginLogEntry { instance_id: string } +export enum BatchExportLogEntryLevel { + Debug = 'DEBUG', + Log = 'LOG', + Info = 'INFO', + Warning = 'WARNING', + Error = 'ERROR', +} + +export interface BatchExportLogEntry { + team_id: number + batch_export_id: number + run_id: number + timestamp: string + level: BatchExportLogEntryLevel + message: string +} + export enum AnnotationScope { Insight = 'dashboard_item', Project = 'project', diff --git a/playwright/e2e-vrt/layout/Navigation.spec.ts-snapshots/Navigation-App-Page-With-Side-Bar-Hidden-Mobile-1-chromium-linux.png b/playwright/e2e-vrt/layout/Navigation.spec.ts-snapshots/Navigation-App-Page-With-Side-Bar-Hidden-Mobile-1-chromium-linux.png index 1be473f7dba29..47e977ba601e4 100644 Binary files a/playwright/e2e-vrt/layout/Navigation.spec.ts-snapshots/Navigation-App-Page-With-Side-Bar-Hidden-Mobile-1-chromium-linux.png and b/playwright/e2e-vrt/layout/Navigation.spec.ts-snapshots/Navigation-App-Page-With-Side-Bar-Hidden-Mobile-1-chromium-linux.png differ