From 4302503d470d5f307d47e5c51f3fb02580e4ee41 Mon Sep 17 00:00:00 2001 From: Eric Duong Date: Tue, 16 Jul 2024 14:40:04 -0400 Subject: [PATCH] feat(data-warehouse): log entries by schema (#23686) Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- frontend/src/lib/api.ts | 10 ++ .../DataWarehouseSourceSettingsScene.tsx | 2 +- .../data-warehouse/settings/source/Logs.tsx | 86 +++++++++++ .../data-warehouse/settings/source/Syncs.tsx | 14 ++ .../settings/source/schemaLogLogic.ts | 144 ++++++++++++++++++ 5 files changed, 255 insertions(+), 1 deletion(-) create mode 100644 frontend/src/scenes/data-warehouse/settings/source/Logs.tsx create mode 100644 frontend/src/scenes/data-warehouse/settings/source/schemaLogLogic.ts diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 2a085ddd7ece2..5ff15cd185f89 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -2044,6 +2044,16 @@ const api = { async incremental_fields(schemaId: ExternalDataSourceSchema['id']): Promise { return await new ApiRequest().externalDataSourceSchema(schemaId).withAction('incremental_fields').create() }, + async logs( + schemaId: ExternalDataSourceSchema['id'], + params: LogEntryRequestParams = {} + ): Promise> { + return await new ApiRequest() + .externalDataSourceSchema(schemaId) + .withAction('logs') + .withQueryString(params) + .get() + }, }, dataWarehouseViewLinks: { diff --git a/frontend/src/scenes/data-warehouse/settings/source/DataWarehouseSourceSettingsScene.tsx b/frontend/src/scenes/data-warehouse/settings/source/DataWarehouseSourceSettingsScene.tsx index f3635b8df718f..b40d298e5abd2 100644 --- a/frontend/src/scenes/data-warehouse/settings/source/DataWarehouseSourceSettingsScene.tsx +++ b/frontend/src/scenes/data-warehouse/settings/source/DataWarehouseSourceSettingsScene.tsx @@ -62,7 +62,7 @@ export function DataWarehouseSourceSettingsScene(): JSX.Element { activeKey={currentTab} onChange={(tab) => setCurrentTab(tab as DataWarehouseSourceSettingsTabs)} tabs={Object.entries(TabContent).map(([tab, ContentComponent]) => ({ - label: FriendlyTabNames[tab], + label: FriendlyTabNames[tab as DataWarehouseSourceSettingsTabs], key: tab, content: , }))} diff --git a/frontend/src/scenes/data-warehouse/settings/source/Logs.tsx b/frontend/src/scenes/data-warehouse/settings/source/Logs.tsx new file mode 100644 index 0000000000000..7d9bfe982901f --- /dev/null +++ b/frontend/src/scenes/data-warehouse/settings/source/Logs.tsx @@ -0,0 +1,86 @@ +import { LemonButton, LemonTable, LemonTableColumns } from '@posthog/lemon-ui' +import { useActions, useValues } from 'kea' +import { LOGS_PORTION_LIMIT } from 'lib/constants' +import { dayjs } from 'lib/dayjs' +import { pluralize } from 'lib/utils' +import { LogLevelDisplay } from 'scenes/pipeline/utils' + +import { ExternalDataJob, LogEntry } from '~/types' + +import { schemaLogLogic } from './schemaLogLogic' + +const columns: LemonTableColumns = [ + { + title: 'Timestamp', + key: 'timestamp', + dataIndex: 'timestamp', + width: 1, + render: (_, entry) => dayjs(entry.timestamp).format('YYYY-MM-DD HH:mm:ss.SSS UTC'), + }, + { + title: 'Level', + key: 'level', + dataIndex: 'level', + width: 1, + render: (_, entry) => LogLevelDisplay(entry.level), + }, + { + title: 'Run ID', + key: 'run_id', + dataIndex: 'instance_id', + width: 1, + render: (_, entry) => entry.instance_id, + }, + { + title: 'Message', + key: 'message', + dataIndex: 'message', + width: 6, + }, +] + +interface LogsTableProps { + job: ExternalDataJob +} + +export const LogsView = ({ job }: LogsTableProps): JSX.Element => { + const logic = schemaLogLogic({ job }) + const { logs, logsLoading, logsBackground, isThereMoreToLoad } = useValues(logic) + const { revealBackground, loadSchemaLogsMore } = useActions(logic) + + return ( +
+ + {logsBackground.length + ? `Load ${pluralize(logsBackground.length, 'newer entry', 'newer entries')}` + : 'No new entries'} + + + {!!logs.length && ( + + {isThereMoreToLoad ? `Load up to ${LOGS_PORTION_LIMIT} older entries` : 'No older entries'} + + )} +
+ ) +} diff --git a/frontend/src/scenes/data-warehouse/settings/source/Syncs.tsx b/frontend/src/scenes/data-warehouse/settings/source/Syncs.tsx index c53f1a3d4c096..a8568e478bbc7 100644 --- a/frontend/src/scenes/data-warehouse/settings/source/Syncs.tsx +++ b/frontend/src/scenes/data-warehouse/settings/source/Syncs.tsx @@ -5,6 +5,7 @@ import { useValues } from 'kea' import { ExternalDataJob } from '~/types' import { dataWarehouseSourceSettingsLogic } from './dataWarehouseSourceSettingsLogic' +import { LogsView } from './Logs' const StatusTagSetting: Record = { Running: 'primary', @@ -46,6 +47,19 @@ export const Syncs = (): JSX.Element => { }, }, ]} + expandable={ + jobs.length > 0 + ? { + expandedRowRender: (job) => ( +
+ +
+ ), + rowExpandable: () => true, + noIndent: true, + } + : undefined + } /> ) } diff --git a/frontend/src/scenes/data-warehouse/settings/source/schemaLogLogic.ts b/frontend/src/scenes/data-warehouse/settings/source/schemaLogLogic.ts new file mode 100644 index 0000000000000..79a6470d1e5e4 --- /dev/null +++ b/frontend/src/scenes/data-warehouse/settings/source/schemaLogLogic.ts @@ -0,0 +1,144 @@ +import { actions, events, kea, key, listeners, path, props, reducers, selectors } from 'kea' +import { loaders } from 'kea-loaders' +import api from 'lib/api' +import { LOGS_PORTION_LIMIT } from 'lib/constants' + +import { ExternalDataJob, ExternalDataSourceSchema, LogEntry, LogEntryLevel } from '~/types' + +import type { schemaLogLogicType } from './schemaLogLogicType' + +export const ALL_LOG_LEVELS: LogEntryLevel[] = ['DEBUG', 'LOG', 'INFO', 'WARNING', 'ERROR'] +export const DEFAULT_LOG_LEVELS: LogEntryLevel[] = ['DEBUG', 'LOG', 'INFO', 'WARNING', 'ERROR'] + +export interface SchemaLogLogicProps { + job: ExternalDataJob +} + +export const schemaLogLogic = kea([ + path(['scenes', 'data-warehouse', 'settings', 'source', 'schemaLogLogic']), + props({} as SchemaLogLogicProps), + key(({ job }) => job.id), + actions({ + clearBackgroundLogs: true, + setLogLevelFilters: (levelFilters: LogEntryLevel[]) => ({ levelFilters }), + setSearchTerm: (searchTerm: string) => ({ searchTerm }), + setSchema: (schemaId: ExternalDataSourceSchema['id']) => ({ schemaId }), + markLogsEnd: true, + }), + loaders(({ values, actions, cache, props }) => ({ + logs: { + __default: [] as LogEntry[], + loadSchemaLogs: async () => { + const response = await api.externalDataSchemas.logs(props.job.schema.id, { + level: values.levelFilters.join(','), + search: values.searchTerm, + instance_id: props.job.id, + }) + + if (!cache.pollingInterval) { + cache.pollingInterval = setInterval(actions.loadSchemaLogsBackgroundPoll, 2000) + } + + return response.results + }, + loadSchemaLogsMore: async () => { + if (!values.selectedSchemaId) { + return [] + } + const response = await api.externalDataSchemas.logs(values.selectedSchemaId, { + level: values.levelFilters.join(','), + search: values.searchTerm, + instance_id: props.job.id, + before: values.leadingEntry?.timestamp, + }) + + if (response.results.length < LOGS_PORTION_LIMIT) { + actions.markLogsEnd() + } + + return [...values.logs, ...response.results] + }, + revealBackground: () => { + const newArray = [...values.logsBackground, ...values.logs] + actions.clearBackgroundLogs() + return newArray + }, + }, + logsBackground: { + __default: [] as LogEntry[], + loadSchemaLogsBackgroundPoll: async () => { + const response = await api.externalDataSchemas.logs(props.job.schema.id, { + level: values.levelFilters.join(','), + search: values.searchTerm, + instance_id: props.job.id, + after: values.leadingEntry?.timestamp, + }) + + return [...response.results, ...values.logsBackground] + }, + }, + })), + reducers({ + levelFilters: [ + DEFAULT_LOG_LEVELS, + { + setLogLevelFilters: (_, { levelFilters }) => levelFilters, + }, + ], + searchTerm: [ + '', + { + setSearchTerm: (_, { searchTerm }) => searchTerm, + }, + ], + selectedSchemaId: [ + null as string | null, + { + setSchema: (_, { schemaId }) => schemaId, + }, + ], + isThereMoreToLoad: [ + true, + { + loadSchemaLogsSuccess: (_, { logs }) => logs.length >= LOGS_PORTION_LIMIT, + markLogsEnd: () => false, + }, + ], + }), + selectors({ + leadingEntry: [ + (s) => [s.logs, s.logsBackground], + (logs, logsBackground): LogEntry | null => { + if (logsBackground.length) { + return logsBackground[0] + } + if (logs.length) { + return logs[0] + } + return null + }, + ], + }), + listeners(({ actions }) => ({ + setLogLevelFilters: () => { + actions.loadSchemaLogs() + }, + setSearchTerm: async ({ searchTerm }, breakpoint) => { + if (searchTerm) { + await breakpoint(1000) + } + actions.loadSchemaLogs() + }, + setSchema: () => { + actions.loadSchemaLogs() + }, + })), + events(({ actions, cache }) => ({ + afterMount: () => { + actions.loadSchemaLogs() + }, + beforeUnmount: () => { + clearInterval(cache.pollingInterval) + }, + })), +])