From ddc07c53a541e1e322ec238c554635f905ccc38a Mon Sep 17 00:00:00 2001 From: mohamedhamed-ahmed Date: Mon, 13 Nov 2023 13:51:33 +0000 Subject: [PATCH] [Log Stream] Support date nanos (#170308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closes #88290 ## 📝 Summary As described in #88290 we need to add `date_nanos` support to the Stream UI page. In this PR the necessary changes have been made all over the Stream UI and the usages of it. ## ✅ Testing ⚠️ Testing the Stream UI with old timestamp indices is important to make sure that the behavior is still as before and not affected at all. This can be done by running local env from the PR and simulating all interactions on edge-lit cluster for example, to make sure that the behavior is not changed. For testing the new changes with support of `date_nano`: 1. You can use [the steps here](https://github.com/elastic/kibana/issues/88290#issuecomment-1713400873) to create and ingest documents with nano precision. 2. Navigate to the stream UI and the documents should be displayed properly. 3. Sync with the URL state should be changed from timestamp to ISO string date. 4. Changing time ranges should behave as before, as well as Text Highlights. 5. Open the logs flyout and you should see the timestamp in nano seconds. 6. Play around with the minimap, it should behave exactly as before. ### Stream UI: Screenshot 2023-11-02 at 14 15 49 - The stream UI has been affected in many areas: - The logPosition key in the URL should now be in ISO string, but still backward compatible incase the user has bookmarks with timestamps. - The minimap should still behave as before in terms of navigation onClick and also highlighting displayed areas ### Stream UI Flyout: Screenshot 2023-11-02 at 14 15 23 - The logs flyout should now display the date in nanos format if the document is ingested using a nano format. ### Anomalies: Screenshot 2023-11-01 at 10 37 22 -Anomalies tab under logs as a navigation to stream UI which should still behave as before passing the filtration and time. ### Categories: Screenshot 2023-11-01 at 10 38 19 -Categories tab under logs as a navigation to stream UI which should still behave as before passing the filtration and time. ### External Links To Stream: - All links to the Stream UI should still work as before: - APM Links for traces, containers, hosts - Infra links in Inventory and Hosts views ## 🎥 Demo https://github.com/elastic/kibana/assets/11225826/9a39bc5a-ba37-49e0-b7f2-e73260fb01f0 --- packages/kbn-io-ts-utils/index.ts | 8 + .../kbn-io-ts-utils/src/time_key_rt/index.ts | 53 +++ .../infra/common/locators/locators.test.ts | 2 +- x-pack/plugins/infra/common/time/time_key.ts | 3 +- .../infra/common/url_state_storage_service.ts | 6 +- .../logging/log_minimap/log_minimap.tsx | 7 +- .../log_entry_categories/module_descriptor.ts | 2 +- .../log_entry_rate/module_descriptor.ts | 2 +- .../log_stream_page/state/src/types.ts | 2 +- .../src/state_machine.ts | 16 +- .../src/url_state_storage_service.ts | 11 - .../src/url_state_storage_service.ts | 5 +- .../category_example_message.tsx | 7 +- .../sections/anomalies/log_entry_example.tsx | 2 +- .../public/pages/logs/stream/page_content.tsx | 2 +- .../pages/logs/stream/page_logs_content.tsx | 10 +- .../infra/public/test_utils/entries.ts | 6 +- x-pack/plugins/logs_shared/common/index.ts | 2 + .../logs_shared/common/log_entry/log_entry.ts | 4 +- .../common/log_entry/log_entry_cursor.ts | 4 +- .../logs_shared/common/time/time_key.ts | 20 +- .../common/utils/date_helpers.test.ts | 45 ++ .../logs_shared/common/utils/date_helpers.ts | 29 ++ .../plugins/logs_shared/common/utils/index.ts | 8 + .../public/components/formatted_time.tsx | 4 +- .../log_stream/log_stream.stories.tsx | 2 +- .../components/log_stream/log_stream.tsx | 2 +- .../log_entry_actions_menu.test.tsx | 17 +- .../log_entry_fields_table.tsx | 2 +- .../log_entry_flyout/log_entry_flyout.tsx | 2 +- .../log_text_stream/column_headers.tsx | 2 +- .../logging/log_text_stream/item.ts | 9 +- .../logging/log_text_stream/log_date_row.tsx | 6 +- .../log_text_stream/log_entry_column.tsx | 3 +- .../logging/log_text_stream/log_entry_row.tsx | 2 +- .../log_entry_timestamp_column.tsx | 4 +- .../scrollable_log_text_stream_view.tsx | 17 +- .../log_highlights/log_entry_highlights.tsx | 2 +- .../logs/log_highlights/log_highlights.tsx | 3 +- .../logs/log_highlights/next_and_previous.tsx | 3 +- .../logs/log_position/use_log_position.ts | 6 +- .../use_fetch_log_entries_around.ts | 3 +- .../logs_shared/public/test_utils/entries.ts | 6 +- .../utils/log_column_render_configuration.tsx | 2 +- .../public/utils/log_entry/log_entry.ts | 5 +- .../log_entries/kibana_log_entries_adapter.ts | 21 +- .../log_entries_domain/log_entries_domain.ts | 5 +- .../log_entries_search_strategy.test.ts | 9 +- .../log_entries_search_strategy.ts | 2 +- .../log_entry_search_strategy.test.ts | 9 +- .../services/log_entries/queries/common.ts | 6 +- .../log_entries/queries/log_entries.ts | 4 +- .../services/log_entries/queries/log_entry.ts | 13 +- .../apis/metrics_ui/log_entry_highlights.ts | 5 +- x-pack/test/functional/apps/infra/index.ts | 1 + x-pack/test/functional/apps/infra/link_to.ts | 7 +- .../apps/infra/log_stream_date_nano.ts | 98 ++++ .../infra/logs_with_nano_date/data.json.gz | Bin 0 -> 338 bytes .../infra/logs_with_nano_date/mappings.json | 419 ++++++++++++++++++ .../functional/services/logs_ui/log_stream.ts | 26 ++ 60 files changed, 850 insertions(+), 133 deletions(-) create mode 100644 packages/kbn-io-ts-utils/src/time_key_rt/index.ts create mode 100644 x-pack/plugins/logs_shared/common/utils/date_helpers.test.ts create mode 100644 x-pack/plugins/logs_shared/common/utils/date_helpers.ts create mode 100644 x-pack/plugins/logs_shared/common/utils/index.ts create mode 100644 x-pack/test/functional/apps/infra/log_stream_date_nano.ts create mode 100644 x-pack/test/functional/es_archives/infra/logs_with_nano_date/data.json.gz create mode 100644 x-pack/test/functional/es_archives/infra/logs_with_nano_date/mappings.json diff --git a/packages/kbn-io-ts-utils/index.ts b/packages/kbn-io-ts-utils/index.ts index 57d9e7bb47713..8f1b974eccdcb 100644 --- a/packages/kbn-io-ts-utils/index.ts +++ b/packages/kbn-io-ts-utils/index.ts @@ -32,3 +32,11 @@ export { export { datemathStringRt } from './src/datemath_string_rt'; export { createPlainError, decodeOrThrow, formatErrors, throwErrors } from './src/decode_or_throw'; + +export { + DateFromStringOrNumber, + minimalTimeKeyRT, + type MinimalTimeKey, + type TimeKey, + type UniqueTimeKey, +} from './src/time_key_rt'; diff --git a/packages/kbn-io-ts-utils/src/time_key_rt/index.ts b/packages/kbn-io-ts-utils/src/time_key_rt/index.ts new file mode 100644 index 0000000000000..1884ea5a52e83 --- /dev/null +++ b/packages/kbn-io-ts-utils/src/time_key_rt/index.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as rt from 'io-ts'; +import moment from 'moment'; +import { pipe } from 'fp-ts/lib/pipeable'; +import { chain } from 'fp-ts/lib/Either'; + +const NANO_DATE_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3,9}Z$/; + +export const DateFromStringOrNumber = new rt.Type( + 'DateFromStringOrNumber', + (input): input is string => typeof input === 'string', + (input, context) => { + if (typeof input === 'string') { + return NANO_DATE_PATTERN.test(input) ? rt.success(input) : rt.failure(input, context); + } + return pipe( + rt.number.validate(input, context), + chain((timestamp) => { + const momentValue = moment(timestamp); + return momentValue.isValid() + ? rt.success(momentValue.toISOString()) + : rt.failure(timestamp, context); + }) + ); + }, + String +); + +export const minimalTimeKeyRT = rt.type({ + time: DateFromStringOrNumber, + tiebreaker: rt.number, +}); +export type MinimalTimeKey = rt.TypeOf; + +const timeKeyRT = rt.intersection([ + minimalTimeKeyRT, + rt.partial({ + gid: rt.string, + fromAutoReload: rt.boolean, + }), +]); +export type TimeKey = rt.TypeOf; + +export interface UniqueTimeKey extends TimeKey { + gid: string; +} diff --git a/x-pack/plugins/infra/common/locators/locators.test.ts b/x-pack/plugins/infra/common/locators/locators.test.ts index fb6f8283b63fd..607fd41b1bab6 100644 --- a/x-pack/plugins/infra/common/locators/locators.test.ts +++ b/x-pack/plugins/infra/common/locators/locators.test.ts @@ -233,7 +233,7 @@ const constructLogView = (logView?: LogViewReference) => { }; const constructLogPosition = (time: number = 1550671089404) => { - return `(position:(tiebreaker:0,time:${time}))`; + return `(position:(tiebreaker:0,time:'${moment(time).toISOString()}'))`; }; const constructLogFilter = ({ diff --git a/x-pack/plugins/infra/common/time/time_key.ts b/x-pack/plugins/infra/common/time/time_key.ts index efc5a8e7b8517..7acbef8b25025 100644 --- a/x-pack/plugins/infra/common/time/time_key.ts +++ b/x-pack/plugins/infra/common/time/time_key.ts @@ -5,12 +5,13 @@ * 2.0. */ +import { DateFromStringOrNumber } from '@kbn/io-ts-utils'; import { ascending, bisector } from 'd3-array'; import * as rt from 'io-ts'; import { pick } from 'lodash'; export const minimalTimeKeyRT = rt.type({ - time: rt.number, + time: DateFromStringOrNumber, tiebreaker: rt.number, }); export type MinimalTimeKey = rt.TypeOf; diff --git a/x-pack/plugins/infra/common/url_state_storage_service.ts b/x-pack/plugins/infra/common/url_state_storage_service.ts index 5d701ad1876bb..ece3d1ccb09bf 100644 --- a/x-pack/plugins/infra/common/url_state_storage_service.ts +++ b/x-pack/plugins/infra/common/url_state_storage_service.ts @@ -24,18 +24,18 @@ export const defaultLogViewKey = 'logView'; const encodeRisonUrlState = (state: any) => encode(state); -// Used by linkTo components +// Used by Locator components export const replaceLogPositionInQueryString = (time?: number) => Number.isNaN(time) || time == null ? (value: string) => value : replaceStateKeyInQueryString(defaultPositionStateKey, { position: { - time, + time: moment(time).toISOString(), tiebreaker: 0, }, }); -// NOTE: Used by link-to components +// NOTE: Used by Locator components export const replaceLogViewInQueryString = (logViewReference: LogViewReference) => replaceStateKeyInQueryString(defaultLogViewKey, logViewReference); diff --git a/x-pack/plugins/infra/public/components/logging/log_minimap/log_minimap.tsx b/x-pack/plugins/infra/public/components/logging/log_minimap/log_minimap.tsx index df1591a96afa9..00c6bcc4a623f 100644 --- a/x-pack/plugins/infra/public/components/logging/log_minimap/log_minimap.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_minimap/log_minimap.tsx @@ -12,6 +12,7 @@ import { LogEntryTime, } from '@kbn/logs-shared-plugin/common'; import { scaleLinear } from 'd3-scale'; +import moment from 'moment'; import * as React from 'react'; import { DensityChart } from './density_chart'; import { HighlightedInterval } from './highlighted_interval'; @@ -67,7 +68,7 @@ export class LogMinimap extends React.Component) => void; jumpToTargetPosition: (targetPosition: TimeKey | null) => void; - jumpToTargetPositionTime: (time: number) => void; + jumpToTargetPositionTime: (time: string) => void; reportVisiblePositions: (visiblePositions: VisiblePositions) => void; startLiveStreaming: () => void; stopLiveStreaming: () => void; diff --git a/x-pack/plugins/infra/public/observability_logs/log_stream_position_state/src/state_machine.ts b/x-pack/plugins/infra/public/observability_logs/log_stream_position_state/src/state_machine.ts index 96b7dbb687851..868f29a5c07e9 100644 --- a/x-pack/plugins/infra/public/observability_logs/log_stream_position_state/src/state_machine.ts +++ b/x-pack/plugins/infra/public/observability_logs/log_stream_position_state/src/state_machine.ts @@ -7,6 +7,8 @@ import { IToasts } from '@kbn/core-notifications-browser'; import { IKbnUrlStateStorage } from '@kbn/kibana-utils-plugin/public'; +import { convertISODateToNanoPrecision } from '@kbn/logs-shared-plugin/common'; +import moment from 'moment'; import { actions, ActorRefFrom, createMachine, EmittedFrom, SpecialTargets } from 'xstate'; import { isSameTimeKey } from '../../../../common/time'; import { OmitDeprecatedState, sendIfDefined } from '../../xstate_helpers'; @@ -159,11 +161,19 @@ export const createPureLogStreamPositionStateMachine = (initialContext: LogStrea updatePositionsFromTimeChange: actions.assign((_context, event) => { if (!('timeRange' in event)) return {}; + const { + timestamps: { startTimestamp, endTimestamp }, + } = event; + // Reset the target position if it doesn't fall within the new range. + const targetPositionNanoTime = + _context.targetPosition && convertISODateToNanoPrecision(_context.targetPosition.time); + const startNanoDate = convertISODateToNanoPrecision(moment(startTimestamp).toISOString()); + const endNanoDate = convertISODateToNanoPrecision(moment(endTimestamp).toISOString()); + const targetPositionShouldReset = - _context.targetPosition && - (event.timestamps.startTimestamp > _context.targetPosition.time || - event.timestamps.endTimestamp < _context.targetPosition.time); + targetPositionNanoTime && + (startNanoDate > targetPositionNanoTime || endNanoDate < targetPositionNanoTime); return { targetPosition: targetPositionShouldReset ? null : _context.targetPosition, diff --git a/x-pack/plugins/infra/public/observability_logs/log_stream_position_state/src/url_state_storage_service.ts b/x-pack/plugins/infra/public/observability_logs/log_stream_position_state/src/url_state_storage_service.ts index c98ab9e147444..5607bf9207054 100644 --- a/x-pack/plugins/infra/public/observability_logs/log_stream_position_state/src/url_state_storage_service.ts +++ b/x-pack/plugins/infra/public/observability_logs/log_stream_position_state/src/url_state_storage_service.ts @@ -10,7 +10,6 @@ import { IKbnUrlStateStorage, withNotifyOnErrors } from '@kbn/kibana-utils-plugi import * as Either from 'fp-ts/lib/Either'; import { pipe } from 'fp-ts/lib/function'; import { InvokeCreator } from 'xstate'; -import { replaceStateKeyInQueryString } from '../../../../common/url_state_storage_service'; import { minimalTimeKeyRT, pickTimeKey } from '../../../../common/time'; import { createPlainError, formatErrors } from '../../../../common/runtime_types'; import type { LogStreamPositionContext, LogStreamPositionEvent } from './types'; @@ -98,13 +97,3 @@ export type PositionStateInUrl = rt.TypeOf; const decodePositionQueryValueFromUrl = (queryValueFromUrl: unknown) => { return positionStateInUrlRT.decode(queryValueFromUrl); }; - -export const replaceLogPositionInQueryString = (time?: number) => - Number.isNaN(time) || time == null - ? (value: string) => value - : replaceStateKeyInQueryString(defaultPositionStateKey, { - position: { - time, - tiebreaker: 0, - }, - }); diff --git a/x-pack/plugins/infra/public/observability_logs/log_stream_query_state/src/url_state_storage_service.ts b/x-pack/plugins/infra/public/observability_logs/log_stream_query_state/src/url_state_storage_service.ts index fb65fcd987a11..1a4daad1832d9 100644 --- a/x-pack/plugins/infra/public/observability_logs/log_stream_query_state/src/url_state_storage_service.ts +++ b/x-pack/plugins/infra/public/observability_logs/log_stream_query_state/src/url_state_storage_service.ts @@ -17,6 +17,7 @@ import { defaultPositionStateKey, DEFAULT_REFRESH_INTERVAL, } from '@kbn/logs-shared-plugin/common'; +import moment from 'moment'; import { getTimeRangeEndFromTime, getTimeRangeStartFromTime, @@ -159,8 +160,8 @@ export const initializeFromUrl = Either.chain(({ position }) => position && position.time ? Either.right({ - from: getTimeRangeStartFromTime(position.time), - to: getTimeRangeEndFromTime(position.time), + from: getTimeRangeStartFromTime(moment(position.time).valueOf()), + to: getTimeRangeEndFromTime(moment(position.time).valueOf()), }) : Either.left(null) ) diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx index 0811ec1708530..babd0e9a1ae3b 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx @@ -58,7 +58,7 @@ export const CategoryExampleMessage: React.FunctionComponent<{ search: { logPosition: encode({ end: moment(timeRange.endTime).format('YYYY-MM-DDTHH:mm:ss.SSSZ'), - position: { tiebreaker, time: timestamp }, + position: { tiebreaker, time: moment(timestamp).toISOString() }, start: moment(timeRange.startTime).format('YYYY-MM-DDTHH:mm:ss.SSSZ'), streamLive: false, }), @@ -128,7 +128,10 @@ export const CategoryExampleMessage: React.FunctionComponent<{ id, index: '', // TODO: use real index when loading via async search context, - cursor: { time: timestamp, tiebreaker }, + cursor: { + time: moment(timestamp).toISOString(), + tiebreaker, + }, columns: [], }; trackMetric({ metric: 'view_in_context__categories' }); diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/log_entry_example.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/log_entry_example.tsx index 288fc3df39c8e..174abbbe60bc4 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/log_entry_example.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/log_entry_example.tsx @@ -102,7 +102,7 @@ export const LogEntryExampleMessage: React.FunctionComponent = ({ search: { logPosition: encode({ end: moment(timeRange.endTime).format('YYYY-MM-DDTHH:mm:ss.SSSZ'), - position: { tiebreaker, time: timestamp }, + position: { tiebreaker, time: moment(timestamp).toISOString() }, start: moment(timeRange.startTime).format('YYYY-MM-DDTHH:mm:ss.SSSZ'), streamLive: false, }), diff --git a/x-pack/plugins/infra/public/pages/logs/stream/page_content.tsx b/x-pack/plugins/infra/public/pages/logs/stream/page_content.tsx index b011931adbd85..e14d52fe1693b 100644 --- a/x-pack/plugins/infra/public/pages/logs/stream/page_content.tsx +++ b/x-pack/plugins/infra/public/pages/logs/stream/page_content.tsx @@ -38,7 +38,7 @@ export const ConnectedStreamPageContent: React.FC = () => { jumpToTargetPosition: (targetPosition: TimeKey | null) => { logStreamPageSend({ type: 'JUMP_TO_TARGET_POSITION', targetPosition }); }, - jumpToTargetPositionTime: (time: number) => { + jumpToTargetPositionTime: (time: string) => { logStreamPageSend({ type: 'JUMP_TO_TARGET_POSITION', targetPosition: { time } }); }, reportVisiblePositions: (visiblePositions: VisiblePositions) => { diff --git a/x-pack/plugins/infra/public/pages/logs/stream/page_logs_content.tsx b/x-pack/plugins/infra/public/pages/logs/stream/page_logs_content.tsx index 14ebe864d5d68..9841553ce9860 100644 --- a/x-pack/plugins/infra/public/pages/logs/stream/page_logs_content.tsx +++ b/x-pack/plugins/infra/public/pages/logs/stream/page_logs_content.tsx @@ -8,7 +8,7 @@ import { EuiSpacer } from '@elastic/eui'; import type { Query } from '@kbn/es-query'; import { euiStyled } from '@kbn/kibana-react-plugin/common'; -import { LogEntry } from '@kbn/logs-shared-plugin/common'; +import { LogEntry, convertISODateToNanoPrecision } from '@kbn/logs-shared-plugin/common'; import { LogEntryFlyout, LogEntryStreamItem, @@ -117,8 +117,12 @@ export const StreamPageLogsContent = React.memo<{ const isCenterPointOutsideLoadedRange = targetPosition != null && - ((topCursor != null && targetPosition.time < topCursor.time) || - (bottomCursor != null && targetPosition.time > bottomCursor.time)); + ((topCursor != null && + convertISODateToNanoPrecision(targetPosition.time) < + convertISODateToNanoPrecision(topCursor.time)) || + (bottomCursor != null && + convertISODateToNanoPrecision(targetPosition.time) > + convertISODateToNanoPrecision(bottomCursor.time))); const hasQueryChanged = filterQuery !== prevFilterQuery; diff --git a/x-pack/plugins/infra/public/test_utils/entries.ts b/x-pack/plugins/infra/public/test_utils/entries.ts index 35dad808cdf4f..0b04c90ee633d 100644 --- a/x-pack/plugins/infra/public/test_utils/entries.ts +++ b/x-pack/plugins/infra/public/test_utils/entries.ts @@ -18,14 +18,16 @@ export function generateFakeEntries( const timestampStep = Math.floor((endTimestamp - startTimestamp) / count); for (let i = 0; i < count; i++) { const timestamp = i === count - 1 ? endTimestamp : startTimestamp + timestampStep * i; + const date = new Date(timestamp).toISOString(); + entries.push({ id: `entry-${i}`, index: 'logs-fake', context: {}, - cursor: { time: timestamp, tiebreaker: i }, + cursor: { time: date, tiebreaker: i }, columns: columns.map((column) => { if ('timestampColumn' in column) { - return { columnId: column.timestampColumn.id, timestamp }; + return { columnId: column.timestampColumn.id, time: date }; } else if ('messageColumn' in column) { return { columnId: column.messageColumn.id, diff --git a/x-pack/plugins/logs_shared/common/index.ts b/x-pack/plugins/logs_shared/common/index.ts index 654e8e2e13335..99fd7c1166863 100644 --- a/x-pack/plugins/logs_shared/common/index.ts +++ b/x-pack/plugins/logs_shared/common/index.ts @@ -44,6 +44,8 @@ export { // eslint-disable-next-line @kbn/eslint/no_export_all export * from './log_entry'; +export { convertISODateToNanoPrecision } from './utils'; + // Http types export type { LogEntriesSummaryBucket, LogEntriesSummaryHighlightsBucket } from './http_api'; diff --git a/x-pack/plugins/logs_shared/common/log_entry/log_entry.ts b/x-pack/plugins/logs_shared/common/log_entry/log_entry.ts index b1fc80634a922..24a04f4c2e38e 100644 --- a/x-pack/plugins/logs_shared/common/log_entry/log_entry.ts +++ b/x-pack/plugins/logs_shared/common/log_entry/log_entry.ts @@ -5,8 +5,8 @@ * 2.0. */ +import { TimeKey } from '@kbn/io-ts-utils'; import * as rt from 'io-ts'; -import { TimeKey } from '../time'; import { jsonArrayRT } from '../typed_json'; import { logEntryCursorRT } from './log_entry_cursor'; @@ -34,7 +34,7 @@ export type LogMessagePart = rt.TypeOf; * columns */ -export const logTimestampColumnRT = rt.type({ columnId: rt.string, timestamp: rt.number }); +export const logTimestampColumnRT = rt.type({ columnId: rt.string, time: rt.string }); export type LogTimestampColumn = rt.TypeOf; export const logFieldColumnRT = rt.type({ diff --git a/x-pack/plugins/logs_shared/common/log_entry/log_entry_cursor.ts b/x-pack/plugins/logs_shared/common/log_entry/log_entry_cursor.ts index 3cd69b4a6178c..158d14152ad4a 100644 --- a/x-pack/plugins/logs_shared/common/log_entry/log_entry_cursor.ts +++ b/x-pack/plugins/logs_shared/common/log_entry/log_entry_cursor.ts @@ -9,7 +9,7 @@ import * as rt from 'io-ts'; import { decodeOrThrow } from '../runtime_types'; export const logEntryCursorRT = rt.type({ - time: rt.number, + time: rt.string, tiebreaker: rt.number, }); export type LogEntryCursor = rt.TypeOf; @@ -29,7 +29,7 @@ export const logEntryAroundCursorRT = rt.type({ }); export type LogEntryAroundCursor = rt.TypeOf; -export const getLogEntryCursorFromHit = (hit: { sort: [number, number] }) => +export const getLogEntryCursorFromHit = (hit: { sort: [string, number] }) => decodeOrThrow(logEntryCursorRT)({ time: hit.sort[0], tiebreaker: hit.sort[1], diff --git a/x-pack/plugins/logs_shared/common/time/time_key.ts b/x-pack/plugins/logs_shared/common/time/time_key.ts index 4c78158dd5bf6..1ff661af69d0a 100644 --- a/x-pack/plugins/logs_shared/common/time/time_key.ts +++ b/x-pack/plugins/logs_shared/common/time/time_key.ts @@ -5,26 +5,8 @@ * 2.0. */ +import type { TimeKey } from '@kbn/io-ts-utils'; import { ascending, bisector } from 'd3-array'; -import * as rt from 'io-ts'; - -export const minimalTimeKeyRT = rt.type({ - time: rt.number, - tiebreaker: rt.number, -}); - -export const timeKeyRT = rt.intersection([ - minimalTimeKeyRT, - rt.partial({ - gid: rt.string, - fromAutoReload: rt.boolean, - }), -]); -export type TimeKey = rt.TypeOf; - -export interface UniqueTimeKey extends TimeKey { - gid: string; -} export type Comparator = (firstValue: any, secondValue: any) => number; diff --git a/x-pack/plugins/logs_shared/common/utils/date_helpers.test.ts b/x-pack/plugins/logs_shared/common/utils/date_helpers.test.ts new file mode 100644 index 0000000000000..fb04f64435611 --- /dev/null +++ b/x-pack/plugins/logs_shared/common/utils/date_helpers.test.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { subtractMillisecondsFromDate } from './date_helpers'; + +describe('Date Helpers', function () { + describe('subtractMillisecondsFromDate', function () { + it('should subtract milliseconds from the nano date correctly', () => { + const inputDate = '2023-10-30T12:00:00.001000000Z'; + const millisecondsToSubtract = 1; + + const result = subtractMillisecondsFromDate(inputDate, millisecondsToSubtract); + + const expectedDate = '2023-10-30T12:00:00.000000000Z'; + + expect(result).toBe(expectedDate); + }); + + it('should subtract seconds from the date if no milliseconds available', () => { + const inputDate = '2023-10-30T12:00:00.000000000Z'; + const millisecondsToSubtract = 1; + + const result = subtractMillisecondsFromDate(inputDate, millisecondsToSubtract); + + const expectedDate = '2023-10-30T11:59:59.999000000Z'; + + expect(result).toBe(expectedDate); + }); + + it('should convert date to nano and subtract milliseconds properly', () => { + const inputDate = '2023-10-30T12:00:00.000Z'; + const millisecondsToSubtract = 1; + + const result = subtractMillisecondsFromDate(inputDate, millisecondsToSubtract); + + const expectedDate = '2023-10-30T11:59:59.999000000Z'; + + expect(result).toBe(expectedDate); + }); + }); +}); diff --git a/x-pack/plugins/logs_shared/common/utils/date_helpers.ts b/x-pack/plugins/logs_shared/common/utils/date_helpers.ts new file mode 100644 index 0000000000000..69af6c74110b5 --- /dev/null +++ b/x-pack/plugins/logs_shared/common/utils/date_helpers.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import dateMath from '@kbn/datemath'; + +export function convertISODateToNanoPrecision(date: string): string { + const dateParts = date.split('.'); + + const fractionSeconds = dateParts.length === 2 ? dateParts[1].replace('Z', '') : ''; + const fractionSecondsInNanos = + fractionSeconds.length !== 9 ? fractionSeconds.padEnd(9, '0') : fractionSeconds; + + return `${dateParts[0]}.${fractionSecondsInNanos}Z`; +} + +export function subtractMillisecondsFromDate(date: string, milliseconds: number): string { + const dateInNano = convertISODateToNanoPrecision(date); + + const dateParts = dateInNano.split('.'); + const nanoPart = dateParts[1].substring(3, dateParts[1].length); // given 123456789Z => 456789Z + + const isoDate = dateMath.parse(date)?.subtract(milliseconds, 'ms').toISOString(); + + return `${isoDate?.replace('Z', nanoPart)}`; +} diff --git a/x-pack/plugins/logs_shared/common/utils/index.ts b/x-pack/plugins/logs_shared/common/utils/index.ts new file mode 100644 index 0000000000000..ba1e5fcefd1a4 --- /dev/null +++ b/x-pack/plugins/logs_shared/common/utils/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './date_helpers'; diff --git a/x-pack/plugins/logs_shared/public/components/formatted_time.tsx b/x-pack/plugins/logs_shared/public/components/formatted_time.tsx index 6c7309d814af4..4b65436a670a6 100644 --- a/x-pack/plugins/logs_shared/public/components/formatted_time.tsx +++ b/x-pack/plugins/logs_shared/public/components/formatted_time.tsx @@ -11,7 +11,7 @@ import { useMemo } from 'react'; import { useKibanaUiSetting } from '../utils/use_kibana_ui_setting'; const getFormattedTime = ( - time: number, + time: string, userFormat: string | undefined, fallbackFormat: string = 'Y-MM-DD HH:mm:ss.SSS' ) => { @@ -26,7 +26,7 @@ interface UseFormattedTimeOptions { } export const useFormattedTime = ( - time: number, + time: string, { format = 'dateTime', fallbackFormat }: UseFormattedTimeOptions = {} ) => { // `dateFormat:scaled` is an array of `[key, format]` tuples. diff --git a/x-pack/plugins/logs_shared/public/components/log_stream/log_stream.stories.tsx b/x-pack/plugins/logs_shared/public/components/log_stream/log_stream.stories.tsx index 10561b24afb05..eeba1e0b448dd 100644 --- a/x-pack/plugins/logs_shared/public/components/log_stream/log_stream.stories.tsx +++ b/x-pack/plugins/logs_shared/public/components/log_stream/log_stream.stories.tsx @@ -38,7 +38,7 @@ export const BasicDateRange = LogStreamStoryTemplate.bind({}); export const CenteredOnLogEntry = LogStreamStoryTemplate.bind({}); CenteredOnLogEntry.args = { - center: { time: 1595146275000, tiebreaker: 150 }, + center: { time: '2020-07-19T08:11:15.000Z', tiebreaker: 150 }, }; export const HighlightedLogEntry = LogStreamStoryTemplate.bind({}); diff --git a/x-pack/plugins/logs_shared/public/components/log_stream/log_stream.tsx b/x-pack/plugins/logs_shared/public/components/log_stream/log_stream.tsx index 50871650da268..98be3f7567880 100644 --- a/x-pack/plugins/logs_shared/public/components/log_stream/log_stream.tsx +++ b/x-pack/plugins/logs_shared/public/components/log_stream/log_stream.tsx @@ -40,7 +40,7 @@ interface CommonColumnDefinition { interface TimestampColumnDefinition extends CommonColumnDefinition { type: 'timestamp'; /** Timestamp renderer. Takes a epoch_millis and returns a valid `ReactNode` */ - render?: (timestamp: number) => React.ReactNode; + render?: (timestamp: string) => React.ReactNode; } interface MessageColumnDefinition extends CommonColumnDefinition { diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx b/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx index 5a6639a1f2298..a6957524b2f97 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx +++ b/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx @@ -24,6 +24,7 @@ const ProviderWrapper: React.FC = ({ children }) => { }; describe('LogEntryActionsMenu component', () => { + const time = new Date().toISOString(); describe('uptime link', () => { it('renders with a host ip filter when present in log entry', () => { const elementWrapper = mount( @@ -34,7 +35,7 @@ describe('LogEntryActionsMenu component', () => { id: 'ITEM_ID', index: 'INDEX', cursor: { - time: 0, + time, tiebreaker: 0, }, }} @@ -64,7 +65,7 @@ describe('LogEntryActionsMenu component', () => { id: 'ITEM_ID', index: 'INDEX', cursor: { - time: 0, + time, tiebreaker: 0, }, }} @@ -94,7 +95,7 @@ describe('LogEntryActionsMenu component', () => { id: 'ITEM_ID', index: 'INDEX', cursor: { - time: 0, + time, tiebreaker: 0, }, }} @@ -128,7 +129,7 @@ describe('LogEntryActionsMenu component', () => { id: 'ITEM_ID', index: 'INDEX', cursor: { - time: 0, + time, tiebreaker: 0, }, }} @@ -160,7 +161,7 @@ describe('LogEntryActionsMenu component', () => { id: 'ITEM_ID', index: 'INDEX', cursor: { - time: 0, + time, tiebreaker: 0, }, }} @@ -194,7 +195,7 @@ describe('LogEntryActionsMenu component', () => { id: 'ITEM_ID', index: 'INDEX', cursor: { - time: 0, + time, tiebreaker: 0, }, }} @@ -228,7 +229,7 @@ describe('LogEntryActionsMenu component', () => { id: 'ITEM_ID', index: 'INDEX', cursor: { - time: 0, + time, tiebreaker: 0, }, }} @@ -258,7 +259,7 @@ describe('LogEntryActionsMenu component', () => { id: 'ITEM_ID', index: 'INDEX', cursor: { - time: 0, + time, tiebreaker: 0, }, }} diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_fields_table.tsx b/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_fields_table.tsx index 09b8381859354..dbde4085da289 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_fields_table.tsx +++ b/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_fields_table.tsx @@ -10,9 +10,9 @@ import { EuiBasicTableColumn, EuiInMemoryTable } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import * as rt from 'io-ts'; import React, { useMemo } from 'react'; +import { TimeKey } from '@kbn/io-ts-utils'; import { LogEntryField } from '../../../../common/log_entry'; import { LogEntry } from '../../../../common/search_strategies/log_entries/log_entry'; -import { TimeKey } from '../../../../common/time'; import { JsonScalar, jsonScalarRT } from '../../../../common/typed_json'; import { FieldValue } from '../log_text_stream/field_value'; diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_flyout.tsx b/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_flyout.tsx index b66e864c2a499..99b89f0c1c432 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_flyout.tsx +++ b/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_flyout.tsx @@ -21,9 +21,9 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { createKibanaReactContext } from '@kbn/kibana-react-plugin/public'; import React, { useCallback, useEffect, useRef } from 'react'; +import { TimeKey } from '@kbn/io-ts-utils'; import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; import { LogViewReference } from '../../../../common/log_views'; -import { TimeKey } from '../../../../common/time'; import { useLogEntry } from '../../../containers/logs/log_entry'; import { CenteredEuiFlyoutBody } from '../../centered_flyout_body'; import { DataSearchErrorCallout } from '../../data_search_error_callout'; diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/column_headers.tsx b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/column_headers.tsx index 3fd275ea56dc2..a1b1cbf8fe770 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/column_headers.tsx +++ b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/column_headers.tsx @@ -41,7 +41,7 @@ export const LogColumnHeaders: React.FunctionComponent<{ columnHeader = columnConfiguration.timestampColumn.header; } else { columnHeader = firstVisiblePosition - ? localizedDate(firstVisiblePosition.time) + ? localizedDate(new Date(firstVisiblePosition.time)) : i18n.translate('xpack.logsShared.logs.stream.timestampColumnTitle', { defaultMessage: 'Timestamp', }); diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/item.ts b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/item.ts index d6dc5fdbef4f6..86e2baa9d6f93 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/item.ts +++ b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/item.ts @@ -5,9 +5,10 @@ * 2.0. */ +import type { TimeKey } from '@kbn/io-ts-utils'; import { bisector } from 'd3-array'; -import { compareToTimeKey, TimeKey } from '../../../../common/time'; import { LogEntry } from '../../../../common/log_entry'; +import { compareToTimeKey } from '../../../../common/time'; export type StreamItem = LogEntryStreamItem; @@ -27,17 +28,17 @@ export function getStreamItemTimeKey(item: StreamItem) { export function getStreamItemId(item: StreamItem) { switch (item.kind) { case 'logEntry': - return `${item.logEntry.cursor.time}:${item.logEntry.cursor.tiebreaker}:${item.logEntry.id}`; + return `${item.logEntry.cursor.time}/${item.logEntry.cursor.tiebreaker}/${item.logEntry.id}`; } } export function parseStreamItemId(id: string) { - const idFragments = id.split(':'); + const idFragments = id.split('/'); return { gid: idFragments.slice(2).join(':'), tiebreaker: parseInt(idFragments[1], 10), - time: parseInt(idFragments[0], 10), + time: idFragments[0], }; } diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_date_row.tsx b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_date_row.tsx index cd499b3ae3425..9bc6e29429ce5 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_date_row.tsx +++ b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_date_row.tsx @@ -10,14 +10,14 @@ import { EuiFlexGroup, EuiFlexItem, EuiHorizontalRule, EuiTitle } from '@elastic import { localizedDate } from '../../../../common/formatters/datetime'; interface LogDateRowProps { - timestamp: number; + time: string; } /** * Show a row with the date in the log stream */ -export const LogDateRow: React.FC = ({ timestamp }) => { - const formattedDate = localizedDate(timestamp); +export const LogDateRow: React.FC = ({ time }) => { + const formattedDate = localizedDate(new Date(time)); return ( diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_column.tsx b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_column.tsx index 81d6a43254ecb..7df56fb79c320 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_column.tsx +++ b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_column.tsx @@ -8,6 +8,7 @@ import { useMemo } from 'react'; import { euiStyled } from '@kbn/kibana-react-plugin/common'; +import moment from 'moment'; import { TextScale } from '../../../../common/log_text_scale'; import { LogColumnRenderConfiguration, @@ -142,7 +143,7 @@ export const useColumnWidths = ({ timeFormat?: TimeFormat; }) => { const { CharacterDimensionsProbe, dimensions } = useMeasuredCharacterDimensions(scale); - const referenceTime = useMemo(() => Date.now(), []); + const referenceTime = useMemo(() => moment().toISOString(), []); const formattedCurrentDate = useFormattedTime(referenceTime, { format: timeFormat }); const columnWidths = useMemo( () => getColumnWidths(columnConfigurations, dimensions.width, formattedCurrentDate.length), diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_row.tsx b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_row.tsx index 5b100c4df4beb..b73da833032f4 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_row.tsx +++ b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_row.tsx @@ -189,7 +189,7 @@ export const LogEntryRow = memo( > {isTimestampColumn(column) ? ( ) : null} diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_timestamp_column.tsx b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_timestamp_column.tsx index 22a94f05ea265..0b145503ddf54 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_timestamp_column.tsx +++ b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_timestamp_column.tsx @@ -13,8 +13,8 @@ import { LogEntryColumnContent } from './log_entry_column'; interface LogEntryTimestampColumnProps { format?: TimeFormat; - time: number; - render?: (timestamp: number) => React.ReactNode; + time: string; + render?: (time: string) => React.ReactNode; } export const LogEntryTimestampColumn = memo( diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/scrollable_log_text_stream_view.tsx b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/scrollable_log_text_stream_view.tsx index c92347ce8c0bd..feae9b8b21d8a 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/scrollable_log_text_stream_view.tsx +++ b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/scrollable_log_text_stream_view.tsx @@ -9,10 +9,9 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import React, { Fragment, GetDerivedStateFromProps } from 'react'; import moment from 'moment'; - import { euiStyled } from '@kbn/kibana-react-plugin/common'; +import { TimeKey, UniqueTimeKey } from '@kbn/io-ts-utils'; import { TextScale } from '../../../../common/log_text_scale'; -import { TimeKey, UniqueTimeKey } from '../../../../common/time'; import { callWithoutRepeats } from '../../../utils/handlers'; import { AutoSizer } from '../../auto_sizer'; import { NoData } from '../../empty_states'; @@ -216,7 +215,7 @@ export class ScrollableLogTextStreamView extends React.PureComponent< position="start" isLoading={isLoadingMore} hasMore={hasMoreBeforeStart} - timestamp={items[0].logEntry.cursor.time} + timestamp={moment(items[0].logEntry.cursor.time).valueOf()} isStreaming={false} startDateExpression={startDateExpression} endDateExpression={endDateExpression} @@ -225,17 +224,17 @@ export class ScrollableLogTextStreamView extends React.PureComponent< } /> {items.map((item, idx) => { - const currentTimestamp = item.logEntry.cursor.time; + const currentTime = item.logEntry.cursor.time; let showDate = false; if (idx > 0) { - const prevTimestamp = items[idx - 1].logEntry.cursor.time; - showDate = !moment(currentTimestamp).isSame(prevTimestamp, 'day'); + const prevTime = items[idx - 1].logEntry.cursor.time; + showDate = !moment(currentTime).isSame(prevTime, 'day'); } return ( - {showDate && } + {showDate && } void; - jumpToTargetPositionTime: (time: number) => void; + jumpToTargetPositionTime: (time: string) => void; reportVisiblePositions: (visPos: VisiblePositions) => void; startLiveStreaming: () => void; stopLiveStreaming: () => void; @@ -62,7 +62,7 @@ export interface LogPositionCallbacks { export interface LogStreamPageCallbacks { updateTimeRange: (timeRange: Partial) => void; jumpToTargetPosition: (targetPosition: TimeKey | null) => void; - jumpToTargetPositionTime: (time: number) => void; + jumpToTargetPositionTime: (time: string) => void; reportVisiblePositions: (visiblePositions: VisiblePositions) => void; startLiveStreaming: () => void; stopLiveStreaming: () => void; diff --git a/x-pack/plugins/logs_shared/public/containers/logs/log_stream/use_fetch_log_entries_around.ts b/x-pack/plugins/logs_shared/public/containers/logs/log_stream/use_fetch_log_entries_around.ts index a6bd8ba794288..6dd2566392ee0 100644 --- a/x-pack/plugins/logs_shared/public/containers/logs/log_stream/use_fetch_log_entries_around.ts +++ b/x-pack/plugins/logs_shared/public/containers/logs/log_stream/use_fetch_log_entries_around.ts @@ -8,6 +8,7 @@ import { useCallback } from 'react'; import { combineLatest, Observable, ReplaySubject } from 'rxjs'; import { last, map, startWith, switchMap } from 'rxjs/operators'; +import { subtractMillisecondsFromDate } from '../../../../common/utils'; import { LogEntryCursor } from '../../../../common/log_entry'; import { LogViewColumnConfiguration, LogViewReference } from '../../../../common/log_views'; import { LogEntriesSearchRequestQuery } from '../../../../common/search_strategies/log_entries/log_entries'; @@ -73,7 +74,7 @@ export const useFetchLogEntriesAround = ({ last(), // in the future we could start earlier if we receive partial results already map((lastBeforeSearchResponse) => { const cursorAfter = lastBeforeSearchResponse.response.data?.bottomCursor ?? { - time: cursor.time - 1, + time: subtractMillisecondsFromDate(cursor.time, 1), tiebreaker: 0, }; diff --git a/x-pack/plugins/logs_shared/public/test_utils/entries.ts b/x-pack/plugins/logs_shared/public/test_utils/entries.ts index 4dc3732fd49d5..5277f49b1175e 100644 --- a/x-pack/plugins/logs_shared/public/test_utils/entries.ts +++ b/x-pack/plugins/logs_shared/public/test_utils/entries.ts @@ -27,14 +27,16 @@ export function generateFakeEntries( const timestampStep = Math.floor((endTimestamp - startTimestamp) / count); for (let i = 0; i < count; i++) { const timestamp = i === count - 1 ? endTimestamp : startTimestamp + timestampStep * i; + const date = new Date(timestamp).toISOString(); + entries.push({ id: `entry-${i}`, index: 'logs-fake', context: {}, - cursor: { time: timestamp, tiebreaker: i }, + cursor: { time: date, tiebreaker: i }, columns: columns.map((column) => { if ('timestampColumn' in column) { - return { columnId: column.timestampColumn.id, timestamp }; + return { columnId: column.timestampColumn.id, time: date }; } else if ('messageColumn' in column) { return { columnId: column.messageColumn.id, diff --git a/x-pack/plugins/logs_shared/public/utils/log_column_render_configuration.tsx b/x-pack/plugins/logs_shared/public/utils/log_column_render_configuration.tsx index ff4a24f1498a6..a1b23c9a0e3dd 100644 --- a/x-pack/plugins/logs_shared/public/utils/log_column_render_configuration.tsx +++ b/x-pack/plugins/logs_shared/public/utils/log_column_render_configuration.tsx @@ -19,7 +19,7 @@ interface CommonRenderConfiguration { interface TimestampColumnRenderConfiguration { timestampColumn: CommonRenderConfiguration & { - render?: (timestamp: number) => ReactNode; + render?: (time: string) => ReactNode; }; } diff --git a/x-pack/plugins/logs_shared/public/utils/log_entry/log_entry.ts b/x-pack/plugins/logs_shared/public/utils/log_entry/log_entry.ts index 18d118750fdeb..e1027aeeb5799 100644 --- a/x-pack/plugins/logs_shared/public/utils/log_entry/log_entry.ts +++ b/x-pack/plugins/logs_shared/public/utils/log_entry/log_entry.ts @@ -5,8 +5,9 @@ * 2.0. */ +import type { TimeKey, UniqueTimeKey } from '@kbn/io-ts-utils'; import { bisector } from 'd3-array'; -import { compareToTimeKey, getIndexAtTimeKey, TimeKey, UniqueTimeKey } from '../../../common/time'; +import { compareToTimeKey, getIndexAtTimeKey } from '../../../common/time'; import { LogEntry, LogColumn, @@ -38,7 +39,7 @@ export const getLogEntryAtTime = (entries: LogEntry[], time: TimeKey) => { }; export const isTimestampColumn = (column: LogColumn): column is LogTimestampColumn => - column != null && 'timestamp' in column; + column != null && 'time' in column; export const isMessageColumn = (column: LogColumn): column is LogMessageColumn => column != null && 'message' in column; diff --git a/x-pack/plugins/logs_shared/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts b/x-pack/plugins/logs_shared/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts index 66de5699cfb3e..e4eed9b61d349 100644 --- a/x-pack/plugins/logs_shared/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts +++ b/x-pack/plugins/logs_shared/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts @@ -62,7 +62,11 @@ export class LogsSharedKibanaLogEntriesAdapter implements LogEntriesAdapter { : {}; const sort = { - [TIMESTAMP_FIELD]: sortDirection, + [TIMESTAMP_FIELD]: { + order: sortDirection, + format: 'strict_date_optional_time_nanos', + numeric_type: 'date_nanos', + }, [TIEBREAKER_FIELD]: sortDirection, }; @@ -155,7 +159,16 @@ export class LogsSharedKibanaLogEntriesAdapter implements LogEntriesAdapter { top_hits_by_key: { top_hits: { size: 1, - sort: [{ [TIMESTAMP_FIELD]: 'asc' }, { [TIEBREAKER_FIELD]: 'asc' }], + sort: [ + { + [TIMESTAMP_FIELD]: { + order: 'asc', + format: 'strict_date_optional_time_nanos', + numeric_type: 'date_nanos', + }, + }, + { [TIEBREAKER_FIELD]: 'asc' }, + ], _source: false, }, }, @@ -265,7 +278,7 @@ const createQueryFilterClauses = (filterQuery: LogEntryQuery | undefined) => function processCursor(cursor: LogEntriesParams['cursor']): { sortDirection: 'asc' | 'desc'; - searchAfterClause: { search_after?: readonly [number, number] }; + searchAfterClause: { search_after?: readonly [string, number] }; } { if (cursor) { if ('before' in cursor) { @@ -295,7 +308,7 @@ const LogSummaryDateRangeBucketRuntimeType = runtimeTypes.intersection([ hits: runtimeTypes.type({ hits: runtimeTypes.array( runtimeTypes.type({ - sort: runtimeTypes.tuple([runtimeTypes.number, runtimeTypes.number]), + sort: runtimeTypes.tuple([runtimeTypes.string, runtimeTypes.number]), }) ), }), diff --git a/x-pack/plugins/logs_shared/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/plugins/logs_shared/server/lib/domains/log_entries_domain/log_entries_domain.ts index 92829c676b935..2601167f2d988 100644 --- a/x-pack/plugins/logs_shared/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/plugins/logs_shared/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -7,6 +7,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { JsonObject } from '@kbn/utility-types'; +import { subtractMillisecondsFromDate } from '../../../../common/utils'; import { LogEntriesSummaryBucket, LogEntriesSummaryHighlightsBucket, @@ -147,7 +148,7 @@ export class LogsSharedLogEntriesDomain implements ILogsSharedLogEntriesDomain { const cursorAfter = entriesBefore.length > 0 ? entriesBefore[entriesBefore.length - 1].cursor - : { time: center.time - 1, tiebreaker: 0 }; + : { time: subtractMillisecondsFromDate(center.time, 1), tiebreaker: 0 }; const { entries: entriesAfter, hasMoreAfter } = await this.getLogEntries( requestContext, @@ -200,7 +201,7 @@ export class LogsSharedLogEntriesDomain implements ILogsSharedLogEntriesDomain { if ('timestampColumn' in column) { return { columnId: column.timestampColumn.id, - timestamp: doc.cursor.time, + time: doc.cursor.time, }; } else if ('messageColumn' in column) { return { diff --git a/x-pack/plugins/logs_shared/server/services/log_entries/log_entries_search_strategy.test.ts b/x-pack/plugins/logs_shared/server/services/log_entries/log_entries_search_strategy.test.ts index 305f6292deb28..177e6091dc06b 100644 --- a/x-pack/plugins/logs_shared/server/services/log_entries/log_entries_search_strategy.test.ts +++ b/x-pack/plugins/logs_shared/server/services/log_entries/log_entries_search_strategy.test.ts @@ -95,6 +95,7 @@ describe('LogEntries search strategy', () => { }); it('handles subsequent polling requests', async () => { + const date = new Date(1605116827143).toISOString(); const esSearchStrategyMock = createEsSearchStrategyMock({ id: 'ASYNC_REQUEST_ID', isRunning: false, @@ -112,12 +113,12 @@ describe('LogEntries search strategy', () => { _score: 0, _source: null, fields: { - '@timestamp': [1605116827143], + '@timestamp': [date], 'event.dataset': ['HIT_DATASET'], message: ['HIT_MESSAGE'], 'container.id': ['HIT_CONTAINER_ID'], }, - sort: [1605116827143 as any, 1 as any], // incorrectly typed as string upstream + sort: [date as any, 1 as any], // incorrectly typed as string upstream }, ], }, @@ -164,13 +165,13 @@ describe('LogEntries search strategy', () => { id: 'HIT_ID', index: 'HIT_INDEX', cursor: { - time: 1605116827143, + time: date, tiebreaker: 1, }, columns: [ { columnId: 'TIMESTAMP_COLUMN_ID', - timestamp: 1605116827143, + time: date, }, { columnId: 'DATASET_COLUMN_ID', diff --git a/x-pack/plugins/logs_shared/server/services/log_entries/log_entries_search_strategy.ts b/x-pack/plugins/logs_shared/server/services/log_entries/log_entries_search_strategy.ts index f0f5c6304d615..fb305d9ae73e4 100644 --- a/x-pack/plugins/logs_shared/server/services/log_entries/log_entries_search_strategy.ts +++ b/x-pack/plugins/logs_shared/server/services/log_entries/log_entries_search_strategy.ts @@ -190,7 +190,7 @@ const getLogEntryFromHit = if ('timestampColumn' in column) { return { columnId: column.timestampColumn.id, - timestamp: cursor.time, + time: cursor.time, }; } else if ('messageColumn' in column) { return { diff --git a/x-pack/plugins/logs_shared/server/services/log_entries/log_entry_search_strategy.test.ts b/x-pack/plugins/logs_shared/server/services/log_entries/log_entry_search_strategy.test.ts index 19d5345122374..a123536d5bb27 100644 --- a/x-pack/plugins/logs_shared/server/services/log_entries/log_entry_search_strategy.test.ts +++ b/x-pack/plugins/logs_shared/server/services/log_entries/log_entry_search_strategy.test.ts @@ -99,6 +99,7 @@ describe('LogEntry search strategy', () => { }); it('handles subsequent polling requests', async () => { + const date = new Date(1605116827143).toISOString(); const esSearchStrategyMock = createEsSearchStrategyMock({ id: 'ASYNC_REQUEST_ID', isRunning: false, @@ -116,10 +117,10 @@ describe('LogEntry search strategy', () => { _score: 0, _source: null, fields: { - '@timestamp': [1605116827143], + '@timestamp': [date], message: ['HIT_MESSAGE'], }, - sort: [1605116827143 as any, 1 as any], // incorrectly typed as string upstream + sort: [date as any, 1 as any], // incorrectly typed as string upstream }, ], }, @@ -163,11 +164,11 @@ describe('LogEntry search strategy', () => { id: 'HIT_ID', index: 'HIT_INDEX', cursor: { - time: 1605116827143, + time: date, tiebreaker: 1, }, fields: [ - { field: '@timestamp', value: [1605116827143] }, + { field: '@timestamp', value: [date] }, { field: 'message', value: ['HIT_MESSAGE'] }, ], }); diff --git a/x-pack/plugins/logs_shared/server/services/log_entries/queries/common.ts b/x-pack/plugins/logs_shared/server/services/log_entries/queries/common.ts index 296ac9e4d2f34..c40e2fb9418f2 100644 --- a/x-pack/plugins/logs_shared/server/services/log_entries/queries/common.ts +++ b/x-pack/plugins/logs_shared/server/services/log_entries/queries/common.ts @@ -11,7 +11,11 @@ export const createSortClause = ( tiebreakerField: string ) => ({ sort: { - [timestampField]: sortDirection, + [timestampField]: { + order: sortDirection, + format: 'strict_date_optional_time_nanos', + numeric_type: 'date_nanos', + }, [tiebreakerField]: sortDirection, }, }); diff --git a/x-pack/plugins/logs_shared/server/services/log_entries/queries/log_entries.ts b/x-pack/plugins/logs_shared/server/services/log_entries/queries/log_entries.ts index 18ee4b0104366..18992448fdfcc 100644 --- a/x-pack/plugins/logs_shared/server/services/log_entries/queries/log_entries.ts +++ b/x-pack/plugins/logs_shared/server/services/log_entries/queries/log_entries.ts @@ -68,7 +68,7 @@ export const getSortDirection = ( const createSearchAfterClause = ( cursor: LogEntryBeforeCursor | LogEntryAfterCursor | null | undefined -): { search_after?: [number, number] } => { +): { search_after?: [string, number] } => { if (logEntryBeforeCursorRT.is(cursor) && cursor.before !== 'last') { return { search_after: [cursor.before.time, cursor.before.tiebreaker], @@ -122,7 +122,7 @@ const createHighlightQuery = ( export const logEntryHitRT = rt.intersection([ commonHitFieldsRT, rt.type({ - sort: rt.tuple([rt.number, rt.number]), + sort: rt.tuple([rt.string, rt.number]), }), rt.partial({ fields: rt.record(rt.string, jsonArrayRT), diff --git a/x-pack/plugins/logs_shared/server/services/log_entries/queries/log_entry.ts b/x-pack/plugins/logs_shared/server/services/log_entries/queries/log_entry.ts index 57a644c79f01f..575541f3f7931 100644 --- a/x-pack/plugins/logs_shared/server/services/log_entries/queries/log_entry.ts +++ b/x-pack/plugins/logs_shared/server/services/log_entries/queries/log_entry.ts @@ -33,7 +33,16 @@ export const createGetLogEntryQuery = ( }, fields: ['*'], runtime_mappings: runtimeMappings, - sort: [{ [timestampField]: 'desc' }, { [tiebreakerField]: 'desc' }], + sort: [ + { + [timestampField]: { + order: 'desc', + format: 'strict_date_optional_time_nanos', + numeric_type: 'date_nanos', + }, + }, + { [tiebreakerField]: 'desc' }, + ], _source: false, }, }); @@ -41,7 +50,7 @@ export const createGetLogEntryQuery = ( export const logEntryHitRT = rt.intersection([ commonHitFieldsRT, rt.type({ - sort: rt.tuple([rt.number, rt.number]), + sort: rt.tuple([rt.string, rt.number]), }), rt.partial({ fields: rt.record(rt.string, jsonArrayRT), diff --git a/x-pack/test/api_integration/apis/metrics_ui/log_entry_highlights.ts b/x-pack/test/api_integration/apis/metrics_ui/log_entry_highlights.ts index 1e565f63b0733..9910f4724045a 100644 --- a/x-pack/test/api_integration/apis/metrics_ui/log_entry_highlights.ts +++ b/x-pack/test/api_integration/apis/metrics_ui/log_entry_highlights.ts @@ -19,6 +19,7 @@ import { logEntriesHighlightsResponseRT, } from '@kbn/logs-shared-plugin/common'; +import moment from 'moment'; import { FtrProviderContext } from '../../ftr_provider_context'; const KEY_BEFORE_START = { @@ -112,8 +113,8 @@ export default function ({ getService }: FtrProviderContext) { // Entries fall within range // @kbn/expect doesn't have a `lessOrEqualThan` or `moreOrEqualThan` comparators - expect(firstEntry.cursor.time >= KEY_BEFORE_START.time).to.be(true); - expect(lastEntry.cursor.time <= KEY_AFTER_END.time).to.be(true); + expect(firstEntry.cursor.time >= moment(KEY_BEFORE_START.time).toISOString()).to.be(true); + expect(lastEntry.cursor.time <= moment(KEY_AFTER_END.time).toISOString()).to.be(true); // All entries contain the highlights entries.forEach((entry) => { diff --git a/x-pack/test/functional/apps/infra/index.ts b/x-pack/test/functional/apps/infra/index.ts index b389d56b9032c..948d99f7cce5f 100644 --- a/x-pack/test/functional/apps/infra/index.ts +++ b/x-pack/test/functional/apps/infra/index.ts @@ -26,6 +26,7 @@ export default ({ loadTestFile }: FtrProviderContext) => { loadTestFile(require.resolve('./log_entry_categories_tab')); loadTestFile(require.resolve('./log_entry_rate_tab')); loadTestFile(require.resolve('./logs_source_configuration')); + loadTestFile(require.resolve('./log_stream_date_nano')); loadTestFile(require.resolve('./link_to')); loadTestFile(require.resolve('./log_stream')); }); diff --git a/x-pack/test/functional/apps/infra/link_to.ts b/x-pack/test/functional/apps/infra/link_to.ts index 7ad37696f5a46..f0e8da90d14d7 100644 --- a/x-pack/test/functional/apps/infra/link_to.ts +++ b/x-pack/test/functional/apps/infra/link_to.ts @@ -18,7 +18,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { const retry = getService('retry'); const browser = getService('browser'); - const timestamp = Date.now(); + const date = new Date(); + const timestamp = date.getTime(); const startDate = new Date(timestamp - ONE_HOUR).toISOString(); const endDate = new Date(timestamp + ONE_HOUR).toISOString(); @@ -52,7 +53,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { `(query:(language:kuery,query:\'trace.id:${traceId}'),refreshInterval:(pause:!t,value:5000),timeRange:(from:'${startDate}',to:'${endDate}'))` ); expect(parsedUrl.searchParams.get('logPosition')).to.be( - `(position:(tiebreaker:0,time:${timestamp}))` + `(position:(tiebreaker:0,time:'${date.toISOString()}'))` ); expect(parsedUrl.searchParams.get('logView')).to.be(LOG_VIEW_REFERENCE); expect(documentTitle).to.contain('Stream - Logs - Observability - Elastic'); @@ -87,7 +88,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { `(query:(language:kuery,query:\'(kubernetes.pod.uid: 1234) and (trace.id:${traceId})\'),refreshInterval:(pause:!t,value:5000),timeRange:(from:'${startDate}',to:'${endDate}'))` ); expect(parsedUrl.searchParams.get('logPosition')).to.be( - `(position:(tiebreaker:0,time:${timestamp}))` + `(position:(tiebreaker:0,time:'${date.toISOString()}'))` ); expect(parsedUrl.searchParams.get('logView')).to.be(LOG_VIEW_REFERENCE); expect(documentTitle).to.contain('Stream - Logs - Observability - Elastic'); diff --git a/x-pack/test/functional/apps/infra/log_stream_date_nano.ts b/x-pack/test/functional/apps/infra/log_stream_date_nano.ts new file mode 100644 index 0000000000000..99541e29a2330 --- /dev/null +++ b/x-pack/test/functional/apps/infra/log_stream_date_nano.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { URL } from 'url'; +import { FtrProviderContext } from '../../ftr_provider_context'; +import { DATES } from './constants'; + +export default ({ getPageObjects, getService }: FtrProviderContext) => { + const retry = getService('retry'); + const browser = getService('browser'); + const esArchiver = getService('esArchiver'); + const logsUi = getService('logsUi'); + const find = getService('find'); + const logFilter = { + timeRange: { + from: DATES.metricsAndLogs.stream.startWithData, + to: DATES.metricsAndLogs.stream.endWithData, + }, + }; + + describe('Log stream supports nano precision', function () { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/infra/logs_with_nano_date'); + }); + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/infra/logs_with_nano_date'); + }); + + it('should display logs entries containing date_nano timestamps properly ', async () => { + await logsUi.logStreamPage.navigateTo({ logFilter }); + + const logStreamEntries = await logsUi.logStreamPage.getStreamEntries(); + + expect(logStreamEntries.length).to.be(4); + }); + + it('should render timestamp column properly', async () => { + await logsUi.logStreamPage.navigateTo({ logFilter }); + + await retry.try(async () => { + const columnHeaderLabels = await logsUi.logStreamPage.getColumnHeaderLabels(); + expect(columnHeaderLabels[0]).to.eql('Oct 17, 2018'); + }); + }); + + it('should render timestamp column values properly', async () => { + await logsUi.logStreamPage.navigateTo({ logFilter }); + + const logStreamEntries = await logsUi.logStreamPage.getStreamEntries(); + + const firstLogStreamEntry = logStreamEntries[0]; + + const entryTimestamp = await logsUi.logStreamPage.getLogEntryColumnValueByName( + firstLogStreamEntry, + 'timestampLogColumn' + ); + + expect(entryTimestamp).to.be('19:43:22.111'); + }); + + it('should properly sync logPosition in url', async () => { + const currentUrl = await browser.getCurrentUrl(); + const parsedUrl = new URL(currentUrl); + + expect(parsedUrl.searchParams.get('logPosition')).to.be( + `(position:(tiebreaker:3,time:\'2018-10-17T19:46:22.333333333Z\'))` + ); + }); + + it('should properly render timestamp in flyout with nano precision', async () => { + await logsUi.logStreamPage.navigateTo({ logFilter }); + + const logStreamEntries = await logsUi.logStreamPage.getStreamEntries(); + const firstLogStreamEntry = logStreamEntries[0]; + + await logsUi.logStreamPage.openLogEntryDetailsFlyout(firstLogStreamEntry); + + const cells = await find.allByCssSelector('.euiTableCellContent'); + + let isFound = false; + + for (const cell of cells) { + const cellText = await cell.getVisibleText(); + if (cellText === '2018-10-17T19:43:22.111111111Z') { + isFound = true; + return; + } + } + + expect(isFound).to.be(true); + }); + }); +}; diff --git a/x-pack/test/functional/es_archives/infra/logs_with_nano_date/data.json.gz b/x-pack/test/functional/es_archives/infra/logs_with_nano_date/data.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..68e1284e7d1307a866a7cd0484dfaa19b2bf1642 GIT binary patch literal 338 zcmV-Y0j>TYiwFpn4M=4G17u-zVJ>QOZ*BnPlFMp@FcgOOK1FadnGwP}oN~eVm zUAXF|r0RsU&ys^CvlA{PvFBZ#D;9Pl__j z4_1US1%I-F6jO$9h%kfL1{qsb)$G%cdzs9omPM{KFKX*ZF$p2TkgO6C#XgHEMP!V} zlgqdFjAo`#a^qI>;S6_r+Ns`8{YIp?7(^tZ2qTJo3LseATqnSjg{dj;S)i9%fOV2Q zcm&}Epcn({uh{b4uy^e09j7}F0yYN(zC}RCI4=Sk1QUZTZ)IAQ$b7yKU`rPM3V`i; k4gdlR0Ef{(`};o)-~X%OFYWNz7;{DN2D}6`<^u)*0D%CVo&W#< literal 0 HcmV?d00001 diff --git a/x-pack/test/functional/es_archives/infra/logs_with_nano_date/mappings.json b/x-pack/test/functional/es_archives/infra/logs_with_nano_date/mappings.json new file mode 100644 index 0000000000000..294ee1f204e93 --- /dev/null +++ b/x-pack/test/functional/es_archives/infra/logs_with_nano_date/mappings.json @@ -0,0 +1,419 @@ +{ + "type": "data_stream", + "value": { + "data_stream": "logs-gaming-activity", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-gaming-activity" + ], + "name": "logs-gaming-activity", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} + +{ + "type": "data_stream", + "value": { + "data_stream": "logs-gaming-events", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-gaming-events" + ], + "name": "logs-gaming-events", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} + +{ + "type": "data_stream", + "value": { + "data_stream": "logs-gaming-scores", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-gaming-scores" + ], + "name": "logs-gaming-scores", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} + +{ + "type": "data_stream", + "value": { + "data_stream": "logs-manufacturing-downtime", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-manufacturing-downtime" + ], + "name": "logs-manufacturing-downtime", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} + +{ + "type": "data_stream", + "value": { + "data_stream": "logs-manufacturing-output", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-manufacturing-output" + ], + "name": "logs-manufacturing-output", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} + +{ + "type": "data_stream", + "value": { + "data_stream": "logs-manufacturing-quality", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-manufacturing-quality" + ], + "name": "logs-manufacturing-quality", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} + +{ + "type": "data_stream", + "value": { + "data_stream": "logs-retail-customers", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-retail-customers" + ], + "name": "logs-retail-customers", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} + +{ + "type": "data_stream", + "value": { + "data_stream": "logs-retail-inventory", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-retail-inventory" + ], + "name": "logs-retail-inventory", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} + +{ + "type": "data_stream", + "value": { + "data_stream": "logs-retail-promotions", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-retail-promotions" + ], + "name": "logs-retail-promotions", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} + +{ + "type": "data_stream", + "value": { + "data_stream": "logs-retail-sales", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-retail-sales" + ], + "name": "logs-retail-sales", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/x-pack/test/functional/services/logs_ui/log_stream.ts b/x-pack/test/functional/services/logs_ui/log_stream.ts index 214290bd21ef4..1a068439a2d2d 100644 --- a/x-pack/test/functional/services/logs_ui/log_stream.ts +++ b/x-pack/test/functional/services/logs_ui/log_stream.ts @@ -12,6 +12,7 @@ import { TabsParams } from '../../page_objects/infra_logs_page'; export function LogStreamPageProvider({ getPageObjects, getService }: FtrProviderContext) { const pageObjects = getPageObjects(['infraLogs']); const retry = getService('retry'); + const find = getService('find'); const testSubjects = getService('testSubjects'); return { @@ -43,6 +44,31 @@ export function LogStreamPageProvider({ getPageObjects, getService }: FtrProvide return await testSubjects.findAllDescendant('~logColumn', entryElement); }, + async getLogEntryColumnValueByName( + entryElement: WebElementWrapper, + column: string + ): Promise { + const columnElement = await testSubjects.findDescendant(`~${column}`, entryElement); + + const contentElement = await columnElement.findByCssSelector( + `[data-test-subj='LogEntryColumnContent']` + ); + + return await contentElement.getVisibleText(); + }, + + async openLogEntryDetailsFlyout(entryElement: WebElementWrapper) { + await entryElement.click(); + + const menuButton = await testSubjects.findDescendant( + `~infraLogEntryContextMenuButton`, + entryElement + ); + await menuButton.click(); + + await find.clickByButtonText('View details'); + }, + async getNoLogsIndicesPrompt() { return await testSubjects.find('noLogsIndicesPrompt'); },