diff --git a/packages/kbn-io-ts-utils/index.ts b/packages/kbn-io-ts-utils/index.ts index 57d9e7bb4771..8f1b974eccdc 100644 --- a/packages/kbn-io-ts-utils/index.ts +++ b/packages/kbn-io-ts-utils/index.ts @@ -32,3 +32,11 @@ export { export { datemathStringRt } from './src/datemath_string_rt'; export { createPlainError, decodeOrThrow, formatErrors, throwErrors } from './src/decode_or_throw'; + +export { + DateFromStringOrNumber, + minimalTimeKeyRT, + type MinimalTimeKey, + type TimeKey, + type UniqueTimeKey, +} from './src/time_key_rt'; diff --git a/packages/kbn-io-ts-utils/src/time_key_rt/index.ts b/packages/kbn-io-ts-utils/src/time_key_rt/index.ts new file mode 100644 index 000000000000..1884ea5a52e8 --- /dev/null +++ b/packages/kbn-io-ts-utils/src/time_key_rt/index.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as rt from 'io-ts'; +import moment from 'moment'; +import { pipe } from 'fp-ts/lib/pipeable'; +import { chain } from 'fp-ts/lib/Either'; + +const NANO_DATE_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3,9}Z$/; + +export const DateFromStringOrNumber = new rt.Type( + 'DateFromStringOrNumber', + (input): input is string => typeof input === 'string', + (input, context) => { + if (typeof input === 'string') { + return NANO_DATE_PATTERN.test(input) ? rt.success(input) : rt.failure(input, context); + } + return pipe( + rt.number.validate(input, context), + chain((timestamp) => { + const momentValue = moment(timestamp); + return momentValue.isValid() + ? rt.success(momentValue.toISOString()) + : rt.failure(timestamp, context); + }) + ); + }, + String +); + +export const minimalTimeKeyRT = rt.type({ + time: DateFromStringOrNumber, + tiebreaker: rt.number, +}); +export type MinimalTimeKey = rt.TypeOf; + +const timeKeyRT = rt.intersection([ + minimalTimeKeyRT, + rt.partial({ + gid: rt.string, + fromAutoReload: rt.boolean, + }), +]); +export type TimeKey = rt.TypeOf; + +export interface UniqueTimeKey extends TimeKey { + gid: string; +} diff --git a/x-pack/plugins/infra/common/locators/locators.test.ts b/x-pack/plugins/infra/common/locators/locators.test.ts index fb6f8283b63f..607fd41b1bab 100644 --- a/x-pack/plugins/infra/common/locators/locators.test.ts +++ b/x-pack/plugins/infra/common/locators/locators.test.ts @@ -233,7 +233,7 @@ const constructLogView = (logView?: LogViewReference) => { }; const constructLogPosition = (time: number = 1550671089404) => { - return `(position:(tiebreaker:0,time:${time}))`; + return `(position:(tiebreaker:0,time:'${moment(time).toISOString()}'))`; }; const constructLogFilter = ({ diff --git a/x-pack/plugins/infra/common/time/time_key.ts b/x-pack/plugins/infra/common/time/time_key.ts index efc5a8e7b851..7acbef8b2502 100644 --- a/x-pack/plugins/infra/common/time/time_key.ts +++ b/x-pack/plugins/infra/common/time/time_key.ts @@ -5,12 +5,13 @@ * 2.0. */ +import { DateFromStringOrNumber } from '@kbn/io-ts-utils'; import { ascending, bisector } from 'd3-array'; import * as rt from 'io-ts'; import { pick } from 'lodash'; export const minimalTimeKeyRT = rt.type({ - time: rt.number, + time: DateFromStringOrNumber, tiebreaker: rt.number, }); export type MinimalTimeKey = rt.TypeOf; diff --git a/x-pack/plugins/infra/common/url_state_storage_service.ts b/x-pack/plugins/infra/common/url_state_storage_service.ts index 5d701ad1876b..ece3d1ccb09b 100644 --- a/x-pack/plugins/infra/common/url_state_storage_service.ts +++ b/x-pack/plugins/infra/common/url_state_storage_service.ts @@ -24,18 +24,18 @@ export const defaultLogViewKey = 'logView'; const encodeRisonUrlState = (state: any) => encode(state); -// Used by linkTo components +// Used by Locator components export const replaceLogPositionInQueryString = (time?: number) => Number.isNaN(time) || time == null ? (value: string) => value : replaceStateKeyInQueryString(defaultPositionStateKey, { position: { - time, + time: moment(time).toISOString(), tiebreaker: 0, }, }); -// NOTE: Used by link-to components +// NOTE: Used by Locator components export const replaceLogViewInQueryString = (logViewReference: LogViewReference) => replaceStateKeyInQueryString(defaultLogViewKey, logViewReference); diff --git a/x-pack/plugins/infra/public/components/logging/log_minimap/log_minimap.tsx b/x-pack/plugins/infra/public/components/logging/log_minimap/log_minimap.tsx index df1591a96afa..00c6bcc4a623 100644 --- a/x-pack/plugins/infra/public/components/logging/log_minimap/log_minimap.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_minimap/log_minimap.tsx @@ -12,6 +12,7 @@ import { LogEntryTime, } from '@kbn/logs-shared-plugin/common'; import { scaleLinear } from 'd3-scale'; +import moment from 'moment'; import * as React from 'react'; import { DensityChart } from './density_chart'; import { HighlightedInterval } from './highlighted_interval'; @@ -67,7 +68,7 @@ export class LogMinimap extends React.Component) => void; jumpToTargetPosition: (targetPosition: TimeKey | null) => void; - jumpToTargetPositionTime: (time: number) => void; + jumpToTargetPositionTime: (time: string) => void; reportVisiblePositions: (visiblePositions: VisiblePositions) => void; startLiveStreaming: () => void; stopLiveStreaming: () => void; diff --git a/x-pack/plugins/infra/public/observability_logs/log_stream_position_state/src/state_machine.ts b/x-pack/plugins/infra/public/observability_logs/log_stream_position_state/src/state_machine.ts index 96b7dbb68785..868f29a5c07e 100644 --- a/x-pack/plugins/infra/public/observability_logs/log_stream_position_state/src/state_machine.ts +++ b/x-pack/plugins/infra/public/observability_logs/log_stream_position_state/src/state_machine.ts @@ -7,6 +7,8 @@ import { IToasts } from '@kbn/core-notifications-browser'; import { IKbnUrlStateStorage } from '@kbn/kibana-utils-plugin/public'; +import { convertISODateToNanoPrecision } from '@kbn/logs-shared-plugin/common'; +import moment from 'moment'; import { actions, ActorRefFrom, createMachine, EmittedFrom, SpecialTargets } from 'xstate'; import { isSameTimeKey } from '../../../../common/time'; import { OmitDeprecatedState, sendIfDefined } from '../../xstate_helpers'; @@ -159,11 +161,19 @@ export const createPureLogStreamPositionStateMachine = (initialContext: LogStrea updatePositionsFromTimeChange: actions.assign((_context, event) => { if (!('timeRange' in event)) return {}; + const { + timestamps: { startTimestamp, endTimestamp }, + } = event; + // Reset the target position if it doesn't fall within the new range. + const targetPositionNanoTime = + _context.targetPosition && convertISODateToNanoPrecision(_context.targetPosition.time); + const startNanoDate = convertISODateToNanoPrecision(moment(startTimestamp).toISOString()); + const endNanoDate = convertISODateToNanoPrecision(moment(endTimestamp).toISOString()); + const targetPositionShouldReset = - _context.targetPosition && - (event.timestamps.startTimestamp > _context.targetPosition.time || - event.timestamps.endTimestamp < _context.targetPosition.time); + targetPositionNanoTime && + (startNanoDate > targetPositionNanoTime || endNanoDate < targetPositionNanoTime); return { targetPosition: targetPositionShouldReset ? null : _context.targetPosition, diff --git a/x-pack/plugins/infra/public/observability_logs/log_stream_position_state/src/url_state_storage_service.ts b/x-pack/plugins/infra/public/observability_logs/log_stream_position_state/src/url_state_storage_service.ts index c98ab9e14744..5607bf920705 100644 --- a/x-pack/plugins/infra/public/observability_logs/log_stream_position_state/src/url_state_storage_service.ts +++ b/x-pack/plugins/infra/public/observability_logs/log_stream_position_state/src/url_state_storage_service.ts @@ -10,7 +10,6 @@ import { IKbnUrlStateStorage, withNotifyOnErrors } from '@kbn/kibana-utils-plugi import * as Either from 'fp-ts/lib/Either'; import { pipe } from 'fp-ts/lib/function'; import { InvokeCreator } from 'xstate'; -import { replaceStateKeyInQueryString } from '../../../../common/url_state_storage_service'; import { minimalTimeKeyRT, pickTimeKey } from '../../../../common/time'; import { createPlainError, formatErrors } from '../../../../common/runtime_types'; import type { LogStreamPositionContext, LogStreamPositionEvent } from './types'; @@ -98,13 +97,3 @@ export type PositionStateInUrl = rt.TypeOf; const decodePositionQueryValueFromUrl = (queryValueFromUrl: unknown) => { return positionStateInUrlRT.decode(queryValueFromUrl); }; - -export const replaceLogPositionInQueryString = (time?: number) => - Number.isNaN(time) || time == null - ? (value: string) => value - : replaceStateKeyInQueryString(defaultPositionStateKey, { - position: { - time, - tiebreaker: 0, - }, - }); diff --git a/x-pack/plugins/infra/public/observability_logs/log_stream_query_state/src/url_state_storage_service.ts b/x-pack/plugins/infra/public/observability_logs/log_stream_query_state/src/url_state_storage_service.ts index fb65fcd987a1..1a4daad1832d 100644 --- a/x-pack/plugins/infra/public/observability_logs/log_stream_query_state/src/url_state_storage_service.ts +++ b/x-pack/plugins/infra/public/observability_logs/log_stream_query_state/src/url_state_storage_service.ts @@ -17,6 +17,7 @@ import { defaultPositionStateKey, DEFAULT_REFRESH_INTERVAL, } from '@kbn/logs-shared-plugin/common'; +import moment from 'moment'; import { getTimeRangeEndFromTime, getTimeRangeStartFromTime, @@ -159,8 +160,8 @@ export const initializeFromUrl = Either.chain(({ position }) => position && position.time ? Either.right({ - from: getTimeRangeStartFromTime(position.time), - to: getTimeRangeEndFromTime(position.time), + from: getTimeRangeStartFromTime(moment(position.time).valueOf()), + to: getTimeRangeEndFromTime(moment(position.time).valueOf()), }) : Either.left(null) ) diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx index 0811ec170853..babd0e9a1ae3 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_example_message.tsx @@ -58,7 +58,7 @@ export const CategoryExampleMessage: React.FunctionComponent<{ search: { logPosition: encode({ end: moment(timeRange.endTime).format('YYYY-MM-DDTHH:mm:ss.SSSZ'), - position: { tiebreaker, time: timestamp }, + position: { tiebreaker, time: moment(timestamp).toISOString() }, start: moment(timeRange.startTime).format('YYYY-MM-DDTHH:mm:ss.SSSZ'), streamLive: false, }), @@ -128,7 +128,10 @@ export const CategoryExampleMessage: React.FunctionComponent<{ id, index: '', // TODO: use real index when loading via async search context, - cursor: { time: timestamp, tiebreaker }, + cursor: { + time: moment(timestamp).toISOString(), + tiebreaker, + }, columns: [], }; trackMetric({ metric: 'view_in_context__categories' }); diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/log_entry_example.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/log_entry_example.tsx index 288fc3df39c8..174abbbe60bc 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/log_entry_example.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/log_entry_example.tsx @@ -102,7 +102,7 @@ export const LogEntryExampleMessage: React.FunctionComponent = ({ search: { logPosition: encode({ end: moment(timeRange.endTime).format('YYYY-MM-DDTHH:mm:ss.SSSZ'), - position: { tiebreaker, time: timestamp }, + position: { tiebreaker, time: moment(timestamp).toISOString() }, start: moment(timeRange.startTime).format('YYYY-MM-DDTHH:mm:ss.SSSZ'), streamLive: false, }), diff --git a/x-pack/plugins/infra/public/pages/logs/stream/page_content.tsx b/x-pack/plugins/infra/public/pages/logs/stream/page_content.tsx index b011931adbd8..e14d52fe1693 100644 --- a/x-pack/plugins/infra/public/pages/logs/stream/page_content.tsx +++ b/x-pack/plugins/infra/public/pages/logs/stream/page_content.tsx @@ -38,7 +38,7 @@ export const ConnectedStreamPageContent: React.FC = () => { jumpToTargetPosition: (targetPosition: TimeKey | null) => { logStreamPageSend({ type: 'JUMP_TO_TARGET_POSITION', targetPosition }); }, - jumpToTargetPositionTime: (time: number) => { + jumpToTargetPositionTime: (time: string) => { logStreamPageSend({ type: 'JUMP_TO_TARGET_POSITION', targetPosition: { time } }); }, reportVisiblePositions: (visiblePositions: VisiblePositions) => { diff --git a/x-pack/plugins/infra/public/pages/logs/stream/page_logs_content.tsx b/x-pack/plugins/infra/public/pages/logs/stream/page_logs_content.tsx index 14ebe864d5d6..9841553ce986 100644 --- a/x-pack/plugins/infra/public/pages/logs/stream/page_logs_content.tsx +++ b/x-pack/plugins/infra/public/pages/logs/stream/page_logs_content.tsx @@ -8,7 +8,7 @@ import { EuiSpacer } from '@elastic/eui'; import type { Query } from '@kbn/es-query'; import { euiStyled } from '@kbn/kibana-react-plugin/common'; -import { LogEntry } from '@kbn/logs-shared-plugin/common'; +import { LogEntry, convertISODateToNanoPrecision } from '@kbn/logs-shared-plugin/common'; import { LogEntryFlyout, LogEntryStreamItem, @@ -117,8 +117,12 @@ export const StreamPageLogsContent = React.memo<{ const isCenterPointOutsideLoadedRange = targetPosition != null && - ((topCursor != null && targetPosition.time < topCursor.time) || - (bottomCursor != null && targetPosition.time > bottomCursor.time)); + ((topCursor != null && + convertISODateToNanoPrecision(targetPosition.time) < + convertISODateToNanoPrecision(topCursor.time)) || + (bottomCursor != null && + convertISODateToNanoPrecision(targetPosition.time) > + convertISODateToNanoPrecision(bottomCursor.time))); const hasQueryChanged = filterQuery !== prevFilterQuery; diff --git a/x-pack/plugins/infra/public/test_utils/entries.ts b/x-pack/plugins/infra/public/test_utils/entries.ts index 35dad808cdf4..0b04c90ee633 100644 --- a/x-pack/plugins/infra/public/test_utils/entries.ts +++ b/x-pack/plugins/infra/public/test_utils/entries.ts @@ -18,14 +18,16 @@ export function generateFakeEntries( const timestampStep = Math.floor((endTimestamp - startTimestamp) / count); for (let i = 0; i < count; i++) { const timestamp = i === count - 1 ? endTimestamp : startTimestamp + timestampStep * i; + const date = new Date(timestamp).toISOString(); + entries.push({ id: `entry-${i}`, index: 'logs-fake', context: {}, - cursor: { time: timestamp, tiebreaker: i }, + cursor: { time: date, tiebreaker: i }, columns: columns.map((column) => { if ('timestampColumn' in column) { - return { columnId: column.timestampColumn.id, timestamp }; + return { columnId: column.timestampColumn.id, time: date }; } else if ('messageColumn' in column) { return { columnId: column.messageColumn.id, diff --git a/x-pack/plugins/logs_shared/common/index.ts b/x-pack/plugins/logs_shared/common/index.ts index 654e8e2e1333..99fd7c116686 100644 --- a/x-pack/plugins/logs_shared/common/index.ts +++ b/x-pack/plugins/logs_shared/common/index.ts @@ -44,6 +44,8 @@ export { // eslint-disable-next-line @kbn/eslint/no_export_all export * from './log_entry'; +export { convertISODateToNanoPrecision } from './utils'; + // Http types export type { LogEntriesSummaryBucket, LogEntriesSummaryHighlightsBucket } from './http_api'; diff --git a/x-pack/plugins/logs_shared/common/log_entry/log_entry.ts b/x-pack/plugins/logs_shared/common/log_entry/log_entry.ts index b1fc80634a92..24a04f4c2e38 100644 --- a/x-pack/plugins/logs_shared/common/log_entry/log_entry.ts +++ b/x-pack/plugins/logs_shared/common/log_entry/log_entry.ts @@ -5,8 +5,8 @@ * 2.0. */ +import { TimeKey } from '@kbn/io-ts-utils'; import * as rt from 'io-ts'; -import { TimeKey } from '../time'; import { jsonArrayRT } from '../typed_json'; import { logEntryCursorRT } from './log_entry_cursor'; @@ -34,7 +34,7 @@ export type LogMessagePart = rt.TypeOf; * columns */ -export const logTimestampColumnRT = rt.type({ columnId: rt.string, timestamp: rt.number }); +export const logTimestampColumnRT = rt.type({ columnId: rt.string, time: rt.string }); export type LogTimestampColumn = rt.TypeOf; export const logFieldColumnRT = rt.type({ diff --git a/x-pack/plugins/logs_shared/common/log_entry/log_entry_cursor.ts b/x-pack/plugins/logs_shared/common/log_entry/log_entry_cursor.ts index 3cd69b4a6178..158d14152ad4 100644 --- a/x-pack/plugins/logs_shared/common/log_entry/log_entry_cursor.ts +++ b/x-pack/plugins/logs_shared/common/log_entry/log_entry_cursor.ts @@ -9,7 +9,7 @@ import * as rt from 'io-ts'; import { decodeOrThrow } from '../runtime_types'; export const logEntryCursorRT = rt.type({ - time: rt.number, + time: rt.string, tiebreaker: rt.number, }); export type LogEntryCursor = rt.TypeOf; @@ -29,7 +29,7 @@ export const logEntryAroundCursorRT = rt.type({ }); export type LogEntryAroundCursor = rt.TypeOf; -export const getLogEntryCursorFromHit = (hit: { sort: [number, number] }) => +export const getLogEntryCursorFromHit = (hit: { sort: [string, number] }) => decodeOrThrow(logEntryCursorRT)({ time: hit.sort[0], tiebreaker: hit.sort[1], diff --git a/x-pack/plugins/logs_shared/common/time/time_key.ts b/x-pack/plugins/logs_shared/common/time/time_key.ts index 4c78158dd5bf..1ff661af69d0 100644 --- a/x-pack/plugins/logs_shared/common/time/time_key.ts +++ b/x-pack/plugins/logs_shared/common/time/time_key.ts @@ -5,26 +5,8 @@ * 2.0. */ +import type { TimeKey } from '@kbn/io-ts-utils'; import { ascending, bisector } from 'd3-array'; -import * as rt from 'io-ts'; - -export const minimalTimeKeyRT = rt.type({ - time: rt.number, - tiebreaker: rt.number, -}); - -export const timeKeyRT = rt.intersection([ - minimalTimeKeyRT, - rt.partial({ - gid: rt.string, - fromAutoReload: rt.boolean, - }), -]); -export type TimeKey = rt.TypeOf; - -export interface UniqueTimeKey extends TimeKey { - gid: string; -} export type Comparator = (firstValue: any, secondValue: any) => number; diff --git a/x-pack/plugins/logs_shared/common/utils/date_helpers.test.ts b/x-pack/plugins/logs_shared/common/utils/date_helpers.test.ts new file mode 100644 index 000000000000..fb04f6443561 --- /dev/null +++ b/x-pack/plugins/logs_shared/common/utils/date_helpers.test.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { subtractMillisecondsFromDate } from './date_helpers'; + +describe('Date Helpers', function () { + describe('subtractMillisecondsFromDate', function () { + it('should subtract milliseconds from the nano date correctly', () => { + const inputDate = '2023-10-30T12:00:00.001000000Z'; + const millisecondsToSubtract = 1; + + const result = subtractMillisecondsFromDate(inputDate, millisecondsToSubtract); + + const expectedDate = '2023-10-30T12:00:00.000000000Z'; + + expect(result).toBe(expectedDate); + }); + + it('should subtract seconds from the date if no milliseconds available', () => { + const inputDate = '2023-10-30T12:00:00.000000000Z'; + const millisecondsToSubtract = 1; + + const result = subtractMillisecondsFromDate(inputDate, millisecondsToSubtract); + + const expectedDate = '2023-10-30T11:59:59.999000000Z'; + + expect(result).toBe(expectedDate); + }); + + it('should convert date to nano and subtract milliseconds properly', () => { + const inputDate = '2023-10-30T12:00:00.000Z'; + const millisecondsToSubtract = 1; + + const result = subtractMillisecondsFromDate(inputDate, millisecondsToSubtract); + + const expectedDate = '2023-10-30T11:59:59.999000000Z'; + + expect(result).toBe(expectedDate); + }); + }); +}); diff --git a/x-pack/plugins/logs_shared/common/utils/date_helpers.ts b/x-pack/plugins/logs_shared/common/utils/date_helpers.ts new file mode 100644 index 000000000000..69af6c74110b --- /dev/null +++ b/x-pack/plugins/logs_shared/common/utils/date_helpers.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import dateMath from '@kbn/datemath'; + +export function convertISODateToNanoPrecision(date: string): string { + const dateParts = date.split('.'); + + const fractionSeconds = dateParts.length === 2 ? dateParts[1].replace('Z', '') : ''; + const fractionSecondsInNanos = + fractionSeconds.length !== 9 ? fractionSeconds.padEnd(9, '0') : fractionSeconds; + + return `${dateParts[0]}.${fractionSecondsInNanos}Z`; +} + +export function subtractMillisecondsFromDate(date: string, milliseconds: number): string { + const dateInNano = convertISODateToNanoPrecision(date); + + const dateParts = dateInNano.split('.'); + const nanoPart = dateParts[1].substring(3, dateParts[1].length); // given 123456789Z => 456789Z + + const isoDate = dateMath.parse(date)?.subtract(milliseconds, 'ms').toISOString(); + + return `${isoDate?.replace('Z', nanoPart)}`; +} diff --git a/x-pack/plugins/logs_shared/common/utils/index.ts b/x-pack/plugins/logs_shared/common/utils/index.ts new file mode 100644 index 000000000000..ba1e5fcefd1a --- /dev/null +++ b/x-pack/plugins/logs_shared/common/utils/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './date_helpers'; diff --git a/x-pack/plugins/logs_shared/public/components/formatted_time.tsx b/x-pack/plugins/logs_shared/public/components/formatted_time.tsx index 6c7309d814af..4b65436a670a 100644 --- a/x-pack/plugins/logs_shared/public/components/formatted_time.tsx +++ b/x-pack/plugins/logs_shared/public/components/formatted_time.tsx @@ -11,7 +11,7 @@ import { useMemo } from 'react'; import { useKibanaUiSetting } from '../utils/use_kibana_ui_setting'; const getFormattedTime = ( - time: number, + time: string, userFormat: string | undefined, fallbackFormat: string = 'Y-MM-DD HH:mm:ss.SSS' ) => { @@ -26,7 +26,7 @@ interface UseFormattedTimeOptions { } export const useFormattedTime = ( - time: number, + time: string, { format = 'dateTime', fallbackFormat }: UseFormattedTimeOptions = {} ) => { // `dateFormat:scaled` is an array of `[key, format]` tuples. diff --git a/x-pack/plugins/logs_shared/public/components/log_stream/log_stream.stories.tsx b/x-pack/plugins/logs_shared/public/components/log_stream/log_stream.stories.tsx index 10561b24afb0..eeba1e0b448d 100644 --- a/x-pack/plugins/logs_shared/public/components/log_stream/log_stream.stories.tsx +++ b/x-pack/plugins/logs_shared/public/components/log_stream/log_stream.stories.tsx @@ -38,7 +38,7 @@ export const BasicDateRange = LogStreamStoryTemplate.bind({}); export const CenteredOnLogEntry = LogStreamStoryTemplate.bind({}); CenteredOnLogEntry.args = { - center: { time: 1595146275000, tiebreaker: 150 }, + center: { time: '2020-07-19T08:11:15.000Z', tiebreaker: 150 }, }; export const HighlightedLogEntry = LogStreamStoryTemplate.bind({}); diff --git a/x-pack/plugins/logs_shared/public/components/log_stream/log_stream.tsx b/x-pack/plugins/logs_shared/public/components/log_stream/log_stream.tsx index 50871650da26..98be3f756788 100644 --- a/x-pack/plugins/logs_shared/public/components/log_stream/log_stream.tsx +++ b/x-pack/plugins/logs_shared/public/components/log_stream/log_stream.tsx @@ -40,7 +40,7 @@ interface CommonColumnDefinition { interface TimestampColumnDefinition extends CommonColumnDefinition { type: 'timestamp'; /** Timestamp renderer. Takes a epoch_millis and returns a valid `ReactNode` */ - render?: (timestamp: number) => React.ReactNode; + render?: (timestamp: string) => React.ReactNode; } interface MessageColumnDefinition extends CommonColumnDefinition { diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx b/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx index 5a6639a1f229..a6957524b2f9 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx +++ b/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_actions_menu.test.tsx @@ -24,6 +24,7 @@ const ProviderWrapper: React.FC = ({ children }) => { }; describe('LogEntryActionsMenu component', () => { + const time = new Date().toISOString(); describe('uptime link', () => { it('renders with a host ip filter when present in log entry', () => { const elementWrapper = mount( @@ -34,7 +35,7 @@ describe('LogEntryActionsMenu component', () => { id: 'ITEM_ID', index: 'INDEX', cursor: { - time: 0, + time, tiebreaker: 0, }, }} @@ -64,7 +65,7 @@ describe('LogEntryActionsMenu component', () => { id: 'ITEM_ID', index: 'INDEX', cursor: { - time: 0, + time, tiebreaker: 0, }, }} @@ -94,7 +95,7 @@ describe('LogEntryActionsMenu component', () => { id: 'ITEM_ID', index: 'INDEX', cursor: { - time: 0, + time, tiebreaker: 0, }, }} @@ -128,7 +129,7 @@ describe('LogEntryActionsMenu component', () => { id: 'ITEM_ID', index: 'INDEX', cursor: { - time: 0, + time, tiebreaker: 0, }, }} @@ -160,7 +161,7 @@ describe('LogEntryActionsMenu component', () => { id: 'ITEM_ID', index: 'INDEX', cursor: { - time: 0, + time, tiebreaker: 0, }, }} @@ -194,7 +195,7 @@ describe('LogEntryActionsMenu component', () => { id: 'ITEM_ID', index: 'INDEX', cursor: { - time: 0, + time, tiebreaker: 0, }, }} @@ -228,7 +229,7 @@ describe('LogEntryActionsMenu component', () => { id: 'ITEM_ID', index: 'INDEX', cursor: { - time: 0, + time, tiebreaker: 0, }, }} @@ -258,7 +259,7 @@ describe('LogEntryActionsMenu component', () => { id: 'ITEM_ID', index: 'INDEX', cursor: { - time: 0, + time, tiebreaker: 0, }, }} diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_fields_table.tsx b/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_fields_table.tsx index 09b838185935..dbde4085da28 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_fields_table.tsx +++ b/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_fields_table.tsx @@ -10,9 +10,9 @@ import { EuiBasicTableColumn, EuiInMemoryTable } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import * as rt from 'io-ts'; import React, { useMemo } from 'react'; +import { TimeKey } from '@kbn/io-ts-utils'; import { LogEntryField } from '../../../../common/log_entry'; import { LogEntry } from '../../../../common/search_strategies/log_entries/log_entry'; -import { TimeKey } from '../../../../common/time'; import { JsonScalar, jsonScalarRT } from '../../../../common/typed_json'; import { FieldValue } from '../log_text_stream/field_value'; diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_flyout.tsx b/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_flyout.tsx index b66e864c2a49..99b89f0c1c43 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_flyout.tsx +++ b/x-pack/plugins/logs_shared/public/components/logging/log_entry_flyout/log_entry_flyout.tsx @@ -21,9 +21,9 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { createKibanaReactContext } from '@kbn/kibana-react-plugin/public'; import React, { useCallback, useEffect, useRef } from 'react'; +import { TimeKey } from '@kbn/io-ts-utils'; import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; import { LogViewReference } from '../../../../common/log_views'; -import { TimeKey } from '../../../../common/time'; import { useLogEntry } from '../../../containers/logs/log_entry'; import { CenteredEuiFlyoutBody } from '../../centered_flyout_body'; import { DataSearchErrorCallout } from '../../data_search_error_callout'; diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/column_headers.tsx b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/column_headers.tsx index 3fd275ea56dc..a1b1cbf8fe77 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/column_headers.tsx +++ b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/column_headers.tsx @@ -41,7 +41,7 @@ export const LogColumnHeaders: React.FunctionComponent<{ columnHeader = columnConfiguration.timestampColumn.header; } else { columnHeader = firstVisiblePosition - ? localizedDate(firstVisiblePosition.time) + ? localizedDate(new Date(firstVisiblePosition.time)) : i18n.translate('xpack.logsShared.logs.stream.timestampColumnTitle', { defaultMessage: 'Timestamp', }); diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/item.ts b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/item.ts index d6dc5fdbef4f..86e2baa9d6f9 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/item.ts +++ b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/item.ts @@ -5,9 +5,10 @@ * 2.0. */ +import type { TimeKey } from '@kbn/io-ts-utils'; import { bisector } from 'd3-array'; -import { compareToTimeKey, TimeKey } from '../../../../common/time'; import { LogEntry } from '../../../../common/log_entry'; +import { compareToTimeKey } from '../../../../common/time'; export type StreamItem = LogEntryStreamItem; @@ -27,17 +28,17 @@ export function getStreamItemTimeKey(item: StreamItem) { export function getStreamItemId(item: StreamItem) { switch (item.kind) { case 'logEntry': - return `${item.logEntry.cursor.time}:${item.logEntry.cursor.tiebreaker}:${item.logEntry.id}`; + return `${item.logEntry.cursor.time}/${item.logEntry.cursor.tiebreaker}/${item.logEntry.id}`; } } export function parseStreamItemId(id: string) { - const idFragments = id.split(':'); + const idFragments = id.split('/'); return { gid: idFragments.slice(2).join(':'), tiebreaker: parseInt(idFragments[1], 10), - time: parseInt(idFragments[0], 10), + time: idFragments[0], }; } diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_date_row.tsx b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_date_row.tsx index cd499b3ae342..9bc6e29429ce 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_date_row.tsx +++ b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_date_row.tsx @@ -10,14 +10,14 @@ import { EuiFlexGroup, EuiFlexItem, EuiHorizontalRule, EuiTitle } from '@elastic import { localizedDate } from '../../../../common/formatters/datetime'; interface LogDateRowProps { - timestamp: number; + time: string; } /** * Show a row with the date in the log stream */ -export const LogDateRow: React.FC = ({ timestamp }) => { - const formattedDate = localizedDate(timestamp); +export const LogDateRow: React.FC = ({ time }) => { + const formattedDate = localizedDate(new Date(time)); return ( diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_column.tsx b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_column.tsx index 81d6a43254ec..7df56fb79c32 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_column.tsx +++ b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_column.tsx @@ -8,6 +8,7 @@ import { useMemo } from 'react'; import { euiStyled } from '@kbn/kibana-react-plugin/common'; +import moment from 'moment'; import { TextScale } from '../../../../common/log_text_scale'; import { LogColumnRenderConfiguration, @@ -142,7 +143,7 @@ export const useColumnWidths = ({ timeFormat?: TimeFormat; }) => { const { CharacterDimensionsProbe, dimensions } = useMeasuredCharacterDimensions(scale); - const referenceTime = useMemo(() => Date.now(), []); + const referenceTime = useMemo(() => moment().toISOString(), []); const formattedCurrentDate = useFormattedTime(referenceTime, { format: timeFormat }); const columnWidths = useMemo( () => getColumnWidths(columnConfigurations, dimensions.width, formattedCurrentDate.length), diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_row.tsx b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_row.tsx index 5b100c4df4be..b73da833032f 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_row.tsx +++ b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_row.tsx @@ -189,7 +189,7 @@ export const LogEntryRow = memo( > {isTimestampColumn(column) ? ( ) : null} diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_timestamp_column.tsx b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_timestamp_column.tsx index 22a94f05ea26..0b145503ddf5 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_timestamp_column.tsx +++ b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/log_entry_timestamp_column.tsx @@ -13,8 +13,8 @@ import { LogEntryColumnContent } from './log_entry_column'; interface LogEntryTimestampColumnProps { format?: TimeFormat; - time: number; - render?: (timestamp: number) => React.ReactNode; + time: string; + render?: (time: string) => React.ReactNode; } export const LogEntryTimestampColumn = memo( diff --git a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/scrollable_log_text_stream_view.tsx b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/scrollable_log_text_stream_view.tsx index c92347ce8c0b..feae9b8b21d8 100644 --- a/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/scrollable_log_text_stream_view.tsx +++ b/x-pack/plugins/logs_shared/public/components/logging/log_text_stream/scrollable_log_text_stream_view.tsx @@ -9,10 +9,9 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import React, { Fragment, GetDerivedStateFromProps } from 'react'; import moment from 'moment'; - import { euiStyled } from '@kbn/kibana-react-plugin/common'; +import { TimeKey, UniqueTimeKey } from '@kbn/io-ts-utils'; import { TextScale } from '../../../../common/log_text_scale'; -import { TimeKey, UniqueTimeKey } from '../../../../common/time'; import { callWithoutRepeats } from '../../../utils/handlers'; import { AutoSizer } from '../../auto_sizer'; import { NoData } from '../../empty_states'; @@ -216,7 +215,7 @@ export class ScrollableLogTextStreamView extends React.PureComponent< position="start" isLoading={isLoadingMore} hasMore={hasMoreBeforeStart} - timestamp={items[0].logEntry.cursor.time} + timestamp={moment(items[0].logEntry.cursor.time).valueOf()} isStreaming={false} startDateExpression={startDateExpression} endDateExpression={endDateExpression} @@ -225,17 +224,17 @@ export class ScrollableLogTextStreamView extends React.PureComponent< } /> {items.map((item, idx) => { - const currentTimestamp = item.logEntry.cursor.time; + const currentTime = item.logEntry.cursor.time; let showDate = false; if (idx > 0) { - const prevTimestamp = items[idx - 1].logEntry.cursor.time; - showDate = !moment(currentTimestamp).isSame(prevTimestamp, 'day'); + const prevTime = items[idx - 1].logEntry.cursor.time; + showDate = !moment(currentTime).isSame(prevTime, 'day'); } return ( - {showDate && } + {showDate && } void; - jumpToTargetPositionTime: (time: number) => void; + jumpToTargetPositionTime: (time: string) => void; reportVisiblePositions: (visPos: VisiblePositions) => void; startLiveStreaming: () => void; stopLiveStreaming: () => void; @@ -62,7 +62,7 @@ export interface LogPositionCallbacks { export interface LogStreamPageCallbacks { updateTimeRange: (timeRange: Partial) => void; jumpToTargetPosition: (targetPosition: TimeKey | null) => void; - jumpToTargetPositionTime: (time: number) => void; + jumpToTargetPositionTime: (time: string) => void; reportVisiblePositions: (visiblePositions: VisiblePositions) => void; startLiveStreaming: () => void; stopLiveStreaming: () => void; diff --git a/x-pack/plugins/logs_shared/public/containers/logs/log_stream/use_fetch_log_entries_around.ts b/x-pack/plugins/logs_shared/public/containers/logs/log_stream/use_fetch_log_entries_around.ts index a6bd8ba79428..6dd2566392ee 100644 --- a/x-pack/plugins/logs_shared/public/containers/logs/log_stream/use_fetch_log_entries_around.ts +++ b/x-pack/plugins/logs_shared/public/containers/logs/log_stream/use_fetch_log_entries_around.ts @@ -8,6 +8,7 @@ import { useCallback } from 'react'; import { combineLatest, Observable, ReplaySubject } from 'rxjs'; import { last, map, startWith, switchMap } from 'rxjs/operators'; +import { subtractMillisecondsFromDate } from '../../../../common/utils'; import { LogEntryCursor } from '../../../../common/log_entry'; import { LogViewColumnConfiguration, LogViewReference } from '../../../../common/log_views'; import { LogEntriesSearchRequestQuery } from '../../../../common/search_strategies/log_entries/log_entries'; @@ -73,7 +74,7 @@ export const useFetchLogEntriesAround = ({ last(), // in the future we could start earlier if we receive partial results already map((lastBeforeSearchResponse) => { const cursorAfter = lastBeforeSearchResponse.response.data?.bottomCursor ?? { - time: cursor.time - 1, + time: subtractMillisecondsFromDate(cursor.time, 1), tiebreaker: 0, }; diff --git a/x-pack/plugins/logs_shared/public/test_utils/entries.ts b/x-pack/plugins/logs_shared/public/test_utils/entries.ts index 4dc3732fd49d..5277f49b1175 100644 --- a/x-pack/plugins/logs_shared/public/test_utils/entries.ts +++ b/x-pack/plugins/logs_shared/public/test_utils/entries.ts @@ -27,14 +27,16 @@ export function generateFakeEntries( const timestampStep = Math.floor((endTimestamp - startTimestamp) / count); for (let i = 0; i < count; i++) { const timestamp = i === count - 1 ? endTimestamp : startTimestamp + timestampStep * i; + const date = new Date(timestamp).toISOString(); + entries.push({ id: `entry-${i}`, index: 'logs-fake', context: {}, - cursor: { time: timestamp, tiebreaker: i }, + cursor: { time: date, tiebreaker: i }, columns: columns.map((column) => { if ('timestampColumn' in column) { - return { columnId: column.timestampColumn.id, timestamp }; + return { columnId: column.timestampColumn.id, time: date }; } else if ('messageColumn' in column) { return { columnId: column.messageColumn.id, diff --git a/x-pack/plugins/logs_shared/public/utils/log_column_render_configuration.tsx b/x-pack/plugins/logs_shared/public/utils/log_column_render_configuration.tsx index ff4a24f1498a..a1b23c9a0e3d 100644 --- a/x-pack/plugins/logs_shared/public/utils/log_column_render_configuration.tsx +++ b/x-pack/plugins/logs_shared/public/utils/log_column_render_configuration.tsx @@ -19,7 +19,7 @@ interface CommonRenderConfiguration { interface TimestampColumnRenderConfiguration { timestampColumn: CommonRenderConfiguration & { - render?: (timestamp: number) => ReactNode; + render?: (time: string) => ReactNode; }; } diff --git a/x-pack/plugins/logs_shared/public/utils/log_entry/log_entry.ts b/x-pack/plugins/logs_shared/public/utils/log_entry/log_entry.ts index 18d118750fde..e1027aeeb579 100644 --- a/x-pack/plugins/logs_shared/public/utils/log_entry/log_entry.ts +++ b/x-pack/plugins/logs_shared/public/utils/log_entry/log_entry.ts @@ -5,8 +5,9 @@ * 2.0. */ +import type { TimeKey, UniqueTimeKey } from '@kbn/io-ts-utils'; import { bisector } from 'd3-array'; -import { compareToTimeKey, getIndexAtTimeKey, TimeKey, UniqueTimeKey } from '../../../common/time'; +import { compareToTimeKey, getIndexAtTimeKey } from '../../../common/time'; import { LogEntry, LogColumn, @@ -38,7 +39,7 @@ export const getLogEntryAtTime = (entries: LogEntry[], time: TimeKey) => { }; export const isTimestampColumn = (column: LogColumn): column is LogTimestampColumn => - column != null && 'timestamp' in column; + column != null && 'time' in column; export const isMessageColumn = (column: LogColumn): column is LogMessageColumn => column != null && 'message' in column; diff --git a/x-pack/plugins/logs_shared/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts b/x-pack/plugins/logs_shared/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts index 66de5699cfb3..e4eed9b61d34 100644 --- a/x-pack/plugins/logs_shared/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts +++ b/x-pack/plugins/logs_shared/server/lib/adapters/log_entries/kibana_log_entries_adapter.ts @@ -62,7 +62,11 @@ export class LogsSharedKibanaLogEntriesAdapter implements LogEntriesAdapter { : {}; const sort = { - [TIMESTAMP_FIELD]: sortDirection, + [TIMESTAMP_FIELD]: { + order: sortDirection, + format: 'strict_date_optional_time_nanos', + numeric_type: 'date_nanos', + }, [TIEBREAKER_FIELD]: sortDirection, }; @@ -155,7 +159,16 @@ export class LogsSharedKibanaLogEntriesAdapter implements LogEntriesAdapter { top_hits_by_key: { top_hits: { size: 1, - sort: [{ [TIMESTAMP_FIELD]: 'asc' }, { [TIEBREAKER_FIELD]: 'asc' }], + sort: [ + { + [TIMESTAMP_FIELD]: { + order: 'asc', + format: 'strict_date_optional_time_nanos', + numeric_type: 'date_nanos', + }, + }, + { [TIEBREAKER_FIELD]: 'asc' }, + ], _source: false, }, }, @@ -265,7 +278,7 @@ const createQueryFilterClauses = (filterQuery: LogEntryQuery | undefined) => function processCursor(cursor: LogEntriesParams['cursor']): { sortDirection: 'asc' | 'desc'; - searchAfterClause: { search_after?: readonly [number, number] }; + searchAfterClause: { search_after?: readonly [string, number] }; } { if (cursor) { if ('before' in cursor) { @@ -295,7 +308,7 @@ const LogSummaryDateRangeBucketRuntimeType = runtimeTypes.intersection([ hits: runtimeTypes.type({ hits: runtimeTypes.array( runtimeTypes.type({ - sort: runtimeTypes.tuple([runtimeTypes.number, runtimeTypes.number]), + sort: runtimeTypes.tuple([runtimeTypes.string, runtimeTypes.number]), }) ), }), diff --git a/x-pack/plugins/logs_shared/server/lib/domains/log_entries_domain/log_entries_domain.ts b/x-pack/plugins/logs_shared/server/lib/domains/log_entries_domain/log_entries_domain.ts index 92829c676b93..2601167f2d98 100644 --- a/x-pack/plugins/logs_shared/server/lib/domains/log_entries_domain/log_entries_domain.ts +++ b/x-pack/plugins/logs_shared/server/lib/domains/log_entries_domain/log_entries_domain.ts @@ -7,6 +7,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { JsonObject } from '@kbn/utility-types'; +import { subtractMillisecondsFromDate } from '../../../../common/utils'; import { LogEntriesSummaryBucket, LogEntriesSummaryHighlightsBucket, @@ -147,7 +148,7 @@ export class LogsSharedLogEntriesDomain implements ILogsSharedLogEntriesDomain { const cursorAfter = entriesBefore.length > 0 ? entriesBefore[entriesBefore.length - 1].cursor - : { time: center.time - 1, tiebreaker: 0 }; + : { time: subtractMillisecondsFromDate(center.time, 1), tiebreaker: 0 }; const { entries: entriesAfter, hasMoreAfter } = await this.getLogEntries( requestContext, @@ -200,7 +201,7 @@ export class LogsSharedLogEntriesDomain implements ILogsSharedLogEntriesDomain { if ('timestampColumn' in column) { return { columnId: column.timestampColumn.id, - timestamp: doc.cursor.time, + time: doc.cursor.time, }; } else if ('messageColumn' in column) { return { diff --git a/x-pack/plugins/logs_shared/server/services/log_entries/log_entries_search_strategy.test.ts b/x-pack/plugins/logs_shared/server/services/log_entries/log_entries_search_strategy.test.ts index 305f6292deb2..177e6091dc06 100644 --- a/x-pack/plugins/logs_shared/server/services/log_entries/log_entries_search_strategy.test.ts +++ b/x-pack/plugins/logs_shared/server/services/log_entries/log_entries_search_strategy.test.ts @@ -95,6 +95,7 @@ describe('LogEntries search strategy', () => { }); it('handles subsequent polling requests', async () => { + const date = new Date(1605116827143).toISOString(); const esSearchStrategyMock = createEsSearchStrategyMock({ id: 'ASYNC_REQUEST_ID', isRunning: false, @@ -112,12 +113,12 @@ describe('LogEntries search strategy', () => { _score: 0, _source: null, fields: { - '@timestamp': [1605116827143], + '@timestamp': [date], 'event.dataset': ['HIT_DATASET'], message: ['HIT_MESSAGE'], 'container.id': ['HIT_CONTAINER_ID'], }, - sort: [1605116827143 as any, 1 as any], // incorrectly typed as string upstream + sort: [date as any, 1 as any], // incorrectly typed as string upstream }, ], }, @@ -164,13 +165,13 @@ describe('LogEntries search strategy', () => { id: 'HIT_ID', index: 'HIT_INDEX', cursor: { - time: 1605116827143, + time: date, tiebreaker: 1, }, columns: [ { columnId: 'TIMESTAMP_COLUMN_ID', - timestamp: 1605116827143, + time: date, }, { columnId: 'DATASET_COLUMN_ID', diff --git a/x-pack/plugins/logs_shared/server/services/log_entries/log_entries_search_strategy.ts b/x-pack/plugins/logs_shared/server/services/log_entries/log_entries_search_strategy.ts index f0f5c6304d61..fb305d9ae73e 100644 --- a/x-pack/plugins/logs_shared/server/services/log_entries/log_entries_search_strategy.ts +++ b/x-pack/plugins/logs_shared/server/services/log_entries/log_entries_search_strategy.ts @@ -190,7 +190,7 @@ const getLogEntryFromHit = if ('timestampColumn' in column) { return { columnId: column.timestampColumn.id, - timestamp: cursor.time, + time: cursor.time, }; } else if ('messageColumn' in column) { return { diff --git a/x-pack/plugins/logs_shared/server/services/log_entries/log_entry_search_strategy.test.ts b/x-pack/plugins/logs_shared/server/services/log_entries/log_entry_search_strategy.test.ts index 19d534512237..a123536d5bb2 100644 --- a/x-pack/plugins/logs_shared/server/services/log_entries/log_entry_search_strategy.test.ts +++ b/x-pack/plugins/logs_shared/server/services/log_entries/log_entry_search_strategy.test.ts @@ -99,6 +99,7 @@ describe('LogEntry search strategy', () => { }); it('handles subsequent polling requests', async () => { + const date = new Date(1605116827143).toISOString(); const esSearchStrategyMock = createEsSearchStrategyMock({ id: 'ASYNC_REQUEST_ID', isRunning: false, @@ -116,10 +117,10 @@ describe('LogEntry search strategy', () => { _score: 0, _source: null, fields: { - '@timestamp': [1605116827143], + '@timestamp': [date], message: ['HIT_MESSAGE'], }, - sort: [1605116827143 as any, 1 as any], // incorrectly typed as string upstream + sort: [date as any, 1 as any], // incorrectly typed as string upstream }, ], }, @@ -163,11 +164,11 @@ describe('LogEntry search strategy', () => { id: 'HIT_ID', index: 'HIT_INDEX', cursor: { - time: 1605116827143, + time: date, tiebreaker: 1, }, fields: [ - { field: '@timestamp', value: [1605116827143] }, + { field: '@timestamp', value: [date] }, { field: 'message', value: ['HIT_MESSAGE'] }, ], }); diff --git a/x-pack/plugins/logs_shared/server/services/log_entries/queries/common.ts b/x-pack/plugins/logs_shared/server/services/log_entries/queries/common.ts index 296ac9e4d2f3..c40e2fb9418f 100644 --- a/x-pack/plugins/logs_shared/server/services/log_entries/queries/common.ts +++ b/x-pack/plugins/logs_shared/server/services/log_entries/queries/common.ts @@ -11,7 +11,11 @@ export const createSortClause = ( tiebreakerField: string ) => ({ sort: { - [timestampField]: sortDirection, + [timestampField]: { + order: sortDirection, + format: 'strict_date_optional_time_nanos', + numeric_type: 'date_nanos', + }, [tiebreakerField]: sortDirection, }, }); diff --git a/x-pack/plugins/logs_shared/server/services/log_entries/queries/log_entries.ts b/x-pack/plugins/logs_shared/server/services/log_entries/queries/log_entries.ts index 18ee4b010436..18992448fdfc 100644 --- a/x-pack/plugins/logs_shared/server/services/log_entries/queries/log_entries.ts +++ b/x-pack/plugins/logs_shared/server/services/log_entries/queries/log_entries.ts @@ -68,7 +68,7 @@ export const getSortDirection = ( const createSearchAfterClause = ( cursor: LogEntryBeforeCursor | LogEntryAfterCursor | null | undefined -): { search_after?: [number, number] } => { +): { search_after?: [string, number] } => { if (logEntryBeforeCursorRT.is(cursor) && cursor.before !== 'last') { return { search_after: [cursor.before.time, cursor.before.tiebreaker], @@ -122,7 +122,7 @@ const createHighlightQuery = ( export const logEntryHitRT = rt.intersection([ commonHitFieldsRT, rt.type({ - sort: rt.tuple([rt.number, rt.number]), + sort: rt.tuple([rt.string, rt.number]), }), rt.partial({ fields: rt.record(rt.string, jsonArrayRT), diff --git a/x-pack/plugins/logs_shared/server/services/log_entries/queries/log_entry.ts b/x-pack/plugins/logs_shared/server/services/log_entries/queries/log_entry.ts index 57a644c79f01..575541f3f793 100644 --- a/x-pack/plugins/logs_shared/server/services/log_entries/queries/log_entry.ts +++ b/x-pack/plugins/logs_shared/server/services/log_entries/queries/log_entry.ts @@ -33,7 +33,16 @@ export const createGetLogEntryQuery = ( }, fields: ['*'], runtime_mappings: runtimeMappings, - sort: [{ [timestampField]: 'desc' }, { [tiebreakerField]: 'desc' }], + sort: [ + { + [timestampField]: { + order: 'desc', + format: 'strict_date_optional_time_nanos', + numeric_type: 'date_nanos', + }, + }, + { [tiebreakerField]: 'desc' }, + ], _source: false, }, }); @@ -41,7 +50,7 @@ export const createGetLogEntryQuery = ( export const logEntryHitRT = rt.intersection([ commonHitFieldsRT, rt.type({ - sort: rt.tuple([rt.number, rt.number]), + sort: rt.tuple([rt.string, rt.number]), }), rt.partial({ fields: rt.record(rt.string, jsonArrayRT), diff --git a/x-pack/test/api_integration/apis/metrics_ui/log_entry_highlights.ts b/x-pack/test/api_integration/apis/metrics_ui/log_entry_highlights.ts index 1e565f63b073..9910f4724045 100644 --- a/x-pack/test/api_integration/apis/metrics_ui/log_entry_highlights.ts +++ b/x-pack/test/api_integration/apis/metrics_ui/log_entry_highlights.ts @@ -19,6 +19,7 @@ import { logEntriesHighlightsResponseRT, } from '@kbn/logs-shared-plugin/common'; +import moment from 'moment'; import { FtrProviderContext } from '../../ftr_provider_context'; const KEY_BEFORE_START = { @@ -112,8 +113,8 @@ export default function ({ getService }: FtrProviderContext) { // Entries fall within range // @kbn/expect doesn't have a `lessOrEqualThan` or `moreOrEqualThan` comparators - expect(firstEntry.cursor.time >= KEY_BEFORE_START.time).to.be(true); - expect(lastEntry.cursor.time <= KEY_AFTER_END.time).to.be(true); + expect(firstEntry.cursor.time >= moment(KEY_BEFORE_START.time).toISOString()).to.be(true); + expect(lastEntry.cursor.time <= moment(KEY_AFTER_END.time).toISOString()).to.be(true); // All entries contain the highlights entries.forEach((entry) => { diff --git a/x-pack/test/functional/apps/infra/index.ts b/x-pack/test/functional/apps/infra/index.ts index b389d56b9032..948d99f7cce5 100644 --- a/x-pack/test/functional/apps/infra/index.ts +++ b/x-pack/test/functional/apps/infra/index.ts @@ -26,6 +26,7 @@ export default ({ loadTestFile }: FtrProviderContext) => { loadTestFile(require.resolve('./log_entry_categories_tab')); loadTestFile(require.resolve('./log_entry_rate_tab')); loadTestFile(require.resolve('./logs_source_configuration')); + loadTestFile(require.resolve('./log_stream_date_nano')); loadTestFile(require.resolve('./link_to')); loadTestFile(require.resolve('./log_stream')); }); diff --git a/x-pack/test/functional/apps/infra/link_to.ts b/x-pack/test/functional/apps/infra/link_to.ts index 7ad37696f5a4..f0e8da90d14d 100644 --- a/x-pack/test/functional/apps/infra/link_to.ts +++ b/x-pack/test/functional/apps/infra/link_to.ts @@ -18,7 +18,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { const retry = getService('retry'); const browser = getService('browser'); - const timestamp = Date.now(); + const date = new Date(); + const timestamp = date.getTime(); const startDate = new Date(timestamp - ONE_HOUR).toISOString(); const endDate = new Date(timestamp + ONE_HOUR).toISOString(); @@ -52,7 +53,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { `(query:(language:kuery,query:\'trace.id:${traceId}'),refreshInterval:(pause:!t,value:5000),timeRange:(from:'${startDate}',to:'${endDate}'))` ); expect(parsedUrl.searchParams.get('logPosition')).to.be( - `(position:(tiebreaker:0,time:${timestamp}))` + `(position:(tiebreaker:0,time:'${date.toISOString()}'))` ); expect(parsedUrl.searchParams.get('logView')).to.be(LOG_VIEW_REFERENCE); expect(documentTitle).to.contain('Stream - Logs - Observability - Elastic'); @@ -87,7 +88,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { `(query:(language:kuery,query:\'(kubernetes.pod.uid: 1234) and (trace.id:${traceId})\'),refreshInterval:(pause:!t,value:5000),timeRange:(from:'${startDate}',to:'${endDate}'))` ); expect(parsedUrl.searchParams.get('logPosition')).to.be( - `(position:(tiebreaker:0,time:${timestamp}))` + `(position:(tiebreaker:0,time:'${date.toISOString()}'))` ); expect(parsedUrl.searchParams.get('logView')).to.be(LOG_VIEW_REFERENCE); expect(documentTitle).to.contain('Stream - Logs - Observability - Elastic'); diff --git a/x-pack/test/functional/apps/infra/log_stream_date_nano.ts b/x-pack/test/functional/apps/infra/log_stream_date_nano.ts new file mode 100644 index 000000000000..99541e29a233 --- /dev/null +++ b/x-pack/test/functional/apps/infra/log_stream_date_nano.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { URL } from 'url'; +import { FtrProviderContext } from '../../ftr_provider_context'; +import { DATES } from './constants'; + +export default ({ getPageObjects, getService }: FtrProviderContext) => { + const retry = getService('retry'); + const browser = getService('browser'); + const esArchiver = getService('esArchiver'); + const logsUi = getService('logsUi'); + const find = getService('find'); + const logFilter = { + timeRange: { + from: DATES.metricsAndLogs.stream.startWithData, + to: DATES.metricsAndLogs.stream.endWithData, + }, + }; + + describe('Log stream supports nano precision', function () { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/infra/logs_with_nano_date'); + }); + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/infra/logs_with_nano_date'); + }); + + it('should display logs entries containing date_nano timestamps properly ', async () => { + await logsUi.logStreamPage.navigateTo({ logFilter }); + + const logStreamEntries = await logsUi.logStreamPage.getStreamEntries(); + + expect(logStreamEntries.length).to.be(4); + }); + + it('should render timestamp column properly', async () => { + await logsUi.logStreamPage.navigateTo({ logFilter }); + + await retry.try(async () => { + const columnHeaderLabels = await logsUi.logStreamPage.getColumnHeaderLabels(); + expect(columnHeaderLabels[0]).to.eql('Oct 17, 2018'); + }); + }); + + it('should render timestamp column values properly', async () => { + await logsUi.logStreamPage.navigateTo({ logFilter }); + + const logStreamEntries = await logsUi.logStreamPage.getStreamEntries(); + + const firstLogStreamEntry = logStreamEntries[0]; + + const entryTimestamp = await logsUi.logStreamPage.getLogEntryColumnValueByName( + firstLogStreamEntry, + 'timestampLogColumn' + ); + + expect(entryTimestamp).to.be('19:43:22.111'); + }); + + it('should properly sync logPosition in url', async () => { + const currentUrl = await browser.getCurrentUrl(); + const parsedUrl = new URL(currentUrl); + + expect(parsedUrl.searchParams.get('logPosition')).to.be( + `(position:(tiebreaker:3,time:\'2018-10-17T19:46:22.333333333Z\'))` + ); + }); + + it('should properly render timestamp in flyout with nano precision', async () => { + await logsUi.logStreamPage.navigateTo({ logFilter }); + + const logStreamEntries = await logsUi.logStreamPage.getStreamEntries(); + const firstLogStreamEntry = logStreamEntries[0]; + + await logsUi.logStreamPage.openLogEntryDetailsFlyout(firstLogStreamEntry); + + const cells = await find.allByCssSelector('.euiTableCellContent'); + + let isFound = false; + + for (const cell of cells) { + const cellText = await cell.getVisibleText(); + if (cellText === '2018-10-17T19:43:22.111111111Z') { + isFound = true; + return; + } + } + + expect(isFound).to.be(true); + }); + }); +}; diff --git a/x-pack/test/functional/es_archives/infra/logs_with_nano_date/data.json.gz b/x-pack/test/functional/es_archives/infra/logs_with_nano_date/data.json.gz new file mode 100644 index 000000000000..68e1284e7d13 Binary files /dev/null and b/x-pack/test/functional/es_archives/infra/logs_with_nano_date/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/infra/logs_with_nano_date/mappings.json b/x-pack/test/functional/es_archives/infra/logs_with_nano_date/mappings.json new file mode 100644 index 000000000000..294ee1f204e9 --- /dev/null +++ b/x-pack/test/functional/es_archives/infra/logs_with_nano_date/mappings.json @@ -0,0 +1,419 @@ +{ + "type": "data_stream", + "value": { + "data_stream": "logs-gaming-activity", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-gaming-activity" + ], + "name": "logs-gaming-activity", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} + +{ + "type": "data_stream", + "value": { + "data_stream": "logs-gaming-events", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-gaming-events" + ], + "name": "logs-gaming-events", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} + +{ + "type": "data_stream", + "value": { + "data_stream": "logs-gaming-scores", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-gaming-scores" + ], + "name": "logs-gaming-scores", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} + +{ + "type": "data_stream", + "value": { + "data_stream": "logs-manufacturing-downtime", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-manufacturing-downtime" + ], + "name": "logs-manufacturing-downtime", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} + +{ + "type": "data_stream", + "value": { + "data_stream": "logs-manufacturing-output", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-manufacturing-output" + ], + "name": "logs-manufacturing-output", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} + +{ + "type": "data_stream", + "value": { + "data_stream": "logs-manufacturing-quality", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-manufacturing-quality" + ], + "name": "logs-manufacturing-quality", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} + +{ + "type": "data_stream", + "value": { + "data_stream": "logs-retail-customers", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-retail-customers" + ], + "name": "logs-retail-customers", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} + +{ + "type": "data_stream", + "value": { + "data_stream": "logs-retail-inventory", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-retail-inventory" + ], + "name": "logs-retail-inventory", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} + +{ + "type": "data_stream", + "value": { + "data_stream": "logs-retail-promotions", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-retail-promotions" + ], + "name": "logs-retail-promotions", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} + +{ + "type": "data_stream", + "value": { + "data_stream": "logs-retail-sales", + "template": { + "_meta": { + "description": "Template for my time series data", + "my-custom-meta-field": "More arbitrary metadata" + }, + "data_stream": { + "allow_custom_routing": false, + "hidden": false + }, + "index_patterns": [ + "logs-retail-sales" + ], + "name": "logs-retail-sales", + "priority": 500, + "template": { + "mappings": { + "properties": { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "type": "date_nanos" + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + } + } + }, + "message": { + "type": "wildcard" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/x-pack/test/functional/services/logs_ui/log_stream.ts b/x-pack/test/functional/services/logs_ui/log_stream.ts index 214290bd21ef..1a068439a2d2 100644 --- a/x-pack/test/functional/services/logs_ui/log_stream.ts +++ b/x-pack/test/functional/services/logs_ui/log_stream.ts @@ -12,6 +12,7 @@ import { TabsParams } from '../../page_objects/infra_logs_page'; export function LogStreamPageProvider({ getPageObjects, getService }: FtrProviderContext) { const pageObjects = getPageObjects(['infraLogs']); const retry = getService('retry'); + const find = getService('find'); const testSubjects = getService('testSubjects'); return { @@ -43,6 +44,31 @@ export function LogStreamPageProvider({ getPageObjects, getService }: FtrProvide return await testSubjects.findAllDescendant('~logColumn', entryElement); }, + async getLogEntryColumnValueByName( + entryElement: WebElementWrapper, + column: string + ): Promise { + const columnElement = await testSubjects.findDescendant(`~${column}`, entryElement); + + const contentElement = await columnElement.findByCssSelector( + `[data-test-subj='LogEntryColumnContent']` + ); + + return await contentElement.getVisibleText(); + }, + + async openLogEntryDetailsFlyout(entryElement: WebElementWrapper) { + await entryElement.click(); + + const menuButton = await testSubjects.findDescendant( + `~infraLogEntryContextMenuButton`, + entryElement + ); + await menuButton.click(); + + await find.clickByButtonText('View details'); + }, + async getNoLogsIndicesPrompt() { return await testSubjects.find('noLogsIndicesPrompt'); },