From 9e1829eacde6a6632ad6b4d7fa737848b0aecd1d Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 1 Oct 2024 00:15:45 +1000 Subject: [PATCH] [8.x] [ML] Anomaly explorer: Show Data Gaps and Connect Anomalous Points on Single Metric Charts (#194119) (#194426) # Backport This will backport the following commits from `main` to `8.x`: - [[ML] Anomaly explorer: Show Data Gaps and Connect Anomalous Points on Single Metric Charts (#194119)](https://github.com/elastic/kibana/pull/194119) ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) Co-authored-by: Robert Jaszczurek <92210485+rbrtj@users.noreply.github.com> --- .../explorer_chart_single_metric.js | 22 +++++++++++++--- .../models/results_service/anomaly_charts.ts | 26 +++++++++++++++---- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_chart_single_metric.js b/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_chart_single_metric.js index e358c381288d3..1b746647ef38d 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_chart_single_metric.js +++ b/x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_chart_single_metric.js @@ -368,6 +368,7 @@ export class ExplorerChartSingleMetric extends React.Component { // These are used for displaying tooltips on mouseover. // Don't render dots where value=null (data gaps, with no anomalies) // or for multi-bucket anomalies. + // Except for scheduled events. const dots = lineChartGroup .append('g') .attr('class', 'chart-markers') @@ -375,7 +376,9 @@ export class ExplorerChartSingleMetric extends React.Component { .data( data.filter( (d) => - (d.value !== null || typeof d.anomalyScore === 'number') && + (d.value !== null || + typeof d.anomalyScore === 'number' || + d.scheduledEvents !== undefined) && !showMultiBucketAnomalyMarker(d) ) ); @@ -407,7 +410,11 @@ export class ExplorerChartSingleMetric extends React.Component { // Update all dots to new positions. dots .attr('cx', (d) => lineChartXScale(d.date)) - .attr('cy', (d) => lineChartYScale(d.value)) + // Fallback with domain's min value if value is null + // To ensure event markers are rendered properly at the bottom of the chart + .attr('cy', (d) => + lineChartYScale(d.value !== null ? d.value : lineChartYScale.domain()[0]) + ) .attr('class', (d) => { let markerClass = 'metric-value'; if (isAnomalyVisible(d)) { @@ -470,7 +477,14 @@ export class ExplorerChartSingleMetric extends React.Component { // Update all markers to new positions. scheduledEventMarkers .attr('x', (d) => lineChartXScale(d.date) - LINE_CHART_ANOMALY_RADIUS) - .attr('y', (d) => lineChartYScale(d.value) - SCHEDULED_EVENT_SYMBOL_HEIGHT / 2); + .attr( + 'y', + (d) => + // Fallback with domain's min value if value is null + // To ensure event markers are rendered properly at the bottom of the chart + lineChartYScale(d.value !== null ? d.value : lineChartYScale.domain()[0]) - + SCHEDULED_EVENT_SYMBOL_HEIGHT / 2 + ); } function showAnomalyPopover(marker, circle) { @@ -596,7 +610,7 @@ export class ExplorerChartSingleMetric extends React.Component { }); } } - } else { + } else if (marker.value !== null) { tooltipData.push({ label: i18n.translate( 'xpack.ml.explorer.singleMetricChart.valueWithoutAnomalyScoreLabel', diff --git a/x-pack/plugins/ml/server/models/results_service/anomaly_charts.ts b/x-pack/plugins/ml/server/models/results_service/anomaly_charts.ts index 1690e2db74164..771e40fc13336 100644 --- a/x-pack/plugins/ml/server/models/results_service/anomaly_charts.ts +++ b/x-pack/plugins/ml/server/models/results_service/anomaly_charts.ts @@ -1055,9 +1055,11 @@ export function anomalyChartsDataProvider(mlClient: MlClient, client: IScopedClu // differently because of how the source data is structured. // For rare chart values we are only interested wether a value is either `0` or not, // `0` acts like a flag in the chart whether to display the dot/marker. - // All other charts (single metric, population) are metric based and with + // For single metric chart, we need to pass null values to display data gaps. + // All other charts are distribution based and with // those a value of `null` acts as the flag to hide a data point. if ( + chartType === CHART_TYPE.SINGLE_METRIC || (chartType === CHART_TYPE.EVENT_DISTRIBUTION && value > 0) || (chartType !== CHART_TYPE.EVENT_DISTRIBUTION && value !== null) ) { @@ -1079,6 +1081,7 @@ export function anomalyChartsDataProvider(mlClient: MlClient, client: IScopedClu // Iterate through the anomaly records, adding anomalyScore properties // to the chartData entries for anomalous buckets. const chartDataForPointSearch = getChartDataForPointSearch(chartData, records[0], chartType); + let shouldSortChartData = false; each(records, (record) => { // Look for a chart point with the same time as the record. // If none found, insert a point for anomalies due to a gap in the data. @@ -1087,10 +1090,17 @@ export function anomalyChartsDataProvider(mlClient: MlClient, client: IScopedClu if (chartPoint === undefined) { chartPoint = { date: recordTime, value: null }; chartData.push(chartPoint); + shouldSortChartData = true; } if (chartPoint !== undefined) { chartPoint.anomalyScore = record.record_score; + // If it is an empty chart point, set the value to the actual value + // To properly display the anomaly marker on the chart + if (chartPoint.value === null) { + chartPoint.value = Array.isArray(record.actual) ? record.actual[0] : record.actual; + } + if (record.actual !== undefined) { chartPoint.actual = record.actual; chartPoint.typical = record.typical; @@ -1119,6 +1129,12 @@ export function anomalyChartsDataProvider(mlClient: MlClient, client: IScopedClu } }); + // Chart data is sorted by default, but if we added points for anomalies, + // we need to sort again to ensure the points are in the correct order. + if (shouldSortChartData) { + chartData.sort((a, b) => a.date - b.date); + } + // Add a scheduledEvents property to any points in the chart data set // which correspond to times of scheduled events for the job. if (scheduledEvents !== undefined) { @@ -1126,14 +1142,14 @@ export function anomalyChartsDataProvider(mlClient: MlClient, client: IScopedClu const chartPoint = findChartPointForTime(chartDataForPointSearch, Number(time)); if (chartPoint !== undefined) { chartPoint.scheduledEvents = events; - // We do not want to create additional points for single metric charts - // as it could break the chart. - } else if (chartType !== CHART_TYPE.SINGLE_METRIC) { + } else { // If there's no underlying metric data point for the scheduled event, // create a new chart point with a value of 0. + // Except for Single Metric Charts, where we want to create a point at the bottom of the chart. + // Which is not always `0`. const eventChartPoint: ChartPoint = { date: Number(time), - value: 0, + value: chartType === CHART_TYPE.SINGLE_METRIC ? null : 0, entity: SCHEDULE_EVENT_MARKER_ENTITY, scheduledEvents: events, };