Trace Analytics support for custom sources

common/constants/trace_analytics.ts

@@ -7,14 +7,20 @@ export const JAEGER_INDEX_NAME = '*jaeger-span-*';
 export const JAEGER_SERVICE_INDEX_NAME = '*jaeger-service*';
 export const DATA_PREPPER_INDEX_NAME = 'otel-v1-apm-span-*';
 export const DATA_PREPPER_SERVICE_INDEX_NAME = 'otel-v1-apm-service-map*';
-export const TRACE_ANALYTICS_DATE_FORMAT = 'MM/DD/YYYY HH:mm:ss';
-export const TRACE_ANALYTICS_PLOTS_DATE_FORMAT = 'MMM D, YYYY HH:mm:ss';
+export const TRACE_ANALYTICS_DATE_FORMAT = 'MM/DD/YYYY HH:mm:ss.SSS';
+export const TRACE_ANALYTICS_PLOTS_DATE_FORMAT = 'MMM D, YYYY HH:mm:ss.SSS';
 export const SERVICE_MAP_MAX_NODES = 500;
 // size limit when requesting edge related queries, not necessarily the number of edges
 export const SERVICE_MAP_MAX_EDGES = 1000;
 export const TRACES_MAX_NUM = 3000;
-export const TRACE_ANALYTICS_DOCUMENTATION_LINK = 'https://opensearch.org/docs/latest/observability-plugin/trace/index/';
+export const TRACE_ANALYTICS_DOCUMENTATION_LINK =
+ 'https://opensearch.org/docs/latest/observability-plugin/trace/index/';
 
-export const TRACE_ANALYTICS_JAEGER_INDICES_ROUTE = '/api/observability/trace_analytics/jaeger_indices';
-export const TRACE_ANALYTICS_DATA_PREPPER_INDICES_ROUTE = '/api/observability/trace_analytics/data_prepper_indices';
+export const TRACE_ANALYTICS_JAEGER_INDICES_ROUTE =
+ '/api/observability/trace_analytics/jaeger_indices';
+export const TRACE_ANALYTICS_DATA_PREPPER_INDICES_ROUTE =
+ '/api/observability/trace_analytics/data_prepper_indices';
 export const TRACE_ANALYTICS_DSL_ROUTE = '/api/observability/trace_analytics/query';
+
+export const TRACE_CUSTOM_SPAN_INDEX_SETTING = 'observability:traceAnalyticsSpanIndices';
+export const TRACE_CUSTOM_SERVICE_INDEX_SETTING = 'observability:traceAnalyticsServiceIndices';

common/types/trace_analytics.ts

@@ -52,3 +52,5 @@ export interface GraphVisEdge {
  to: number;
  color: string;
 }
+
+export type TraceAnalyticsMode = 'jaeger' | 'data_prepper' | 'custom_data_prepper';

public/components/application_analytics/components/application.tsx

@@ -134,7 +134,7 @@
  const [serviceFlyoutName, setServiceFlyoutName] = useState<string>('');
  const [traceFlyoutId, setTraceFlyoutId] = useState<string>('');
  const [spanFlyoutId, setSpanFlyoutId] = useState<string>('');
  const [spanDSL, setSpanDSL] = useState<any>({});
  const [totalSpans, setTotalSpans] = useState<number>(0);
  const [editVizId, setEditVizId] = useState<string>('');
  const [visWithAvailability, setVisWithAvailability] = useState<EuiSelectOption[]>([]);
@@ -157,7 +157,7 @@
  sessionStorage.setItem(`${application.name}EndTime`, newEndTime);
  };
 
  const addSpanFilter = (field: string, value: any) => {
  const newFilters = [...filters];
  const index = newFilters.findIndex(({ field: filterField }) => field === filterField);
  if (index === -1) {
@@ -286,7 +286,7 @@
  },
  ];
 
  const nameColumnAction = (item: any) => openServiceFlyout(item);
  const traceColumnAction = () => switchToTrace();
 
  const getService = () => {
@@ -314,8 +314,6 @@
  setSelectedTab(TAB_TRACE_ID);
  };
 
- const traceIdColumnAction = (item: any) => openTraceFlyout(item);
-
  const getTrace = () => {
  return (
  <>
@@ -325,7 +323,7 @@
  page="app"
  parentBreadcrumb={parentBreadcrumbs[0]}
  childBreadcrumbs={childBreadcrumbs}
- traceIdColumnAction={traceIdColumnAction}
+ openTraceFlyout={openTraceFlyout}
  startTime={appStartTime}
  endTime={appEndTime}
  setStartTime={setStartTimeForApp}

public/components/application_analytics/components/flyout_components/service_detail_flyout.tsx

@@ -49,10 +49,10 @@
  openSpanFlyout,
  mode,
  } = props;
  const [fields, setFields] = useState<any>({});
  const [serviceMap, setServiceMap] = useState<ServiceObject>({});
  const [total, setTotal] = useState(0);
  const [DSL, setDSL] = useState<any>({});
  const [serviceMapIdSelected, setServiceMapIdSelected] = useState<
  'latency' | 'error_rate' | 'throughput'
  >('latency');
@@ -133,7 +133,7 @@
  handleServiceViewRequest(serviceName, http, serviceDSL, setFields, mode);
  handleServiceMapRequest(http, serviceDSL, mode, '', setServiceMap, serviceName);
  const spanDSL = filtersToDsl(mode, filters, query, startTime, endTime, 'app', appConfigs);
- spanDSL.query.bool.must.push({
+ spanDSL.query.bool.filter.push({
  term: {
  serviceName,
  },

public/components/application_analytics/components/flyout_components/trace_detail_render.tsx

@@ -3,28 +3,33 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import { EuiText, EuiSpacer, EuiHorizontalRule, EuiCodeBlock } from '@elastic/eui';
+import { EuiCodeBlock, EuiHorizontalRule, EuiSpacer, EuiText } from '@elastic/eui';
 import React, { useEffect, useMemo, useState } from 'react';
+import { HttpStart } from '../../../../../../../src/core/public';
+import { TraceAnalyticsMode } from '../../../../../common/types/trace_analytics';
 import { ServiceBreakdownPanel } from '../../../trace_analytics/components/traces/service_breakdown_panel';
 import { SpanDetailPanel } from '../../../trace_analytics/components/traces/span_detail_panel';
 import {
- handleTraceViewRequest,
- handleServicesPieChartRequest,
  handlePayloadRequest,
+ handleServicesPieChartRequest,
+ handleTraceViewRequest,
 } from '../../../trace_analytics/requests/traces_request_handler';
-import { HttpStart } from '../../../../../../../src/core/public';
 import { getListItem } from '../../helpers/utils';
-import { TraceAnalyticsMode } from '../../../../../public/components/trace_analytics/home';
 
 interface TraceDetailRenderProps {
  traceId: string;
  http: HttpStart;
  openSpanFlyout: (spanId: string) => void;
- mode : TraceAnalyticsMode
+ mode: TraceAnalyticsMode;
 }
 
-export const TraceDetailRender = ({ traceId, http, openSpanFlyout, mode }: TraceDetailRenderProps) => {
+export const TraceDetailRender = ({
+ traceId,
+ http,
+ openSpanFlyout,
+ mode,
+}: TraceDetailRenderProps) => {
  const [fields, setFields] = useState<any>({});
  const [serviceBreakdownData, setServiceBreakdownData] = useState([]);
  const [payloadData, setPayloadData] = useState('');
  const [colorMap, setColorMap] = useState({});
@@ -83,13 +88,13 @@
  ) : null}
  </>
  );
  }, [traceId, fields, serviceBreakdownData, colorMap, payloadData]);
 
  useEffect(() => {
  handleTraceViewRequest(traceId, http, fields, setFields, mode);
  handleServicesPieChartRequest(traceId, http, setServiceBreakdownData, setColorMap, mode);
  handlePayloadRequest(traceId, http, payloadData, setPayloadData, mode);
  }, [traceId]);
 
  return renderContent;
 };

public/components/trace_analytics/components/common/__tests__/helper_functions.test.tsx

@@ -5,7 +5,6 @@
 
 import { configure, mount, shallow } from 'enzyme';
 import Adapter from 'enzyme-adapter-react-16';
-import { TraceAnalyticsMode } from 'public/components/trace_analytics/home';
 import React from 'react';
 import { TEST_SERVICE_MAP, TEST_SERVICE_MAP_GRAPH } from '../../../../../../test/constants';
 import {
@@ -38,7 +37,9 @@ describe('Helper functions', () => {
 
  it('renders no match and missing configuration messages', () => {
  const noMatchMessage = shallow(<NoMatchMessage size="s" />);
- const missingConfigurationMessage = shallow(<MissingConfigurationMessage mode='data_prepper'/>)
+ const missingConfigurationMessage = shallow(
+ <MissingConfigurationMessage mode="data_prepper" />
+ );
  expect(noMatchMessage).toMatchSnapshot();
  expect(missingConfigurationMessage).toMatchSnapshot();
  });
@@ -137,16 +138,16 @@ describe('Helper functions', () => {
  );
  const existsDSL = getTestDslFromFilters();
  expect(JSON.stringify(existsDSL)).toEqual(
- '{"query":{"bool":{"must":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}},{"exists":{"field":"traceGroup"}}],"filter":[],"should":[],"must_not":[]}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":[]}}}}}'
+ '{"query":{"bool":{"must":[{"exists":{"field":"traceGroup"}}],"filter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}}],"should":[],"must_not":[]}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":[]}}}}}'
  );
 
  const isDSL = getTestDslFromFilters('traceGroup', 'is');
  expect(JSON.stringify(isDSL)).toEqual(
- '{"query":{"bool":{"must":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}},{"term":{"traceGroup":{"from":"100","to":"∞"}}}],"filter":[],"should":[],"must_not":[]}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":[]}}}}}'
+ '{"query":{"bool":{"must":[{"term":{"traceGroup":{"from":"100","to":"∞"}}}],"filter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}}],"should":[],"must_not":[]}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":[]}}}}}'
  );
  const isBetweenDSL = getTestDslFromFilters('durationInNanos', 'is between');
  expect(JSON.stringify(isBetweenDSL)).toEqual(
- `{"query":{"bool":{"must":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}},{"range":{"durationInNanos":{"gte":"100"}}}],"filter":[],"should":[],"must_not":[]}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":[]}}}}}`
+ '{"query":{"bool":{"must":[{"range":{"durationInNanos":{"gte":"100"}}}],"filter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}}],"should":[],"must_not":[]}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":[]}}}}}'
  );
 
  const customDSL = filtersToDsl(
@@ -166,7 +167,7 @@ describe('Helper functions', () => {
  'now'
  );
  expect(JSON.stringify(customDSL)).toEqual(
- `{"query":{"bool":{"must":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}}],"filter":[],"should":["test"],"must_not":[],"minimum_should_match":1}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":["test"],"minimum_should_match":1}}}}}`
+ '{"query":{"bool":{"must":[],"filter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}},{"query_string":{"query":"order"}}],"should":["test"],"must_not":[],"minimum_should_match":1}},"custom":{"timeFilter":[{"range":{"startTime":{"gte":"now-5m","lte":"now"}}}],"serviceNames":[],"serviceNamesExclude":[],"traceGroup":[],"traceGroupExclude":[],"percentiles":{"query":{"bool":{"should":["test"],"minimum_should_match":1}}}}}'
  );
  });
 });

public/components/trace_analytics/components/common/custom_index_flyout.tsx

@@ -0,0 +1,167 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import {
+ EuiCallOut,
+ EuiCompressedFieldText,
+ EuiDescribedFormGroup,
+ EuiFlexGroup,
+ EuiFlexItem,
+ EuiFlyout,
+ EuiFlyoutBody,
+ EuiFlyoutFooter,
+ EuiFlyoutHeader,
+ EuiFormRow,
+ EuiLink,
+ EuiSmallButton,
+ EuiSmallButtonEmpty,
+ EuiTitle,
+} from '@elastic/eui';
+import React, { Fragment, useEffect, useState } from 'react';
+import {
+ TRACE_CUSTOM_SERVICE_INDEX_SETTING,
+ TRACE_CUSTOM_SPAN_INDEX_SETTING,
+} from '../../../../../common/constants/trace_analytics';
+import { uiSettingsService } from '../../../../../common/utils';
+import { useToast } from '../../../common/toast';
+
+interface CustomIndexFlyoutProps {
+ isFlyoutVisible: boolean;
+ setIsFlyoutVisible: React.Dispatch<React.SetStateAction<boolean>>;
+}
+
+export const CustomIndexFlyout = ({
+ isFlyoutVisible,
+ setIsFlyoutVisible,
+}: CustomIndexFlyoutProps) => {
+ const { setToast } = useToast();
+ const [spanIndices, setSpanIndices] = useState('');
+ const [serviceIndices, setServiceIndices] = useState('');
+ const [isLoading, setIsLoading] = useState(false);
+
+ const onChangeSpanIndices = (e: { target: { value: React.SetStateAction<string> } }) => {
+ setSpanIndices(e.target.value);
+ };
+
+ const onChangeServiceIndices = (e: { target: { value: React.SetStateAction<string> } }) => {
+ setServiceIndices(e.target.value);
+ };
+
+ useEffect(() => {
+ setSpanIndices(uiSettingsService.get(TRACE_CUSTOM_SPAN_INDEX_SETTING));
+ setServiceIndices(uiSettingsService.get(TRACE_CUSTOM_SERVICE_INDEX_SETTING));
+ }, [uiSettingsService]);
+
+ const onSaveIndices = async () => {
+ try {
+ setIsLoading(true);
+ await uiSettingsService.set(TRACE_CUSTOM_SPAN_INDEX_SETTING, spanIndices);
+ await uiSettingsService.set(TRACE_CUSTOM_SERVICE_INDEX_SETTING, serviceIndices);
+ setIsLoading(false);
+ setToast('Updated trace analytics sources successfully', 'success');
+ } catch (error) {
+ console.error(error);
+ setToast('Failed to update trace analytics sources', 'danger');
+ }
+ setIsLoading(false);
+ };
+
+ const callout = (
+ <EuiCallOut
+ title="Custom indices in trace analytics is an experimental feature"
+ color="warning"
+ iconType="help"
+ >
+ <p>
+ This feature is experimental, all indices added here should adhere to data prepper index
+ mappings. For more information on mappings, visit{' '}
+ <EuiLink
+ href="https://github.com/opensearch-project/data-prepper/tree/main/docs/schemas/trace-analytics"
+ target="_blank"
+ >
+ schema documentation
+ </EuiLink>
+ .
+ </p>
+ </EuiCallOut>
+ );
+ let flyout;
+
+ if (isFlyoutVisible) {
+ flyout = (
+ <EuiFlyout ownFocus onClose={() => setIsFlyoutVisible(false)} aria-labelledby="flyoutTitle">
+ <EuiFlyoutHeader hasBorder>
+ <EuiTitle size="m">
+ <h3 id="flyoutTitle">Manage custom trace source</h3>
+ </EuiTitle>
+ </EuiFlyoutHeader>
+ <EuiFlyoutBody banner={callout}>
+ <EuiDescribedFormGroup
+ title={<h3>Custom span indices</h3>}
+ description={
+ <Fragment>
+ Configure custom span indices to be used by the trace analytics plugin
+ </Fragment>
+ }
+ >
+ <EuiFormRow label="Custom span indices">
+ <EuiCompressedFieldText
+ name="spanIndices"
+ aria-label="spanIndices"
+ placeholder="index1,cluster1:index2,cluster:index3"
+ value={spanIndices}
+ onChange={onChangeSpanIndices}
+ />
+ </EuiFormRow>
+ </EuiDescribedFormGroup>
+ <EuiDescribedFormGroup
+ title={<h3>Custom service indices</h3>}
+ description={
+ <Fragment>
+ Configure custom service indices to be used by the trace analytics plugin
+ </Fragment>
+ }
+ >
+ <EuiFormRow label="Custom service indices">
+ <EuiCompressedFieldText
+ name="serviceIndices"
+ aria-label="serviceIndices"
+ placeholder="index1,cluster1:index2,cluster:index3"
+ value={serviceIndices}
+ onChange={onChangeServiceIndices}
+ />
+ </EuiFormRow>
+ </EuiDescribedFormGroup>
+ </EuiFlyoutBody>
+ <EuiFlyoutFooter>
+ <EuiFlexGroup justifyContent="spaceBetween">
+ <EuiFlexItem grow={false}>
+ <EuiSmallButtonEmpty
+ iconType="cross"
+ onClick={() => setIsFlyoutVisible(false)}
+ flush="left"
+ >
+ Close
+ </EuiSmallButtonEmpty>
+ </EuiFlexItem>
+ <EuiFlexItem grow={false}>
+ <EuiSmallButton
+ onClick={async () => {
+ await onSaveIndices();
+ setIsFlyoutVisible(false);
+ }}
+ fill
+ isLoading={isLoading}
+ >
+ Save
+ </EuiSmallButton>
+ </EuiFlexItem>
+ </EuiFlexGroup>
+ </EuiFlyoutFooter>
+ </EuiFlyout>
+ );
+ }
+ return <div>{flyout}</div>;
+};

public/components/trace_analytics/components/common/filters/filter_helpers.tsx

@@ -6,20 +6,20 @@
 import {
  EuiCompressedComboBox,
  EuiCompressedFieldText,
- EuiFormControlLayoutDelimited,
  EuiCompressedFormRow,
+ EuiFormControlLayoutDelimited,
  EuiSpacer,
 } from '@elastic/eui';
 import get from 'lodash/get';
-import { TraceAnalyticsMode } from 'public/components/trace_analytics/home';
 import React from 'react';
+import { TraceAnalyticsMode } from '../../../../../../common/types/trace_analytics';
 
 const getFields = (
  mode: TraceAnalyticsMode,
  page: 'dashboard' | 'traces' | 'services' | 'app',
  attributesFilterFields: string[]
 ) =>
- mode === 'data_prepper'
+ mode === 'data_prepper' || mode === 'custom_data_prepper'
  ? {
  dashboard: ['traceGroup', 'serviceName', 'error', 'status.message', 'latency'],
  traces: [
@@ -151,7 +151,7 @@
 export const getValueComponent = (
  field: string,
  operator: string,
  value: any,
  setValue: (v: any) => void
 ) => {
  const textField = (

public/components/trace_analytics/components/common/filters/filters.tsx

@@ -6,18 +6,18 @@
 import {
  EuiBadge,
  EuiButtonEmpty,
- EuiSmallButtonIcon,
  EuiContextMenu,
  EuiContextMenuPanelDescriptor,
  EuiFlexGroup,
  EuiFlexItem,
  EuiIcon,
  EuiPopover,
  EuiPopoverTitle,
+ EuiSmallButtonIcon,
  EuiTextColor,
 } from '@elastic/eui';
-import { TraceAnalyticsMode } from 'public/components/trace_analytics/home';
 import React, { useMemo, useState } from 'react';
+import { TraceAnalyticsMode } from '../../../../../../common/types/trace_analytics';
 import { FilterEditPopover } from './filter_edit_popover';
 import { getFilterFields, getValidFilterFields } from './filter_helpers';