WIP

x-pack/plugins/observability_log_explorer/public/components/ingest_pathways/graph_visualization.tsx

@@ -7,10 +7,10 @@
 
 import { css } from '@emotion/react';
 import { useActor } from '@xstate/react';
-import cytoscape, { CytoscapeOptions, NodeSingular } from 'cytoscape';
+import cytoscape, { CytoscapeOptions, EdgeSingular, NodeSingular } from 'cytoscape';
 import dagre from 'cytoscape-dagre';
 import React, { useEffect, useState } from 'react';
-import { useIngestPathwaysPageStateContext } from '../../state_machines/ingest_pathways';
+import { Agent, useIngestPathwaysPageStateContext } from '../../state_machines/ingest_pathways';
 
 export const ConnectedGraphVisualization = React.memo(() => {
   const [state] = useActor(useIngestPathwaysPageStateContext());
@@ -64,13 +64,24 @@ const initialGraphOptions: CytoscapeOptions & { layout: Record<string, any> } =
         },
         'text-wrap': 'wrap',
         shape: 'ellipse',
+        'text-valign': 'center',
+        'text-halign': 'left',
       },
     },
     {
       selector: 'node.dataStream',
       style: {
         label: 'data(dataStream.id)',
         shape: 'hexagon',
+        'text-valign': 'center',
+        'text-halign': 'right',
+      },
+    },
+    {
+      selector: 'node.ingestPipeline',
+      style: {
+        label: 'data(ingestPipeline.id)',
+        shape: 'diamond',
       },
     },
     {
@@ -81,18 +92,33 @@ const initialGraphOptions: CytoscapeOptions & { layout: Record<string, any> } =
       },
     },
     {
-      selector: 'edge.agentShipsTo',
+      selector: 'edge.shipsTo',
       style: {
-        label: 'data(relation.signalCount)',
         'target-arrow-shape': 'chevron',
         'target-arrow-fill': 'filled',
       },
     },
+    {
+      selector: 'edge.agentShipsTo',
+      style: {
+        // label: 'data(shipsTo.signalCount)',
+        width: (edge: EdgeSingular) => {
+          const agent: Agent = edge.data('agent');
+          const totalSignalCount = agent.shipsTo.reduce(
+            (accumulatedSignalCount, { signalCount }) => accumulatedSignalCount + signalCount,
+            0
+          );
+          return 1 + (9.0 / totalSignalCount) * edge.data('shipsTo').signalCount;
+        },
+      },
+    },
   ],
   layout: {
     name: 'dagre',
     rankDir: 'LR',
     rankSep: 300,
+    nodeSep: 30,
+    ranker: 'longest-path',
   },
 };
 

x-pack/plugins/observability_log_explorer/public/components/ingest_pathways/ingest_pathways.tsx

@@ -10,14 +10,15 @@ import React, { useMemo } from 'react';
 import { ObservabilityLogExplorerPageTemplate } from '../page_template';
 import { ConnectedGraphVisualization } from './graph_visualization';
 import { ConnectedLoadingIndicator } from './loading_indicator';
+import { ConnectedReloadButton } from './reload_button';
 
 export const IngestPathways = React.memo(() => {
   const pageHeaderProps = useMemo(
     (): EuiPageHeaderProps => ({
       alignItems: 'center',
       bottomBorder: 'extended',
       pageTitle: 'Ingest Pathways',
-      rightSideItems: [<ConnectedLoadingIndicator />],
+      rightSideItems: [<ConnectedLoadingIndicator />, <ConnectedReloadButton />],
     }),
     []
   );

x-pack/plugins/observability_log_explorer/public/components/ingest_pathways/reload_button.tsx

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { EuiButton } from '@elastic/eui';
+import { useActor } from '@xstate/react';
+import React from 'react';
+import { useIngestPathwaysPageStateContext } from '../../state_machines/ingest_pathways';
+
+export const ConnectedReloadButton = React.memo(() => {
+  const [, send] = useActor(useIngestPathwaysPageStateContext());
+
+  return (
+    <EuiButton
+      onClick={() => {
+        send({
+          type: 'load',
+        });
+      }}
+    >
+      Reload
+    </EuiButton>
+  );
+});

x-pack/plugins/observability_log_explorer/public/state_machines/ingest_pathways/graph/calculate_graph.ts

@@ -6,25 +6,34 @@
  */
 
 import { ElementDefinition } from 'cytoscape';
-import { Agent, DataStream, IngestPathwaysData, Relation } from '../types';
+import {
+  Agent,
+  DataStreamEntry,
+  IndexTemplate,
+  IngestPathwaysData,
+  IngestPipelineEntry,
+} from '../types';
 
 export const calculateGraph = ({
   agents,
   dataStreams,
-  relations,
+  indexTemplates,
+  ingestPipelines,
 }: IngestPathwaysData): { elements: ElementDefinition[] } => {
   const dataStreamElements = Object.values(dataStreams).flatMap(convertDataStreamToGraphElements);
-  const agentElements = Object.values(agents).flatMap(convertAgentToGraphElements);
-  const relationElements = relations.flatMap(
-    convertRelationToGraphElements({ agents, dataStreams })
+  const agentElements = Object.values(agents).flatMap(
+    convertAgentToGraphElements({ dataStreams, indexTemplates })
+  );
+  const ingestPipelineElements = Object.values(ingestPipelines).flatMap(
+    convertIngestPipelineToGraphElements
   );
 
   return {
-    elements: [...dataStreamElements, ...agentElements, ...relationElements],
+    elements: [...dataStreamElements, ...agentElements, ...ingestPipelineElements],
   };
 };
 
-const convertDataStreamToGraphElements = (dataStream: DataStream): ElementDefinition[] => [
+const convertDataStreamToGraphElements = (dataStream: DataStreamEntry): ElementDefinition[] => [
   {
     group: 'nodes',
     classes: 'dataStream',
@@ -35,49 +44,106 @@ const convertDataStreamToGraphElements = (dataStream: DataStream): ElementDefini
   },
 ];
 
-const convertAgentToGraphElements = (agent: Agent): ElementDefinition[] => [
-  {
-    group: 'nodes',
-    classes: 'agent',
-    data: {
-      id: getAgentElementId(agent),
-      agent,
-    },
-  },
-];
-
-const convertRelationToGraphElements =
+const convertAgentToGraphElements =
   ({
-    agents,
     dataStreams,
+    indexTemplates,
   }: {
-    agents: Record<string, Agent>;
-    dataStreams: Record<string, DataStream>;
+    dataStreams: Record<string, DataStreamEntry>;
+    indexTemplates: Record<string, IndexTemplate>;
   }) =>
-  (relation: Relation): ElementDefinition[] => {
-    if (relation.type === 'agent-ships-to-data-stream') {
-      const agent = agents[relation.agentId];
-      const dataStream = dataStreams[relation.dataStreamId];
-
-      return [
-        {
+  (agent: Agent): ElementDefinition[] =>
+    [
+      {
+        group: 'nodes',
+        classes: 'agent',
+        data: {
+          id: getAgentElementId(agent),
+          agent,
+        },
+      },
+      ...agent.shipsTo.flatMap((shipsTo): ElementDefinition[] => {
+        const dataStream = dataStreams[shipsTo.dataStreamId];
+        const source = getAgentElementId(agent);
+        const target = getDataStreamElementId({
+          type: 'dataStreamStub',
+          id: shipsTo.dataStreamId,
+        });
+        const agentDataStreamEdge: ElementDefinition = {
           group: 'edges',
-          classes: 'agentShipsTo',
+          classes: 'shipsTo agentShipsTo',
           data: {
-            id: `relation-agent-${relation.agentId}-ships-to-dataStream-${relation.dataStreamId}`,
-            source: getAgentElementId(agent),
-            target: getDataStreamElementId(dataStream),
-            relation,
+            id: `relation-${source}-ships-to-${target}`,
+            source,
+            target,
             agent,
-            dataStream,
+            shipsTo,
           },
-        },
-      ];
-    } else {
-      return [];
-    }
-  };
+        };
+
+        if (dataStream.type === 'dataStream') {
+          const indexTemplate = indexTemplates[dataStream.indexTemplateId];
 
-const getDataStreamElementId = ({ id }: DataStream) => `dataStream-${id}`;
+          return indexTemplate.ingestPipelineIds.reduce<ElementDefinition[]>(
+            (edges, ingestPipelineId, ingestPipelineIndex, ingestPipelineIds) => {
+              const lastEdge = edges[edges.length - 1];
+              const leadingEdges = edges.slice(0, -1);
+
+              const ingestPipelineElementId = getIngestPipelineElementId({
+                type: 'ingestPipelineStub',
+                id: ingestPipelineId,
+              });
+
+              const splitEdges: ElementDefinition[] = [
+                {
+                  group: 'edges',
+                  classes: lastEdge.classes,
+                  data: {
+                    id: `relation-${lastEdge.data.source}-ships-to-${ingestPipelineElementId}`,
+                    source: lastEdge.data.source,
+                    target: ingestPipelineElementId,
+                    shipsTo: lastEdge.data.shipsTo,
+                    agent,
+                  },
+                },
+                {
+                  group: 'edges',
+                  classes: 'shipsTo',
+                  data: {
+                    id: `relation-${ingestPipelineElementId}-ships-to-${lastEdge.data.target}`,
+                    source: ingestPipelineElementId,
+                    target: lastEdge.data.target,
+                    shipsTo: {},
+                    agent,
+                  },
+                },
+              ];
+
+              return [...leadingEdges, ...splitEdges];
+            },
+            [agentDataStreamEdge]
+          );
+        } else {
+          return [agentDataStreamEdge];
+        }
+      }),
+    ];
+
+const convertIngestPipelineToGraphElements = (
+  ingestPipeline: IngestPipelineEntry
+): ElementDefinition[] => [
+  {
+    group: 'nodes',
+    classes: 'ingestPipeline',
+    data: {
+      id: getIngestPipelineElementId(ingestPipeline),
+      ingestPipeline,
+    },
+  },
+];
+
+const getDataStreamElementId = ({ id }: DataStreamEntry) => `dataStream-${id}`;
 
 const getAgentElementId = ({ id }: Agent) => `agent-${id}`;
+
+const getIngestPipelineElementId = ({ id }: IngestPipelineEntry) => `ingestPipeline-${id}`;