Minor: address dq feedback and move lineage tracing in observability …

…layer (#18751) * Minor: address dq feedback and move lineage tracing in observability layer * fixed: failing unit test
open-metadata · Nov 23, 2024 · 9edf379 · 9edf379
1 parent 0465913
commit 9edf379
Show file tree

Hide file tree

Showing 8 changed files with 81 additions and 47 deletions.
diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/no-dimension-icon.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/no-dimension-icon.svg
diff --git a/...urces/ui/src/components/DataQuality/ChartWidgets/StatusCardWidget/status-card-widget.less b/...urces/ui/src/components/DataQuality/ChartWidgets/StatusCardWidget/status-card-widget.less
@@ -15,7 +15,9 @@
 .status-count-container {
   border: @global-border;
   height: 30px;
-  width: 30px;
+  width: 100%;
+  min-width: 50px;
+  padding: 0 12px;
   display: flex;
   align-items: center;
   justify-content: center;

diff --git a/...ata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx b/...ata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx
@@ -90,10 +90,21 @@ export const CustomEdge = ({
     activeLayer,
     onAddPipelineClick,
     onColumnEdgeRemove,
+    dataQualityLineage,
   } = useLineageProvider();
 
   const { theme } = useApplicationStore();
 
+  const showDqTracing = useMemo(() => {
+    return (
+      (activeLayer.includes(LineageLayer.DataObservability) &&
+        dataQualityLineage?.edges?.some(
+          (dqEdge) => dqEdge?.doc_id === edge?.doc_id
+        )) ??
+      false
+    );
+  }, [activeLayer, dataQualityLineage, edge]);
+
   const isColumnHighlighted = useMemo(() => {
     if (!isColumnLineage) {
       return false;
@@ -154,7 +165,7 @@ export const CustomEdge = ({
 
     let stroke = isStrokeNeeded ? theme.primaryColor : undefined;
 
-    if (edge?.isDqTestFailure) {
+    if (showDqTracing) {
       stroke = RED_3;
     }
 
@@ -172,6 +183,7 @@ export const CustomEdge = ({
     isColumnHighlighted,
     isColumnLineage,
     tracedColumns,
+    showDqTracing,
   ]);
 
   const isPipelineEdgeAllowed = (

diff --git a/...a-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx b/...a-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx
@@ -25,6 +25,7 @@ import {
 import { ReactComponent as IconTimesCircle } from '../../../assets/svg/ic-times-circle.svg';
 import { useLineageProvider } from '../../../context/LineageProvider/LineageProvider';
 import { EntityLineageNodeType } from '../../../enums/entity.enum';
+import { LineageLayer } from '../../../generated/configuration/lineageSettings';
 import { checkUpstreamDownstream } from '../../../utils/EntityLineageUtils';
 import './custom-node.less';
 import { getCollapseHandle, getExpandHandle } from './CustomNode.utils';
@@ -46,10 +47,20 @@ const CustomNodeV1 = (props: NodeProps) => {
     onNodeCollapse,
     removeNodeHandler,
     loadChildNodesHandler,
+    activeLayer,
+    dataQualityLineage,
   } = useLineageProvider();
 
   const { label, isNewNode, node = {}, isRootNode } = data;
 
+  const showDqTracing = useMemo(() => {
+    return (
+      (activeLayer.includes(LineageLayer.DataObservability) &&
+        dataQualityLineage?.nodes?.some((dqNode) => dqNode.id === node?.id)) ??
+      false
+    );
+  }, [activeLayer, dataQualityLineage, node]);
+
   const nodeType = isEditMode ? EntityLineageNodeType.DEFAULT : type;
   const isSelected = selectedNode === node;
   const { id, lineage, fullyQualifiedName } = node;
@@ -258,7 +269,9 @@ const CustomNodeV1 = (props: NodeProps) => {
       className={classNames(
         'lineage-node p-0',
         isSelected ? 'custom-node-header-active' : 'custom-node-header-normal',
-        { 'data-quality-failed-custom-node-header': node?.isDqTestFailure },
+        {
+          'data-quality-failed-custom-node-header': showDqTracing,
+        },
         { 'custom-node-header-tracing': isTraced }
       )}
       data-testid={`lineage-node-${fullyQualifiedName}`}>

diff --git a/...tadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.interface.tsx b/...tadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.interface.tsx
@@ -47,6 +47,7 @@ export type UpstreamDownstreamData = {
 
 export interface LineageContextType {
   reactFlowInstance?: ReactFlowInstance;
+  dataQualityLineage?: EntityLineageResponse;
   nodes: Node[];
   edges: Edge[];
   tracedNodes: string[];

diff --git a/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.test.tsx b/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.test.tsx
@@ -11,6 +11,7 @@
  *  limitations under the License.
  */
 import { act, fireEvent, render, screen } from '@testing-library/react';
+import QueryString from 'qs';
 import React, { useEffect } from 'react';
 import { Edge } from 'reactflow';
 import { EdgeTypeEnum } from '../../components/Entity/EntityLineage/EntityLineage.interface';
@@ -162,6 +163,9 @@ describe('LineageProvider', () => {
   });
 
   it('getDataQualityLineage should be called if alert is supported', async () => {
+    mockLocation.search = QueryString.stringify({
+      layers: ['DataObservability'],
+    });
     mockIsAlertSupported = true;
     (getLineageDataByFQN as jest.Mock).mockImplementationOnce(() =>
       Promise.resolve({
@@ -190,6 +194,7 @@ describe('LineageProvider', () => {
     );
 
     mockIsAlertSupported = false;
+    mockLocation.search = '';
   });
 
   it('should call loadChildNodesHandler', async () => {

diff --git a/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/context/LineageProvider/LineageProvider.tsx
@@ -147,6 +147,8 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
     edges: [],
     entity: {} as EntityReference,
   });
+  const [dataQualityLineage, setDataQualityLineage] =
+    useState<EntityLineageResponse>();
   const [updatedEntityLineage, setUpdatedEntityLineage] =
     useState<EntityLineageResponse | null>(null);
   const [showDeleteModal, setShowDeleteModal] = useState<boolean>(false);
@@ -221,6 +223,25 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
     [entityLineage]
   );
 
+  const fetchDataQualityLineage = async (
+    fqn: string,
+    config?: LineageConfig
+  ) => {
+    if (isTourOpen || !tableClassBase.getAlertEnableStatus()) {
+      return;
+    }
+    try {
+      const dqLineageResp = await getDataQualityLineage(
+        fqn,
+        config,
+        queryFilter
+      );
+      setDataQualityLineage(dqLineageResp);
+    } catch (error) {
+      setDataQualityLineage(undefined);
+    }
+  };
+
   const fetchLineageData = useCallback(
     async (fqn: string, entityType: string, config?: LineageConfig) => {
       if (isTourOpen) {
@@ -237,61 +258,25 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
           config,
           queryFilter
         );
-
-        const dqLineageResp = tableClassBase.getAlertEnableStatus()
-          ? await getDataQualityLineage(fqn, config, queryFilter)
-          : { nodes: [], edges: [] };
-
         if (res) {
-          const { nodes = [], entity, edges } = res;
+          const { nodes = [], entity } = res;
           const allNodes = uniqWith(
             [...nodes, entity].filter(Boolean),
             isEqual
-          ).map((node) => {
-            return {
-              ...node,
-              isDqTestFailure:
-                dqLineageResp.nodes?.some((dqNode) => dqNode.id === node.id) ??
-                false,
-            };
-          });
-
-          const updatedEntity = {
-            ...entity,
-            isDqTestFailure:
-              dqLineageResp.nodes?.some((dqNode) => dqNode.id === entity.id) ??
-              false,
-          };
-
-          const updatedEdges = edges?.map((edge) => {
-            return {
-              ...edge,
-              isDqTestFailure:
-                dqLineageResp.edges?.some(
-                  (dqEdge) => dqEdge?.doc_id === edge?.doc_id
-                ) ?? false,
-            };
-          });
+          );
 
           if (
             entityType !== EntityType.PIPELINE &&
             entityType !== EntityType.STORED_PROCEDURE
           ) {
             const { map: childMapObj } = getChildMap(
-              {
-                ...res,
-                nodes: allNodes,
-                edges: updatedEdges,
-                entity: updatedEntity,
-              },
+              { ...res, nodes: allNodes },
               decodedFqn
             );
             setChildMap(childMapObj);
             const { nodes: newNodes, edges: newEdges } = getPaginatedChildMap(
               {
                 ...res,
-                entity: updatedEntity,
-                edges: updatedEdges,
                 nodes: allNodes,
               },
               childMapObj,
@@ -301,14 +286,12 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
 
             setEntityLineage({
               ...res,
-              entity: updatedEntity,
               nodes: newNodes,
-              edges: [...(updatedEdges ?? []), ...newEdges],
+              edges: [...(res.edges ?? []), ...newEdges],
             });
           } else {
             setEntityLineage({
               ...res,
-              entity: updatedEntity,
               nodes: allNodes,
             });
           }
@@ -1299,8 +1282,10 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
       onAddPipelineClick,
       onUpdateLayerView,
       onExportClick,
+      dataQualityLineage,
     };
   }, [
+    dataQualityLineage,
     isDrawerOpen,
     loading,
     isEditMode,
@@ -1363,6 +1348,12 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
     }
   }, [lineageLayer]);
 
+  useEffect(() => {
+    if (activeLayer.includes(LineageLayer.DataObservability)) {
+      fetchDataQualityLineage(decodedFqn, lineageConfig);
+    }
+  }, [activeLayer, decodedFqn, lineageConfig]);
+
   return (
     <LineageContext.Provider value={activityFeedContextValues}>
       <div

diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/DataQuality/DataQualityUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/DataQuality/DataQualityUtils.ts
@@ -11,14 +11,14 @@
  *  limitations under the License.
  */
 import { cloneDeep, isArray, isUndefined, omit, omitBy } from 'lodash';
-import { ReactComponent as TestCaseIcon } from '../../assets/svg/all-activity-v2.svg';
 import { ReactComponent as AccuracyIcon } from '../../assets/svg/ic-accuracy.svg';
 import { ReactComponent as CompletenessIcon } from '../../assets/svg/ic-completeness.svg';
 import { ReactComponent as ConsistencyIcon } from '../../assets/svg/ic-consistency.svg';
 import { ReactComponent as IntegrityIcon } from '../../assets/svg/ic-integrity.svg';
 import { ReactComponent as SqlIcon } from '../../assets/svg/ic-sql.svg';
 import { ReactComponent as UniquenessIcon } from '../../assets/svg/ic-uniqueness.svg';
 import { ReactComponent as ValidityIcon } from '../../assets/svg/ic-validity.svg';
+import { ReactComponent as NoDimensionIcon } from '../../assets/svg/no-dimension-icon.svg';
 import { StatusData } from '../../components/DataQuality/ChartWidgets/StatusCardWidget/StatusCardWidget.interface';
 import { TestCaseSearchParams } from '../../components/DataQuality/DataQuality.interface';
 import {
@@ -235,6 +235,6 @@ export const getDimensionIcon = (dimension: DataQualityDimensions) => {
     case DataQualityDimensions.Validity:
       return ValidityIcon;
     default:
-      return TestCaseIcon;
+      return NoDimensionIcon;
   }
 };