Merge branch 'master' into ps-add-recipe-deploy

datahub-project · Aug 1, 2023 · 13a314d · 13a314d
2 parents b2fb924 + 2e2a674
commit 13a314d
Show file tree

Hide file tree

Showing 56 changed files with 55,445 additions and 687 deletions.
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -43,7 +43,7 @@ jobs:
           timezoneLinux: ${{ matrix.timezone }}
       - uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:

diff --git a/.github/workflows/check-datahub-jars.yml b/.github/workflows/check-datahub-jars.yml
@@ -38,7 +38,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:

diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
@@ -38,7 +38,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"

diff --git a/.github/workflows/docker-ingestion-base.yml b/.github/workflows/docker-ingestion-base.yml
@@ -30,7 +30,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Build and Push image
         uses: ./.github/actions/docker-custom-build-and-push
         with:

diff --git a/.github/workflows/docker-ingestion-smoke.yml b/.github/workflows/docker-ingestion-smoke.yml
@@ -53,7 +53,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:

diff --git a/.github/workflows/docker-ingestion.yml b/.github/workflows/docker-ingestion.yml
@@ -57,7 +57,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:

diff --git a/.github/workflows/docker-postgres-setup.yml b/.github/workflows/docker-postgres-setup.yml
@@ -48,7 +48,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:

diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -64,7 +64,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :metadata-service:war:build -x test --parallel
@@ -122,7 +122,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :metadata-jobs:mae-consumer-job:build -x test --parallel
@@ -180,7 +180,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :metadata-jobs:mce-consumer-job:build -x test --parallel
@@ -238,7 +238,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :datahub-upgrade:build -x test --parallel
@@ -296,7 +296,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Pre-build artifacts for docker image
         run: |
           ./gradlew :datahub-frontend:dist -x test -x yarnTest -x yarnLint --parallel
@@ -356,7 +356,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:
@@ -378,7 +378,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:
@@ -400,7 +400,7 @@ jobs:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Build and push
         uses: ./.github/actions/docker-custom-build-and-push
         with:

diff --git a/.github/workflows/publish-datahub-jars.yml b/.github/workflows/publish-datahub-jars.yml
@@ -50,7 +50,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 800
       - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:

diff --git a/datahub-web-react/src/app/entity/shared/__tests__/siblingsUtils.test.ts b/datahub-web-react/src/app/entity/shared/__tests__/siblingsUtils.test.ts
@@ -195,6 +195,7 @@ const searchResultWithSiblings = [
     {
         entity: {
             urn: 'urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)',
+            exists: true,
             type: 'DATASET',
             name: 'cypress_project.jaffle_shop.raw_orders',
             origin: 'PROD',
@@ -328,6 +329,7 @@ const searchResultWithSiblings = [
                 siblings: [
                     {
                         urn: 'urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)',
+                        exists: true,
                         type: 'DATASET',
                         platform: {
                             urn: 'urn:li:dataPlatform:dbt',
@@ -376,6 +378,7 @@ const searchResultWithSiblings = [
     {
         entity: {
             urn: 'urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)',
+            exists: true,
             type: 'DATASET',
             name: 'cypress_project.jaffle_shop.raw_orders',
             origin: 'PROD',
@@ -513,6 +516,169 @@ const searchResultWithSiblings = [
     },
 ];
 
+const searchResultWithGhostSiblings = [
+    {
+        entity: {
+            urn: 'urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)',
+            exists: true,
+            type: 'DATASET',
+            name: 'cypress_project.jaffle_shop.raw_orders',
+            origin: 'PROD',
+            uri: null,
+            platform: {
+                urn: 'urn:li:dataPlatform:bigquery',
+                type: 'DATA_PLATFORM',
+                name: 'bigquery',
+                properties: {
+                    type: 'RELATIONAL_DB',
+                    displayName: 'BigQuery',
+                    datasetNameDelimiter: '.',
+                    logoUrl: '/assets/platforms/bigquerylogo.png',
+                    __typename: 'DataPlatformProperties',
+                },
+                displayName: null,
+                info: null,
+                __typename: 'DataPlatform',
+            },
+            dataPlatformInstance: null,
+            editableProperties: null,
+            platformNativeType: null,
+            properties: {
+                name: 'raw_orders',
+                description: null,
+                qualifiedName: null,
+                customProperties: [],
+                __typename: 'DatasetProperties',
+            },
+            ownership: null,
+            globalTags: null,
+            glossaryTerms: null,
+            subTypes: {
+                typeNames: ['table'],
+                __typename: 'SubTypes',
+            },
+            domain: null,
+            container: {
+                urn: 'urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb',
+                platform: {
+                    urn: 'urn:li:dataPlatform:bigquery',
+                    type: 'DATA_PLATFORM',
+                    name: 'bigquery',
+                    properties: {
+                        type: 'RELATIONAL_DB',
+                        displayName: 'BigQuery',
+                        datasetNameDelimiter: '.',
+                        logoUrl: '/assets/platforms/bigquerylogo.png',
+                        __typename: 'DataPlatformProperties',
+                    },
+                    displayName: null,
+                    info: null,
+                    __typename: 'DataPlatform',
+                },
+                properties: {
+                    name: 'jaffle_shop',
+                    __typename: 'ContainerProperties',
+                },
+                subTypes: {
+                    typeNames: ['Dataset'],
+                    __typename: 'SubTypes',
+                },
+                deprecation: null,
+                __typename: 'Container',
+            },
+            parentContainers: {
+                count: 2,
+                containers: [
+                    {
+                        urn: 'urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb',
+                        platform: {
+                            urn: 'urn:li:dataPlatform:bigquery',
+                            type: 'DATA_PLATFORM',
+                            name: 'bigquery',
+                            properties: {
+                                type: 'RELATIONAL_DB',
+                                displayName: 'BigQuery',
+                                datasetNameDelimiter: '.',
+                                logoUrl: '/assets/platforms/bigquerylogo.png',
+                                __typename: 'DataPlatformProperties',
+                            },
+                            displayName: null,
+                            info: null,
+                            __typename: 'DataPlatform',
+                        },
+                        properties: {
+                            name: 'jaffle_shop',
+                            __typename: 'ContainerProperties',
+                        },
+                        subTypes: {
+                            typeNames: ['Dataset'],
+                            __typename: 'SubTypes',
+                        },
+                        deprecation: null,
+                        __typename: 'Container',
+                    },
+                    {
+                        urn: 'urn:li:container:b5e95fce839e7d78151ed7e0a7420d84',
+                        platform: {
+                            urn: 'urn:li:dataPlatform:bigquery',
+                            type: 'DATA_PLATFORM',
+                            name: 'bigquery',
+                            properties: {
+                                type: 'RELATIONAL_DB',
+                                displayName: 'BigQuery',
+                                datasetNameDelimiter: '.',
+                                logoUrl: '/assets/platforms/bigquerylogo.png',
+                                __typename: 'DataPlatformProperties',
+                            },
+                            displayName: null,
+                            info: null,
+                            __typename: 'DataPlatform',
+                        },
+                        properties: {
+                            name: 'cypress_project',
+                            __typename: 'ContainerProperties',
+                        },
+                        subTypes: {
+                            typeNames: ['Project'],
+                            __typename: 'SubTypes',
+                        },
+                        deprecation: null,
+                        __typename: 'Container',
+                    },
+                ],
+                __typename: 'ParentContainersResult',
+            },
+            deprecation: null,
+            siblings: {
+                isPrimary: false,
+                siblings: [
+                    {
+                        urn: 'urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)',
+                        exists: false,
+                        type: 'DATASET',
+                    },
+                ],
+                __typename: 'SiblingProperties',
+            },
+            __typename: 'Dataset',
+        },
+        matchedFields: [
+            {
+                name: 'name',
+                value: 'raw_orders',
+                __typename: 'MatchedField',
+            },
+            {
+                name: 'id',
+                value: 'cypress_project.jaffle_shop.raw_orders',
+                __typename: 'MatchedField',
+            },
+        ],
+        insights: [],
+        __typename: 'SearchResult',
+    },
+];
+
 describe('siblingUtils', () => {
     describe('combineEntityDataWithSiblings', () => {
         it('combines my metadata with my siblings as primary', () => {
@@ -564,6 +730,18 @@ describe('siblingUtils', () => {
             expect(result?.[0]?.matchedEntities?.[1]?.urn).toEqual(
                 'urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)',
             );
+
+            expect(result?.[0]?.matchedEntities).toHaveLength(2);
+        });
+
+        it('will not combine an entity with a ghost node', () => {
+            const result = combineSiblingsInSearchResults(searchResultWithGhostSiblings as any);
+
+            expect(result).toHaveLength(1);
+            expect(result?.[0]?.matchedEntities?.[0]?.urn).toEqual(
+                'urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)',
+            );
+            expect(result?.[0]?.matchedEntities).toHaveLength(1);
         });
     });
 

diff --git a/...hub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarSiblingsSection.tsx b/...hub-web-react/src/app/entity/shared/containers/profile/sidebar/SidebarSiblingsSection.tsx
@@ -4,7 +4,7 @@ import styled from 'styled-components';
 import { useDataNotCombinedWithSiblings, useEntityData } from '../../../EntityContext';
 import { SidebarHeader } from './SidebarHeader';
 import { CompactEntityNameList } from '../../../../../recommendations/renderer/component/CompactEntityNameList';
-import { Entity } from '../../../../../../types.generated';
+import { Dataset, Entity } from '../../../../../../types.generated';
 import { SEPARATE_SIBLINGS_URL_PARAM, stripSiblingsFromEntity, useIsSeparateSiblingsMode } from '../../../siblingUtils';
 import { GetDatasetQuery } from '../../../../../../graphql/dataset.generated';
 
@@ -36,14 +36,22 @@ export const SidebarSiblingsSection = () => {
     const siblingEntities = entityData?.siblings?.siblings || [];
     const entityDataWithoutSiblings = stripSiblingsFromEntity(dataNotCombinedWithSiblings.dataset);
 
-    const allSiblingsInGroup = [...siblingEntities, entityDataWithoutSiblings] as Entity[];
+    const allSiblingsInGroup = [...siblingEntities, entityDataWithoutSiblings] as Dataset[];
+
+    const allSiblingsInGroupThatExist = allSiblingsInGroup.filter((sibling) => sibling.exists);
+
+    // you are always going to be in the sibling group, so if the sibling group is just you do not render.
+    // The less than case is likely not neccessary but just there as a safety case for unexpected scenarios
+    if (allSiblingsInGroupThatExist.length <= 1) {
+        return <></>;
+    }
 
     return (
         <div>
             <SidebarHeader title="Composed Of" />
             <EntityListContainer>
                 <CompactEntityNameList
-                    entities={allSiblingsInGroup}
+                    entities={allSiblingsInGroupThatExist}
                     linkUrlParams={{ [SEPARATE_SIBLINGS_URL_PARAM]: true }}
                     showTooltips
                 />