Merge branch 'datahub-project:master' into master

.github/workflows/docker-unified.yml

@@ -496,6 +496,9 @@ jobs:
         if: ${{ needs.setup.outputs.publish != 'true' }}
         with:
           image: ${{ env.DATAHUB_UPGRADE_IMAGE }}:${{ needs.setup.outputs.unique_tag }}
+      - name: Disable datahub-actions
+        run: |
+          yq -i 'del(.services.datahub-actions)' docker/quickstart/docker-compose-without-neo4j.quickstart.yml
       - name: run quickstart
         env:
           DATAHUB_TELEMETRY_ENABLED: false
@@ -507,6 +510,20 @@ jobs:
           # we are doing this because gms takes time to get ready
           # and we don't have a better readiness check when bootstrap is done
           sleep 60s
+      - name: Disable ES Disk Threshold
+        run: |
+          curl -XPUT "http://localhost:9200/_cluster/settings" \
+          -H 'Content-Type: application/json' -d'{
+            "persistent": {
+              "cluster": {
+                "routing": {
+                  "allocation.disk.threshold_enabled": false
+                }
+              }
+            }
+          }'
+      - name: Remove Source Code
+        run: find ./*/* ! -path "./metadata-ingestion*" ! -path "./smoke-test*" ! -path "./gradle*" -delete
       - name: Smoke test
         env:
           RUN_QUICKSTART: false

datahub-frontend/app/controllers/Application.java

@@ -136,11 +136,11 @@ public CompletableFuture<Result> proxy(String path, Http.Request request) throws
             .stream()
             // Remove X-DataHub-Actor to prevent malicious delegation.
             .filter(entry -> !AuthenticationConstants.LEGACY_X_DATAHUB_ACTOR_HEADER.equalsIgnoreCase(entry.getKey()))
-            .filter(entry -> !Http.HeaderNames.CONTENT_LENGTH.equals(entry.getKey()))
-            .filter(entry -> !Http.HeaderNames.CONTENT_TYPE.equals(entry.getKey()))
-            .filter(entry -> !Http.HeaderNames.AUTHORIZATION.equals(entry.getKey()))
+            .filter(entry -> !Http.HeaderNames.CONTENT_LENGTH.equalsIgnoreCase(entry.getKey()))
+            .filter(entry -> !Http.HeaderNames.CONTENT_TYPE.equalsIgnoreCase(entry.getKey()))
+            .filter(entry -> !Http.HeaderNames.AUTHORIZATION.equalsIgnoreCase(entry.getKey()))
             // Remove Host s.th. service meshes do not route to wrong host
-            .filter(entry -> !Http.HeaderNames.HOST.equals(entry.getKey()))
+            .filter(entry -> !Http.HeaderNames.HOST.equalsIgnoreCase(entry.getKey()))
             .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))
         )
         .addHeader(Http.HeaderNames.AUTHORIZATION, authorizationHeaderValue)
@@ -152,8 +152,8 @@ public CompletableFuture<Result> proxy(String path, Http.Request request) throws
           final ResponseHeader header = new ResponseHeader(apiResponse.getStatus(), apiResponse.getHeaders()
               .entrySet()
               .stream()
-              .filter(entry -> !Http.HeaderNames.CONTENT_LENGTH.equals(entry.getKey()))
-              .filter(entry -> !Http.HeaderNames.CONTENT_TYPE.equals(entry.getKey()))
+              .filter(entry -> !Http.HeaderNames.CONTENT_LENGTH.equalsIgnoreCase(entry.getKey()))
+              .filter(entry -> !Http.HeaderNames.CONTENT_TYPE.equalsIgnoreCase(entry.getKey()))
               .map(entry -> Pair.of(entry.getKey(), String.join(";", entry.getValue())))
               .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)));
           final HttpEntity body = new HttpEntity.Strict(apiResponse.getBodyAsBytes(), Optional.ofNullable(apiResponse.getContentType()));

datahub-web-react/src/App.tsx

@@ -48,7 +48,8 @@ const errorLink = onError((error) => {
         if (serverError.statusCode === 401) {
             isLoggedInVar(false);
             Cookies.remove(GlobalCfg.CLIENT_AUTH_COOKIE);
-            window.location.replace(PageRoutes.AUTHENTICATE);
+            const currentPath = window.location.pathname + window.location.search;
+            window.location.replace(`${PageRoutes.AUTHENTICATE}?redirect_uri=${encodeURIComponent(currentPath)}`);
         }
     }
     if (graphQLErrors && graphQLErrors.length) {

datahub-web-react/src/app/context/UserContextProvider.tsx

@@ -3,7 +3,6 @@ import { useGetMeLazyQuery } from '../../graphql/me.generated';
 import { useGetGlobalViewsSettingsLazyQuery } from '../../graphql/app.generated';
 import { CorpUser, PlatformPrivileges } from '../../types.generated';
 import { UserContext, LocalState, DEFAULT_STATE, State } from './userContext';
-import { useInitialRedirect } from './useInitialRedirect';
 
 // TODO: Migrate all usage of useAuthenticatedUser to using this provider.
 
@@ -125,11 +124,6 @@ const UserContextProvider = ({ children }: { children: React.ReactNode }) => {
         }
     }, [state, localState.selectedViewUrn, setDefaultSelectedView]);
 
-    /**
-     * Route to the most recently visited path once on first load of home page, if present in local storage.
-     */
-    useInitialRedirect(state, localState, setState, updateLocalState);
-
     return (
         <UserContext.Provider
             value={{

datahub-web-react/src/app/context/userContext.tsx

@@ -22,10 +22,6 @@ export type State = {
         loadedPersonalDefaultViewUrn: boolean;
         hasSetDefaultView: boolean;
     };
-    /**
-     * Whether the initial page path has been loaded.
-     */
-    loadedInitialPath: boolean;
 };
 
 /**
@@ -54,7 +50,6 @@ export const DEFAULT_STATE: State = {
         loadedPersonalDefaultViewUrn: false,
         hasSetDefaultView: false,
     },
-    loadedInitialPath: false,
 };
 
 export const DEFAULT_CONTEXT = {

datahub-web-react/src/app/entity/group/AddGroupMembersModal.tsx

@@ -83,9 +83,9 @@ export const AddGroupMembersModal = ({ urn, visible, onCloseModal, onSubmit }: P
         setSelectedMembers(newUsers);
     };
 
-    const onDeselectMember = (memberUrn: string) => {
+    const onDeselectMember = (memberUrn: { key: string; label: React.ReactNode; value: string }) => {
         setInputValue('');
-        const newUserActors = selectedMembers.filter((user) => user !== memberUrn);
+        const newUserActors = selectedMembers.filter((user) => user.value !== memberUrn.value);
         setSelectedMembers(newUserActors);
     };
 

datahub-web-react/src/app/entity/shared/__tests__/siblingsUtils.test.ts

@@ -195,6 +195,7 @@ const searchResultWithSiblings = [
     {
         entity: {
             urn: 'urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)',
+            exists: true,
             type: 'DATASET',
             name: 'cypress_project.jaffle_shop.raw_orders',
             origin: 'PROD',
@@ -328,6 +329,7 @@ const searchResultWithSiblings = [
                 siblings: [
                     {
                         urn: 'urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)',
+                        exists: true,
                         type: 'DATASET',
                         platform: {
                             urn: 'urn:li:dataPlatform:dbt',
@@ -376,6 +378,7 @@ const searchResultWithSiblings = [
     {
         entity: {
             urn: 'urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)',
+            exists: true,
             type: 'DATASET',
             name: 'cypress_project.jaffle_shop.raw_orders',
             origin: 'PROD',
@@ -513,6 +516,169 @@ const searchResultWithSiblings = [
     },
 ];
 
+const searchResultWithGhostSiblings = [
+    {
+        entity: {
+            urn: 'urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)',
+            exists: true,
+            type: 'DATASET',
+            name: 'cypress_project.jaffle_shop.raw_orders',
+            origin: 'PROD',
+            uri: null,
+            platform: {
+                urn: 'urn:li:dataPlatform:bigquery',
+                type: 'DATA_PLATFORM',
+                name: 'bigquery',
+                properties: {
+                    type: 'RELATIONAL_DB',
+                    displayName: 'BigQuery',
+                    datasetNameDelimiter: '.',
+                    logoUrl: '/assets/platforms/bigquerylogo.png',
+                    __typename: 'DataPlatformProperties',
+                },
+                displayName: null,
+                info: null,
+                __typename: 'DataPlatform',
+            },
+            dataPlatformInstance: null,
+            editableProperties: null,
+            platformNativeType: null,
+            properties: {
+                name: 'raw_orders',
+                description: null,
+                qualifiedName: null,
+                customProperties: [],
+                __typename: 'DatasetProperties',
+            },
+            ownership: null,
+            globalTags: null,
+            glossaryTerms: null,
+            subTypes: {
+                typeNames: ['table'],
+                __typename: 'SubTypes',
+            },
+            domain: null,
+            container: {
+                urn: 'urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb',
+                platform: {
+                    urn: 'urn:li:dataPlatform:bigquery',
+                    type: 'DATA_PLATFORM',
+                    name: 'bigquery',
+                    properties: {
+                        type: 'RELATIONAL_DB',
+                        displayName: 'BigQuery',
+                        datasetNameDelimiter: '.',
+                        logoUrl: '/assets/platforms/bigquerylogo.png',
+                        __typename: 'DataPlatformProperties',
+                    },
+                    displayName: null,
+                    info: null,
+                    __typename: 'DataPlatform',
+                },
+                properties: {
+                    name: 'jaffle_shop',
+                    __typename: 'ContainerProperties',
+                },
+                subTypes: {
+                    typeNames: ['Dataset'],
+                    __typename: 'SubTypes',
+                },
+                deprecation: null,
+                __typename: 'Container',
+            },
+            parentContainers: {
+                count: 2,
+                containers: [
+                    {
+                        urn: 'urn:li:container:348c96555971d3f5c1ffd7dd2e7446cb',
+                        platform: {
+                            urn: 'urn:li:dataPlatform:bigquery',
+                            type: 'DATA_PLATFORM',
+                            name: 'bigquery',
+                            properties: {
+                                type: 'RELATIONAL_DB',
+                                displayName: 'BigQuery',
+                                datasetNameDelimiter: '.',
+                                logoUrl: '/assets/platforms/bigquerylogo.png',
+                                __typename: 'DataPlatformProperties',
+                            },
+                            displayName: null,
+                            info: null,
+                            __typename: 'DataPlatform',
+                        },
+                        properties: {
+                            name: 'jaffle_shop',
+                            __typename: 'ContainerProperties',
+                        },
+                        subTypes: {
+                            typeNames: ['Dataset'],
+                            __typename: 'SubTypes',
+                        },
+                        deprecation: null,
+                        __typename: 'Container',
+                    },
+                    {
+                        urn: 'urn:li:container:b5e95fce839e7d78151ed7e0a7420d84',
+                        platform: {
+                            urn: 'urn:li:dataPlatform:bigquery',
+                            type: 'DATA_PLATFORM',
+                            name: 'bigquery',
+                            properties: {
+                                type: 'RELATIONAL_DB',
+                                displayName: 'BigQuery',
+                                datasetNameDelimiter: '.',
+                                logoUrl: '/assets/platforms/bigquerylogo.png',
+                                __typename: 'DataPlatformProperties',
+                            },
+                            displayName: null,
+                            info: null,
+                            __typename: 'DataPlatform',
+                        },
+                        properties: {
+                            name: 'cypress_project',
+                            __typename: 'ContainerProperties',
+                        },
+                        subTypes: {
+                            typeNames: ['Project'],
+                            __typename: 'SubTypes',
+                        },
+                        deprecation: null,
+                        __typename: 'Container',
+                    },
+                ],
+                __typename: 'ParentContainersResult',
+            },
+            deprecation: null,
+            siblings: {
+                isPrimary: false,
+                siblings: [
+                    {
+                        urn: 'urn:li:dataset:(urn:li:dataPlatform:dbt,cypress_project.jaffle_shop.raw_orders,PROD)',
+                        exists: false,
+                        type: 'DATASET',
+                    },
+                ],
+                __typename: 'SiblingProperties',
+            },
+            __typename: 'Dataset',
+        },
+        matchedFields: [
+            {
+                name: 'name',
+                value: 'raw_orders',
+                __typename: 'MatchedField',
+            },
+            {
+                name: 'id',
+                value: 'cypress_project.jaffle_shop.raw_orders',
+                __typename: 'MatchedField',
+            },
+        ],
+        insights: [],
+        __typename: 'SearchResult',
+    },
+];
+
 describe('siblingUtils', () => {
     describe('combineEntityDataWithSiblings', () => {
         it('combines my metadata with my siblings as primary', () => {
@@ -564,6 +730,18 @@ describe('siblingUtils', () => {
             expect(result?.[0]?.matchedEntities?.[1]?.urn).toEqual(
                 'urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)',
             );
+
+            expect(result?.[0]?.matchedEntities).toHaveLength(2);
+        });
+
+        it('will not combine an entity with a ghost node', () => {
+            const result = combineSiblingsInSearchResults(searchResultWithGhostSiblings as any);
+
+            expect(result).toHaveLength(1);
+            expect(result?.[0]?.matchedEntities?.[0]?.urn).toEqual(
+                'urn:li:dataset:(urn:li:dataPlatform:bigquery,cypress_project.jaffle_shop.raw_orders,PROD)',
+            );
+            expect(result?.[0]?.matchedEntities).toHaveLength(1);
         });
     });