Merge branch 'datahub-project:master' into master

acryldata · Mar 12, 2024 · c47471f · c47471f
2 parents 3f780cd + 5937472
commit c47471f
Show file tree

Hide file tree

Showing 72 changed files with 910 additions and 292 deletions.
diff --git a/build.gradle b/build.gradle
@@ -30,25 +30,25 @@ buildscript {
 
   ext.junitJupiterVersion = '5.6.1'
   // Releases: https://github.com/linkedin/rest.li/blob/master/CHANGELOG.md
-  ext.pegasusVersion = '29.51.0'
+  ext.pegasusVersion = '29.51.6'
   ext.mavenVersion = '3.6.3'
-  ext.springVersion = '6.1.2'
-  ext.springBootVersion = '3.2.1'
-  ext.springKafkaVersion = '3.1.1'
+  ext.springVersion = '6.1.4'
+  ext.springBootVersion = '3.2.3'
+  ext.springKafkaVersion = '3.1.2'
   ext.openTelemetryVersion = '1.18.0'
   ext.neo4jVersion = '5.14.0'
   ext.neo4jTestVersion = '5.14.0'
   ext.neo4jApocVersion = '5.14.0'
   ext.testContainersVersion = '1.17.4'
-  ext.elasticsearchVersion = '2.9.0' // ES 7.10, Opensearch 1.x, 2.x
+  ext.elasticsearchVersion = '2.11.1' // ES 7.10, Opensearch 1.x, 2.x
   ext.jacksonVersion = '2.15.3'
   ext.jettyVersion = '11.0.19'
   ext.playVersion = '2.8.21'
   ext.log4jVersion = '2.19.0'
   ext.slf4jVersion = '1.7.36'
   ext.logbackClassic = '1.4.14'
   ext.hadoop3Version = '3.3.5'
-  ext.kafkaVersion = '2.3.0'
+  ext.kafkaVersion = '5.5.15'
   ext.hazelcastVersion = '5.3.6'
   ext.ebeanVersion = '12.16.1'
   ext.googleJavaFormatVersion = '1.18.1'
@@ -135,7 +135,7 @@ project.ext.externalDependency = [
     'gson': 'com.google.code.gson:gson:2.8.9',
     'guice': 'com.google.inject:guice:7.0.0',
     'guicePlay': 'com.google.inject:guice:5.0.1', // Used for frontend while still on old Play version
-    'guava': 'com.google.guava:guava:32.1.2-jre',
+    'guava': 'com.google.guava:guava:32.1.3-jre',
     'h2': 'com.h2database:h2:2.2.224',
     'hadoopCommon':'org.apache.hadoop:hadoop-common:2.7.2',
     'hadoopMapreduceClient':'org.apache.hadoop:hadoop-mapreduce-client-core:2.7.2',
@@ -157,7 +157,7 @@ project.ext.externalDependency = [
     'javatuples': 'org.javatuples:javatuples:1.2',
     'javaxInject' : 'javax.inject:javax.inject:1',
     'javaxValidation' : 'javax.validation:validation-api:2.0.1.Final',
-    'jerseyCore': 'org.glassfish.jersey.core:jersey-client:2.25.1',
+    'jerseyCore': 'org.glassfish.jersey.core:jersey-client:2.41',
     'jerseyGuava': 'org.glassfish.jersey.bundles.repackaged:jersey-guava:2.25.1',
     'jettyJaas': "org.eclipse.jetty:jetty-jaas:$jettyVersion",
     'jettyClient': "org.eclipse.jetty:jetty-client:$jettyVersion",
@@ -173,9 +173,9 @@ project.ext.externalDependency = [
     'junitJupiterParams': "org.junit.jupiter:junit-jupiter-params:$junitJupiterVersion",
     'junitJupiterEngine': "org.junit.jupiter:junit-jupiter-engine:$junitJupiterVersion",
     // avro-serde includes dependencies for `kafka-avro-serializer` `kafka-schema-registry-client` and `avro`
-    'kafkaAvroSerde': 'io.confluent:kafka-streams-avro-serde:5.5.1',
+    'kafkaAvroSerde': "io.confluent:kafka-streams-avro-serde:$kafkaVersion",
     'kafkaAvroSerializer': 'io.confluent:kafka-avro-serializer:5.1.4',
-    'kafkaClients': "org.apache.kafka:kafka-clients:$kafkaVersion",
+    'kafkaClients': "org.apache.kafka:kafka-clients:$kafkaVersion-ccs",
     'snappy': 'org.xerial.snappy:snappy-java:1.1.10.4',
     'logbackClassic': "ch.qos.logback:logback-classic:$logbackClassic",
     'logbackClassicJava8' : "ch.qos.logback:logback-classic:$logbackClassicJava8",
@@ -192,7 +192,7 @@ project.ext.externalDependency = [
     'mockitoInline': 'org.mockito:mockito-inline:4.11.0',
     'mockServer': 'org.mock-server:mockserver-netty:5.11.2',
     'mockServerClient': 'org.mock-server:mockserver-client-java:5.11.2',
-    'mysqlConnector': 'mysql:mysql-connector-java:8.0.20',
+    'mysqlConnector': 'mysql:mysql-connector-java:8.0.28',
     'neo4jHarness': 'org.neo4j.test:neo4j-harness:' + neo4jTestVersion,
     'neo4jJavaDriver': 'org.neo4j.driver:neo4j-java-driver:' + neo4jVersion,
     'neo4jTestJavaDriver': 'org.neo4j.driver:neo4j-java-driver:' + neo4jTestVersion,
@@ -216,7 +216,7 @@ project.ext.externalDependency = [
     'playFilters': "com.typesafe.play:filters-helpers_2.12:$playVersion",
     'pac4j': 'org.pac4j:pac4j-oidc:4.5.7',
     'playPac4j': 'org.pac4j:play-pac4j_2.12:9.0.2',
-    'postgresql': 'org.postgresql:postgresql:42.3.8',
+    'postgresql': 'org.postgresql:postgresql:42.3.9',
     'protobuf': 'com.google.protobuf:protobuf-java:3.19.6',
     'grpcProtobuf': 'io.grpc:grpc-protobuf:1.53.0',
     'rangerCommons': 'org.apache.ranger:ranger-plugins-common:2.3.0',
@@ -378,7 +378,7 @@ subprojects {
       constraints {
         implementation("com.google.googlejavaformat:google-java-format:$googleJavaFormatVersion")
         implementation('io.netty:netty-all:4.1.100.Final')
-        implementation('org.apache.commons:commons-compress:1.21')
+        implementation('org.apache.commons:commons-compress:1.26.0')
         implementation('org.apache.velocity:velocity-engine-core:2.3')
         implementation('org.hibernate:hibernate-validator:6.0.20.Final')
         implementation("com.fasterxml.jackson.core:jackson-databind:$jacksonVersion")

diff --git a/datahub-upgrade/build.gradle b/datahub-upgrade/build.gradle
@@ -24,7 +24,7 @@ dependencies {
     exclude group: 'net.minidev', module: 'json-smart'
     exclude group: 'com.nimbusds', module: 'nimbus-jose-jwt'
     exclude group: "org.apache.htrace", module: "htrace-core4"
-    exclude group: "org.eclipse.jetty", module: "jetty-util"
+    exclude group: "org.eclipse.jetty"
     exclude group: "org.apache.hadoop.thirdparty", module: "hadoop-shaded-protobuf_3_7"
     exclude group: "com.charleskorn.kaml", module:"kaml"
 
@@ -43,13 +43,16 @@ dependencies {
     implementation(externalDependency.jettison) {
       because("previous versions are vulnerable")
     }
+    implementation(externalDependency.guava) {
+      because("CVE-2023-2976")
+    }
   }
 
 
   // mock internal schema registry
   implementation externalDependency.kafkaAvroSerde
   implementation externalDependency.kafkaAvroSerializer
-  implementation "org.apache.kafka:kafka_2.12:$kafkaVersion"
+  implementation "org.apache.kafka:kafka_2.12:3.7.0"
 
   implementation externalDependency.slf4jApi
   compileOnly externalDependency.lombok

diff --git a/.../main/java/com/linkedin/datahub/upgrade/system/policyfields/BackfillPolicyFieldsStep.java b/.../main/java/com/linkedin/datahub/upgrade/system/policyfields/BackfillPolicyFieldsStep.java
@@ -25,13 +25,16 @@
 import com.linkedin.metadata.search.ScrollResult;
 import com.linkedin.metadata.search.SearchEntity;
 import com.linkedin.metadata.search.SearchService;
-import com.linkedin.metadata.utils.GenericRecordUtils;
-import com.linkedin.mxe.MetadataChangeProposal;
 import com.linkedin.mxe.SystemMetadata;
 import com.linkedin.policy.DataHubPolicyInfo;
 import io.datahubproject.metadata.context.OperationContext;
 import java.net.URISyntaxException;
 import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
 import java.util.function.Function;
 import lombok.extern.slf4j.Slf4j;
 import org.jetbrains.annotations.NotNull;
@@ -144,9 +147,10 @@ private String backfillPolicies(AuditStamp auditStamp, String scrollId) {
       return null;
     }
 
+    List<Future<?>> futures = new LinkedList<>();
     for (SearchEntity searchEntity : scrollResult.getEntities()) {
       try {
-        ingestPolicyFields(searchEntity.getEntity(), auditStamp);
+        ingestPolicyFields(searchEntity.getEntity(), auditStamp).ifPresent(futures::add);
       } catch (Exception e) {
         // don't stop the whole step because of one bad urn or one bad ingestion
         log.error(
@@ -157,6 +161,15 @@ private String backfillPolicies(AuditStamp auditStamp, String scrollId) {
       }
     }
 
+    futures.forEach(
+        f -> {
+          try {
+            f.get();
+          } catch (InterruptedException | ExecutionException e) {
+            throw new RuntimeException(e);
+          }
+        });
+
     return scrollResult.getScrollId();
   }
 
@@ -174,7 +187,7 @@ private Filter backfillPolicyFieldFilter() {
     return filter;
   }
 
-  private void ingestPolicyFields(Urn urn, AuditStamp auditStamp) {
+  private Optional<Future<?>> ingestPolicyFields(Urn urn, AuditStamp auditStamp) {
     EntityResponse entityResponse = null;
     try {
       entityResponse =
@@ -193,19 +206,30 @@ private void ingestPolicyFields(Urn urn, AuditStamp auditStamp) {
       final DataMap dataMap =
           entityResponse.getAspects().get(DATAHUB_POLICY_INFO_ASPECT_NAME).getValue().data();
       final DataHubPolicyInfo infoAspect = new DataHubPolicyInfo(dataMap);
+
       log.debug("Restating policy information for urn {} with value {}", urn, infoAspect);
-      MetadataChangeProposal proposal = new MetadataChangeProposal();
-      proposal.setEntityUrn(urn);
-      proposal.setEntityType(urn.getEntityType());
-      proposal.setAspectName(DATAHUB_POLICY_INFO_ASPECT_NAME);
-      proposal.setChangeType(ChangeType.RESTATE);
-      proposal.setSystemMetadata(
-          new SystemMetadata()
-              .setRunId(DEFAULT_RUN_ID)
-              .setLastObserved(System.currentTimeMillis()));
-      proposal.setAspect(GenericRecordUtils.serializeAspect(infoAspect));
-      entityService.ingestProposal(proposal, auditStamp, true);
+      return Optional.of(
+          entityService
+              .alwaysProduceMCLAsync(
+                  urn,
+                  urn.getEntityType(),
+                  DATAHUB_POLICY_INFO_ASPECT_NAME,
+                  entityService
+                      .getEntityRegistry()
+                      .getAspectSpecs()
+                      .get(DATAHUB_POLICY_INFO_ASPECT_NAME),
+                  null,
+                  infoAspect,
+                  null,
+                  new SystemMetadata()
+                      .setRunId(DEFAULT_RUN_ID)
+                      .setLastObserved(System.currentTimeMillis()),
+                  auditStamp,
+                  ChangeType.RESTATE)
+              .getFirst());
     }
+
+    return Optional.empty();
   }
 
   @NotNull

diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json
@@ -90,8 +90,8 @@
         "generate": "graphql-codegen --config codegen.yml",
         "lint": "eslint . --ext .ts,.tsx --quiet && yarn type-check",
         "lint-fix": "eslint '*/**/*.{ts,tsx}' --quiet --fix",
-        "type-check": "tsc --noEmit -p tsconfig.test.json",
-        "type-watch": "tsc -w --noEmit -p tsconfig.test.json"
+        "type-check": "tsc --noEmit",
+        "type-watch": "tsc -w --noEmit"
     },
     "browserslist": {
         "production": [

diff --git a/datahub-web-react/src/AppConfigProvider.tsx b/datahub-web-react/src/AppConfigProvider.tsx
@@ -39,7 +39,11 @@ const AppConfigProvider = ({ children }: { children: React.ReactNode }) => {
 
     return (
         <AppConfigContext.Provider
-            value={{ config: appConfigData?.appConfig || DEFAULT_APP_CONFIG, refreshContext: refreshAppConfig }}
+            value={{
+                config: appConfigData?.appConfig || DEFAULT_APP_CONFIG,
+                loaded: !!appConfigData,
+                refreshContext: refreshAppConfig,
+            }}
         >
             {children}
         </AppConfigContext.Provider>

diff --git a/datahub-web-react/src/CustomThemeProvider.tsx b/datahub-web-react/src/CustomThemeProvider.tsx
@@ -4,23 +4,28 @@ import { Theme } from './conf/theme/types';
 import defaultThemeConfig from './conf/theme/theme_light.config.json';
 import { CustomThemeContext } from './customThemeContext';
 
-const CustomThemeProvider = ({ children }: { children: React.ReactNode }) => {
+interface Props {
+    children: React.ReactNode;
+    skipSetTheme?: boolean;
+}
+
+const CustomThemeProvider = ({ children, skipSetTheme }: Props) => {
     const [currentTheme, setTheme] = useState<Theme>(defaultThemeConfig);
 
     useEffect(() => {
         if (import.meta.env.DEV) {
             import(/* @vite-ignore */ `./conf/theme/${import.meta.env.REACT_APP_THEME_CONFIG}`).then((theme) => {
                 setTheme(theme);
             });
-        } else {
+        } else if (!skipSetTheme) {
             // Send a request to the server to get the theme config.
             fetch(`/assets/conf/theme/${import.meta.env.REACT_APP_THEME_CONFIG}`)
                 .then((response) => response.json())
                 .then((theme) => {
                     setTheme(theme);
                 });
         }
-    }, []);
+    }, [skipSetTheme]);
 
     return (
         <CustomThemeContext.Provider value={{ theme: currentTheme, updateTheme: setTheme }}>

diff --git a/datahub-web-react/src/app/ProtectedRoutes.tsx b/datahub-web-react/src/app/ProtectedRoutes.tsx
@@ -3,7 +3,6 @@ import { Switch, Route } from 'react-router-dom';
 import { Layout } from 'antd';
 import { HomePage } from './home/HomePage';
 import { SearchRoutes } from './SearchRoutes';
-import AppProviders from './AppProviders';
 import EmbedRoutes from './EmbedRoutes';
 import { PageRoutes } from '../conf/Global';
 
@@ -12,14 +11,12 @@ import { PageRoutes } from '../conf/Global';
  */
 export const ProtectedRoutes = (): JSX.Element => {
     return (
-        <AppProviders>
-            <Layout>
-                <Switch>
-                    <Route exact path="/" render={() => <HomePage />} />
-                    <Route path={PageRoutes.EMBED} render={() => <EmbedRoutes />} />
-                    <Route path="/*" render={() => <SearchRoutes />} />
-                </Switch>
-            </Layout>
-        </AppProviders>
+        <Layout>
+            <Switch>
+                <Route exact path="/" render={() => <HomePage />} />
+                <Route path={PageRoutes.EMBED} render={() => <EmbedRoutes />} />
+                <Route path="/*" render={() => <SearchRoutes />} />
+            </Switch>
+        </Layout>
     );
 };
diff --git a/datahub-web-react/src/app/Routes.tsx b/datahub-web-react/src/app/Routes.tsx
@@ -1,6 +1,7 @@
 import React from 'react';
 import { Switch, Route, RouteProps } from 'react-router-dom';
 import { useReactiveVar } from '@apollo/client';
+import AppProviders from './AppProviders';
 import { LogIn } from './auth/LogIn';
 import { SignUp } from './auth/SignUp';
 import { ResetCredentials } from './auth/ResetCredentials';
@@ -36,7 +37,14 @@ export const Routes = (): JSX.Element => {
             <Route path={PageRoutes.LOG_IN} component={LogIn} />
             <Route path={PageRoutes.SIGN_UP} component={SignUp} />
             <Route path={PageRoutes.RESET_CREDENTIALS} component={ResetCredentials} />
-            <ProtectedRoute isLoggedIn={isLoggedIn} render={() => <ProtectedRoutes />} />
+            <ProtectedRoute
+                isLoggedIn={isLoggedIn}
+                render={() => (
+                    <AppProviders>
+                        <ProtectedRoutes />
+                    </AppProviders>
+                )}
+            />
             <Route path="/*" component={NoPageFound} />
         </Switch>
     );

diff --git a/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx b/datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx
@@ -149,6 +149,11 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
         }
     };
 
+    const handleBlur = (event: React.FocusEvent<HTMLInputElement>, setterFunction: (value: string) => void) => {
+        const trimmedValue = event.target.value.trim();
+        setterFunction(trimmedValue);
+    };
+
     return (
         <>
             <Form layout="vertical">
@@ -168,6 +173,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
                         placeholder="My Redshift Source #2"
                         value={state.name}
                         onChange={(event) => setName(event.target.value)}
+                        onBlur={(event) => handleBlur(event, setName)}
                     />
                 </Form.Item>
                 <Collapse ghost>
@@ -181,6 +187,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
                                 placeholder="default"
                                 value={state.config?.executorId || ''}
                                 onChange={(event) => setExecutorId(event.target.value)}
+                                onBlur={(event) => handleBlur(event, setExecutorId)}
                             />
                         </Form.Item>
                         <Form.Item label={<Typography.Text strong>CLI Version</Typography.Text>}>
@@ -193,6 +200,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
                                 placeholder="(e.g. 0.12.0)"
                                 value={state.config?.version || ''}
                                 onChange={(event) => setVersion(event.target.value)}
+                                onBlur={(event) => handleBlur(event, setVersion)}
                             />
                         </Form.Item>
                         <Form.Item label={<Typography.Text strong>Debug Mode</Typography.Text>}>
@@ -213,6 +221,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
                                 placeholder='{"MY_CUSTOM_ENV": "my_custom_value2"}'
                                 value={retrieveExtraEnvs()}
                                 onChange={(event) => setExtraEnvs(event.target.value)}
+                                onBlur={(event) => handleBlur(event, setExtraEnvs)}
                             />
                         </Form.Item>
                         <Form.Item label={<Typography.Text strong>Extra DataHub plugins</Typography.Text>}>
@@ -224,6 +233,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
                                 placeholder='["debug"]'
                                 value={retrieveExtraDataHubPlugins()}
                                 onChange={(event) => setExtraDataHubPlugins(event.target.value)}
+                                onBlur={(event) => handleBlur(event, setExtraDataHubPlugins)}
                             />
                         </Form.Item>
                         <Form.Item label={<Typography.Text strong>Extra Pip Libraries</Typography.Text>}>
@@ -235,6 +245,7 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
                                 placeholder='["sqlparse==0.4.3"]'
                                 value={retrieveExtraReqs()}
                                 onChange={(event) => setExtraReqs(event.target.value)}
+                                onBlur={(event) => handleBlur(event, setExtraReqs)}
                             />
                         </Form.Item>
                     </Collapse.Panel>