Merge branch 'datahub-project:master' into master

.github/workflows/metadata-io.yml

@@ -8,6 +8,7 @@ on:
       - "li-utils/**"
       - "metadata-models/**"
       - "metadata-io/**"
+      - ".github/workflows/metadata-io.yml"
   pull_request:
     branches:
       - "**"
@@ -16,6 +17,7 @@ on:
       - "li-utils/**"
       - "metadata-models/**"
       - "metadata-io/**"
+      - ".github/workflows/metadata-io.yml"
   release:
     types: [published]
 
@@ -52,6 +54,8 @@ jobs:
           sudo apt-get remove 'dotnet-*' azure-cli || true
           sudo rm -rf /usr/local/lib/android/ || true
           sudo docker image prune -a -f || true
+      - name: Disk Check
+        run: df -h . && docker images
       - uses: acryldata/sane-checkout-action@v3
       - name: Set up JDK 17
         uses: actions/setup-java@v4

build.gradle

@@ -117,9 +117,9 @@ project.ext.externalDependency = [
     'awsRds':'software.amazon.awssdk:rds:2.18.24',
     'cacheApi': 'javax.cache:cache-api:1.1.0',
     'commonsCli': 'commons-cli:commons-cli:1.5.0',
-    'commonsIo': 'commons-io:commons-io:2.4',
+    'commonsIo': 'commons-io:commons-io:2.17.0',
     'commonsLang': 'commons-lang:commons-lang:2.6',
-    'commonsText': 'org.apache.commons:commons-text:1.10.0',
+    'commonsText': 'org.apache.commons:commons-text:1.12.0',
     'commonsCollections': 'commons-collections:commons-collections:3.2.2',
     'caffeine': 'com.github.ben-manes.caffeine:caffeine:3.1.8',
     'datastaxOssNativeProtocol': 'com.datastax.oss:native-protocol:1.5.1',
@@ -270,12 +270,12 @@ project.ext.externalDependency = [
     'zookeeper': 'org.apache.zookeeper:zookeeper:3.8.4',
     'wire': 'com.squareup.wire:wire-compiler:3.7.1',
     'charle':  'com.charleskorn.kaml:kaml:0.53.0',
-    'common': 'commons-io:commons-io:2.7',
     'jline':'jline:jline:1.4.1',
     'jetbrains':' org.jetbrains.kotlin:kotlin-stdlib:1.6.0',
     'annotationApi': 'javax.annotation:javax.annotation-api:1.3.2',
     'jakartaAnnotationApi': 'jakarta.annotation:jakarta.annotation-api:3.0.0',
     'classGraph': 'io.github.classgraph:classgraph:4.8.172',
+    'mustache': 'com.github.spullara.mustache.java:compiler:0.9.14'
 ]
 
 allprojects {
@@ -391,12 +391,13 @@ subprojects {
       implementation externalDependency.annotationApi
       constraints {
         implementation("com.google.googlejavaformat:google-java-format:$googleJavaFormatVersion")
-        implementation('io.netty:netty-all:4.1.100.Final')
-        implementation('org.apache.commons:commons-compress:1.26.0')
-        implementation('org.apache.velocity:velocity-engine-core:2.3')
+        implementation('io.netty:netty-all:4.1.114.Final')
+        implementation('org.apache.commons:commons-compress:1.27.1')
+        implementation('org.apache.velocity:velocity-engine-core:2.4')
         implementation('org.hibernate:hibernate-validator:6.0.20.Final')
         implementation("com.fasterxml.jackson.core:jackson-databind:$jacksonVersion")
         implementation("com.fasterxml.jackson.core:jackson-dataformat-cbor:$jacksonVersion")
+        implementation(externalDependency.commonsIo)
       }
     }
 

buildSrc/build.gradle

@@ -22,7 +22,7 @@ dependencies {
   implementation 'com.google.guava:guava:32.1.2-jre'
   implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.5'
   implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.5'
-  implementation 'commons-io:commons-io:2.11.0'
+  implementation 'commons-io:commons-io:2.17.0'
 
   compileOnly 'org.projectlombok:lombok:1.18.30'
   annotationProcessor 'org.projectlombok:lombok:1.18.30'

datahub-frontend/play.gradle

@@ -20,7 +20,7 @@ dependencies {
     play('com.nimbusds:nimbus-jose-jwt:8.18')
     play('com.typesafe.akka:akka-actor_2.12:2.6.20')
     play(externalDependency.jsonSmart)
-    play('io.netty:netty-all:4.1.86.Final')
+    play('io.netty:netty-all:4.1.114.Final')
     implementation(externalDependency.commonsText) {
       because("previous versions are vulnerable to CVE-2022-42889")
     }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java

@@ -25,6 +25,7 @@
 import graphql.schema.DataFetchingEnvironment;
 import io.datahubproject.metadata.services.RestrictedService;
 import java.util.HashSet;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
@@ -60,14 +61,16 @@ public CompletableFuture<EntityLineageResult> get(DataFetchingEnvironment enviro
     final LineageInput input = bindArgument(environment.getArgument("input"), LineageInput.class);
 
     final LineageDirection lineageDirection = input.getDirection();
-    @Nullable final Integer start = input.getStart(); // Optional!
-    @Nullable final Integer count = input.getCount(); // Optional!
-    @Nullable final Boolean separateSiblings = input.getSeparateSiblings(); // Optional!
-    @Nullable final Long startTimeMillis = input.getStartTimeMillis(); // Optional!
+    // All inputs are optional
+    @Nullable final Integer start = input.getStart();
+    @Nullable final Integer count = input.getCount();
+    @Nullable final Boolean separateSiblings = input.getSeparateSiblings();
+    @Nullable final Long startTimeMillis = input.getStartTimeMillis();
     @Nullable
     final Long endTimeMillis =
-        ResolverUtils.getLineageEndTimeMillis(
-            input.getStartTimeMillis(), input.getEndTimeMillis()); // Optional!
+        ResolverUtils.getLineageEndTimeMillis(input.getStartTimeMillis(), input.getEndTimeMillis());
+    final Boolean includeGhostEntities =
+        Optional.ofNullable(input.getIncludeGhostEntities()).orElse(false);
 
     com.linkedin.metadata.graph.LineageDirection resolvedDirection =
         com.linkedin.metadata.graph.LineageDirection.valueOf(lineageDirection.toString());
@@ -80,6 +83,8 @@ public CompletableFuture<EntityLineageResult> get(DataFetchingEnvironment enviro
                 _siblingGraphService.getLineage(
                     context
                         .getOperationContext()
+                        .withSearchFlags(
+                            searchFlags -> searchFlags.setIncludeSoftDeleted(includeGhostEntities))
                         .withLineageFlags(
                             flags ->
                                 flags
@@ -91,6 +96,7 @@ public CompletableFuture<EntityLineageResult> get(DataFetchingEnvironment enviro
                     count != null ? count : 100,
                     1,
                     separateSiblings != null ? input.getSeparateSiblings() : false,
+                    input.getIncludeGhostEntities(),
                     new HashSet<>());
 
             Set<Urn> restrictedUrns = new HashSet<>();

datahub-graphql-core/src/main/resources/entity.graphql

@@ -1258,6 +1258,11 @@ input LineageInput {
     An optional ending time to filter on
     """
     endTimeMillis: Long
+
+    """
+    If enabled, include entities that do not exist or are soft deleted.
+    """
+    includeGhostEntities: Boolean = false
 }
 
 """
@@ -2721,6 +2726,11 @@ enum FabricType {
     Designates review fabrics
     """
     RVW
+
+    """
+    Designates sandbox fabrics
+    """
+    SANDBOX
 }
 
 """

datahub-graphql-core/src/main/resources/search.graphql

@@ -521,6 +521,11 @@ enum FilterOperator {
   """
   EQUAL
 
+  """
+  Represent the relation: field = value (case-insensitive), e.g. platform = HDFS
+  """
+  IEQUAL
+
   """
   * Represent the relation: String field is one of the array values to, e.g. name in ["Profile", "Event"]
   """
@@ -575,6 +580,7 @@ enum FilterOperator {
   Represent the relation: URN field matches any nested child or parent in addition to the given URN
   """
   RELATED_INCL
+
 }
 
 """

datahub-upgrade/build.gradle

@@ -19,6 +19,7 @@ dependencies {
   implementation project(':metadata-dao-impl:kafka-producer')
   implementation externalDependency.charle
 
+  implementation externalDependency.mustache
   implementation externalDependency.javaxInject
   implementation(externalDependency.hadoopClient) {
     exclude group: 'net.minidev', module: 'json-smart'
@@ -83,6 +84,7 @@ dependencies {
   testImplementation externalDependency.springBootTest
   testImplementation externalDependency.mockito
   testImplementation externalDependency.testng
+  testImplementation 'uk.org.webcompere:system-stubs-testng:2.1.7'
   testRuntimeOnly externalDependency.logbackClassic
 
   constraints {

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/UpgradeManager.java

@@ -8,7 +8,7 @@
 public interface UpgradeManager {
 
   /** Register an {@link Upgrade} with the manaager. */
-  void register(Upgrade upgrade);
+  UpgradeManager register(Upgrade upgrade);
 
   /** Kick off an {@link Upgrade} by identifier. */
   UpgradeResult execute(

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/BootstrapMCPConfig.java

@@ -0,0 +1,30 @@
+package com.linkedin.datahub.upgrade.config;
+
+import com.linkedin.datahub.upgrade.system.bootstrapmcps.BootstrapMCP;
+import com.linkedin.metadata.entity.EntityService;
+import io.datahubproject.metadata.context.OperationContext;
+import java.io.IOException;
+import javax.annotation.Nonnull;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class BootstrapMCPConfig {
+
+  @Nonnull
+  @Value("${systemUpdate.bootstrap.mcpConfig}")
+  private String bootstrapMCPConfig;
+
+  @Bean(name = "bootstrapMCPNonBlocking")
+  public BootstrapMCP bootstrapMCPNonBlocking(
+      final OperationContext opContext, EntityService<?> entityService) throws IOException {
+    return new BootstrapMCP(opContext, bootstrapMCPConfig, entityService, false);
+  }
+
+  @Bean(name = "bootstrapMCPBlocking")
+  public BootstrapMCP bootstrapMCPBlocking(
+      final OperationContext opContext, EntityService<?> entityService) throws IOException {
+    return new BootstrapMCP(opContext, bootstrapMCPConfig, entityService, true);
+  }
+}

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java

@@ -6,6 +6,7 @@
 import com.linkedin.datahub.upgrade.system.SystemUpdate;
 import com.linkedin.datahub.upgrade.system.SystemUpdateBlocking;
 import com.linkedin.datahub.upgrade.system.SystemUpdateNonBlocking;
+import com.linkedin.datahub.upgrade.system.bootstrapmcps.BootstrapMCP;
 import com.linkedin.datahub.upgrade.system.elasticsearch.steps.DataHubStartupStep;
 import com.linkedin.gms.factory.config.ConfigurationProvider;
 import com.linkedin.gms.factory.kafka.DataHubKafkaProducerFactory;
@@ -31,6 +32,7 @@
 import io.datahubproject.metadata.services.RestrictedService;
 import java.util.List;
 import javax.annotation.Nonnull;
+import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.avro.generic.IndexedRecord;
 import org.apache.kafka.clients.producer.KafkaProducer;
@@ -54,21 +56,31 @@ public class SystemUpdateConfig {
   public SystemUpdate systemUpdate(
       final List<BlockingSystemUpgrade> blockingSystemUpgrades,
       final List<NonBlockingSystemUpgrade> nonBlockingSystemUpgrades,
-      final DataHubStartupStep dataHubStartupStep) {
-    return new SystemUpdate(blockingSystemUpgrades, nonBlockingSystemUpgrades, dataHubStartupStep);
+      final DataHubStartupStep dataHubStartupStep,
+      @Qualifier("bootstrapMCPBlocking") @NonNull final BootstrapMCP bootstrapMCPBlocking,
+      @Qualifier("bootstrapMCPNonBlocking") @NonNull final BootstrapMCP bootstrapMCPNonBlocking) {
+    return new SystemUpdate(
+        blockingSystemUpgrades,
+        nonBlockingSystemUpgrades,
+        dataHubStartupStep,
+        bootstrapMCPBlocking,
+        bootstrapMCPNonBlocking);
   }
 
   @Bean(name = "systemUpdateBlocking")
   public SystemUpdateBlocking systemUpdateBlocking(
       final List<BlockingSystemUpgrade> blockingSystemUpgrades,
-      final DataHubStartupStep dataHubStartupStep) {
-    return new SystemUpdateBlocking(blockingSystemUpgrades, List.of(), dataHubStartupStep);
+      final DataHubStartupStep dataHubStartupStep,
+      @Qualifier("bootstrapMCPBlocking") @NonNull final BootstrapMCP bootstrapMCPBlocking) {
+    return new SystemUpdateBlocking(
+        blockingSystemUpgrades, dataHubStartupStep, bootstrapMCPBlocking);
   }
 
   @Bean(name = "systemUpdateNonBlocking")
   public SystemUpdateNonBlocking systemUpdateNonBlocking(
-      final List<NonBlockingSystemUpgrade> nonBlockingSystemUpgrades) {
-    return new SystemUpdateNonBlocking(List.of(), nonBlockingSystemUpgrades, null);
+      final List<NonBlockingSystemUpgrade> nonBlockingSystemUpgrades,
+      @Qualifier("bootstrapMCPNonBlocking") @NonNull final BootstrapMCP bootstrapMCPNonBlocking) {
+    return new SystemUpdateNonBlocking(nonBlockingSystemUpgrades, bootstrapMCPNonBlocking);
   }
 
   @Value("#{systemEnvironment['DATAHUB_REVISION'] ?: '0'}")

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeContext.java

@@ -24,7 +24,7 @@ public class DefaultUpgradeContext implements UpgradeContext {
   private final List<String> args;
   private final Map<String, Optional<String>> parsedArgs;
 
-  DefaultUpgradeContext(
+  public DefaultUpgradeContext(
       @Nonnull OperationContext opContext,
       Upgrade upgrade,
       UpgradeReport report,

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/impl/DefaultUpgradeManager.java

@@ -26,8 +26,9 @@ public class DefaultUpgradeManager implements UpgradeManager {
   private final Map<String, Upgrade> _upgrades = new HashMap<>();
 
   @Override
-  public void register(@Nonnull Upgrade upgrade) {
+  public UpgradeManager register(@Nonnull Upgrade upgrade) {
     _upgrades.put(upgrade.id(), upgrade);
+    return this;
   }
 
   @Override

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/SystemUpdate.java

@@ -3,6 +3,7 @@
 import com.linkedin.datahub.upgrade.Upgrade;
 import com.linkedin.datahub.upgrade.UpgradeCleanupStep;
 import com.linkedin.datahub.upgrade.UpgradeStep;
+import com.linkedin.datahub.upgrade.system.bootstrapmcps.BootstrapMCP;
 import com.linkedin.datahub.upgrade.system.elasticsearch.steps.DataHubStartupStep;
 import java.util.LinkedList;
 import java.util.List;
@@ -22,7 +23,9 @@ public class SystemUpdate implements Upgrade {
   public SystemUpdate(
       @NonNull final List<BlockingSystemUpgrade> blockingSystemUpgrades,
       @NonNull final List<NonBlockingSystemUpgrade> nonBlockingSystemUpgrades,
-      @Nullable final DataHubStartupStep dataHubStartupStep) {
+      @Nullable final DataHubStartupStep dataHubStartupStep,
+      @Nullable final BootstrapMCP bootstrapMCPBlocking,
+      @Nullable final BootstrapMCP bootstrapMCPNonBlocking) {
 
     steps = new LinkedList<>();
     cleanupSteps = new LinkedList<>();
@@ -32,11 +35,23 @@ public SystemUpdate(
     cleanupSteps.addAll(
         blockingSystemUpgrades.stream().flatMap(up -> up.cleanupSteps().stream()).toList());
 
+    // bootstrap blocking only
+    if (bootstrapMCPBlocking != null) {
+      steps.addAll(bootstrapMCPBlocking.steps());
+      cleanupSteps.addAll(bootstrapMCPBlocking.cleanupSteps());
+    }
+
     // emit system update message if blocking upgrade(s) present
     if (dataHubStartupStep != null && !blockingSystemUpgrades.isEmpty()) {
       steps.add(dataHubStartupStep);
     }
 
+    // bootstrap non-blocking only
+    if (bootstrapMCPNonBlocking != null) {
+      steps.addAll(bootstrapMCPNonBlocking.steps());
+      cleanupSteps.addAll(bootstrapMCPNonBlocking.cleanupSteps());
+    }
+
     // add non-blocking upgrades last
     steps.addAll(nonBlockingSystemUpgrades.stream().flatMap(up -> up.steps().stream()).toList());
     cleanupSteps.addAll(

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/SystemUpdateBlocking.java

@@ -1,16 +1,16 @@
 package com.linkedin.datahub.upgrade.system;
 
+import com.linkedin.datahub.upgrade.system.bootstrapmcps.BootstrapMCP;
 import com.linkedin.datahub.upgrade.system.elasticsearch.steps.DataHubStartupStep;
 import java.util.List;
 import lombok.NonNull;
-import org.jetbrains.annotations.Nullable;
 
 public class SystemUpdateBlocking extends SystemUpdate {
 
   public SystemUpdateBlocking(
       @NonNull List<BlockingSystemUpgrade> blockingSystemUpgrades,
-      @NonNull List<NonBlockingSystemUpgrade> nonBlockingSystemUpgrades,
-      @Nullable DataHubStartupStep dataHubStartupStep) {
-    super(blockingSystemUpgrades, nonBlockingSystemUpgrades, dataHubStartupStep);
+      @NonNull DataHubStartupStep dataHubStartupStep,
+      @NonNull final BootstrapMCP bootstrapMCPBlocking) {
+    super(blockingSystemUpgrades, List.of(), dataHubStartupStep, bootstrapMCPBlocking, null);
   }
 }

datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/system/SystemUpdateNonBlocking.java

@@ -1,16 +1,14 @@
 package com.linkedin.datahub.upgrade.system;
 
-import com.linkedin.datahub.upgrade.system.elasticsearch.steps.DataHubStartupStep;
+import com.linkedin.datahub.upgrade.system.bootstrapmcps.BootstrapMCP;
 import java.util.List;
 import lombok.NonNull;
-import org.jetbrains.annotations.Nullable;
 
 public class SystemUpdateNonBlocking extends SystemUpdate {
 
   public SystemUpdateNonBlocking(
-      @NonNull List<BlockingSystemUpgrade> blockingSystemUpgrades,
       @NonNull List<NonBlockingSystemUpgrade> nonBlockingSystemUpgrades,
-      @Nullable DataHubStartupStep dataHubStartupStep) {
-    super(blockingSystemUpgrades, nonBlockingSystemUpgrades, dataHubStartupStep);
+      final BootstrapMCP bootstrapMCPNonBlocking) {
+    super(List.of(), nonBlockingSystemUpgrades, null, null, bootstrapMCPNonBlocking);
   }
 }