Merge branch 'datahub-project:master' into master

.github/workflows/docker-unified.yml

@@ -989,7 +989,7 @@ jobs:
           DATAHUB_VERSION: ${{ needs.setup.outputs.unique_tag }}
           DATAHUB_ACTIONS_IMAGE: ${{ env.DATAHUB_INGESTION_IMAGE }}
           ACTIONS_VERSION: ${{ needs.datahub_ingestion_slim_build.outputs.tag || 'head-slim' }}
-          ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor]==0.0.13 acryl-datahub-actions==0.0.13 acryl-datahub==0.10.5"
+          ACTIONS_EXTRA_PACKAGES: "acryl-datahub-actions[executor] acryl-datahub-actions"
           ACTIONS_CONFIG: "https://raw.githubusercontent.com/acryldata/datahub-actions/main/docker/config/executor.yaml"
         run: |
           ./smoke-test/run-quickstart.sh

build.gradle

@@ -52,7 +52,7 @@ buildscript {
   ext.hadoop3Version = '3.3.6'
   ext.kafkaVersion = '5.5.15'
   ext.hazelcastVersion = '5.3.6'
-  ext.ebeanVersion = '12.16.1'
+  ext.ebeanVersion = '15.5.2'
   ext.googleJavaFormatVersion = '1.18.1'
   ext.openLineageVersion = '1.19.0'
   ext.logbackClassicJava8 = '1.2.12'
@@ -104,8 +104,8 @@ project.ext.spec = [
 
 project.ext.externalDependency = [
     'akkaHttp': 'com.typesafe.akka:akka-http-core_2.12:10.2.10',
-    'antlr4Runtime': 'org.antlr:antlr4-runtime:4.7.2',
-    'antlr4': 'org.antlr:antlr4:4.7.2',
+    'antlr4Runtime': 'org.antlr:antlr4-runtime:4.9.3',
+    'antlr4': 'org.antlr:antlr4:4.9.3',
     'assertJ': 'org.assertj:assertj-core:3.11.1',
     'avro': 'org.apache.avro:avro:1.11.3',
     'avroCompiler': 'org.apache.avro:avro-compiler:1.11.3',
@@ -129,8 +129,10 @@ project.ext.externalDependency = [
     'dropwizardMetricsCore': 'io.dropwizard.metrics:metrics-core:4.2.3',
     'dropwizardMetricsJmx': 'io.dropwizard.metrics:metrics-jmx:4.2.3',
     'ebean': 'io.ebean:ebean:' + ebeanVersion,
+    'ebeanTest': 'io.ebean:ebean-test:' + ebeanVersion,
     'ebeanAgent': 'io.ebean:ebean-agent:' + ebeanVersion,
     'ebeanDdl': 'io.ebean:ebean-ddl-generator:' + ebeanVersion,
+    'ebeanQueryBean': 'io.ebean:querybean-generator:' + ebeanVersion,
     'elasticSearchRest': 'org.opensearch.client:opensearch-rest-high-level-client:' + elasticsearchVersion,
     'elasticSearchJava': 'org.opensearch.client:opensearch-java:2.6.0',
     'findbugsAnnotations': 'com.google.code.findbugs:annotations:3.0.1',
@@ -359,6 +361,9 @@ configure(subprojects.findAll {! it.name.startsWith('spark-lineage')}) {
     exclude group: "org.slf4j", module: "slf4j-log4j12"
     exclude group: "org.slf4j", module: "slf4j-nop"
     exclude group: "org.slf4j", module: "slf4j-ext"
+
+    resolutionStrategy.force externalDependency.antlr4Runtime
+    resolutionStrategy.force externalDependency.antlr4
   }
 }
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java

@@ -6,22 +6,19 @@
 import com.datahub.authentication.Authentication;
 import com.fasterxml.jackson.core.StreamReadConstraints;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.ImmutableSet;
 import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.data.template.StringArray;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.exception.ValidationException;
 import com.linkedin.datahub.graphql.generated.AndFilterInput;
 import com.linkedin.datahub.graphql.generated.FacetFilterInput;
 import com.linkedin.datahub.graphql.resolvers.search.SearchUtils;
-import com.linkedin.metadata.aspect.AspectRetriever;
 import com.linkedin.metadata.query.filter.Condition;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
 import com.linkedin.metadata.query.filter.Criterion;
 import com.linkedin.metadata.query.filter.CriterionArray;
 import com.linkedin.metadata.query.filter.Filter;
-import com.linkedin.metadata.search.utils.ESUtils;
 import com.linkedin.metadata.service.ViewService;
 import com.linkedin.view.DataHubViewInfo;
 import graphql.schema.DataFetchingEnvironment;
@@ -39,8 +36,6 @@
 
 public class ResolverUtils {
 
-  private static final Set<String> KEYWORD_EXCLUDED_FILTERS =
-      ImmutableSet.of("runId", "_entityType");
   private static final ObjectMapper MAPPER = new ObjectMapper();
 
   static {
@@ -111,11 +106,10 @@ public static Map<String, String> buildFacetFilters(
     return facetFilters;
   }
 
-  public static List<Criterion> criterionListFromAndFilter(
-      List<FacetFilterInput> andFilters, @Nullable AspectRetriever aspectRetriever) {
+  public static List<Criterion> criterionListFromAndFilter(List<FacetFilterInput> andFilters) {
     return andFilters != null && !andFilters.isEmpty()
         ? andFilters.stream()
-            .map(filter -> criterionFromFilter(filter, aspectRetriever))
+            .map(filter -> criterionFromFilter(filter))
             .collect(Collectors.toList())
         : Collections.emptyList();
   }
@@ -124,24 +118,21 @@ public static List<Criterion> criterionListFromAndFilter(
   // conjunctive criterion
   // arrays, rather than just one for the AND case.
   public static ConjunctiveCriterionArray buildConjunctiveCriterionArrayWithOr(
-      @Nonnull List<AndFilterInput> orFilters, @Nullable AspectRetriever aspectRetriever) {
+      @Nonnull List<AndFilterInput> orFilters) {
     return new ConjunctiveCriterionArray(
         orFilters.stream()
             .map(
                 orFilter -> {
                   CriterionArray andCriterionForOr =
-                      new CriterionArray(
-                          criterionListFromAndFilter(orFilter.getAnd(), aspectRetriever));
+                      new CriterionArray(criterionListFromAndFilter(orFilter.getAnd()));
                   return new ConjunctiveCriterion().setAnd(andCriterionForOr);
                 })
             .collect(Collectors.toList()));
   }
 
   @Nullable
   public static Filter buildFilter(
-      @Nullable List<FacetFilterInput> andFilters,
-      @Nullable List<AndFilterInput> orFilters,
-      @Nullable AspectRetriever aspectRetriever) {
+      @Nullable List<FacetFilterInput> andFilters, @Nullable List<AndFilterInput> orFilters) {
     if ((andFilters == null || andFilters.isEmpty())
         && (orFilters == null || orFilters.isEmpty())) {
       return null;
@@ -150,34 +141,21 @@ public static Filter buildFilter(
     // Or filters are the new default. We will check them first.
     // If we have OR filters, we need to build a series of CriterionArrays
     if (orFilters != null && !orFilters.isEmpty()) {
-      return new Filter().setOr(buildConjunctiveCriterionArrayWithOr(orFilters, aspectRetriever));
+      return new Filter().setOr(buildConjunctiveCriterionArrayWithOr(orFilters));
     }
 
     // If or filters are not set, someone may be using the legacy and filters
-    final List<Criterion> andCriterions = criterionListFromAndFilter(andFilters, aspectRetriever);
+    final List<Criterion> andCriterions = criterionListFromAndFilter(andFilters);
     return new Filter()
         .setOr(
             new ConjunctiveCriterionArray(
                 new ConjunctiveCriterion().setAnd(new CriterionArray(andCriterions))));
   }
 
-  public static Criterion criterionFromFilter(
-      final FacetFilterInput filter, @Nullable AspectRetriever aspectRetriever) {
-    return criterionFromFilter(filter, false, aspectRetriever);
-  }
-
   // Translates a FacetFilterInput (graphql input class) into Criterion (our internal model)
-  public static Criterion criterionFromFilter(
-      final FacetFilterInput filter,
-      final Boolean skipKeywordSuffix,
-      @Nullable AspectRetriever aspectRetriever) {
+  public static Criterion criterionFromFilter(final FacetFilterInput filter) {
     Criterion result = new Criterion();
-
-    if (skipKeywordSuffix) {
-      result.setField(filter.getField());
-    } else {
-      result.setField(getFilterField(filter.getField(), skipKeywordSuffix, aspectRetriever));
-    }
+    result.setField(filter.getField());
 
     // `value` is deprecated in place of `values`- this is to support old query patterns. If values
     // is provided,
@@ -210,16 +188,6 @@ public static Criterion criterionFromFilter(
     return result;
   }
 
-  private static String getFilterField(
-      final String originalField,
-      final boolean skipKeywordSuffix,
-      @Nullable AspectRetriever aspectRetriever) {
-    if (KEYWORD_EXCLUDED_FILTERS.contains(originalField)) {
-      return originalField;
-    }
-    return ESUtils.toKeywordField(originalField, skipKeywordSuffix, aspectRetriever);
-  }
-
   public static Filter viewFilter(
       OperationContext opContext, ViewService viewService, String viewUrn) {
     if (viewUrn == null) {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/assertion/AssertionRunEventResolver.java

@@ -12,6 +12,7 @@
 import com.linkedin.datahub.graphql.generated.AssertionRunStatus;
 import com.linkedin.datahub.graphql.generated.FacetFilterInput;
 import com.linkedin.datahub.graphql.generated.FilterInput;
+import com.linkedin.datahub.graphql.resolvers.ResolverUtils;
 import com.linkedin.datahub.graphql.types.dataset.mappers.AssertionRunEventMapper;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.Constants;
@@ -147,7 +148,7 @@ public static Filter buildFilter(
                     .setAnd(
                         new CriterionArray(
                             facetFilters.stream()
-                                .map(filter -> criterionFromFilter(filter, true, aspectRetriever))
+                                .map(ResolverUtils::criterionFromFilter)
                                 .collect(Collectors.toList())))));
   }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolver.java

@@ -71,10 +71,7 @@ public CompletableFuture<ListAccessTokenResult> get(DataFetchingEnvironment envi
                           .withSearchFlags(flags -> flags.setFulltext(true)),
                       Constants.ACCESS_TOKEN_ENTITY_NAME,
                       "",
-                      buildFilter(
-                          filters,
-                          Collections.emptyList(),
-                          context.getOperationContext().getAspectRetriever()),
+                      buildFilter(filters, Collections.emptyList()),
                       sortCriteria,
                       start,
                       count);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/chart/BrowseV2Resolver.java

@@ -74,9 +74,7 @@ public CompletableFuture<BrowseResultsV2> get(DataFetchingEnvironment environmen
                     ? BROWSE_PATH_V2_DELIMITER
                         + String.join(BROWSE_PATH_V2_DELIMITER, input.getPath())
                     : "";
-            final Filter inputFilter =
-                ResolverUtils.buildFilter(
-                    null, input.getOrFilters(), context.getOperationContext().getAspectRetriever());
+            final Filter inputFilter = ResolverUtils.buildFilter(null, input.getOrFilters());
 
             BrowseResultV2 browseResults =
                 _entityClient.browseV2(

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/dataproduct/ListDataProductAssetsResolver.java

@@ -151,11 +151,7 @@ public CompletableFuture<SearchResults> get(DataFetchingEnvironment environment)
             filters.removeIf(f -> f.getField().equals(OUTPUT_PORTS_FILTER_FIELD));
           }
           // add urns from the aspect to our filters
-          final Filter baseFilter =
-              ResolverUtils.buildFilter(
-                  filters,
-                  input.getOrFilters(),
-                  context.getOperationContext().getAspectRetriever());
+          final Filter baseFilter = ResolverUtils.buildFilter(filters, input.getOrFilters());
           final Filter finalFilter =
               buildFilterWithUrns(
                   context.getDataHubAppConfig(), new HashSet<>(urnsToFilterOn), baseFilter);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/DomainEntitiesResolver.java

@@ -79,9 +79,7 @@ public CompletableFuture<SearchResults> get(final DataFetchingEnvironment enviro
                   .getFilters()
                   .forEach(
                       filter -> {
-                        criteria.add(
-                            criterionFromFilter(
-                                filter, true, context.getOperationContext().getAspectRetriever()));
+                        criteria.add(criterionFromFilter(filter));
                       });
             }
 

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/form/CreateDynamicFormAssignmentResolver.java

@@ -33,9 +33,7 @@ public CompletableFuture<Boolean> get(final DataFetchingEnvironment environment)
     final CreateDynamicFormAssignmentInput input =
         bindArgument(environment.getArgument("input"), CreateDynamicFormAssignmentInput.class);
     final Urn formUrn = UrnUtils.getUrn(input.getFormUrn());
-    final DynamicFormAssignment formAssignment =
-        FormUtils.mapDynamicFormAssignment(
-            input, context.getOperationContext().getAspectRetriever());
+    final DynamicFormAssignment formAssignment = FormUtils.mapDynamicFormAssignment(input);
 
     return GraphQLConcurrencyUtils.supplyAsync(
         () -> {

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ingest/source/ListIngestionSourcesResolver.java

@@ -68,10 +68,7 @@ public CompletableFuture<ListIngestionSourcesResult> get(
                           .withSearchFlags(flags -> flags.setFulltext(true)),
                       Constants.INGESTION_SOURCE_ENTITY_NAME,
                       query,
-                      buildFilter(
-                          filters,
-                          Collections.emptyList(),
-                          context.getOperationContext().getAspectRetriever()),
+                      buildFilter(filters, Collections.emptyList()),
                       null,
                       start,
                       count);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/TimeSeriesAspectResolver.java

@@ -9,9 +9,9 @@
 import com.linkedin.datahub.graphql.generated.Entity;
 import com.linkedin.datahub.graphql.generated.FilterInput;
 import com.linkedin.datahub.graphql.generated.TimeSeriesAspect;
+import com.linkedin.datahub.graphql.resolvers.ResolverUtils;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.Constants;
-import com.linkedin.metadata.aspect.AspectRetriever;
 import com.linkedin.metadata.aspect.EnvelopedAspect;
 import com.linkedin.metadata.authorization.PoliciesConfig;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
@@ -120,7 +120,7 @@ public CompletableFuture<List<TimeSeriesAspect>> get(DataFetchingEnvironment env
                     maybeStartTimeMillis,
                     maybeEndTimeMillis,
                     maybeLimit,
-                    buildFilters(maybeFilters, context.getOperationContext().getAspectRetriever()),
+                    buildFilters(maybeFilters),
                     maybeSort);
 
             // Step 2: Bind profiles into GraphQL strong types.
@@ -135,8 +135,7 @@ public CompletableFuture<List<TimeSeriesAspect>> get(DataFetchingEnvironment env
         "get");
   }
 
-  private Filter buildFilters(
-      @Nullable FilterInput maybeFilters, @Nullable AspectRetriever aspectRetriever) {
+  private Filter buildFilters(@Nullable FilterInput maybeFilters) {
     if (maybeFilters == null) {
       return null;
     }
@@ -147,7 +146,7 @@ private Filter buildFilters(
                     .setAnd(
                         new CriterionArray(
                             maybeFilters.getAnd().stream()
-                                .map(filter -> criterionFromFilter(filter, true, aspectRetriever))
+                                .map(ResolverUtils::criterionFromFilter)
                                 .collect(Collectors.toList())))));
   }
 }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/util/FormUtils.java

@@ -18,7 +18,6 @@
 import com.linkedin.form.FormPromptType;
 import com.linkedin.form.FormType;
 import com.linkedin.form.StructuredPropertyParams;
-import com.linkedin.metadata.aspect.AspectRetriever;
 import com.linkedin.metadata.query.filter.Condition;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
 import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray;
@@ -31,7 +30,6 @@
 import java.util.UUID;
 import java.util.stream.Collectors;
 import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
 
 public class FormUtils {
 
@@ -61,16 +59,13 @@ public static PrimitivePropertyValueArray getStructuredPropertyValuesFromInput(
   /** Map a GraphQL CreateDynamicFormAssignmentInput to the GMS DynamicFormAssignment aspect */
   @Nonnull
   public static DynamicFormAssignment mapDynamicFormAssignment(
-      @Nonnull final CreateDynamicFormAssignmentInput input,
-      @Nullable AspectRetriever aspectRetriever) {
+      @Nonnull final CreateDynamicFormAssignmentInput input) {
     Objects.requireNonNull(input, "input must not be null");
 
     final DynamicFormAssignment result = new DynamicFormAssignment();
     final Filter filter =
         new Filter()
-            .setOr(
-                ResolverUtils.buildConjunctiveCriterionArrayWithOr(
-                    input.getOrFilters(), aspectRetriever));
+            .setOr(ResolverUtils.buildConjunctiveCriterionArrayWithOr(input.getOrFilters()));
     result.setFilter(filter);
     return result;
   }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ownership/ListOwnershipTypesResolver.java

@@ -63,10 +63,7 @@ public CompletableFuture<ListOwnershipTypesResult> get(DataFetchingEnvironment e
                     context.getOperationContext().withSearchFlags(flags -> flags.setFulltext(true)),
                     Constants.OWNERSHIP_TYPE_ENTITY_NAME,
                     query,
-                    buildFilter(
-                        filters,
-                        Collections.emptyList(),
-                        context.getOperationContext().getAspectRetriever()),
+                    buildFilter(filters, Collections.emptyList()),
                     Collections.singletonList(DEFAULT_SORT_CRITERION),
                     start,
                     count);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/policy/ListPoliciesResolver.java

@@ -59,11 +59,7 @@ public CompletableFuture<ListPoliciesResult> get(final DataFetchingEnvironment e
       log.debug(
           "User {} listing policies with filters {}", context.getActorUrn(), filters.toString());
 
-      final Filter filter =
-          ResolverUtils.buildFilter(
-              facetFilters,
-              Collections.emptyList(),
-              context.getOperationContext().getAspectRetriever());
+      final Filter filter = ResolverUtils.buildFilter(facetFilters, Collections.emptyList());
 
       return _policyFetcher
           .fetchPolicies(context.getOperationContext(), start, query, count, filter)