From ea5282b2386c0488fc4eef1bc6f6855797b5a38e Mon Sep 17 00:00:00 2001 From: David Leifker Date: Wed, 2 Oct 2024 16:39:39 -0500 Subject: [PATCH 1/8] debug(ci): debug ci failure in metadata-io test --- .github/workflows/metadata-io.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml index 7018b42949e89..5cce02f8c14c1 100644 --- a/.github/workflows/metadata-io.yml +++ b/.github/workflows/metadata-io.yml @@ -70,7 +70,7 @@ jobs: cache: "pip" - name: Gradle build (and test) run: | - ./gradlew :metadata-io:test + ./gradlew :metadata-io:test --no-daemon - uses: actions/upload-artifact@v3 if: always() with: From d3a86dc75ebb742e58c6b22d4499dbfc76309190 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Wed, 2 Oct 2024 17:28:44 -0500 Subject: [PATCH 2/8] test if schema fields effects test --- .../linkedin/datahub/graphql/resolvers/search/SearchUtils.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java index 04777c3fcdb4e..f8c9e6df7fe1b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java @@ -73,8 +73,7 @@ private SearchUtils() {} EntityType.DOMAIN, EntityType.DATA_PRODUCT, EntityType.NOTEBOOK, - EntityType.BUSINESS_ATTRIBUTE, - EntityType.SCHEMA_FIELD); + EntityType.BUSINESS_ATTRIBUTE); /** Entities that are part of autocomplete by default in Auto Complete Across Entities */ public static final List AUTO_COMPLETE_ENTITY_TYPES = From 8878bba0838675db71fdbd239db8c8b9eff0f230 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Wed, 2 Oct 2024 17:42:21 -0500 Subject: [PATCH 3/8] Revert "test if schema fields effects test" This reverts commit e7545da19f2c683bec6a43e912821b01764adc5b. --- .../linkedin/datahub/graphql/resolvers/search/SearchUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java index f8c9e6df7fe1b..04777c3fcdb4e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java @@ -73,7 +73,8 @@ private SearchUtils() {} EntityType.DOMAIN, EntityType.DATA_PRODUCT, EntityType.NOTEBOOK, - EntityType.BUSINESS_ATTRIBUTE); + EntityType.BUSINESS_ATTRIBUTE, + EntityType.SCHEMA_FIELD); /** Entities that are part of autocomplete by default in Auto Complete Across Entities */ public static final List AUTO_COMPLETE_ENTITY_TYPES = From 0bd25b95ba8c2b2e84006fcef06598350215af5e Mon Sep 17 00:00:00 2001 From: David Leifker Date: Wed, 2 Oct 2024 17:42:52 -0500 Subject: [PATCH 4/8] fix test warnings test gradle caching disabled --- .github/workflows/metadata-io.yml | 2 +- gradle.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml index 5cce02f8c14c1..e74b97c9ec737 100644 --- a/.github/workflows/metadata-io.yml +++ b/.github/workflows/metadata-io.yml @@ -70,7 +70,7 @@ jobs: cache: "pip" - name: Gradle build (and test) run: | - ./gradlew :metadata-io:test --no-daemon + ./gradlew :metadata-io:test --rerun-tasks - uses: actions/upload-artifact@v3 if: always() with: diff --git a/gradle.properties b/gradle.properties index e42e18dab677b..00e6e8a3a9d3a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ org.gradle.configureondemand=true org.gradle.parallel=true -org.gradle.caching=true +org.gradle.caching=false # Cycle daemons after 30m org.gradle.daemon.idletimeout=1800000 From 7ed2d95c1512988107fe07f6c143b2c5fa72ada4 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Wed, 2 Oct 2024 18:03:16 -0500 Subject: [PATCH 5/8] revert cache test --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 00e6e8a3a9d3a..e42e18dab677b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ org.gradle.configureondemand=true org.gradle.parallel=true -org.gradle.caching=false +org.gradle.caching=true # Cycle daemons after 30m org.gradle.daemon.idletimeout=1800000 From b639cfb94dd1484ffb2f9eb36ec4929553993ada Mon Sep 17 00:00:00 2001 From: David Leifker Date: Wed, 2 Oct 2024 18:04:02 -0500 Subject: [PATCH 6/8] revert rerun-tasks --- .github/workflows/metadata-io.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml index e74b97c9ec737..7018b42949e89 100644 --- a/.github/workflows/metadata-io.yml +++ b/.github/workflows/metadata-io.yml @@ -70,7 +70,7 @@ jobs: cache: "pip" - name: Gradle build (and test) run: | - ./gradlew :metadata-io:test --rerun-tasks + ./gradlew :metadata-io:test - uses: actions/upload-artifact@v3 if: always() with: From 72aa42be6689fd6865831a5e6ab526286e96009d Mon Sep 17 00:00:00 2001 From: David Leifker Date: Wed, 2 Oct 2024 19:36:51 -0500 Subject: [PATCH 7/8] test fix - search should() handling --- .../graph/elastic/ESGraphQueryDAO.java | 22 +++++++++++++++++-- .../graph/elastic/GraphFilterUtils.java | 11 ++++++---- .../query/filter/BaseQueryFilterRewriter.java | 5 ++++- .../request/AutocompleteRequestHandler.java | 13 ++++++----- .../query/request/SearchQueryBuilder.java | 15 ++++++++++--- .../metadata/search/utils/ESUtils.java | 9 ++++---- .../ContainerExpansionRewriterTest.java | 11 +++++++--- .../filter/DomainExpansionRewriterTest.java | 20 ++++++++++++----- .../metadata/search/utils/ESUtilsTest.java | 8 +++++++ .../test/fixtures/search/LineageExporter.java | 8 ++++--- .../lineage_query_filters_full.json | 6 +++++ ...eage_query_filters_full_empty_filters.json | 2 ++ ...e_query_filters_full_multiple_filters.json | 6 +++++ .../lineage_query_filters_limited.json | 1 + .../lineage_time_query_filters_1.json | 4 ++++ 15 files changed, 109 insertions(+), 32 deletions(-) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java index 8c7f0e3256cf8..a801cab81c952 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java @@ -128,6 +128,9 @@ private static void addFilterToQueryBuilder( criterion.getValues()))); orQuery.should(andQuery); } + if (!orQuery.should().isEmpty()) { + orQuery.minimumShouldMatch(1); + } rootQuery.filter(orQuery); } @@ -177,21 +180,26 @@ private SearchResponse executeGroupByLineageSearchQuery( // directions for lineage // set up filters for each relationship type in the correct direction to limit buckets BoolQueryBuilder sourceFilterQuery = QueryBuilders.boolQuery(); - sourceFilterQuery.minimumShouldMatch(1); + validEdges.stream() .filter(pair -> RelationshipDirection.OUTGOING.equals(pair.getValue().getDirection())) .forEach( pair -> sourceFilterQuery.should( getAggregationFilter(pair, RelationshipDirection.OUTGOING))); + if (!sourceFilterQuery.should().isEmpty()) { + sourceFilterQuery.minimumShouldMatch(1); + } BoolQueryBuilder destFilterQuery = QueryBuilders.boolQuery(); - destFilterQuery.minimumShouldMatch(1); validEdges.stream() .filter(pair -> RelationshipDirection.INCOMING.equals(pair.getValue().getDirection())) .forEach( pair -> destFilterQuery.should(getAggregationFilter(pair, RelationshipDirection.INCOMING))); + if (!destFilterQuery.should().isEmpty()) { + destFilterQuery.minimumShouldMatch(1); + } FilterAggregationBuilder sourceRelationshipTypeFilters = AggregationBuilders.filter(FILTER_BY_SOURCE_RELATIONSHIP, sourceFilterQuery); @@ -347,6 +355,9 @@ public static BoolQueryBuilder buildQuery( relationshipType -> relationshipQuery.should( QueryBuilders.termQuery(RELATIONSHIP_TYPE, relationshipType))); + if (!relationshipQuery.should().isEmpty()) { + relationshipQuery.minimumShouldMatch(1); + } finalQuery.filter(relationshipQuery); } @@ -697,6 +708,9 @@ public static QueryBuilder getLineageQuery( urns, edgesPerEntityType.get(entityType), graphFilters)); } }); + if (!entityTypeQueries.should().isEmpty()) { + entityTypeQueries.minimumShouldMatch(1); + } BoolQueryBuilder finalQuery = QueryBuilders.boolQuery(); @@ -741,6 +755,10 @@ static QueryBuilder getLineageQueryForEntityType( query.should(getIncomingEdgeQuery(urns, incomingEdges, graphFilters)); } + if (!query.should().isEmpty()) { + query.minimumShouldMatch(1); + } + return query; } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/GraphFilterUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/GraphFilterUtils.java index 982bcae9b5fd9..b57b5b0b4b5eb 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/GraphFilterUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/GraphFilterUtils.java @@ -37,11 +37,14 @@ public static QueryBuilder getUrnStatusQuery( if (removed) { finalQuery.filter(QueryBuilders.termQuery(statusField, removed.toString())); } else { - finalQuery.minimumShouldMatch(1); finalQuery.should(QueryBuilders.termQuery(statusField, removed.toString())); finalQuery.should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(statusField))); } + if (!finalQuery.should().isEmpty()) { + finalQuery.minimumShouldMatch(1); + } + return finalQuery; } @@ -102,7 +105,7 @@ public static QueryBuilder getEdgeTimeFilterQuery( * 2. The createdOn and updatedOn window does not exist on the edge at all (support legacy cases) * 3. Special lineage case: The edge is marked as a "manual" edge, meaning that the time filters should NOT be applied. */ - BoolQueryBuilder timeFilterQuery = QueryBuilders.boolQuery(); + BoolQueryBuilder timeFilterQuery = QueryBuilders.boolQuery().minimumShouldMatch(1); timeFilterQuery.should(buildTimeWindowFilter(startTimeMillis, endTimeMillis)); timeFilterQuery.should(buildTimestampsMissingFilter()); timeFilterQuery.should(buildManualLineageFilter()); @@ -158,7 +161,7 @@ public static QueryBuilder getEdgeTimeFilterQuery( */ private static QueryBuilder buildTimeWindowFilter( final long startTimeMillis, final long endTimeMillis) { - final BoolQueryBuilder timeWindowQuery = QueryBuilders.boolQuery(); + final BoolQueryBuilder timeWindowQuery = QueryBuilders.boolQuery().minimumShouldMatch(1); /* * To perform comparison: @@ -198,7 +201,7 @@ private static QueryBuilder buildTimestampsMissingFilter() { private static QueryBuilder buildNotExistsFilter(String fieldName) { // This filter returns 'true' if the field DOES NOT EXIST or it exists but is equal to 0. - final BoolQueryBuilder notExistsFilter = QueryBuilders.boolQuery(); + final BoolQueryBuilder notExistsFilter = QueryBuilders.boolQuery().minimumShouldMatch(1); notExistsFilter.should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(fieldName))); notExistsFilter.should(QueryBuilders.boolQuery().must(QueryBuilders.termQuery(fieldName, 0L))); return notExistsFilter; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/filter/BaseQueryFilterRewriter.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/filter/BaseQueryFilterRewriter.java index 367705d369c7c..d545f60a1ee8f 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/filter/BaseQueryFilterRewriter.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/filter/BaseQueryFilterRewriter.java @@ -110,9 +110,12 @@ private BoolQueryBuilder handleNestedFilters( mustNotQueryBuilders.forEach(expandedQueryBuilder::mustNot); expandedQueryBuilder.queryName(boolQueryBuilder.queryName()); expandedQueryBuilder.adjustPureNegative(boolQueryBuilder.adjustPureNegative()); - expandedQueryBuilder.minimumShouldMatch(boolQueryBuilder.minimumShouldMatch()); expandedQueryBuilder.boost(boolQueryBuilder.boost()); + if (!expandedQueryBuilder.should().isEmpty()) { + expandedQueryBuilder.minimumShouldMatch(1); + } + return expandedQueryBuilder; } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java index b7a04f2064d9b..294efb069a904 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/AutocompleteRequestHandler.java @@ -115,8 +115,7 @@ public SearchRequest getSearchRequest( QueryConfiguration customQueryConfig = customizedQueryHandler.lookupQueryConfig(input).orElse(null); - BoolQueryBuilder baseQuery = QueryBuilders.boolQuery(); - baseQuery.minimumShouldMatch(1); + BoolQueryBuilder baseQuery = QueryBuilders.boolQuery().minimumShouldMatch(1); // Initial query with input filters BoolQueryBuilder filterQuery = @@ -176,12 +175,15 @@ public BoolQueryBuilder getQuery( BoolQueryBuilder finalQuery = Optional.ofNullable(customAutocompleteConfig) .flatMap(cac -> CustomizedQueryHandler.boolQueryBuilder(objectMapper, cac, query)) - .orElse(QueryBuilders.boolQuery()) - .minimumShouldMatch(1); + .orElse(QueryBuilders.boolQuery()); getAutocompleteQuery(customAutocompleteConfig, autocompleteFields, query) .ifPresent(finalQuery::should); + if (!finalQuery.should().isEmpty()) { + finalQuery.minimumShouldMatch(1); + } + return finalQuery; } @@ -200,8 +202,7 @@ private Optional getAutocompleteQuery( private static BoolQueryBuilder defaultQuery( List autocompleteFields, @Nonnull String query) { - BoolQueryBuilder finalQuery = QueryBuilders.boolQuery(); - finalQuery.minimumShouldMatch(1); + BoolQueryBuilder finalQuery = QueryBuilders.boolQuery().minimumShouldMatch(1); // Search for exact matches with higher boost and ngram matches MultiMatchQueryBuilder autocompleteQueryBuilder = diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java index 3e76d3600d6a6..529c13c7d71ef 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchQueryBuilder.java @@ -103,8 +103,7 @@ private QueryBuilder buildInternalQuery( cqc -> CustomizedQueryHandler.boolQueryBuilder( opContext.getObjectMapper(), cqc, sanitizedQuery)) - .orElse(QueryBuilders.boolQuery()) - .minimumShouldMatch(1); + .orElse(QueryBuilders.boolQuery()); if (fulltext && !query.startsWith(STRUCTURED_QUERY_PREFIX)) { getSimpleQuery(opContext.getEntityRegistry(), customQueryConfig, entitySpecs, sanitizedQuery) @@ -135,6 +134,10 @@ private QueryBuilder buildInternalQuery( } } + if (!finalQuery.should().isEmpty()) { + finalQuery.minimumShouldMatch(1); + } + return finalQuery; } @@ -368,6 +371,10 @@ private Optional getSimpleQuery( simplePerField.should(simpleBuilder); }); + if (!simplePerField.should().isEmpty()) { + simplePerField.minimumShouldMatch(1); + } + result = Optional.of(simplePerField); } @@ -454,7 +461,9 @@ private Optional getPrefixAndExactMatchQuery( } }); - return finalQuery.should().size() > 0 ? Optional.of(finalQuery) : Optional.empty(); + return finalQuery.should().size() > 0 + ? Optional.of(finalQuery.minimumShouldMatch(1)) + : Optional.empty(); } private Optional getStructuredQuery( diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java index ace7fa2bc197c..3c825640ae3ca 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java @@ -166,8 +166,6 @@ public static BoolQueryBuilder buildFilterQuery( searchableFieldTypes, opContext, queryFilterRewriteChain))); - // The default is not always 1 (ensure consistent default) - finalQueryBuilder.minimumShouldMatch(1); } else if (filter.getCriteria() != null) { // Otherwise, build boolean query from the deprecated "criteria" field. log.warn("Received query Filter with a deprecated field 'criteria'. Use 'or' instead."); @@ -187,7 +185,8 @@ public static BoolQueryBuilder buildFilterQuery( } }); finalQueryBuilder.should(andQueryBuilder); - // The default is not always 1 (ensure consistent default) + } + if (!finalQueryBuilder.should().isEmpty()) { finalQueryBuilder.minimumShouldMatch(1); } return finalQueryBuilder; @@ -533,7 +532,7 @@ private static QueryBuilder getQueryBuilderFromCriterionForFieldToExpand( final Map> searchableFieldTypes, @Nonnull OperationContext opContext, @Nonnull QueryFilterRewriteChain queryFilterRewriteChain) { - final BoolQueryBuilder orQueryBuilder = new BoolQueryBuilder(); + final BoolQueryBuilder orQueryBuilder = new BoolQueryBuilder().minimumShouldMatch(1); for (String field : fields) { orQueryBuilder.should( getQueryBuilderFromCriterionForSingleField( @@ -619,7 +618,7 @@ private static QueryBuilder buildWildcardQueryWithMultipleValues( @Nullable String queryName, @Nonnull AspectRetriever aspectRetriever, String wildcardPattern) { - BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); + BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().minimumShouldMatch(1); for (String value : criterion.getValues()) { boolQuery.should( diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/ContainerExpansionRewriterTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/ContainerExpansionRewriterTest.java index f91e3a28f1bd6..5246e4dbe5bf9 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/ContainerExpansionRewriterTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/ContainerExpansionRewriterTest.java @@ -311,7 +311,7 @@ public void testNestedBoolQueryRewrite() { new RelatedEntities( "IsPartOf", childUrn, parentUrn, RelationshipDirection.OUTGOING, null)))); - BoolQueryBuilder testQuery = QueryBuilders.boolQuery(); + BoolQueryBuilder testQuery = QueryBuilders.boolQuery().minimumShouldMatch(1); testQuery.filter( QueryBuilders.boolQuery() .filter( @@ -319,8 +319,11 @@ public void testNestedBoolQueryRewrite() { testQuery.filter(QueryBuilders.existsQuery("someField")); testQuery.should( QueryBuilders.boolQuery() + .minimumShouldMatch(1) .should( - QueryBuilders.boolQuery().should(QueryBuilders.termsQuery(FIELD_NAME, childUrn)))); + QueryBuilders.boolQuery() + .minimumShouldMatch(1) + .should(QueryBuilders.termsQuery(FIELD_NAME, childUrn)))); testQuery.should(QueryBuilders.existsQuery("someField")); testQuery.must( QueryBuilders.boolQuery() @@ -332,7 +335,7 @@ public void testNestedBoolQueryRewrite() { QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(FIELD_NAME, childUrn)))); testQuery.mustNot(QueryBuilders.existsQuery("someField")); - BoolQueryBuilder expectedRewrite = QueryBuilders.boolQuery(); + BoolQueryBuilder expectedRewrite = QueryBuilders.boolQuery().minimumShouldMatch(1); expectedRewrite.filter( QueryBuilders.boolQuery() .filter( @@ -341,8 +344,10 @@ public void testNestedBoolQueryRewrite() { expectedRewrite.filter(QueryBuilders.existsQuery("someField")); expectedRewrite.should( QueryBuilders.boolQuery() + .minimumShouldMatch(1) .should( QueryBuilders.boolQuery() + .minimumShouldMatch(1) .should(QueryBuilders.termsQuery(FIELD_NAME, childUrn, parentUrn)))); expectedRewrite.should(QueryBuilders.existsQuery("someField")); expectedRewrite.must( diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/DomainExpansionRewriterTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/DomainExpansionRewriterTest.java index 76e650f405456..edc6449438581 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/DomainExpansionRewriterTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/filter/DomainExpansionRewriterTest.java @@ -312,7 +312,7 @@ public void testNestedBoolQueryRewrite() { new RelatedEntities( "IsPartOf", childUrn, parentUrn, RelationshipDirection.INCOMING, null)))); - BoolQueryBuilder testQuery = QueryBuilders.boolQuery(); + BoolQueryBuilder testQuery = QueryBuilders.boolQuery().minimumShouldMatch(1); testQuery.filter( QueryBuilders.boolQuery() .filter( @@ -320,9 +320,15 @@ public void testNestedBoolQueryRewrite() { testQuery.filter(QueryBuilders.boolQuery().filter(QueryBuilders.existsQuery("someField"))); testQuery.should( QueryBuilders.boolQuery() + .minimumShouldMatch(1) .should( - QueryBuilders.boolQuery().should(QueryBuilders.termsQuery(FIELD_NAME, parentUrn)))); - testQuery.should(QueryBuilders.boolQuery().should(QueryBuilders.existsQuery("someField"))); + QueryBuilders.boolQuery() + .minimumShouldMatch(1) + .should(QueryBuilders.termsQuery(FIELD_NAME, parentUrn)))); + testQuery.should( + QueryBuilders.boolQuery() + .minimumShouldMatch(1) + .should(QueryBuilders.existsQuery("someField"))); testQuery.must( QueryBuilders.boolQuery() .must(QueryBuilders.boolQuery().must(QueryBuilders.termsQuery(FIELD_NAME, parentUrn)))); @@ -334,7 +340,7 @@ public void testNestedBoolQueryRewrite() { .mustNot(QueryBuilders.termsQuery(FIELD_NAME, parentUrn)))); testQuery.mustNot(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("someField"))); - BoolQueryBuilder expectedRewrite = QueryBuilders.boolQuery(); + BoolQueryBuilder expectedRewrite = QueryBuilders.boolQuery().minimumShouldMatch(1); expectedRewrite.filter( QueryBuilders.boolQuery() .filter( @@ -344,11 +350,15 @@ public void testNestedBoolQueryRewrite() { QueryBuilders.boolQuery().filter(QueryBuilders.existsQuery("someField"))); expectedRewrite.should( QueryBuilders.boolQuery() + .minimumShouldMatch(1) .should( QueryBuilders.boolQuery() + .minimumShouldMatch(1) .should(QueryBuilders.termsQuery(FIELD_NAME, childUrn, parentUrn)))); expectedRewrite.should( - QueryBuilders.boolQuery().should(QueryBuilders.existsQuery("someField"))); + QueryBuilders.boolQuery() + .minimumShouldMatch(1) + .should(QueryBuilders.existsQuery("someField"))); expectedRewrite.must( QueryBuilders.boolQuery() .must( diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java index 03d104b9e7bfb..cd6f0551a68e1 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/utils/ESUtilsTest.java @@ -179,6 +179,7 @@ public void testGetQueryBuilderFromCriterionContain() { + " }\n" + " ],\n" + " \"adjust_pure_negative\" : true,\n" + + " \"minimum_should_match\" : \"1\",\n" + " \"boost\" : 1.0\n" + " }\n" + "}"; @@ -222,6 +223,7 @@ public void testGetQueryBuilderFromCriterionContain() { + " }\n" + " ],\n" + " \"adjust_pure_negative\" : true,\n" + + " \"minimum_should_match\" : \"1\",\n" + " \"boost\" : 1.0\n" + " }\n" + "}"; @@ -258,6 +260,7 @@ public void testWildcardQueryBuilderFromCriterionWhenStartsWith() { + " }\n" + " ],\n" + " \"adjust_pure_negative\" : true,\n" + + " \"minimum_should_match\" : \"1\",\n" + " \"boost\" : 1.0\n" + " }\n" + "}"; @@ -301,6 +304,7 @@ public void testWildcardQueryBuilderFromCriterionWhenStartsWith() { + " }\n" + " ],\n" + " \"adjust_pure_negative\" : true,\n" + + " \"minimum_should_match\" : \"1\",\n" + " \"boost\" : 1.0\n" + " }\n" + "}"; @@ -337,6 +341,7 @@ public void testWildcardQueryBuilderFromCriterionWhenEndsWith() { + " }\n" + " ],\n" + " \"adjust_pure_negative\" : true,\n" + + " \"minimum_should_match\" : \"1\",\n" + " \"boost\" : 1.0\n" + " }\n" + "}"; @@ -379,6 +384,7 @@ public void testWildcardQueryBuilderFromCriterionWhenEndsWith() { + " }\n" + " ],\n" + " \"adjust_pure_negative\" : true,\n" + + " \"minimum_should_match\" : \"1\",\n" + " \"boost\" : 1.0\n" + " }\n" + "}"; @@ -540,6 +546,7 @@ public void testGetQueryBuilderFromCriterionFieldToExpand() { + " }\n" + " ],\n" + " \"adjust_pure_negative\" : true,\n" + + " \"minimum_should_match\" : \"1\",\n" + " \"boost\" : 1.0\n" + " }\n" + "}"; @@ -582,6 +589,7 @@ public void testGetQueryBuilderFromCriterionFieldToExpand() { + " }\n" + " ],\n" + " \"adjust_pure_negative\" : true,\n" + + " \"minimum_should_match\" : \"1\",\n" + " \"boost\" : 1.0\n" + " }\n" + "}"; diff --git a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/LineageExporter.java b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/LineageExporter.java index 4b7d81aa04416..4a2411138ed67 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/LineageExporter.java +++ b/metadata-io/src/test/java/io/datahubproject/test/fixtures/search/LineageExporter.java @@ -58,7 +58,7 @@ public void exportGraphIndex( Set urns, Set visitedUrns, Set visitedIds, int hops) { Set nextIds = new HashSet<>(); if (!urns.isEmpty()) { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery().minimumShouldMatch(1); boolQueryBuilder.must(QueryBuilders.termQuery("relationshipType", "DownstreamOf")); @@ -70,7 +70,6 @@ public void exportGraphIndex( boolQueryBuilder.should( QueryBuilders.termsQuery("destination.urn", batch.toArray(String[]::new))); }); - boolQueryBuilder.minimumShouldMatch(1); // Exclude visited Lists.partition(Arrays.asList(visitedIds.toArray(String[]::new)), queryStatementSize) @@ -144,7 +143,10 @@ public void exportEntityIndex(Set ids, Set visitedIds, int hops) batch -> boolQueryBuilder.should( QueryBuilders.idsQuery().addIds(batch.toArray(String[]::new)))); - boolQueryBuilder.minimumShouldMatch(1); + + if (!boolQueryBuilder.should().isEmpty()) { + boolQueryBuilder.minimumShouldMatch(1); + } // Exclude visited Lists.partition(Arrays.asList(visitedIds.toArray(String[]::new)), queryStatementSize) diff --git a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full.json b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full.json index 0a1cee08414a9..1aa1b4b5088a1 100644 --- a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full.json +++ b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full.json @@ -33,11 +33,13 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } }, @@ -115,6 +117,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } }, @@ -156,6 +159,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } }, @@ -194,6 +198,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } @@ -212,6 +217,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } diff --git a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full_empty_filters.json b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full_empty_filters.json index ab2841d6602d8..5ba0e36456889 100644 --- a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full_empty_filters.json +++ b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full_empty_filters.json @@ -33,11 +33,13 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } }, diff --git a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full_multiple_filters.json b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full_multiple_filters.json index 39f595e0e8dd2..938d878a9c8d1 100644 --- a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full_multiple_filters.json +++ b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_full_multiple_filters.json @@ -36,11 +36,13 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } }, @@ -122,6 +124,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } }, @@ -163,6 +166,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } }, @@ -201,6 +205,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } @@ -219,6 +224,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } diff --git a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_limited.json b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_limited.json index 95d468ec3dac8..24fbb56065ebf 100644 --- a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_limited.json +++ b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_query_filters_limited.json @@ -27,6 +27,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } \ No newline at end of file diff --git a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_time_query_filters_1.json b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_time_query_filters_1.json index 327f1d4ff9338..13eb02fb61a4e 100644 --- a/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_time_query_filters_1.json +++ b/metadata-io/src/test/resources/elasticsearch/sample_filters/lineage_time_query_filters_1.json @@ -56,6 +56,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } }, @@ -97,6 +98,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } }, @@ -135,6 +137,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } @@ -153,6 +156,7 @@ } ], "adjust_pure_negative" : true, + "minimum_should_match" : "1", "boost" : 1.0 } } \ No newline at end of file From 9bca617d8bf8e16ce718009de87435a4d16088a0 Mon Sep 17 00:00:00 2001 From: David Leifker Date: Thu, 3 Oct 2024 13:32:37 -0500 Subject: [PATCH 8/8] test updates --- .../com/linkedin/metadata/search/fixtures/GoldenTestBase.java | 1 - 1 file changed, 1 deletion(-) diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/GoldenTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/GoldenTestBase.java index 1ebcc03eb690b..052daeece8cd0 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/GoldenTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/fixtures/GoldenTestBase.java @@ -4,7 +4,6 @@ import static com.linkedin.metadata.utils.CriterionUtils.buildCriterion; import static io.datahubproject.test.search.SearchTestUtils.searchAcrossEntities; import static org.testng.Assert.*; -import static org.testng.AssertJUnit.assertNotNull; import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.Urn;