MINOR: Governance Workflows fixes (#18744)

* Add annotation to turn delete_recursive test into the last one to be executed * Improve Fetching Entities by only fetching the FQN * Cut back default batchsize to 500
open-metadata · Nov 24, 2024 · b3d765d · b3d765d
1 parent 31c2dee
commit b3d765d
Show file tree

Hide file tree

Showing 8 changed files with 28 additions and 3 deletions.
diff --git a/...enmetadata/service/governance/workflows/elements/triggers/PeriodicBatchEntityTrigger.java b/...enmetadata/service/governance/workflows/elements/triggers/PeriodicBatchEntityTrigger.java
@@ -53,6 +53,7 @@ public PeriodicBatchEntityTrigger(
 
     StartEvent startEvent =
         new StartEventBuilder().id(getFlowableElementId(triggerWorkflowId, "startEvent")).build();
+    startEvent.setAsynchronousLeave(true);
     oTimerDefinition.ifPresent(startEvent::addEventDefinition);
     process.addFlowElement(startEvent);
 

diff --git a/...g/openmetadata/service/governance/workflows/elements/triggers/impl/FetchEntitiesImpl.java b/...g/openmetadata/service/governance/workflows/elements/triggers/impl/FetchEntitiesImpl.java
@@ -64,7 +64,13 @@ private SearchClient.SearchResultListMapper fetchEntities(
 
     try {
       return searchRepository.listWithDeepPagination(
-          entityType, null, searchFilter, searchSortFilter, batchSize, searchAfter);
+          entityType,
+          null,
+          searchFilter,
+          new String[] {"fullyQualifiedName"},
+          searchSortFilter,
+          batchSize,
+          searchAfter);
     } catch (IOException e) {
       throw new RuntimeException(e);
     }

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java
@@ -174,6 +174,7 @@ SearchResultListMapper listWithDeepPagination(
       String index,
       String query,
       String filter,
+      String[] fields,
       SearchSortFilter searchSortFilter,
       int size,
       Object[] searchAfter)

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java
@@ -841,13 +841,20 @@ public SearchClient.SearchResultListMapper listWithDeepPagination(
       String entityType,
       String query,
       String filter,
+      String[] fields,
       SearchSortFilter searchSortFilter,
       int size,
       Object[] searchAfter)
       throws IOException {
     IndexMapping index = entityIndexMap.get(entityType);
     return searchClient.listWithDeepPagination(
-        index.getIndexName(clusterAlias), query, filter, searchSortFilter, size, searchAfter);
+        index.getIndexName(clusterAlias),
+        query,
+        filter,
+        fields,
+        searchSortFilter,
+        size,
+        searchAfter);
   }
 
   public Response searchBySourceUrl(String sourceUrl) throws IOException {

diff --git a/...vice/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/...vice/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
@@ -700,6 +700,7 @@ public SearchResultListMapper listWithDeepPagination(
       String index,
       String query,
       String filter,
+      String[] fields,
       SearchSortFilter searchSortFilter,
       int size,
       Object[] searchAfter)
@@ -710,6 +711,10 @@ public SearchResultListMapper listWithDeepPagination(
     if (!nullOrEmpty(query)) {
       searchSourceBuilder = getSearchSourceBuilder(index, query, 0, size);
     }
+    if (!nullOrEmpty(fields)) {
+      searchSourceBuilder.fetchSource(fields, null);
+    }
+
     if (Optional.ofNullable(filter).isPresent()) {
       getSearchFilter(filter, searchSourceBuilder, !nullOrEmpty(query));
     }

diff --git a/...ta-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/...ta-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
@@ -691,6 +691,7 @@ public SearchResultListMapper listWithDeepPagination(
       String index,
       String query,
       String filter,
+      String[] fields,
       SearchSortFilter searchSortFilter,
       int size,
       Object[] searchAfter)
@@ -699,6 +700,9 @@ public SearchResultListMapper listWithDeepPagination(
     if (!nullOrEmpty(query)) {
       searchSourceBuilder = getSearchSourceBuilder(index, query, 0, size);
     }
+    if (!nullOrEmpty(fields)) {
+      searchSourceBuilder.fetchSource(fields, null);
+    }
 
     List<Map<String, Object>> results = new ArrayList<>();
 

diff --git a/...e/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java b/...e/src/test/java/org/openmetadata/service/resources/glossary/GlossaryTermResourceTest.java
@@ -731,6 +731,7 @@ void patch_addDeleteStyle(TestInfo test) throws IOException {
   }
 
   @Test
+  @Order(Integer.MAX_VALUE)
   void delete_recursive(TestInfo test) throws IOException {
     Glossary g1 = createGlossary(test, null, emptyList());
 

diff --git a/...ources/json/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.json b/...ources/json/schema/governance/workflows/elements/triggers/periodicBatchEntityTrigger.json
@@ -25,7 +25,7 @@
         "batchSize": {
           "description": "Number of Entities to process at once.",
           "type": "integer",
-          "default": 1000
+          "default": 500
         }
       },
       "required": ["schedule", "entityType", "filters"],