Update the indexes to not have changeDescription

open-metadata · May 3, 2024 · 31b949c · 31b949c
1 parent 881c4bb
commit 31b949c
Show file tree

Hide file tree

Showing 46 changed files with 366 additions and 225 deletions.
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java
@@ -3,7 +3,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
+import java.util.Set;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.TagLabel;
 
@@ -15,15 +15,39 @@ public static List<String> parseFollowers(List<EntityReference> followersRef) {
     if (followersRef == null) {
       return Collections.emptyList();
     }
-    return followersRef.stream().map(item -> item.getId().toString()).collect(Collectors.toList());
+    return followersRef.stream().map(item -> item.getId().toString()).toList();
   }
 
-  public static void removeNonIndexableFields(Map<String, Object> doc, List<String> fields) {
+  public static void removeNonIndexableFields(Map<String, Object> doc, Set<String> fields) {
     for (String key : fields) {
-      doc.remove(key);
+      if (key.contains(".")) {
+        removeFieldByPath(doc, key);
+      } else {
+        doc.remove(key);
+      }
     }
   }
 
+  public static void removeFieldByPath(Map<String, Object> jsonMap, String path) {
+    String[] pathElements = path.split("\\.");
+    Map<String, Object> currentMap = jsonMap;
+
+    for (int i = 0; i < pathElements.length - 1; i++) {
+      String key = pathElements[i];
+      Object value = currentMap.get(key);
+      if (value instanceof Map) {
+        currentMap = (Map<String, Object>) value;
+      } else {
+        // Path Not Found
+        return;
+      }
+    }
+
+    // Remove the field at the last path element
+    String lastKey = pathElements[pathElements.length - 1];
+    currentMap.remove(lastKey);
+  }
+
   public static List<TagLabel> parseTags(List<TagLabel> tags) {
     if (tags == null) {
       return Collections.emptyList();

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java
@@ -252,7 +252,7 @@ public void createEntity(EntityInterface entity) {
       try {
         IndexMapping indexMapping = entityIndexMap.get(entityType);
         SearchIndex index = searchIndexFactory.buildIndex(entityType, entity);
-        String doc = JsonUtils.pojoToJson(index.buildESDoc());
+        String doc = JsonUtils.pojoToJson(index.buildSearchIndexDoc());
         searchClient.createEntity(indexMapping.getIndexName(clusterAlias), entityId, doc);
       } catch (Exception ie) {
         LOG.error(
@@ -280,7 +280,7 @@ public void createTimeSeriesEntity(EntityTimeSeriesInterface entity) {
       try {
         IndexMapping indexMapping = entityIndexMap.get(entityType);
         SearchIndex index = searchIndexFactory.buildIndex(entityType, entity);
-        String doc = JsonUtils.pojoToJson(index.buildESDoc());
+        String doc = JsonUtils.pojoToJson(index.buildSearchIndexDoc());
         searchClient.createTimeSeriesEntity(indexMapping.getIndexName(clusterAlias), entityId, doc);
       } catch (Exception ie) {
         LOG.error(
@@ -309,7 +309,7 @@ public void updateEntity(EntityInterface entity) {
           scriptTxt = getScriptWithParams(entity, doc);
         } else {
           SearchIndex elasticSearchIndex = searchIndexFactory.buildIndex(entityType, entity);
-          doc = elasticSearchIndex.buildESDoc();
+          doc = elasticSearchIndex.buildSearchIndexDoc();
         }
         searchClient.updateEntity(
             indexMapping.getIndexName(clusterAlias), entityId, doc, scriptTxt);

diff --git a/...java/org/openmetadata/service/search/elasticsearch/ElasticSearchDataInsightProcessor.java b/...java/org/openmetadata/service/search/elasticsearch/ElasticSearchDataInsightProcessor.java
@@ -86,7 +86,8 @@ private UpdateRequest getUpdateRequest(String entityType, ReportData reportData)
             indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()),
             reportData.getId().toString());
     updateRequest.doc(
-        JsonUtils.pojoToJson(new ReportDataIndexes(reportData).buildESDoc()), XContentType.JSON);
+        JsonUtils.pojoToJson(new ReportDataIndexes(reportData).buildSearchIndexDoc()),
+        XContentType.JSON);
     updateRequest.docAsUpsert(true);
     return updateRequest;
   }

diff --git a/...in/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntitiesProcessor.java b/...in/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntitiesProcessor.java
@@ -89,7 +89,8 @@ public static UpdateRequest getUpdateRequest(String entityType, EntityInterface
             entity.getId().toString());
     updateRequest.doc(
         JsonUtils.pojoToJson(
-            Objects.requireNonNull(Entity.buildSearchIndex(entityType, entity)).buildESDoc()),
+            Objects.requireNonNull(Entity.buildSearchIndex(entityType, entity))
+                .buildSearchIndexDoc()),
         XContentType.JSON);
     updateRequest.docAsUpsert(true);
     return updateRequest;

diff --git a/...n/java/org/openmetadata/service/search/indexes/AggregatedCostAnalysisReportDataIndex.java b/...n/java/org/openmetadata/service/search/indexes/AggregatedCostAnalysisReportDataIndex.java
@@ -2,13 +2,16 @@
 
 import java.util.Map;
 import org.openmetadata.schema.analytics.ReportData;
-import org.openmetadata.service.util.JsonUtils;
 
 public record AggregatedCostAnalysisReportDataIndex(ReportData reportData) implements SearchIndex {
 
   @Override
-  public Map<String, Object> buildESDoc() {
-    Map<String, Object> doc = JsonUtils.getMap(reportData);
+  public Object getEntity() {
+    return reportData;
+  }
+
+  @Override
+  public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     doc.put("entityType", "aggregatedCostAnalysisReportData");
     return doc;
   }

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ChartIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ChartIndex.java
@@ -8,14 +8,11 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record ChartIndex(Chart chart) implements SearchIndex {
-  private static final List<String> excludeFields = List.of("changeDescription");
 
-  public Map<String, Object> buildESDoc() {
-    Map<String, Object> doc = JsonUtils.getMap(chart);
-    SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+  @Override
+  public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     List<SearchSuggest> suggest = new ArrayList<>();
     suggest.add(SearchSuggest.builder().input(chart.getName()).weight(10).build());
     suggest.add(SearchSuggest.builder().input(chart.getFullyQualifiedName()).weight(5).build());
@@ -35,4 +32,9 @@ public Map<String, Object> buildESDoc() {
             : chart.getVotes().getUpVotes() - chart.getVotes().getDownVotes());
     return doc;
   }
+
+  @Override
+  public Object getEntity() {
+    return chart;
+  }
 }
diff --git a/...ta-service/src/main/java/org/openmetadata/service/search/indexes/ClassificationIndex.java b/...ta-service/src/main/java/org/openmetadata/service/search/indexes/ClassificationIndex.java
@@ -7,14 +7,11 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record ClassificationIndex(Classification classification) implements SearchIndex {
-  private static final List<String> excludeFields = List.of("changeDescription");
 
-  public Map<String, Object> buildESDoc() {
-    Map<String, Object> doc = JsonUtils.getMap(classification);
-    SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+  @Override
+  public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     List<SearchSuggest> suggest = new ArrayList<>();
     suggest.add(SearchSuggest.builder().input(classification.getName()).weight(10).build());
     suggest.add(
@@ -30,4 +27,9 @@ public Map<String, Object> buildESDoc() {
     doc.put("followers", SearchIndexUtils.parseFollowers(classification.getFollowers()));
     return doc;
   }
+
+  @Override
+  public Object getEntity() {
+    return classification;
+  }
 }
diff --git a/...etadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java b/...etadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java
@@ -15,19 +15,21 @@
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.FlattenColumn;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record ContainerIndex(Container container) implements ColumnIndex {
-  private static final List<String> excludeFields = List.of("changeDescription");
 
-  public Map<String, Object> buildESDoc() {
-    Map<String, Object> doc = JsonUtils.getMap(container);
+  @Override
+  public Object getEntity() {
+    return container;
+  }
+
+  @Override
+  public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     List<SearchSuggest> suggest = new ArrayList<>();
     List<SearchSuggest> columnSuggest = new ArrayList<>();
     List<SearchSuggest> serviceSuggest = new ArrayList<>();
     Set<List<TagLabel>> tagsWithChildren = new HashSet<>();
     List<String> columnsWithChildrenName = new ArrayList<>();
-    SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
     suggest.add(SearchSuggest.builder().input(container.getFullyQualifiedName()).weight(5).build());
     suggest.add(SearchSuggest.builder().input(container.getName()).weight(10).build());
     if (container.getDataModel() != null && container.getDataModel().getColumns() != null) {

diff --git a/...ervice/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java b/...ervice/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java
@@ -15,15 +15,17 @@
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.FlattenColumn;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record DashboardDataModelIndex(DashboardDataModel dashboardDataModel)
     implements ColumnIndex {
-  private static final List<String> excludeFields = List.of("changeDescription");
 
-  public Map<String, Object> buildESDoc() {
-    Map<String, Object> doc = JsonUtils.getMap(dashboardDataModel);
-    SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+  @Override
+  public Object getEntity() {
+    return dashboardDataModel;
+  }
+
+  @Override
+  public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     List<SearchSuggest> suggest = new ArrayList<>();
     List<SearchSuggest> columnSuggest = new ArrayList<>();
     suggest.add(SearchSuggest.builder().input(dashboardDataModel.getName()).weight(10).build());
@@ -34,7 +36,6 @@ public Map<String, Object> buildESDoc() {
             .build());
     Set<List<TagLabel>> tagsWithChildren = new HashSet<>();
     List<String> columnsWithChildrenName = new ArrayList<>();
-    SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
     if (dashboardDataModel.getColumns() != null) {
       List<FlattenColumn> cols = new ArrayList<>();
       parseColumns(dashboardDataModel.getColumns(), cols, null);

diff --git a/...etadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java b/...etadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java
@@ -12,19 +12,21 @@
 import org.openmetadata.service.search.ParseTags;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public class DashboardIndex implements SearchIndex {
   final Dashboard dashboard;
-  final List<String> excludeFields = List.of("changeDescription");
 
   public DashboardIndex(Dashboard dashboard) {
     this.dashboard = dashboard;
   }
 
-  public Map<String, Object> buildESDoc() {
-    Map<String, Object> doc = JsonUtils.getMap(dashboard);
-    SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+  @Override
+  public Object getEntity() {
+    return dashboard;
+  }
+
+  @Override
+  public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     List<SearchSuggest> suggest = new ArrayList<>();
     List<SearchSuggest> serviceSuggest = new ArrayList<>();
     List<SearchSuggest> chartSuggest = new ArrayList<>();

diff --git a/...-service/src/main/java/org/openmetadata/service/search/indexes/DashboardServiceIndex.java b/...-service/src/main/java/org/openmetadata/service/search/indexes/DashboardServiceIndex.java
@@ -7,15 +7,16 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record DashboardServiceIndex(DashboardService dashboardService) implements SearchIndex {
 
-  private static final List<String> excludeFields = List.of("changeDescription");
+  @Override
+  public Object getEntity() {
+    return dashboardService;
+  }
 
-  public Map<String, Object> buildESDoc() {
-    Map<String, Object> doc = JsonUtils.getMap(dashboardService);
-    SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+  @Override
+  public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     List<SearchSuggest> suggest = new ArrayList<>();
     suggest.add(SearchSuggest.builder().input(dashboardService.getName()).weight(5).build());
     suggest.add(

diff --git a/...adata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java b/...adata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java
@@ -7,14 +7,16 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record DataProductIndex(DataProduct dataProduct) implements SearchIndex {
-  private static final List<String> excludeFields = List.of("changeDescription");
 
-  public Map<String, Object> buildESDoc() {
-    Map<String, Object> doc = JsonUtils.getMap(dataProduct);
-    SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+  @Override
+  public Object getEntity() {
+    return dataProduct;
+  }
+
+  @Override
+  public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     List<SearchSuggest> suggest = new ArrayList<>();
     suggest.add(SearchSuggest.builder().input(dataProduct.getName()).weight(5).build());
     suggest.add(

diff --git a/...metadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseIndex.java b/...metadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseIndex.java
@@ -8,14 +8,16 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record DatabaseIndex(Database database) implements SearchIndex {
-  private static final List<String> excludeFields = List.of("changeDescription");
 
-  public Map<String, Object> buildESDoc() {
-    Map<String, Object> doc = JsonUtils.getMap(database);
-    SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+  @Override
+  public Object getEntity() {
+    return database;
+  }
+
+  @Override
+  public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     List<SearchSuggest> suggest = new ArrayList<>();
     suggest.add(SearchSuggest.builder().input(database.getName()).weight(5).build());
     suggest.add(SearchSuggest.builder().input(database.getFullyQualifiedName()).weight(5).build());

diff --git a/...ta-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseSchemaIndex.java b/...ta-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseSchemaIndex.java
@@ -8,14 +8,16 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record DatabaseSchemaIndex(DatabaseSchema databaseSchema) implements SearchIndex {
-  private static final List<String> excludeFields = List.of("changeDescription");
 
-  public Map<String, Object> buildESDoc() {
-    Map<String, Object> doc = JsonUtils.getMap(databaseSchema);
-    SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+  @Override
+  public Object getEntity() {
+    return databaseSchema;
+  }
+
+  @Override
+  public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     List<SearchSuggest> suggest = new ArrayList<>();
     suggest.add(SearchSuggest.builder().input(databaseSchema.getName()).weight(5).build());
     suggest.add(

diff --git a/...a-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseServiceIndex.java b/...a-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseServiceIndex.java
@@ -7,14 +7,16 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record DatabaseServiceIndex(DatabaseService databaseService) implements SearchIndex {
-  private static final List<String> excludeFields = List.of("changeDescription");
 
-  public Map<String, Object> buildESDoc() {
-    Map<String, Object> doc = JsonUtils.getMap(databaseService);
-    SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+  @Override
+  public Object getEntity() {
+    return databaseService;
+  }
+
+  @Override
+  public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     List<SearchSuggest> suggest = new ArrayList<>();
     suggest.add(SearchSuggest.builder().input(databaseService.getName()).weight(5).build());
     suggest.add(

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DomainIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DomainIndex.java
@@ -7,14 +7,16 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
-import org.openmetadata.service.util.JsonUtils;
 
 public record DomainIndex(Domain domain) implements SearchIndex {
-  private static final List<String> excludeFields = List.of("changeDescription");
 
-  public Map<String, Object> buildESDoc() {
-    Map<String, Object> doc = JsonUtils.getMap(domain);
-    SearchIndexUtils.removeNonIndexableFields(doc, excludeFields);
+  @Override
+  public Object getEntity() {
+    return domain;
+  }
+
+  @Override
+  public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     List<SearchSuggest> suggest = new ArrayList<>();
     suggest.add(SearchSuggest.builder().input(domain.getName()).weight(5).build());
     suggest.add(SearchSuggest.builder().input(domain.getFullyQualifiedName()).weight(5).build());