From 31b949c5109c5f2cf1724ddde96c23cedbe40bb9 Mon Sep 17 00:00:00 2001 From: Mohit Yadav <105265192+mohityadav766@users.noreply.github.com> Date: Fri, 3 May 2024 11:46:02 +0530 Subject: [PATCH] Update the indexes to not have changeDescription --- .../service/search/SearchIndexUtils.java | 32 +++++++++++++-- .../service/search/SearchRepository.java | 6 +-- .../ElasticSearchDataInsightProcessor.java | 3 +- .../ElasticSearchEntitiesProcessor.java | 3 +- ...AggregatedCostAnalysisReportDataIndex.java | 9 ++-- .../service/search/indexes/ChartIndex.java | 12 +++--- .../search/indexes/ClassificationIndex.java | 12 +++--- .../search/indexes/ContainerIndex.java | 12 +++--- .../indexes/DashboardDataModelIndex.java | 13 +++--- .../search/indexes/DashboardIndex.java | 12 +++--- .../search/indexes/DashboardServiceIndex.java | 11 ++--- .../search/indexes/DataProductIndex.java | 12 +++--- .../service/search/indexes/DatabaseIndex.java | 12 +++--- .../search/indexes/DatabaseSchemaIndex.java | 12 +++--- .../search/indexes/DatabaseServiceIndex.java | 12 +++--- .../service/search/indexes/DomainIndex.java | 12 +++--- .../search/indexes/EntityReportDataIndex.java | 9 ++-- .../service/search/indexes/GlossaryIndex.java | 12 +++--- .../search/indexes/GlossaryTermIndex.java | 12 +++--- .../indexes/IngestionPipelineIndex.java | 20 ++++++--- .../search/indexes/MessagingServiceIndex.java | 12 +++--- .../search/indexes/MetadataServiceIndex.java | 12 +++--- .../service/search/indexes/MlModelIndex.java | 12 +++--- .../search/indexes/MlModelServiceIndex.java | 12 +++--- .../service/search/indexes/PipelineIndex.java | 12 +++--- .../search/indexes/PipelineServiceIndex.java | 12 +++--- .../service/search/indexes/QueryIndex.java | 13 +++--- .../RawCostAnalysisReportDataIndex.java | 10 +++-- .../search/indexes/ReportDataIndexes.java | 10 +++-- .../search/indexes/SearchEntityIndex.java | 12 +++--- .../service/search/indexes/SearchIndex.java | 25 ++++++++++- .../search/indexes/SearchServiceIndex.java | 12 +++--- .../search/indexes/StorageServiceIndex.java | 12 +++--- .../search/indexes/StoredProcedureIndex.java | 12 +++--- .../service/search/indexes/TableIndex.java | 22 ++++++---- .../service/search/indexes/TagIndex.java | 12 +++--- .../service/search/indexes/TeamIndex.java | 19 ++++++--- .../service/search/indexes/TestCaseIndex.java | 41 ++++--------------- .../TestCaseResolutionStatusIndex.java | 7 +++- .../search/indexes/TestSuiteIndex.java | 12 +++--- .../service/search/indexes/TopicIndex.java | 18 +++++--- .../service/search/indexes/UserIndex.java | 20 ++++++--- .../WebAnalyticEntityViewReportDataIndex.java | 9 ++-- ...ebAnalyticUserActivityReportDataIndex.java | 9 ++-- .../OpenSearchDataInsightProcessor.java | 3 +- .../OpenSearchEntitiesProcessor.java | 3 +- 46 files changed, 366 insertions(+), 225 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java index f8fc750ebbc8..cffaa6a1f916 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java @@ -3,7 +3,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; +import java.util.Set; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.TagLabel; @@ -15,15 +15,39 @@ public static List parseFollowers(List followersRef) { if (followersRef == null) { return Collections.emptyList(); } - return followersRef.stream().map(item -> item.getId().toString()).collect(Collectors.toList()); + return followersRef.stream().map(item -> item.getId().toString()).toList(); } - public static void removeNonIndexableFields(Map doc, List fields) { + public static void removeNonIndexableFields(Map doc, Set fields) { for (String key : fields) { - doc.remove(key); + if (key.contains(".")) { + removeFieldByPath(doc, key); + } else { + doc.remove(key); + } } } + public static void removeFieldByPath(Map jsonMap, String path) { + String[] pathElements = path.split("\\."); + Map currentMap = jsonMap; + + for (int i = 0; i < pathElements.length - 1; i++) { + String key = pathElements[i]; + Object value = currentMap.get(key); + if (value instanceof Map) { + currentMap = (Map) value; + } else { + // Path Not Found + return; + } + } + + // Remove the field at the last path element + String lastKey = pathElements[pathElements.length - 1]; + currentMap.remove(lastKey); + } + public static List parseTags(List tags) { if (tags == null) { return Collections.emptyList(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index 353d93f5133f..220bcc788cf3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -252,7 +252,7 @@ public void createEntity(EntityInterface entity) { try { IndexMapping indexMapping = entityIndexMap.get(entityType); SearchIndex index = searchIndexFactory.buildIndex(entityType, entity); - String doc = JsonUtils.pojoToJson(index.buildESDoc()); + String doc = JsonUtils.pojoToJson(index.buildSearchIndexDoc()); searchClient.createEntity(indexMapping.getIndexName(clusterAlias), entityId, doc); } catch (Exception ie) { LOG.error( @@ -280,7 +280,7 @@ public void createTimeSeriesEntity(EntityTimeSeriesInterface entity) { try { IndexMapping indexMapping = entityIndexMap.get(entityType); SearchIndex index = searchIndexFactory.buildIndex(entityType, entity); - String doc = JsonUtils.pojoToJson(index.buildESDoc()); + String doc = JsonUtils.pojoToJson(index.buildSearchIndexDoc()); searchClient.createTimeSeriesEntity(indexMapping.getIndexName(clusterAlias), entityId, doc); } catch (Exception ie) { LOG.error( @@ -309,7 +309,7 @@ public void updateEntity(EntityInterface entity) { scriptTxt = getScriptWithParams(entity, doc); } else { SearchIndex elasticSearchIndex = searchIndexFactory.buildIndex(entityType, entity); - doc = elasticSearchIndex.buildESDoc(); + doc = elasticSearchIndex.buildSearchIndexDoc(); } searchClient.updateEntity( indexMapping.getIndexName(clusterAlias), entityId, doc, scriptTxt); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchDataInsightProcessor.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchDataInsightProcessor.java index c4fe74ee6ef7..d10afe6a18bd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchDataInsightProcessor.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchDataInsightProcessor.java @@ -86,7 +86,8 @@ private UpdateRequest getUpdateRequest(String entityType, ReportData reportData) indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()), reportData.getId().toString()); updateRequest.doc( - JsonUtils.pojoToJson(new ReportDataIndexes(reportData).buildESDoc()), XContentType.JSON); + JsonUtils.pojoToJson(new ReportDataIndexes(reportData).buildSearchIndexDoc()), + XContentType.JSON); updateRequest.docAsUpsert(true); return updateRequest; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntitiesProcessor.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntitiesProcessor.java index 7d07972c8900..13313aeb034e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntitiesProcessor.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchEntitiesProcessor.java @@ -89,7 +89,8 @@ public static UpdateRequest getUpdateRequest(String entityType, EntityInterface entity.getId().toString()); updateRequest.doc( JsonUtils.pojoToJson( - Objects.requireNonNull(Entity.buildSearchIndex(entityType, entity)).buildESDoc()), + Objects.requireNonNull(Entity.buildSearchIndex(entityType, entity)) + .buildSearchIndexDoc()), XContentType.JSON); updateRequest.docAsUpsert(true); return updateRequest; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/AggregatedCostAnalysisReportDataIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/AggregatedCostAnalysisReportDataIndex.java index 8341391d20fc..354105c19406 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/AggregatedCostAnalysisReportDataIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/AggregatedCostAnalysisReportDataIndex.java @@ -2,13 +2,16 @@ import java.util.Map; import org.openmetadata.schema.analytics.ReportData; -import org.openmetadata.service.util.JsonUtils; public record AggregatedCostAnalysisReportDataIndex(ReportData reportData) implements SearchIndex { @Override - public Map buildESDoc() { - Map doc = JsonUtils.getMap(reportData); + public Object getEntity() { + return reportData; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { doc.put("entityType", "aggregatedCostAnalysisReportData"); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ChartIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ChartIndex.java index 64deeb63a43f..721f57e4b83a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ChartIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ChartIndex.java @@ -8,14 +8,11 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record ChartIndex(Chart chart) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(chart); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(chart.getName()).weight(10).build()); suggest.add(SearchSuggest.builder().input(chart.getFullyQualifiedName()).weight(5).build()); @@ -35,4 +32,9 @@ public Map buildESDoc() { : chart.getVotes().getUpVotes() - chart.getVotes().getDownVotes()); return doc; } + + @Override + public Object getEntity() { + return chart; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ClassificationIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ClassificationIndex.java index b7d71dafeddf..b2215af369ef 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ClassificationIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ClassificationIndex.java @@ -7,14 +7,11 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record ClassificationIndex(Classification classification) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(classification); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(classification.getName()).weight(10).build()); suggest.add( @@ -30,4 +27,9 @@ public Map buildESDoc() { doc.put("followers", SearchIndexUtils.parseFollowers(classification.getFollowers())); return doc; } + + @Override + public Object getEntity() { + return classification; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java index 97b200caf9dc..a3694ea8655a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java @@ -15,19 +15,21 @@ import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.FlattenColumn; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record ContainerIndex(Container container) implements ColumnIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(container); + @Override + public Object getEntity() { + return container; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); List columnSuggest = new ArrayList<>(); List serviceSuggest = new ArrayList<>(); Set> tagsWithChildren = new HashSet<>(); List columnsWithChildrenName = new ArrayList<>(); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); suggest.add(SearchSuggest.builder().input(container.getFullyQualifiedName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(container.getName()).weight(10).build()); if (container.getDataModel() != null && container.getDataModel().getColumns() != null) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java index 21f136c0602c..538688f89b64 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java @@ -15,15 +15,17 @@ import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.FlattenColumn; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record DashboardDataModelIndex(DashboardDataModel dashboardDataModel) implements ColumnIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(dashboardDataModel); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return dashboardDataModel; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); List columnSuggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(dashboardDataModel.getName()).weight(10).build()); @@ -34,7 +36,6 @@ public Map buildESDoc() { .build()); Set> tagsWithChildren = new HashSet<>(); List columnsWithChildrenName = new ArrayList<>(); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); if (dashboardDataModel.getColumns() != null) { List cols = new ArrayList<>(); parseColumns(dashboardDataModel.getColumns(), cols, null); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java index 635767c1c675..5ce774c99651 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java @@ -12,19 +12,21 @@ import org.openmetadata.service.search.ParseTags; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public class DashboardIndex implements SearchIndex { final Dashboard dashboard; - final List excludeFields = List.of("changeDescription"); public DashboardIndex(Dashboard dashboard) { this.dashboard = dashboard; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(dashboard); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return dashboard; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); List serviceSuggest = new ArrayList<>(); List chartSuggest = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardServiceIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardServiceIndex.java index 20026af1108f..cdabefd7d96a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardServiceIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardServiceIndex.java @@ -7,15 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record DashboardServiceIndex(DashboardService dashboardService) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); + @Override + public Object getEntity() { + return dashboardService; + } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(dashboardService); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(dashboardService.getName()).weight(5).build()); suggest.add( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java index d8584bd524ff..310d7ca3907e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java @@ -7,14 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record DataProductIndex(DataProduct dataProduct) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(dataProduct); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return dataProduct; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(dataProduct.getName()).weight(5).build()); suggest.add( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseIndex.java index f7523b55c083..0ed613b05493 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseIndex.java @@ -8,14 +8,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record DatabaseIndex(Database database) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(database); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return database; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(database.getName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(database.getFullyQualifiedName()).weight(5).build()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseSchemaIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseSchemaIndex.java index c54f9a1ad85d..39984999f845 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseSchemaIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseSchemaIndex.java @@ -8,14 +8,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record DatabaseSchemaIndex(DatabaseSchema databaseSchema) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(databaseSchema); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return databaseSchema; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(databaseSchema.getName()).weight(5).build()); suggest.add( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseServiceIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseServiceIndex.java index f007fa8871dc..7297d5f4ec13 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseServiceIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseServiceIndex.java @@ -7,14 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record DatabaseServiceIndex(DatabaseService databaseService) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(databaseService); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return databaseService; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(databaseService.getName()).weight(5).build()); suggest.add( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DomainIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DomainIndex.java index 9d2bcbb5c27d..bac0c7e6c616 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DomainIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DomainIndex.java @@ -7,14 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record DomainIndex(Domain domain) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(domain); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return domain; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(domain.getName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(domain.getFullyQualifiedName()).weight(5).build()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/EntityReportDataIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/EntityReportDataIndex.java index 857323ee299d..384c4479497a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/EntityReportDataIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/EntityReportDataIndex.java @@ -2,13 +2,16 @@ import java.util.Map; import org.openmetadata.schema.analytics.ReportData; -import org.openmetadata.service.util.JsonUtils; public record EntityReportDataIndex(ReportData reportData) implements SearchIndex { @Override - public Map buildESDoc() { - Map doc = JsonUtils.getMap(reportData); + public Object getEntity() { + return reportData; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { doc.put("entityType", "entityReportData"); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryIndex.java index 7acb2588e6b1..b773c34bffa6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryIndex.java @@ -8,19 +8,21 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public class GlossaryIndex implements SearchIndex { final Glossary glossary; - final List excludeFields = List.of("changeDescription"); public GlossaryIndex(Glossary glossary) { this.glossary = glossary; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(glossary); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return glossary; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(glossary.getName()).weight(5).build()); if (glossary.getDisplayName() != null && !glossary.getDisplayName().isEmpty()) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java index dff89c3cf0e1..7a1d48dc1c50 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java @@ -8,19 +8,21 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public class GlossaryTermIndex implements SearchIndex { final GlossaryTerm glossaryTerm; - final List excludeFields = List.of("changeDescription"); public GlossaryTermIndex(GlossaryTerm glossaryTerm) { this.glossaryTerm = glossaryTerm; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(glossaryTerm); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return glossaryTerm; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(glossaryTerm.getName()).weight(5).build()); if (glossaryTerm.getDisplayName() != null && !glossaryTerm.getDisplayName().isEmpty()) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java index 2432cfe82714..c69a40962f9d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java @@ -3,26 +3,34 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline; import org.openmetadata.service.Entity; import org.openmetadata.service.search.ParseTags; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public class IngestionPipelineIndex implements SearchIndex { final IngestionPipeline ingestionPipeline; - final List excludeFields = - List.of("changeDescription", "sourceConfig", "openMetadataServerConnection", "airflowConfig"); + final Set excludeFields = + Set.of("sourceConfig", "openMetadataServerConnection", "airflowConfig"); public IngestionPipelineIndex(IngestionPipeline ingestionPipeline) { this.ingestionPipeline = ingestionPipeline; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(ingestionPipeline); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return ingestionPipeline; + } + + public Set getExcludedFields() { + return excludeFields; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); List serviceSuggest = new ArrayList<>(); suggest.add( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MessagingServiceIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MessagingServiceIndex.java index 8a6cb3b1684b..adcc4c360f51 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MessagingServiceIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MessagingServiceIndex.java @@ -7,14 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record MessagingServiceIndex(MessagingService messagingService) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(messagingService); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return messagingService; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(messagingService.getName()).weight(5).build()); suggest.add( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MetadataServiceIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MetadataServiceIndex.java index cd982521a5b1..7c3042d82831 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MetadataServiceIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MetadataServiceIndex.java @@ -7,14 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record MetadataServiceIndex(MetadataService metadataService) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(metadataService); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return metadataService; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(metadataService.getName()).weight(5).build()); suggest.add( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java index f9d890e51aff..095d57af6c3f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java @@ -9,20 +9,22 @@ import org.openmetadata.service.search.ParseTags; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public class MlModelIndex implements SearchIndex { final MlModel mlModel; - final List excludeFields = List.of("changeDescription"); public MlModelIndex(MlModel mlModel) { this.mlModel = mlModel; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(mlModel); + @Override + public Object getEntity() { + return mlModel; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); suggest.add(SearchSuggest.builder().input(mlModel.getFullyQualifiedName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(mlModel.getName()).weight(10).build()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelServiceIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelServiceIndex.java index bb8bd2884397..d618de74f18f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelServiceIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelServiceIndex.java @@ -7,14 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record MlModelServiceIndex(MlModelService mlModelService) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(mlModelService); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return mlModelService; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(mlModelService.getName()).weight(5).build()); suggest.add( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java index 7d44114491e5..0ac6c5521e53 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java @@ -10,19 +10,21 @@ import org.openmetadata.service.search.ParseTags; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public class PipelineIndex implements SearchIndex { final Pipeline pipeline; - final List excludeFields = List.of("changeDescription"); public PipelineIndex(Pipeline pipeline) { this.pipeline = pipeline; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(pipeline); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return pipeline; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); List serviceSuggest = new ArrayList<>(); List taskSuggest = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineServiceIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineServiceIndex.java index 4c9625d163fa..0c84ca16a21d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineServiceIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineServiceIndex.java @@ -7,14 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record PipelineServiceIndex(PipelineService pipelineService) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(pipelineService); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return pipelineService; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(pipelineService.getName()).weight(5).build()); suggest.add( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/QueryIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/QueryIndex.java index e19964a21fcc..8bf73d06b681 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/QueryIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/QueryIndex.java @@ -12,24 +12,25 @@ import org.openmetadata.service.search.ParseTags; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public class QueryIndex implements SearchIndex { - final List excludeTopicFields = List.of("changeDescription"); final Query query; public QueryIndex(Query query) { this.query = query; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(query); + @Override + public Object getEntity() { + return query; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); if (query.getDisplayName() != null) { suggest.add(SearchSuggest.builder().input(query.getName()).weight(10).build()); } - SearchIndexUtils.removeNonIndexableFields(doc, excludeTopicFields); - ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.QUERY, query)); doc.put("displayName", query.getDisplayName() != null ? query.getDisplayName() : ""); doc.put("tags", parseTags.getTags()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/RawCostAnalysisReportDataIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/RawCostAnalysisReportDataIndex.java index 440ef9e6590d..3af6cc7df4b9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/RawCostAnalysisReportDataIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/RawCostAnalysisReportDataIndex.java @@ -2,12 +2,16 @@ import java.util.Map; import org.openmetadata.schema.analytics.ReportData; -import org.openmetadata.service.util.JsonUtils; public record RawCostAnalysisReportDataIndex(ReportData reportData) implements SearchIndex { + + @Override + public Object getEntity() { + return reportData; + } + @Override - public Map buildESDoc() { - Map doc = JsonUtils.getMap(reportData); + public Map buildSearchIndexDocInternal(Map doc) { doc.put("entityType", "rawCostAnalysisReportData"); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ReportDataIndexes.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ReportDataIndexes.java index 3ae8e1197605..8345a3d8d8e8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ReportDataIndexes.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ReportDataIndexes.java @@ -2,12 +2,16 @@ import java.util.Map; import org.openmetadata.schema.analytics.ReportData; -import org.openmetadata.service.util.JsonUtils; public record ReportDataIndexes(ReportData reportData) implements SearchIndex { + + @Override + public Object getEntity() { + return reportData; + } + @Override - public Map buildESDoc() { - Map doc = JsonUtils.getMap(reportData); + public Map buildSearchIndexDocInternal(Map doc) { doc.put("id", null); doc.put("timestamp", reportData.getTimestamp()); doc.put("reportDataType", reportData.getReportDataType()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java index 06261885d8ed..f29d96fa50b2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java @@ -8,15 +8,17 @@ import org.openmetadata.service.search.ParseTags; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record SearchEntityIndex(org.openmetadata.schema.entity.data.SearchIndex searchIndex) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(searchIndex); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return searchIndex; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(searchIndex.getName()).weight(5).build()); suggest.add( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java index a20174ee9b32..abe104404598 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java @@ -11,6 +11,7 @@ import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -24,11 +25,33 @@ import org.openmetadata.schema.type.Relationship; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; public interface SearchIndex { - Map buildESDoc(); + Set DEFAULT_EXCLUDED_FIELDS = Set.of("changeDescription"); + + default Map buildSearchIndexDoc() { + Map esDoc = JsonUtils.getMap(getEntity()); + + // Remove non indexable fields + SearchIndexUtils.removeNonIndexableFields(esDoc, DEFAULT_EXCLUDED_FIELDS); + + // Remove Entity Specific Field + SearchIndexUtils.removeNonIndexableFields(esDoc, getExcludedFields()); + + // Build Index Doc + return this.buildSearchIndexDocInternal(esDoc); + } + + Object getEntity(); + + default Set getExcludedFields() { + return Collections.emptySet(); + } + + Map buildSearchIndexDocInternal(Map esDoc); default Set getFQNParts(String fqn, List fqnSplits) { Set fqnParts = new HashSet<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchServiceIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchServiceIndex.java index 35faf4611180..5c994b42eafd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchServiceIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchServiceIndex.java @@ -7,14 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record SearchServiceIndex(SearchService searchService) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(searchService); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return searchService; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(searchService.getName()).weight(5).build()); suggest.add( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StorageServiceIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StorageServiceIndex.java index 60e5e33c098e..f34ac67c5f9b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StorageServiceIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StorageServiceIndex.java @@ -7,14 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record StorageServiceIndex(StorageService storageService) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(storageService); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return storageService; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(storageService.getName()).weight(5).build()); suggest.add( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java index cd1910e20e31..0e5c8e1724e3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/StoredProcedureIndex.java @@ -9,14 +9,16 @@ import org.openmetadata.service.search.ParseTags; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record StoredProcedureIndex(StoredProcedure storedProcedure) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(storedProcedure); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return storedProcedure; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add( SearchSuggest.builder().input(storedProcedure.getFullyQualifiedName()).weight(5).build()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java index c1a029e0783d..56d7a293e8ff 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java @@ -16,19 +16,28 @@ import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.FlattenColumn; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record TableIndex(Table table) implements ColumnIndex { - private static final List excludeFields = - List.of( + private static final Set excludeFields = + Set.of( "sampleData", "tableProfile", "joins", - "changeDescription", + "testSuite.changeDescription", "viewDefinition, tableProfilerConfig, profile, location, tableQueries, tests, dataModel"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(table); + @Override + public Object getEntity() { + return table; + } + + @Override + public Set getExcludedFields() { + return excludeFields; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); List columnSuggest = new ArrayList<>(); List schemaSuggest = new ArrayList<>(); @@ -36,7 +45,6 @@ public Map buildESDoc() { List serviceSuggest = new ArrayList<>(); Set> tagsWithChildren = new HashSet<>(); List columnsWithChildrenName = new ArrayList<>(); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); if (table.getColumns() != null) { List cols = new ArrayList<>(); parseColumns(table.getColumns(), cols, null); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TagIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TagIndex.java index 7f5336df96b8..d6394e0de980 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TagIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TagIndex.java @@ -7,14 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record TagIndex(Tag tag) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(tag); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return tag; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(tag.getFullyQualifiedName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(tag.getName()).weight(10).build()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TeamIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TeamIndex.java index 2f4bd4fa9efe..5045747ee245 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TeamIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TeamIndex.java @@ -3,24 +3,33 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public class TeamIndex implements SearchIndex { final Team team; - final List excludeFields = List.of("owns", "changeDescription"); + final Set excludeFields = Set.of("owns"); public TeamIndex(Team team) { this.team = team; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(team); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return team; + } + + @Override + public Set getExcludedFields() { + return excludeFields; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(team.getName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(team.getDisplayName()).weight(10).build()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java index d27a6dee8466..6f73dca464c3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java @@ -3,22 +3,22 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.UUID; -import lombok.SneakyThrows; import org.openmetadata.schema.tests.TestCase; import org.openmetadata.schema.tests.TestSuite; -import org.openmetadata.schema.type.EntityReference; -import org.openmetadata.schema.type.Include; import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; import org.openmetadata.service.util.JsonUtils; public record TestCaseIndex(TestCase testCase) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - @SneakyThrows - public Map buildESDoc() { + @Override + public Object getEntity() { + return testCase; + } + + @Override + public Map buildSearchIndexDocInternal(Map esDoc) { List testSuiteArray = new ArrayList<>(); if (testCase.getTestSuites() != null) { for (TestSuite suite : testCase.getTestSuites()) { @@ -28,7 +28,7 @@ public Map buildESDoc() { } testCase.setTestSuites(testSuiteArray); Map doc = JsonUtils.getMap(testCase); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + SearchIndexUtils.removeNonIndexableFields(doc, DEFAULT_EXCLUDED_FIELDS); List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(testCase.getFullyQualifiedName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(testCase.getName()).weight(10).build()); @@ -44,31 +44,6 @@ public Map buildESDoc() { return doc; } - public Map buildESDocForCreate() { - EntityReference testSuiteEntityReference = testCase.getTestSuite(); - TestSuite testSuite = getTestSuite(testSuiteEntityReference.getId()); - List testSuiteArray = new ArrayList<>(); - testSuiteArray.add(testSuite); - Map doc = JsonUtils.getMap(testCase); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); - doc.put("testSuites", testSuiteArray); - return doc; - } - - private TestSuite getTestSuite(UUID testSuiteId) { - TestSuite testSuite = Entity.getEntity(Entity.TEST_SUITE, testSuiteId, "", Include.ALL); - return new TestSuite() - .withId(testSuite.getId()) - .withName(testSuite.getName()) - .withDisplayName(testSuite.getDisplayName()) - .withDescription(testSuite.getDescription()) - .withFullyQualifiedName(testSuite.getFullyQualifiedName()) - .withDeleted(testSuite.getDeleted()) - .withHref(testSuite.getHref()) - .withExecutable(testSuite.getExecutable()) - .withChangeDescription(null); - } - public static Map getFields() { Map fields = SearchIndex.getDefaultFields(); fields.put("testSuite.fullyQualifiedName", 10.0f); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java index ccbd1bac32e2..f6b030156a3f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java @@ -8,7 +8,12 @@ public record TestCaseResolutionStatusIndex(TestCaseResolutionStatus testCaseResolutionStatus) implements SearchIndex { @Override - public Map buildESDoc() { + public Object getEntity() { + return testCaseResolutionStatus; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { return JsonUtils.getMap(testCaseResolutionStatus); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java index 333c5d914a65..9df00d160ac1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java @@ -7,14 +7,16 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public record TestSuiteIndex(TestSuite testSuite) implements SearchIndex { - private static final List excludeFields = List.of("changeDescription"); - public Map buildESDoc() { - Map doc = JsonUtils.getMap(testSuite); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return testSuite; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(testSuite.getFullyQualifiedName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(testSuite.getName()).weight(10).build()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java index 5cda05e2797e..b9ced2a03e9a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java @@ -19,19 +19,27 @@ import org.openmetadata.service.search.models.FlattenSchemaField; import org.openmetadata.service.search.models.SearchSuggest; import org.openmetadata.service.util.FullyQualifiedName; -import org.openmetadata.service.util.JsonUtils; public class TopicIndex implements SearchIndex { - final List excludeTopicFields = - List.of("sampleData", "changeDescription", "messageSchema"); + final Set excludeTopicFields = Set.of("sampleData", "messageSchema"); final Topic topic; public TopicIndex(Topic topic) { this.topic = topic; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(topic); + @Override + public Object getEntity() { + return topic; + } + + @Override + public Set getExcludedFields() { + return excludeTopicFields; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); List fieldSuggest = new ArrayList<>(); List serviceSuggest = new ArrayList<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/UserIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/UserIndex.java index 959847f08ce0..643d4ae985b3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/UserIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/UserIndex.java @@ -3,25 +3,33 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.service.Entity; import org.openmetadata.service.search.SearchIndexUtils; import org.openmetadata.service.search.models.SearchSuggest; -import org.openmetadata.service.util.JsonUtils; public class UserIndex implements SearchIndex { final User user; - final List excludeFields = - List.of("owns", "changeDescription", "follows", "authenticationMechanism"); + final Set excludeFields = Set.of("owns", "follows", "authenticationMechanism"); public UserIndex(User user) { this.user = user; } - public Map buildESDoc() { - Map doc = JsonUtils.getMap(user); - SearchIndexUtils.removeNonIndexableFields(doc, excludeFields); + @Override + public Object getEntity() { + return user; + } + + @Override + public Set getExcludedFields() { + return excludeFields; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { List suggest = new ArrayList<>(); suggest.add(SearchSuggest.builder().input(user.getName()).weight(5).build()); suggest.add(SearchSuggest.builder().input(user.getDisplayName()).weight(10).build()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/WebAnalyticEntityViewReportDataIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/WebAnalyticEntityViewReportDataIndex.java index 7f43a9467030..a06f809da678 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/WebAnalyticEntityViewReportDataIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/WebAnalyticEntityViewReportDataIndex.java @@ -2,12 +2,15 @@ import java.util.Map; import org.openmetadata.schema.analytics.ReportData; -import org.openmetadata.service.util.JsonUtils; public record WebAnalyticEntityViewReportDataIndex(ReportData reportData) implements SearchIndex { @Override - public Map buildESDoc() { - Map doc = JsonUtils.getMap(reportData); + public Object getEntity() { + return reportData; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { doc.put("entityType", "webAnalyticEntityViewReportData"); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/WebAnalyticUserActivityReportDataIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/WebAnalyticUserActivityReportDataIndex.java index 17c3caf0ff64..d4f3b47d134d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/WebAnalyticUserActivityReportDataIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/WebAnalyticUserActivityReportDataIndex.java @@ -2,12 +2,15 @@ import java.util.Map; import org.openmetadata.schema.analytics.ReportData; -import org.openmetadata.service.util.JsonUtils; public record WebAnalyticUserActivityReportDataIndex(ReportData reportData) implements SearchIndex { @Override - public Map buildESDoc() { - Map doc = JsonUtils.getMap(reportData); + public Object getEntity() { + return reportData; + } + + @Override + public Map buildSearchIndexDocInternal(Map doc) { doc.put("entityType", "webAnalyticUserActivityReportData"); return doc; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchDataInsightProcessor.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchDataInsightProcessor.java index 7a353c67ce6a..d6dc8195832e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchDataInsightProcessor.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchDataInsightProcessor.java @@ -86,7 +86,8 @@ private UpdateRequest getUpdateRequest(String entityType, ReportData reportData) indexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()), reportData.getId().toString()); updateRequest.doc( - JsonUtils.pojoToJson(new ReportDataIndexes(reportData).buildESDoc()), XContentType.JSON); + JsonUtils.pojoToJson(new ReportDataIndexes(reportData).buildSearchIndexDoc()), + XContentType.JSON); updateRequest.docAsUpsert(true); return updateRequest; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntitiesProcessor.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntitiesProcessor.java index 38bf872e3ebb..be66b53eeed4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntitiesProcessor.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchEntitiesProcessor.java @@ -89,7 +89,8 @@ public static UpdateRequest getUpdateRequest(String entityType, EntityInterface entity.getId().toString()); updateRequest.doc( JsonUtils.pojoToJson( - Objects.requireNonNull(Entity.buildSearchIndex(entityType, entity)).buildESDoc()), + Objects.requireNonNull(Entity.buildSearchIndex(entityType, entity)) + .buildSearchIndexDoc()), XContentType.JSON); updateRequest.docAsUpsert(true); return updateRequest;