From 6e0b6494be4a860c31bc893c0dded5f398acf9a8 Mon Sep 17 00:00:00 2001 From: sonikashah Date: Mon, 25 Nov 2024 04:26:40 +0530 Subject: [PATCH 1/3] Fix : Knowledge page update Parent patch - search side changes --- .../java/org/openmetadata/service/Entity.java | 1 + .../service/search/SearchClient.java | 2 + .../service/search/SearchRepository.java | 64 +++++++++++++++++-- .../elasticsearch/ElasticSearchClient.java | 37 +++++++++++ .../search/opensearch/OpenSearchClient.java | 36 +++++++++++ 5 files changed, 133 insertions(+), 7 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index bf1a623b90b8..9ee396193102 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -184,6 +184,7 @@ public final class Entity { public static final String WEB_ANALYTIC_EVENT = "webAnalyticEvent"; public static final String DATA_INSIGHT_CUSTOM_CHART = "dataInsightCustomChart"; public static final String DATA_INSIGHT_CHART = "dataInsightChart"; + public static final String PAGE = "page"; // // Policy entity diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java index 837b28cf4ea2..e5ecf353314e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java @@ -266,6 +266,8 @@ void updateChildren( Pair fieldAndValue, Pair> updates); + void updateByFqnPrefix(String indexName, String oldParentFQN, String newParentFQN); + void updateChildren( List indexName, Pair fieldAndValue, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index d37785d17bdb..ba664bf53397 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -54,7 +54,6 @@ import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; -import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.EntityTimeSeriesInterface; import org.openmetadata.schema.analytics.ReportData; @@ -90,11 +89,7 @@ public class SearchRepository { @Getter @Setter public SearchIndexFactory searchIndexFactory = new SearchIndexFactory(); private final List inheritableFields = - List.of( - Entity.FIELD_OWNERS, - Entity.FIELD_DOMAIN, - Entity.FIELD_DISABLED, - Entity.FIELD_TEST_SUITES); + List.of(FIELD_OWNERS, Entity.FIELD_DOMAIN, Entity.FIELD_DISABLED, Entity.FIELD_TEST_SUITES); private final List propagateFields = List.of(Entity.FIELD_TAGS); @Getter private final ElasticSearchConfiguration elasticSearchConfiguration; @@ -371,6 +366,9 @@ public void updateEntity(EntityInterface entity) { entityType, entityId, entity.getChangeDescription(), indexMapping, entity); propagateGlossaryTags( entityType, entity.getFullyQualifiedName(), entity.getChangeDescription()); + + propagatetoRelatedEntities( + entityType, entityId, entity.getChangeDescription(), indexMapping, entity); } catch (Exception ie) { LOG.error( "Issue in Updating the search document for entity [{}] and entityType [{}]. Reason[{}], Cause[{}], Stack [{}]", @@ -454,6 +452,58 @@ public void propagateGlossaryTags( } } + public void propagatetoRelatedEntities( + String entityType, + String entityId, + ChangeDescription changeDescription, + IndexMapping indexMapping, + EntityInterface entity) { + + if (changeDescription != null) { + for (FieldChange field : changeDescription.getFieldsAdded()) { + if (field.getName().contains("parent")) { + if (entityType.equalsIgnoreCase(Entity.PAGE)) { + String indexName = indexMapping.getIndexName(clusterAlias); + String oldParentFQN = entity.getName(); + String newParentFQN = entity.getFullyQualifiedName(); + // Propagate FQN updates to all subchildren + searchClient.updateByFqnPrefix(indexName, oldParentFQN, newParentFQN); + } + } + } + + for (FieldChange field : changeDescription.getFieldsUpdated()) { + if (field.getName().contains("parent")) { + if (entityType.equalsIgnoreCase(Entity.PAGE)) { + String indexName = indexMapping.getIndexName(clusterAlias); + EntityReference newEntityReference = + JsonUtils.readValue(field.getNewValue().toString(), EntityReference.class); + EntityReference entityReferenceBeforeUpdate = + JsonUtils.readValue(field.getOldValue().toString(), EntityReference.class); + // Propagate FQN updates to all subchildren + searchClient.updateByFqnPrefix( + indexName, + entityReferenceBeforeUpdate.getFullyQualifiedName(), + newEntityReference.getFullyQualifiedName()); + } + } + } + + for (FieldChange field : changeDescription.getFieldsDeleted()) { + if (field.getName().contains("parent")) { + if (entityType.equalsIgnoreCase(Entity.PAGE)) { + String indexName = indexMapping.getIndexName(clusterAlias); + EntityReference entityReferenceBeforeUpdate = + JsonUtils.readValue(field.getOldValue().toString(), EntityReference.class); + // Propagate FQN updates to all subchildren + searchClient.updateByFqnPrefix( + indexName, entityReferenceBeforeUpdate.getFullyQualifiedName(), ""); + } + } + } + } + } + private Pair> getInheritedFieldChanges( ChangeDescription changeDescription, EntityInterface entity) { StringBuilder scriptTxt = new StringBuilder(); @@ -979,7 +1029,7 @@ public List getEntitiesContainingFQNFromES( String id = JsonUtils.extractValue(jsonNode, "_source", "id"); String fqn = JsonUtils.extractValue(jsonNode, "_source", "fullyQualifiedName"); String type = JsonUtils.extractValue(jsonNode, "_source", "entityType"); - if (!CommonUtil.nullOrEmpty(fqn) && !CommonUtil.nullOrEmpty(type)) { + if (!nullOrEmpty(fqn) && !nullOrEmpty(type)) { fqns.add( new EntityReference() .withId(UUID.fromString(id)) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 277b1466f0b1..fec119ce0a46 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -2112,6 +2112,43 @@ public void updateChildren( } } + @Override + public void updateByFqnPrefix(String indexName, String oldParentFQN, String newParentFQN) { + // Match all children documents whose fullyQualifiedName starts with the old parent's FQN + PrefixQueryBuilder prefixQuery = + new PrefixQueryBuilder("fullyQualifiedName", oldParentFQN + "."); + + UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName); + updateByQueryRequest.setQuery(prefixQuery); + + Map params = new HashMap<>(); + params.put("oldParentFQN", oldParentFQN); + params.put("newParentFQN", newParentFQN); + + String painlessScript = + "String updatedFQN = ctx._source.fullyQualifiedName.replace(params.oldParentFQN, params.newParentFQN); " + + "ctx._source.fullyQualifiedName = updatedFQN; " + + "ctx._source.fqnDepth = updatedFQN.splitOnToken('.').length; " + + "if (ctx._source.containsKey('parent')) { " + + " if (ctx._source.parent.containsKey('fullyQualifiedName')) { " + + " String parentFQN = ctx._source.parent.fullyQualifiedName; " + + " ctx._source.parent.fullyQualifiedName = parentFQN.replace(params.oldParentFQN, params.newParentFQN); " + + " } " + + "}"; + + Script inlineScript = + new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, painlessScript, params); + + updateByQueryRequest.setScript(inlineScript); + + try { + updateElasticSearchByQuery(updateByQueryRequest); + LOG.info("Successfully propagated FQN updates for parent FQN: {}", oldParentFQN); + } catch (Exception e) { + LOG.error("Error while propagating FQN updates: {}", e.getMessage(), e); + } + } + @Override public void updateChildren( List indexName, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 9029e4842048..dd4b92b7b1d6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -2101,6 +2101,42 @@ public void updateChildren( } } + @Override + public void updateByFqnPrefix(String indexName, String oldParentFQN, String newParentFQN) { + // Match all children documents whose fullyQualifiedName starts with the old parent's FQN + PrefixQueryBuilder prefixQuery = + new PrefixQueryBuilder("fullyQualifiedName", oldParentFQN + "."); + + UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName); + updateByQueryRequest.setQuery(prefixQuery); + + Map params = new HashMap<>(); + params.put("oldParentFQN", oldParentFQN); + params.put("newParentFQN", newParentFQN); + + String painlessScript = + "String updatedFQN = ctx._source.fullyQualifiedName.replace(params.oldParentFQN, params.newParentFQN); " + + "ctx._source.fullyQualifiedName = updatedFQN; " + + "ctx._source.fqnDepth = updatedFQN.splitOnToken('.').length; " + + "if (ctx._source.containsKey('parent')) { " + + " if (ctx._source.parent.containsKey('fullyQualifiedName')) { " + + " String parentFQN = ctx._source.parent.fullyQualifiedName; " + + " ctx._source.parent.fullyQualifiedName = parentFQN.replace(params.oldParentFQN, params.newParentFQN); " + + " } " + + "}"; + Script inlineScript = + new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, painlessScript, params); + + updateByQueryRequest.setScript(inlineScript); + + try { + updateOpenSearchByQuery(updateByQueryRequest); + LOG.info("Successfully propagated FQN updates for parent FQN: {}", oldParentFQN); + } catch (Exception e) { + LOG.error("Error while propagating FQN updates: {}", e.getMessage(), e); + } + } + @Override public void updateLineage( String indexName, Pair fieldAndValue, Map lineagaData) { From a4d92e8298d763a2b3e35850f8472d9c9adedd95 Mon Sep 17 00:00:00 2001 From: sonikashah Date: Mon, 25 Nov 2024 10:49:29 +0530 Subject: [PATCH 2/3] cleanup code --- .../service/search/SearchRepository.java | 48 ++++++++----------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index ba664bf53397..baa825272abe 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -459,46 +459,38 @@ public void propagatetoRelatedEntities( IndexMapping indexMapping, EntityInterface entity) { - if (changeDescription != null) { + if (changeDescription != null && entityType.equalsIgnoreCase(Entity.PAGE)) { + String indexName = indexMapping.getIndexName(clusterAlias); for (FieldChange field : changeDescription.getFieldsAdded()) { if (field.getName().contains("parent")) { - if (entityType.equalsIgnoreCase(Entity.PAGE)) { - String indexName = indexMapping.getIndexName(clusterAlias); - String oldParentFQN = entity.getName(); - String newParentFQN = entity.getFullyQualifiedName(); - // Propagate FQN updates to all subchildren - searchClient.updateByFqnPrefix(indexName, oldParentFQN, newParentFQN); - } + String oldParentFQN = entity.getName(); + String newParentFQN = entity.getFullyQualifiedName(); + // Propagate FQN updates to all subchildren + searchClient.updateByFqnPrefix(indexName, oldParentFQN, newParentFQN); } } for (FieldChange field : changeDescription.getFieldsUpdated()) { if (field.getName().contains("parent")) { - if (entityType.equalsIgnoreCase(Entity.PAGE)) { - String indexName = indexMapping.getIndexName(clusterAlias); - EntityReference newEntityReference = - JsonUtils.readValue(field.getNewValue().toString(), EntityReference.class); - EntityReference entityReferenceBeforeUpdate = - JsonUtils.readValue(field.getOldValue().toString(), EntityReference.class); - // Propagate FQN updates to all subchildren - searchClient.updateByFqnPrefix( - indexName, - entityReferenceBeforeUpdate.getFullyQualifiedName(), - newEntityReference.getFullyQualifiedName()); - } + EntityReference newEntityReference = + JsonUtils.readValue(field.getNewValue().toString(), EntityReference.class); + EntityReference entityReferenceBeforeUpdate = + JsonUtils.readValue(field.getOldValue().toString(), EntityReference.class); + // Propagate FQN updates to all subchildren + searchClient.updateByFqnPrefix( + indexName, + entityReferenceBeforeUpdate.getFullyQualifiedName(), + newEntityReference.getFullyQualifiedName()); } } for (FieldChange field : changeDescription.getFieldsDeleted()) { if (field.getName().contains("parent")) { - if (entityType.equalsIgnoreCase(Entity.PAGE)) { - String indexName = indexMapping.getIndexName(clusterAlias); - EntityReference entityReferenceBeforeUpdate = - JsonUtils.readValue(field.getOldValue().toString(), EntityReference.class); - // Propagate FQN updates to all subchildren - searchClient.updateByFqnPrefix( - indexName, entityReferenceBeforeUpdate.getFullyQualifiedName(), ""); - } + EntityReference entityReferenceBeforeUpdate = + JsonUtils.readValue(field.getOldValue().toString(), EntityReference.class); + // Propagate FQN updates to all subchildren + searchClient.updateByFqnPrefix( + indexName, entityReferenceBeforeUpdate.getFullyQualifiedName(), ""); } } } From 32f86d4f13da37ac2a62b92354247c5058168c70 Mon Sep 17 00:00:00 2001 From: sonikashah Date: Mon, 2 Dec 2024 19:29:53 +0530 Subject: [PATCH 3/3] resolve case when other children under same old parent were getting affected in search --- .../service/search/SearchRepository.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index baa825272abe..442cef3bffad 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -472,15 +472,14 @@ public void propagatetoRelatedEntities( for (FieldChange field : changeDescription.getFieldsUpdated()) { if (field.getName().contains("parent")) { - EntityReference newEntityReference = - JsonUtils.readValue(field.getNewValue().toString(), EntityReference.class); EntityReference entityReferenceBeforeUpdate = JsonUtils.readValue(field.getOldValue().toString(), EntityReference.class); // Propagate FQN updates to all subchildren - searchClient.updateByFqnPrefix( - indexName, - entityReferenceBeforeUpdate.getFullyQualifiedName(), - newEntityReference.getFullyQualifiedName()); + String originalFqn = + String.join( + ".", entityReferenceBeforeUpdate.getFullyQualifiedName(), entity.getName()); + String updatedFqn = entity.getFullyQualifiedName(); + searchClient.updateByFqnPrefix(indexName, originalFqn, updatedFqn); } } @@ -489,8 +488,10 @@ public void propagatetoRelatedEntities( EntityReference entityReferenceBeforeUpdate = JsonUtils.readValue(field.getOldValue().toString(), EntityReference.class); // Propagate FQN updates to all subchildren - searchClient.updateByFqnPrefix( - indexName, entityReferenceBeforeUpdate.getFullyQualifiedName(), ""); + String originalFqn = + String.join( + ".", entityReferenceBeforeUpdate.getFullyQualifiedName(), entity.getName()); + searchClient.updateByFqnPrefix(indexName, originalFqn, ""); } } }