Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix : Knowledge page update Parent patch - search side changes #18760

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,7 @@ public final class Entity {
public static final String WEB_ANALYTIC_EVENT = "webAnalyticEvent";
public static final String DATA_INSIGHT_CUSTOM_CHART = "dataInsightCustomChart";
public static final String DATA_INSIGHT_CHART = "dataInsightChart";
public static final String PAGE = "page";

//
// Policy entity
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,8 @@ void updateChildren(
Pair<String, String> fieldAndValue,
Pair<String, Map<String, Object>> updates);

void updateByFqnPrefix(String indexName, String oldParentFQN, String newParentFQN);

void updateChildren(
List<String> indexName,
Pair<String, String> fieldAndValue,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.EntityTimeSeriesInterface;
import org.openmetadata.schema.analytics.ReportData;
Expand Down Expand Up @@ -90,11 +89,7 @@ public class SearchRepository {
@Getter @Setter public SearchIndexFactory searchIndexFactory = new SearchIndexFactory();

private final List<String> inheritableFields =
List.of(
Entity.FIELD_OWNERS,
Entity.FIELD_DOMAIN,
Entity.FIELD_DISABLED,
Entity.FIELD_TEST_SUITES);
List.of(FIELD_OWNERS, Entity.FIELD_DOMAIN, Entity.FIELD_DISABLED, Entity.FIELD_TEST_SUITES);
private final List<String> propagateFields = List.of(Entity.FIELD_TAGS);

@Getter private final ElasticSearchConfiguration elasticSearchConfiguration;
Expand Down Expand Up @@ -371,6 +366,9 @@ public void updateEntity(EntityInterface entity) {
entityType, entityId, entity.getChangeDescription(), indexMapping, entity);
propagateGlossaryTags(
entityType, entity.getFullyQualifiedName(), entity.getChangeDescription());

propagatetoRelatedEntities(
entityType, entityId, entity.getChangeDescription(), indexMapping, entity);
} catch (Exception ie) {
LOG.error(
"Issue in Updating the search document for entity [{}] and entityType [{}]. Reason[{}], Cause[{}], Stack [{}]",
Expand Down Expand Up @@ -454,6 +452,58 @@ public void propagateGlossaryTags(
}
}

public void propagatetoRelatedEntities(
String entityType,
String entityId,
ChangeDescription changeDescription,
IndexMapping indexMapping,
EntityInterface entity) {

if (changeDescription != null) {
for (FieldChange field : changeDescription.getFieldsAdded()) {
if (field.getName().contains("parent")) {
if (entityType.equalsIgnoreCase(Entity.PAGE)) {
String indexName = indexMapping.getIndexName(clusterAlias);
String oldParentFQN = entity.getName();
String newParentFQN = entity.getFullyQualifiedName();
// Propagate FQN updates to all subchildren
searchClient.updateByFqnPrefix(indexName, oldParentFQN, newParentFQN);
}
}
}

for (FieldChange field : changeDescription.getFieldsUpdated()) {
if (field.getName().contains("parent")) {
if (entityType.equalsIgnoreCase(Entity.PAGE)) {
String indexName = indexMapping.getIndexName(clusterAlias);
EntityReference newEntityReference =
JsonUtils.readValue(field.getNewValue().toString(), EntityReference.class);
EntityReference entityReferenceBeforeUpdate =
JsonUtils.readValue(field.getOldValue().toString(), EntityReference.class);
// Propagate FQN updates to all subchildren
searchClient.updateByFqnPrefix(
indexName,
entityReferenceBeforeUpdate.getFullyQualifiedName(),
newEntityReference.getFullyQualifiedName());
}
}
}

for (FieldChange field : changeDescription.getFieldsDeleted()) {
if (field.getName().contains("parent")) {
if (entityType.equalsIgnoreCase(Entity.PAGE)) {
String indexName = indexMapping.getIndexName(clusterAlias);
EntityReference entityReferenceBeforeUpdate =
JsonUtils.readValue(field.getOldValue().toString(), EntityReference.class);
// Propagate FQN updates to all subchildren
searchClient.updateByFqnPrefix(
indexName, entityReferenceBeforeUpdate.getFullyQualifiedName(), "");
}
}
}
}
}

private Pair<String, Map<String, Object>> getInheritedFieldChanges(
ChangeDescription changeDescription, EntityInterface entity) {
StringBuilder scriptTxt = new StringBuilder();
Expand Down Expand Up @@ -979,7 +1029,7 @@ public List<EntityReference> getEntitiesContainingFQNFromES(
String id = JsonUtils.extractValue(jsonNode, "_source", "id");
String fqn = JsonUtils.extractValue(jsonNode, "_source", "fullyQualifiedName");
String type = JsonUtils.extractValue(jsonNode, "_source", "entityType");
if (!CommonUtil.nullOrEmpty(fqn) && !CommonUtil.nullOrEmpty(type)) {
if (!nullOrEmpty(fqn) && !nullOrEmpty(type)) {
fqns.add(
new EntityReference()
.withId(UUID.fromString(id))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2112,6 +2112,43 @@ public void updateChildren(
}
}

@Override
public void updateByFqnPrefix(String indexName, String oldParentFQN, String newParentFQN) {
// Match all children documents whose fullyQualifiedName starts with the old parent's FQN
PrefixQueryBuilder prefixQuery =
new PrefixQueryBuilder("fullyQualifiedName", oldParentFQN + ".");

UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName);
updateByQueryRequest.setQuery(prefixQuery);

Map<String, Object> params = new HashMap<>();
params.put("oldParentFQN", oldParentFQN);
params.put("newParentFQN", newParentFQN);

String painlessScript =
"String updatedFQN = ctx._source.fullyQualifiedName.replace(params.oldParentFQN, params.newParentFQN); "
+ "ctx._source.fullyQualifiedName = updatedFQN; "
+ "ctx._source.fqnDepth = updatedFQN.splitOnToken('.').length; "
+ "if (ctx._source.containsKey('parent')) { "
+ " if (ctx._source.parent.containsKey('fullyQualifiedName')) { "
+ " String parentFQN = ctx._source.parent.fullyQualifiedName; "
+ " ctx._source.parent.fullyQualifiedName = parentFQN.replace(params.oldParentFQN, params.newParentFQN); "
+ " } "
+ "}";

Script inlineScript =
new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, painlessScript, params);

updateByQueryRequest.setScript(inlineScript);

try {
updateElasticSearchByQuery(updateByQueryRequest);
LOG.info("Successfully propagated FQN updates for parent FQN: {}", oldParentFQN);
} catch (Exception e) {
LOG.error("Error while propagating FQN updates: {}", e.getMessage(), e);
}
}

@Override
public void updateChildren(
List<String> indexName,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2101,6 +2101,42 @@ public void updateChildren(
}
}

@Override
public void updateByFqnPrefix(String indexName, String oldParentFQN, String newParentFQN) {
// Match all children documents whose fullyQualifiedName starts with the old parent's FQN
PrefixQueryBuilder prefixQuery =
new PrefixQueryBuilder("fullyQualifiedName", oldParentFQN + ".");

UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indexName);
updateByQueryRequest.setQuery(prefixQuery);

Map<String, Object> params = new HashMap<>();
params.put("oldParentFQN", oldParentFQN);
params.put("newParentFQN", newParentFQN);

String painlessScript =
"String updatedFQN = ctx._source.fullyQualifiedName.replace(params.oldParentFQN, params.newParentFQN); "
+ "ctx._source.fullyQualifiedName = updatedFQN; "
+ "ctx._source.fqnDepth = updatedFQN.splitOnToken('.').length; "
+ "if (ctx._source.containsKey('parent')) { "
+ " if (ctx._source.parent.containsKey('fullyQualifiedName')) { "
+ " String parentFQN = ctx._source.parent.fullyQualifiedName; "
+ " ctx._source.parent.fullyQualifiedName = parentFQN.replace(params.oldParentFQN, params.newParentFQN); "
+ " } "
+ "}";
Script inlineScript =
new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, painlessScript, params);

updateByQueryRequest.setScript(inlineScript);

try {
updateOpenSearchByQuery(updateByQueryRequest);
LOG.info("Successfully propagated FQN updates for parent FQN: {}", oldParentFQN);
} catch (Exception e) {
LOG.error("Error while propagating FQN updates: {}", e.getMessage(), e);
}
}

@Override
public void updateLineage(
String indexName, Pair<String, String> fieldAndValue, Map<String, Object> lineagaData) {
Expand Down
Loading