diff --git a/bom/build.gradle b/bom/build.gradle index 8286778a1e4..450bedd012f 100644 --- a/bom/build.gradle +++ b/bom/build.gradle @@ -76,7 +76,7 @@ ext { logbackVersion = '1.5.6' lettuceVersion = '6.3.2.RELEASE' // TODO: Version 9 now available - luceneVersion = '8.11.2' + luceneVersion = '8.11.3' openlineageVersion = '1.16.0' ossVersion = '4.16.0' // TODO: Held as data engine breaks diff --git a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/UnityCatalog-catalogs.http b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/UnityCatalog-catalogs.http index 450db905d2e..7d647d9818d 100644 --- a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/UnityCatalog-catalogs.http +++ b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/UnityCatalog-catalogs.http @@ -37,10 +37,11 @@ Content-Type: application/json ### # @name patch catalog -PATCH {{ucURL}}/api/2.1/unity-catalog/catalogs/test4 +PATCH {{ucURL}}/api/2.1/unity-catalog/catalogs/test41 Content-Type: application/json { + "new_name" : "clinical_trials", "comment" : "This is a comment", "properties": { "egeriaGUID" : "c14d6b99-7202-4f04-a05d-49f0fdd7373d" diff --git a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/UnityCatalog-demo-cataloguing-uc.http b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/UnityCatalog-demo-cataloguing-uc.http index 405bfa0d21d..a8a19d22052 100644 --- a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/UnityCatalog-demo-cataloguing-uc.http +++ b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/UnityCatalog-demo-cataloguing-uc.http @@ -116,7 +116,7 @@ Content-Type: application/json ### -@serverGUID=3189db57-1a25-421b-8c18-c428340428b4 +@serverGUID=add server guid here ### # @name getTechnologyTypeElements (UC Server) diff --git a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/ffdc/UCAuditCode.java b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/ffdc/UCAuditCode.java index a4af79014dd..f8a23e60400 100644 --- a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/ffdc/UCAuditCode.java +++ b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/ffdc/UCAuditCode.java @@ -31,13 +31,13 @@ public enum UCAuditCode implements AuditLogMessageSet "Use the details from the error message to determine the cause of the error and retry the request once it is resolved."), /** - * UNITY-CATALOG-CONNECTOR-0002 - The {0} survey action service is unable to retrieve details of any databases for Unity Catalog Server {1} + * UNITY-CATALOG-CONNECTOR-0002 - The {0} connector is unable to retrieve details of any catalogs for Unity Catalog Server {1} ({2}) */ - NO_DATABASES( "UNITY-CATALOG-CONNECTOR-0002", + NO_CATALOGS( "UNITY-CATALOG-CONNECTOR-0002", AuditLogRecordSeverityLevel.INFO, - "The {0} survey action service is unable to retrieve details of any databases for Unity Catalog Server {1} ({2})", - "The survey terminates.", - "This may not be an error if there are no user database on the database server. If there are, check the permissions associated with the database userId."), + "The {0} connector is unable to retrieve details of any catalogs for Unity Catalog Server {1} ({2})", + "The connector terminates.", + "Check that this server is a Unity Catalog (UC) Server. If it is, ensure the permissions are set up so this connector can retrieve information from the server."), /** * UNITY-CATALOG-CONNECTOR-0003 - The {0} integration connector has catalogued data set {1} ({2}) @@ -79,55 +79,100 @@ public enum UCAuditCode implements AuditLogMessageSet "Use the details from the error message to determine the class of the connector. " + "Update the connector type associated with Unity Catalog's Connection in the metadata store."), + /** + * UNITY-CATALOG-CONNECTOR-0007 - The {0} OSS Unity Catalog Connector has been supplied with a friendship connector with GUID {1} + */ FRIENDSHIP_GUID("UNITY-CATALOG-CONNECTOR-0007", AuditLogRecordSeverityLevel.INFO, "The {0} OSS Unity Catalog Connector has been supplied with a friendship connector with GUID {1}", "The friendship connector is an integration connector that is able to synchronize the contents inside a Unity Catalog (UC) connector. Therefore, they will cooperate to synchronize the contents of the Unity Catalog with the open metadata ecosystem.", "No action is required, this message is just to acknowledge that that the two integration connectors are going to collaborate to catalog the entire contents of Unity Catalog"), + /** + * UNITY-CATALOG-CONNECTOR-0008 - The {0} OSS Unity Inside Catalog Synchronizer Connector only works with catalog targets + */ IGNORING_ENDPOINT("UNITY-CATALOG-CONNECTOR-0008", AuditLogRecordSeverityLevel.INFO, "The {0} OSS Unity Inside Catalog Synchronizer Connector only works with catalog targets", "The connector is ignoring the Unity Catalog (UC) server instance that are configured directly through its connection.", "Update the integration connector's configuration to use catalog targets."), + /** + * UNITY-CATALOG-CONNECTOR-0009 - The {0} Connector has added a catalog target relationship {1} from friendship connector {2} to Unity Catalog Server Asset {3} for Unity Catalog (UC) {4} + */ NEW_CATALOG_TARGET("UNITY-CATALOG-CONNECTOR-0009", AuditLogRecordSeverityLevel.INFO, "The {0} Connector has added a catalog target relationship {1} from friendship connector {2} to Unity Catalog Server Asset {3} for Unity Catalog (UC) {4}", "The connector has requested that its friendship connector starts to catalog a new Unity Catalog (UC) catalog.", "Verify that the cataloguing starts the next time that the friendship connector refreshes."), + /** + * UNITY-CATALOG-CONNECTOR-0010 - The {0} Connector has detected a change in the identity of element {1}. The original id was from {2} and now it is {3} in Unity Catalog (UC) {4} + */ IDENTITY_MISMATCH("UNITY-CATALOG-CONNECTOR-0010", AuditLogRecordSeverityLevel.ACTION, - "The {0} Connector has detected a change in the identity of element {1}. The original id was {2} and now it is {3} in Unity Catalog (UC) {4}", - "The connector will continue to synchronize metadata with Unity Catalog (UC) catalog.", + "The {0} Connector has detected a change in the identity of element {1}. The original id was from {2} and now it is {3} in Unity Catalog (UC) {4}", + "The connector will continue to synchronize metadata with Unity Catalog (UC). This element will not be updated in with system and will remain out of sync.", "Verify that the element is being correctly synchronized. It is possible that there are two different elements with the same name. Also investigate why this element changed in Unity Catalog when it is owned by the open metadata ecosystem. It may be that the structure of the element was changed and the UC element was replaced to reflect the new data structure. If this is a planned change then all is ok, if it is unexpected then take steps to repair the data source and prevent it happening again."), - TABLE_UPDATE("UNITY-CATALOG-CONNECTOR-0011", + /** + * UNITY-CATALOG-CONNECTOR-0011 - The {0} Connector has detected a change in the open metadata that controls the definition of catalog {1} but is unable to update the catalog in Unity Catalog (UC) at {2} + */ + CATALOG_UPDATE("UNITY-CATALOG-CONNECTOR-0011", + AuditLogRecordSeverityLevel.ACTION, + "The {0} Connector has detected a change in the open metadata that controls the definition of catalog {1} but is unable to update the catalog in Unity Catalog (UC) at {2}", + "The connector will continue to synchronize metadata with Unity Catalog (UC). However, this catalog's properties will remain unchanged and out of sync with the open metadata ecosystem because Unity Catalog uses the PATCH request which is not supported by Java.", + "Validate that the change in the open metadata ecosystem is intended. If it is, manually update the table in Unity Catalog using the CLI. There may also need to be some data migration in the underlying data source."), + + /** + * UNITY-CATALOG-CONNECTOR-0012 - The {0} Connector has detected a change in the open metadata that controls the definition of schema {1} but is unable to update the schema in Unity Catalog (UC) at {2} + */ + SCHEMA_UPDATE("UNITY-CATALOG-CONNECTOR-0012", + AuditLogRecordSeverityLevel.ACTION, + "The {0} Connector has detected a change in the open metadata that controls the definition of schema {1} but is unable to update the schema in Unity Catalog (UC) at {2}", + "The connector will continue to synchronize metadata with Unity Catalog (UC). However, this schema's properties will remain unchanged and out of sync with the open metadata ecosystem because Unity Catalog uses the PATCH request which is not supported by Java.", + "Validate that the change in the open metadata ecosystem is intended. If it is, manually update the table in Unity Catalog using the CLI. There may also need to be some data migration in the underlying data source."), + + /** + * UNITY-CATALOG-CONNECTOR-0013 - The {0} Connector has detected a change in the open metadata that controls the definition of table {1} but is unable to update the table in Unity Catalog (UC) at {2} + */ + TABLE_UPDATE("UNITY-CATALOG-CONNECTOR-0013", AuditLogRecordSeverityLevel.ACTION, "The {0} Connector has detected a change in the open metadata that controls the definition of table {1} but is unable to update the table in Unity Catalog (UC) at {2}", - "The connector will continue to synchronize metadata with Unity Catalog (UC) catalog. However, this table will remain unchanged and out of sync with the open metadata ecosystem because Unity Catalog does not support an update request.", + "The connector will continue to synchronize metadata with Unity Catalog (UC) catalog. However, this table's properties will remain unchanged and out of sync with the open metadata ecosystem because Unity Catalog does not support an update request.", "Validate that the change in the open metadata ecosystem is intended. If it is, manually update the table in Unity Catalog using the CLI. There may also need to be some data migration in the underlying data source."), - VOLUME_UPDATE("UNITY-CATALOG-CONNECTOR-0012", + /** + * UNITY-CATALOG-CONNECTOR-0014 - The {0} Connector has detected a change in the open metadata element {1} that controls the definition of volume {2} but is unable to update the volume in Unity Catalog (UC) at {3} + */ + VOLUME_UPDATE("UNITY-CATALOG-CONNECTOR-0014", AuditLogRecordSeverityLevel.ACTION, "The {0} Connector has detected a change in the open metadata element {1} that controls the definition of volume {2} but is unable to update the volume in Unity Catalog (UC) at {3}", - "The connector will continue to synchronize metadata with Unity Catalog (UC) catalog. However, this volume will remain unchanged and out of sync with the open metadata ecosystem because Unity Catalog does not support a full update request.", + "The connector will continue to synchronize metadata with Unity Catalog (UC) catalog. However, this volume's properties will remain unchanged and out of sync with the open metadata ecosystem because Unity Catalog does not support a full update request.", "Validate that the change in the open metadata ecosystem is intended. If it is, manually update the volume in Unity Catalog using the CLI. There may also need to be some data migration in the underlying data source."), - FUNCTION_UPDATE("UNITY-CATALOG-CONNECTOR-0013", + /** + * UNITY-CATALOG-CONNECTOR-0015 - The {0} Connector has detected a change in the open metadata element {1} that controls the definition of function {2} but is unable to update the function in Unity Catalog (UC) at {3} + */ + FUNCTION_UPDATE("UNITY-CATALOG-CONNECTOR-0015", AuditLogRecordSeverityLevel.ACTION, "The {0} Connector has detected a change in the open metadata element {1} that controls the definition of function {2} but is unable to update the function in Unity Catalog (UC) at {3}", - "The connector will continue to synchronize metadata with Unity Catalog (UC) catalog. However, this function will remain unchanged and out of sync with the open metadata ecosystem because Unity Catalog does not support a full update request.", + "The connector will continue to synchronize metadata with Unity Catalog (UC) catalog. However, this function's properties will remain unchanged and out of sync with the open metadata ecosystem because Unity Catalog does not support a full update request.", "Validate that the change in the open metadata ecosystem is intended. If it is, manually update the volume in Unity Catalog using the CLI. There may also need to be some data migration in the underlying called data sources."), - UC_ELEMENT_DELETE("UNITY-CATALOG-CONNECTOR-0014", + /** + * UNITY-CATALOG-CONNECTOR-0016 - The {0} Connector has detected that the open metadata element that controls the definition of element {1} in Unity Catalog (UC) at {2} has been deleted + */ + UC_ELEMENT_DELETE("UNITY-CATALOG-CONNECTOR-0016", AuditLogRecordSeverityLevel.INFO, "The {0} Connector has detected that the open metadata element that controls the definition of element {1} in Unity Catalog (UC) at {2} has been deleted", "The connector will delete the element in Unity Catalog (UC) catalog.", "Validate that the change in the open metadata ecosystem is intended. If it is, no additional action is required. If the element is still required, investigate what happened to the element in the open metadata ecosystem. It may have moved zones or metadata collections or its security controls changed, making it invisible to the connector. Or it may have been soft-deleted which means it can be restored."), - MISSING_METHOD("UNITY-CATALOG-CONNECTOR-0015", + /** + * UNITY-CATALOG-CONNECTOR-0017 - The {0} Connector has called method {1} that is not implemented and so is not able to synchronize element {2} with the Unity Catalog (UC) at {3} + */ + MISSING_METHOD("UNITY-CATALOG-CONNECTOR-0017", AuditLogRecordSeverityLevel.ACTION, "The {0} Connector has called method {1} that is not implemented and so is not able to synchronize element {2} with the Unity Catalog (UC) at {3}", "The connector will continue to synchronize metadata with Unity Catalog (UC) catalog. However, the current element will remain unchanged and out of sync with the open metadata ecosystem.", diff --git a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/resource/OSSUnityCatalogResourceConnector.java b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/resource/OSSUnityCatalogResourceConnector.java index f6cfb2276e8..acd9e1aa748 100644 --- a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/resource/OSSUnityCatalogResourceConnector.java +++ b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/resource/OSSUnityCatalogResourceConnector.java @@ -323,7 +323,7 @@ public CatalogInfo updateCatalog(String name, requestBody.setComment(comment); requestBody.setProperties(properties); - if (newName == null) + if ((newName == null) || (newName.equals(name))) { /* * new_name must not be null. diff --git a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncBase.java b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncBase.java index c246524025d..779b40fdcc6 100644 --- a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncBase.java +++ b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncBase.java @@ -5,6 +5,7 @@ import org.odpi.openmetadata.adapters.connectors.unitycatalog.controls.UnityCatalogPlaceholderProperty; +import org.odpi.openmetadata.adapters.connectors.unitycatalog.ffdc.UCAuditCode; import org.odpi.openmetadata.adapters.connectors.unitycatalog.properties.ElementBase; import org.odpi.openmetadata.adapters.connectors.unitycatalog.resource.OSSUnityCatalogResourceConnector; import org.odpi.openmetadata.frameworks.auditlog.AuditLog; @@ -14,6 +15,7 @@ import org.odpi.openmetadata.frameworks.connectors.properties.beans.ElementStatus; import org.odpi.openmetadata.frameworks.governanceaction.controls.PlaceholderProperty; import org.odpi.openmetadata.frameworks.governanceaction.properties.ExternalIdentifierProperties; +import org.odpi.openmetadata.frameworks.governanceaction.properties.MetadataCorrelationHeader; import org.odpi.openmetadata.frameworks.governanceaction.search.ElementProperties; import org.odpi.openmetadata.frameworks.governanceaction.search.PropertyHelper; import org.odpi.openmetadata.frameworks.integration.context.OpenMetadataAccess; @@ -26,9 +28,7 @@ import org.odpi.openmetadata.frameworks.openmetadata.types.OpenMetadataType; import org.odpi.openmetadata.integrationservices.catalog.connector.CatalogIntegratorContext; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; +import java.util.*; /** * Common functions for the synchronizing between Egeria and Unity Catalog. @@ -210,10 +210,11 @@ protected void deleteElementInEgeria(MemberElement memberElement) throws Invalid * @param schemaName name of the schema * @return external identifier properties */ - protected ExternalIdentifierProperties getExternalIdentifierProperties(ElementBase ucElement, - String schemaName, - String elementName, - String id) + protected ExternalIdentifierProperties getExternalIdentifierProperties(ElementBase ucElement, + String schemaName, + String elementName, + String id, + PermittedSynchronization instanceSynchronizationDirection) { ExternalIdentifierProperties externalIdentifierProperties = new ExternalIdentifierProperties(); @@ -232,6 +233,8 @@ protected ExternalIdentifierProperties getExternalIdentifierProperties(ElementBa externalIdentifierProperties.setMappingProperties(mappingProperties); + externalIdentifierProperties.setSynchronizationDirection(instanceSynchronizationDirection); + return externalIdentifierProperties; } @@ -280,6 +283,52 @@ protected void addPropertyFacet(String parentGUID, } + + /** + * Check that the name of the third party element has not changed with respect to Egeria. + * + * @param thirdPartyExternalIdentifier id from Unity Catalog + * @param memberElement element from Egeria + * @return boolean + */ + protected boolean noMismatchInExternalIdentifier(String thirdPartyExternalIdentifier, + MemberElement memberElement) + { + final String methodName = "noMismatchInExternalIdentifier"; + + if ((thirdPartyExternalIdentifier == null) || (memberElement == null) || (memberElement.getExternalIdentifiers() == null)) + { + return true; + } + + List externalIdentifiers = new ArrayList<>(); + + for (MetadataCorrelationHeader correlationHeader : memberElement.getExternalIdentifiers()) + { + if ((correlationHeader != null) && (correlationHeader.getExternalIdentifier() != null)) + { + externalIdentifiers.add(correlationHeader.getExternalIdentifier()); + + if (thirdPartyExternalIdentifier.equals(correlationHeader.getExternalIdentifier())) + { + return true; + } + } + } + + if (externalIdentifiers.isEmpty()) + { + return true; + } + + auditLog.logMessage(methodName, UCAuditCode.IDENTITY_MISMATCH.getMessageDefinition(connectorName, + externalIdentifiers.toString(), + thirdPartyExternalIdentifier, + ucServerEndpoint)); + return false; + } + + /** * Extract the comment for a UC element from the description attribute in an egeria element. * diff --git a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncFunctions.java b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncFunctions.java index 5ff3987a4b4..ae52236eaf3 100644 --- a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncFunctions.java +++ b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncFunctions.java @@ -13,7 +13,6 @@ import org.odpi.openmetadata.frameworks.connectors.ffdc.UserNotAuthorizedException; import org.odpi.openmetadata.frameworks.connectors.properties.beans.ElementStatus; import org.odpi.openmetadata.frameworks.governanceaction.controls.PlaceholderProperty; -import org.odpi.openmetadata.frameworks.governanceaction.properties.RelatedMetadataElement; import org.odpi.openmetadata.frameworks.governanceaction.search.ElementProperties; import org.odpi.openmetadata.frameworks.integration.iterator.IntegrationIterator; import org.odpi.openmetadata.frameworks.integration.iterator.MemberAction; @@ -133,12 +132,12 @@ protected IntegrationIterator refreshEgeria() throws InvalidParameterException, // this is not necessarily an error } - MemberAction memberAction; + MemberAction memberAction = MemberAction.NO_ACTION; if (functionInfo == null) { memberAction = nextElement.getMemberAction(null, null); } - else + else if (noMismatchInExternalIdentifier(functionInfo.getFunction_id(), nextElement)) { memberAction = nextElement.getMemberAction(this.getDateFromLong(functionInfo.getCreated_at()), this.getDateFromLong(functionInfo.getUpdated_at())); @@ -197,7 +196,11 @@ protected void refreshUnityCatalog(IntegrationIterator iterator) throws InvalidP MemberElement memberElement = iterator.getMemberByQualifiedName(ucFunctionQualifiedName); MemberAction memberAction = memberElement.getMemberAction(this.getDateFromLong(functionInfo.getCreated_at()), this.getDateFromLong(functionInfo.getUpdated_at())); - this.takeAction(schemaGUID, functionInfo.getSchema_name(), memberAction, memberElement, functionInfo); + + if (noMismatchInExternalIdentifier(functionInfo.getFunction_id(), memberElement)) + { + this.takeAction(schemaGUID, functionInfo.getSchema_name(), memberAction, memberElement, functionInfo); + } } } } @@ -221,9 +224,9 @@ private void takeAction(String schemaGUID, String schemaName, MemberAction memberAction, MemberElement memberElement, - FunctionInfo functionInfo) throws InvalidParameterException, - PropertyServerException, - UserNotAuthorizedException + FunctionInfo functionInfo) throws InvalidParameterException, + PropertyServerException, + UserNotAuthorizedException { switch (memberAction) { @@ -307,7 +310,8 @@ private void createElementInEgeria(String schemaGUID, this.getExternalIdentifierProperties(functionInfo, functionInfo.getSchema_name(), UnityCatalogPlaceholderProperty.FUNCTION_NAME.getName(), - functionInfo.getFunction_id())); + functionInfo.getFunction_id(), + PermittedSynchronization.FROM_THIRD_PARTY)); this.createSchemaAttributesForUCFunction(ucFunctionGUID, functionInfo); @@ -335,14 +339,9 @@ private void updateElementInEgeria(FunctionInfo functionInfo, false, this.getElementProperties(functionInfo)); - context.updateExternalIdentifier(egeriaFunctionGUID, entityTypeName, this.getExternalIdentifierProperties(functionInfo, - functionInfo.getSchema_name(), - UnityCatalogPlaceholderProperty.FUNCTION_NAME.getName(), - functionInfo.getFunction_id())); - this.updateSchemaAttributesForUCFunction(memberElement, functionInfo); - context.confirmSynchronization(egeriaFunctionGUID, entityTypeName, functionInfo.getName()); + context.confirmSynchronization(egeriaFunctionGUID, entityTypeName, functionInfo.getFunction_id()); } @@ -351,7 +350,7 @@ private void updateElementInEgeria(FunctionInfo functionInfo, * * @param memberElement elements from Egeria * @throws InvalidParameterException parameter error - * @throws PropertyServerException repository error + * @throws PropertyServerException repository error or problem communicating with UC * @throws UserNotAuthorizedException authorization error */ private void createElementInThirdParty(String schemaName, @@ -366,7 +365,15 @@ private void createElementInThirdParty(String schemaName, functionProperties.setName(super.getUCNameFromMember(memberElement)); // todo - complete parameter mapping - ucConnector.createFunction(functionProperties); + FunctionInfo functionInfo = ucConnector.createFunction(functionProperties); + + context.addExternalIdentifier(memberElement.getElement().getElementGUID(), + deployedImplementationType.getAssociatedTypeName(), + this.getExternalIdentifierProperties(functionInfo, + functionInfo.getSchema_name(), + UnityCatalogPlaceholderProperty.FUNCTION_NAME.getName(), + functionInfo.getFunction_id(), + PermittedSynchronization.TO_THIRD_PARTY)); } diff --git a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncSchema.java b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncSchema.java index 3e047b59477..678f5313f8f 100644 --- a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncSchema.java +++ b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncSchema.java @@ -4,6 +4,7 @@ package org.odpi.openmetadata.adapters.connectors.unitycatalog.sync; import org.odpi.openmetadata.adapters.connectors.unitycatalog.controls.UnityCatalogPlaceholderProperty; +import org.odpi.openmetadata.adapters.connectors.unitycatalog.ffdc.UCAuditCode; import org.odpi.openmetadata.adapters.connectors.unitycatalog.properties.SchemaInfo; import org.odpi.openmetadata.adapters.connectors.unitycatalog.resource.OSSUnityCatalogResourceConnector; import org.odpi.openmetadata.frameworks.auditlog.AuditLog; @@ -129,12 +130,12 @@ protected IntegrationIterator refreshEgeria() throws InvalidParameterException, // this is not necessarily an error } - MemberAction memberAction; + MemberAction memberAction = MemberAction.NO_ACTION; if (schemaInfo == null) { memberAction = nextElement.getMemberAction(null, null); } - else + else if (noMismatchInExternalIdentifier(schemaInfo.getSchema_id(), nextElement)) { memberAction = nextElement.getMemberAction(this.getDateFromLong(schemaInfo.getCreated_at()), this.getDateFromLong(schemaInfo.getUpdated_at())); } @@ -175,7 +176,10 @@ protected void refreshUnityCatalog(IntegrationIterator iterator) throws InvalidP MemberElement memberElement = iterator.getMemberByQualifiedName(ucSchemaQualifiedName); MemberAction memberAction = memberElement.getMemberAction(this.getDateFromLong(schemaInfo.getCreated_at()), this.getDateFromLong(schemaInfo.getUpdated_at())); - this.takeAction(memberAction, memberElement, schemaInfo); + if (noMismatchInExternalIdentifier(schemaInfo.getSchema_id(), memberElement)) + { + this.takeAction(memberAction, memberElement, schemaInfo); + } } } } @@ -267,7 +271,8 @@ private void createElementInEgeria(SchemaInfo schemaInfo) throws InvalidParamete this.getExternalIdentifierProperties(schemaInfo, schemaInfo.getName(), PlaceholderProperty.SCHEMA_NAME.getName(), - schemaInfo.getSchema_id())); + schemaInfo.getSchema_id(), + PermittedSynchronization.FROM_THIRD_PARTY)); ucFullNameToEgeriaGUID.put(schemaInfo.getFull_name(), ucSchemaGUID); } @@ -295,16 +300,9 @@ protected void updateElementInEgeria(SchemaInfo schemaInfo, openMetadataAccess.updateMetadataElementInStore(egeriaSchemaGUID, false, getElementProperties(schemaInfo)); - context.updateExternalIdentifier(egeriaSchemaGUID, - deployedImplementationType.getAssociatedTypeName(), - this.getExternalIdentifierProperties(schemaInfo, - schemaInfo.getName(), - PlaceholderProperty.SCHEMA_NAME.getName(), - schemaInfo.getSchema_id())); - context.confirmSynchronization(egeriaSchemaGUID, deployedImplementationType.getAssociatedTypeName(), - schemaInfo.getName()); + schemaInfo.getSchema_id()); } @@ -312,14 +310,26 @@ protected void updateElementInEgeria(SchemaInfo schemaInfo, * Create a schema in UC. * * @param memberElement elements from Egeria - * @throws PropertyServerException problem communicating with UC + * @throws InvalidParameterException parameter error + * @throws PropertyServerException repository error or problem communicating with UC + * @throws UserNotAuthorizedException authorization error */ - private void createElementInThirdParty(MemberElement memberElement) throws PropertyServerException + private void createElementInThirdParty(MemberElement memberElement) throws PropertyServerException, + InvalidParameterException, + UserNotAuthorizedException { - ucConnector.createSchema(super.getUCNameFromMember(memberElement), - catalogName, - super.getUCCommentFomMember(memberElement), - super.getUCPropertiesFomMember(memberElement)); + SchemaInfo schemaInfo = ucConnector.createSchema(super.getUCNameFromMember(memberElement), + catalogName, + super.getUCCommentFomMember(memberElement), + super.getUCPropertiesFomMember(memberElement)); + + context.addExternalIdentifier(memberElement.getElement().getElementGUID(), + deployedImplementationType.getAssociatedTypeName(), + this.getExternalIdentifierProperties(schemaInfo, + schemaInfo.getName(), + PlaceholderProperty.SCHEMA_NAME.getName(), + schemaInfo.getSchema_id(), + PermittedSynchronization.TO_THIRD_PARTY)); } @@ -334,10 +344,13 @@ private void createElementInThirdParty(MemberElement memberElement) throws Prope private void updateElementInThirdParty(SchemaInfo schemaInfo, MemberElement memberElement) throws PropertyServerException { - SchemaInfo newInfo = ucConnector.updateSchema(schemaInfo.getFull_name(), - this.getUCNameFromMember(memberElement), - this.getUCCommentFomMember(memberElement), - this.getUCPropertiesFomMember(memberElement)); + final String methodName = "updateElementInThirdParty"; + + auditLog.logMessage(methodName, + UCAuditCode.SCHEMA_UPDATE.getMessageDefinition(connectorName, + memberElement.getElement().getElementGUID(), + schemaInfo.getCatalog_name() + "." + schemaInfo.getName(), + ucServerEndpoint)); } diff --git a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncTables.java b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncTables.java index 8a2859271d0..ae4b2e5ac07 100644 --- a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncTables.java +++ b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncTables.java @@ -135,12 +135,12 @@ protected IntegrationIterator refreshEgeria() throws InvalidParameterException, // this is not necessarily an error } - MemberAction memberAction; + MemberAction memberAction = MemberAction.NO_ACTION; if (tableInfo == null) { memberAction = nextElement.getMemberAction(null, null); } - else + else if (noMismatchInExternalIdentifier(tableInfo.getTable_id(), nextElement)) { memberAction = nextElement.getMemberAction(this.getDateFromLong(tableInfo.getCreated_at()), this.getDateFromLong(tableInfo.getUpdated_at())); @@ -199,7 +199,10 @@ protected void refreshUnityCatalog(IntegrationIterator iterator) throws InvalidP MemberElement memberElement = iterator.getMemberByQualifiedName(ucTableQualifiedName); MemberAction memberAction = memberElement.getMemberAction(this.getDateFromLong(tableInfo.getCreated_at()), this.getDateFromLong(tableInfo.getUpdated_at())); - this.takeAction(schemaGUID, tableInfo.getSchema_name(), memberAction, memberElement, tableInfo); + if (noMismatchInExternalIdentifier(tableInfo.getTable_id(), memberElement)) + { + this.takeAction(schemaGUID, tableInfo.getSchema_name(), memberAction, memberElement, tableInfo); + } } } } @@ -318,7 +321,8 @@ private void createElementInEgeria(String schemaGUID, this.getExternalIdentifierProperties(tableInfo, tableInfo.getSchema_name(), UnityCatalogPlaceholderProperty.TABLE_NAME.getName(), - tableInfo.getTable_id())); + tableInfo.getTable_id(), + PermittedSynchronization.FROM_THIRD_PARTY)); this.createSchemaAttributesForUCTable(ucTableGUID, tableInfo); @@ -346,14 +350,9 @@ private void updateElementInEgeria(TableInfo tableInfo, false, this.getElementProperties(tableInfo)); - context.updateExternalIdentifier(egeriaTableGUID, entityTypeName, this.getExternalIdentifierProperties(tableInfo, - tableInfo.getSchema_name(), - UnityCatalogPlaceholderProperty.TABLE_NAME.getName(), - tableInfo.getTable_id())); - this.updateSchemaAttributesForUCTable(memberElement, tableInfo); - context.confirmSynchronization(egeriaTableGUID, entityTypeName, tableInfo.getName()); + context.confirmSynchronization(egeriaTableGUID, entityTypeName, tableInfo.getTable_id()); } @@ -362,7 +361,7 @@ private void updateElementInEgeria(TableInfo tableInfo, * * @param memberElement elements from Egeria * @throws InvalidParameterException parameter error - * @throws PropertyServerException repository error + * @throws PropertyServerException repository error or problem communicating with UC * @throws UserNotAuthorizedException authorization error */ private void createElementInThirdParty(String schemaName, @@ -370,15 +369,23 @@ private void createElementInThirdParty(String schemaName, InvalidParameterException, UserNotAuthorizedException { - ucConnector.createTable(super.getUCNameFromMember(memberElement), - catalogName, - schemaName, - super.getUCCommentFomMember(memberElement), - this.getUCTableTypeFromMember(memberElement), - this.getUCDataSourceFormatFromMember(memberElement), - this.getUCColumnsForTable(memberElement), - super.getUCStorageLocationFromMember(memberElement), - super.getUCPropertiesFomMember(memberElement)); + TableInfo tableInfo = ucConnector.createTable(super.getUCNameFromMember(memberElement), + catalogName, + schemaName, + super.getUCCommentFomMember(memberElement), + this.getUCTableTypeFromMember(memberElement), + this.getUCDataSourceFormatFromMember(memberElement), + this.getUCColumnsForTable(memberElement), + super.getUCStorageLocationFromMember(memberElement), + super.getUCPropertiesFomMember(memberElement)); + + context.addExternalIdentifier(memberElement.getElement().getElementGUID(), + deployedImplementationType.getAssociatedTypeName(), + this.getExternalIdentifierProperties(tableInfo, + tableInfo.getSchema_name(), + UnityCatalogPlaceholderProperty.TABLE_NAME.getName(), + tableInfo.getTable_id(), + PermittedSynchronization.FROM_THIRD_PARTY)); } diff --git a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncVolumes.java b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncVolumes.java index ddc46507b63..5c0cb6925ba 100644 --- a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncVolumes.java +++ b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncVolumes.java @@ -135,12 +135,12 @@ protected IntegrationIterator refreshEgeria() throws InvalidParameterException, // this is not necessarily an error } - MemberAction memberAction; + MemberAction memberAction = MemberAction.NO_ACTION; if (volumeInfo == null) { memberAction = nextElement.getMemberAction(null, null); } - else + else if (noMismatchInExternalIdentifier(volumeInfo.getVolume_id(), nextElement)) { memberAction = nextElement.getMemberAction(this.getDateFromLong(volumeInfo.getCreated_at()), this.getDateFromLong(volumeInfo.getUpdated_at())); @@ -199,7 +199,10 @@ protected void refreshUnityCatalog(IntegrationIterator iterator) throws InvalidP MemberElement memberElement = iterator.getMemberByQualifiedName(ucVolumeQualifiedName); MemberAction memberAction = memberElement.getMemberAction(this.getDateFromLong(volumeInfo.getCreated_at()), this.getDateFromLong(volumeInfo.getUpdated_at())); - this.takeAction(schemaGUID, volumeInfo.getSchema_name(), memberAction, memberElement, volumeInfo); + if (noMismatchInExternalIdentifier(volumeInfo.getVolume_id(), memberElement)) + { + this.takeAction(schemaGUID, volumeInfo.getSchema_name(), memberAction, memberElement, volumeInfo); + } } } } @@ -308,7 +311,8 @@ private void createElementInEgeria(String schemaGUID, this.getExternalIdentifierProperties(volumeInfo, volumeInfo.getSchema_name(), UnityCatalogPlaceholderProperty.VOLUME_NAME.getName(), - volumeInfo.getVolume_id())); + volumeInfo.getVolume_id(), + PermittedSynchronization.FROM_THIRD_PARTY)); ucFullNameToEgeriaGUID.put(volumeInfo.getFull_name(), ucVolumeGUID); } @@ -334,11 +338,7 @@ private void updateElementInEgeria(VolumeInfo volumeInfo, false, this.getElementProperties(volumeInfo)); - context.updateExternalIdentifier(egeriaVolumeGUID, entityTypeName, this.getExternalIdentifierProperties(volumeInfo, - volumeInfo.getSchema_name(), - UnityCatalogPlaceholderProperty.VOLUME_NAME.getName(), - volumeInfo.getVolume_id())); - context.confirmSynchronization(egeriaVolumeGUID, entityTypeName, volumeInfo.getName()); + context.confirmSynchronization(egeriaVolumeGUID, entityTypeName, volumeInfo.getVolume_id()); } @@ -346,16 +346,29 @@ private void updateElementInEgeria(VolumeInfo volumeInfo, * Create an equivalent element in UC. * * @param memberElement elements from Egeria + * @throws InvalidParameterException parameter error + * @throws PropertyServerException repository error or problem communicating with UC + * @throws UserNotAuthorizedException authorization error */ private void createElementInThirdParty(String schemaName, - MemberElement memberElement) throws PropertyServerException + MemberElement memberElement) throws PropertyServerException, + InvalidParameterException, + UserNotAuthorizedException { - ucConnector.createVolume(super.getUCNameFromMember(memberElement), - catalogName, - schemaName, - super.getUCCommentFomMember(memberElement), - this.getUCVolumeTypeFromMember(memberElement), - super.getUCStorageLocationFromMember(memberElement)); + VolumeInfo volumeInfo = ucConnector.createVolume(super.getUCNameFromMember(memberElement), + catalogName, + schemaName, + super.getUCCommentFomMember(memberElement), + this.getUCVolumeTypeFromMember(memberElement), + super.getUCStorageLocationFromMember(memberElement)); + + context.addExternalIdentifier(memberElement.getElement().getElementGUID(), + deployedImplementationType.getAssociatedTypeName(), + this.getExternalIdentifierProperties(volumeInfo, + volumeInfo.getSchema_name(), + UnityCatalogPlaceholderProperty.VOLUME_NAME.getName(), + volumeInfo.getVolume_id(), + PermittedSynchronization.FROM_THIRD_PARTY)); } @@ -398,6 +411,7 @@ private void updateElementInThirdParty(VolumeInfo volumeInfo, MemberElement memberElement) throws PropertyServerException { final String methodName = "updateElementInThirdParty"; + auditLog.logMessage(methodName, UCAuditCode.VOLUME_UPDATE.getMessageDefinition(connectorName, memberElement.getElement().getElementGUID(), diff --git a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncCatalog.java b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogServerSyncCatalog.java similarity index 86% rename from open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncCatalog.java rename to open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogServerSyncCatalog.java index 4ee9b857509..b9a56c19ab6 100644 --- a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncCatalog.java +++ b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogServerSyncCatalog.java @@ -30,7 +30,7 @@ /** * Provides the specialist methods for working with Unity Catalog (UC) Catalog. */ -public class OSSUnityCatalogInsideCatalogSyncCatalog extends OSSUnityCatalogInsideCatalogSyncBase +public class OSSUnityCatalogServerSyncCatalog extends OSSUnityCatalogInsideCatalogSyncBase { private final String friendshipConnectorGUID; @@ -54,17 +54,17 @@ public class OSSUnityCatalogInsideCatalogSyncCatalog extends OSSUnityCatalogInsi * @param configurationProperties configuration properties supplied through the catalog target * @param auditLog logging destination */ - public OSSUnityCatalogInsideCatalogSyncCatalog(String connectorName, - CatalogIntegratorContext context, - String catalogTargetName, - String ucServerGUID, - String friendshipConnectorGUID, - PermittedSynchronization targetPermittedSynchronization, - OSSUnityCatalogResourceConnector ucConnector, - String ucServerEndpoint, - Map templates, - Map configurationProperties, - AuditLog auditLog) + public OSSUnityCatalogServerSyncCatalog(String connectorName, + CatalogIntegratorContext context, + String catalogTargetName, + String ucServerGUID, + String friendshipConnectorGUID, + PermittedSynchronization targetPermittedSynchronization, + OSSUnityCatalogResourceConnector ucConnector, + String ucServerEndpoint, + Map templates, + Map configurationProperties, + AuditLog auditLog) { super(connectorName, context, @@ -141,12 +141,12 @@ protected IntegrationIterator refreshEgeria() throws InvalidParameterException, // this is not necessarily an error } - MemberAction memberAction; + MemberAction memberAction = MemberAction.NO_ACTION; if (info == null) { memberAction = nextElement.getMemberAction(null, null); } - else + else if (noMismatchInExternalIdentifier(info.getId(), nextElement)) { memberAction = nextElement.getMemberAction(this.getDateFromLong(info.getCreated_at()), this.getDateFromLong(info.getUpdated_at())); } @@ -187,7 +187,11 @@ protected void refreshUnityCatalog(IntegrationIterator iterator) throws InvalidP MemberElement memberElement = iterator.getMemberByQualifiedName(ucCatalogQualifiedName); MemberAction memberAction = memberElement.getMemberAction(this.getDateFromLong(catalogInfo.getCreated_at()), this.getDateFromLong(catalogInfo.getUpdated_at())); - this.takeAction(memberAction, memberElement, catalogInfo); + + if (noMismatchInExternalIdentifier(catalogInfo.getId(), memberElement)) + { + this.takeAction(memberAction, memberElement, catalogInfo); + } } } } @@ -281,7 +285,8 @@ private void createElementInEgeria(CatalogInfo catalogInfo) throws InvalidParame this.getExternalIdentifierProperties(catalogInfo, null, UnityCatalogPlaceholderProperty.CATALOG_NAME.getName(), - catalogInfo.getId())); + catalogInfo.getId(), + PermittedSynchronization.FROM_THIRD_PARTY)); addCatalogTarget(ucServerGUID, qualifiedName, catalogInfo.getName(), templates, configurationProperties); } @@ -304,18 +309,12 @@ protected void updateElementInEgeria(CatalogInfo catalogInfo, String egeriaCatalogGUID = memberElement.getElement().getElementGUID(); openMetadataAccess.updateMetadataElementInStore(egeriaCatalogGUID, - false, getElementProperties(catalogInfo)); - - context.updateExternalIdentifier(egeriaCatalogGUID, - deployedImplementationType.getAssociatedTypeName(), - this.getExternalIdentifierProperties(catalogInfo, - null, - UnityCatalogPlaceholderProperty.CATALOG_NAME.getName(), - catalogInfo.getId())); + false, + getElementProperties(catalogInfo)); context.confirmSynchronization(egeriaCatalogGUID, deployedImplementationType.getAssociatedTypeName(), - catalogInfo.getName()); + catalogInfo.getId()); } @@ -323,13 +322,25 @@ protected void updateElementInEgeria(CatalogInfo catalogInfo, * Create a catalog in UC. * * @param memberElement elements from Egeria - * @throws PropertyServerException problem communicating with UC + * @throws InvalidParameterException parameter error + * @throws PropertyServerException repository error or problem communicating with UC + * @throws UserNotAuthorizedException authorization error */ - private void createElementInThirdParty(MemberElement memberElement) throws PropertyServerException + private void createElementInThirdParty(MemberElement memberElement) throws PropertyServerException, + InvalidParameterException, + UserNotAuthorizedException { - ucConnector.createCatalog(this.getUCNameFromMember(memberElement), - this.getUCCommentFomMember(memberElement), - this.getUCPropertiesFomMember(memberElement)); + CatalogInfo catalogInfo = ucConnector.createCatalog(this.getUCNameFromMember(memberElement), + this.getUCCommentFomMember(memberElement), + this.getUCPropertiesFomMember(memberElement)); + + context.addExternalIdentifier(memberElement.getElement().getElementGUID(), + OpenMetadataType.CATALOG.typeName, + this.getExternalIdentifierProperties(catalogInfo, + null, + UnityCatalogPlaceholderProperty.CATALOG_NAME.getName(), + catalogInfo.getId(), + PermittedSynchronization.TO_THIRD_PARTY)); } @@ -344,10 +355,13 @@ private void createElementInThirdParty(MemberElement memberElement) throws Prope private void updateElementInThirdParty(CatalogInfo catalogInfo, MemberElement memberElement) throws PropertyServerException { - ucConnector.updateCatalog(catalogInfo.getName(), - this.getUCNameFromMember(memberElement), - this.getUCCommentFomMember(memberElement), - this.getUCPropertiesFomMember(memberElement)); + final String methodName = "updateElementInThirdParty"; + + auditLog.logMessage(methodName, + UCAuditCode.CATALOG_UPDATE.getMessageDefinition(connectorName, + memberElement.getElement().getElementGUID(), + catalogInfo.getName(), + ucServerEndpoint)); } diff --git a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogServerSyncConnector.java b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogServerSyncConnector.java index 4b1a1a106bf..68d88daa05f 100644 --- a/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogServerSyncConnector.java +++ b/open-metadata-implementation/adapters/open-connectors/data-manager-connectors/unity-catalog-connectors/src/main/java/org/odpi/openmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogServerSyncConnector.java @@ -188,17 +188,17 @@ private void catalogCatalogs(String ucServerGUID, String ucServerEndpoint = this.getNetworkAddress(assetConnector); String friendshipConnectorGUID = getFriendshipGUID(configurationProperties); - OSSUnityCatalogInsideCatalogSyncCatalog syncCatalog = new OSSUnityCatalogInsideCatalogSyncCatalog(connectorName, - this.getContext(), - catalogTargetName, - ucServerGUID, - friendshipConnectorGUID, - permittedSynchronization, - assetConnector, - ucServerEndpoint, - templateProperties, - configurationProperties, - auditLog); + OSSUnityCatalogServerSyncCatalog syncCatalog = new OSSUnityCatalogServerSyncCatalog(connectorName, + this.getContext(), + catalogTargetName, + ucServerGUID, + friendshipConnectorGUID, + permittedSynchronization, + assetConnector, + ucServerEndpoint, + templateProperties, + configurationProperties, + auditLog); syncCatalog.refresh(); } diff --git a/open-metadata-implementation/frameworks/open-connector-framework/src/main/java/org/odpi/openmetadata/frameworks/connectors/ConnectorBroker.java b/open-metadata-implementation/frameworks/open-connector-framework/src/main/java/org/odpi/openmetadata/frameworks/connectors/ConnectorBroker.java index 896707e3033..07b4fdc1334 100644 --- a/open-metadata-implementation/frameworks/open-connector-framework/src/main/java/org/odpi/openmetadata/frameworks/connectors/ConnectorBroker.java +++ b/open-metadata-implementation/frameworks/open-connector-framework/src/main/java/org/odpi/openmetadata/frameworks/connectors/ConnectorBroker.java @@ -4,17 +4,23 @@ import org.odpi.openmetadata.frameworks.auditlog.AuditLog; import org.odpi.openmetadata.frameworks.auditlog.AuditLoggingComponent; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.odpi.openmetadata.frameworks.connectors.ffdc.ConnectionCheckedException; import org.odpi.openmetadata.frameworks.connectors.ffdc.ConnectorCheckedException; import org.odpi.openmetadata.frameworks.connectors.ffdc.OCFErrorCode; import org.odpi.openmetadata.frameworks.connectors.ffdc.OCFRuntimeException; -import org.odpi.openmetadata.frameworks.connectors.properties.*; +import org.odpi.openmetadata.frameworks.connectors.properties.ConnectionProperties; +import org.odpi.openmetadata.frameworks.connectors.properties.ConnectorTypeProperties; +import org.odpi.openmetadata.frameworks.connectors.properties.EmbeddedConnectionProperties; +import org.odpi.openmetadata.frameworks.connectors.properties.VirtualConnectionProperties; import org.odpi.openmetadata.frameworks.connectors.properties.beans.Connection; import org.odpi.openmetadata.frameworks.connectors.properties.beans.VirtualConnection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * The ConnectorBroker is a generic factory for Open Connector Framework (OCF) Connectors. diff --git a/open-metadata-implementation/frameworks/open-integration-framework/src/main/java/org/odpi/openmetadata/frameworks/integration/iterator/IntegrationIterator.java b/open-metadata-implementation/frameworks/open-integration-framework/src/main/java/org/odpi/openmetadata/frameworks/integration/iterator/IntegrationIterator.java index 107d885b78b..de729eccc0d 100644 --- a/open-metadata-implementation/frameworks/open-integration-framework/src/main/java/org/odpi/openmetadata/frameworks/integration/iterator/IntegrationIterator.java +++ b/open-metadata-implementation/frameworks/open-integration-framework/src/main/java/org/odpi/openmetadata/frameworks/integration/iterator/IntegrationIterator.java @@ -174,6 +174,8 @@ private MemberElement fillOutMemberElement(OpenMetadataElement element, PropertyServerException, UserNotAuthorizedException { + final String methodName = "fillOutMemberElement"; + List correlationHeaders = null; RelatedMetadataElement rootSchemaType = null; Map> vendorProperties = null; @@ -187,6 +189,16 @@ private MemberElement fillOutMemberElement(OpenMetadataElement element, element.getElementGUID(), element.getType().getTypeName()); } + else if (propertyHelper.isTypeOf(element, OpenMetadataType.SOFTWARE_CAPABILITY.typeName)) + { + correlationHeaders = openMetadataAccess.getMetadataCorrelationHeaders(element.getElementGUID(), + propertyHelper.getStringProperty(element.getElementGUID(), + OpenMetadataProperty.QUALIFIED_NAME.name, + element.getElementProperties(), + methodName), + element.getElementGUID(), + element.getType().getTypeName()); + } vendorProperties = openMetadataAccess.getVendorProperties(element.getElementGUID(), element.getType().getTypeName()); diff --git a/open-metadata-implementation/frameworks/open-integration-framework/src/main/java/org/odpi/openmetadata/frameworks/integration/iterator/MemberElement.java b/open-metadata-implementation/frameworks/open-integration-framework/src/main/java/org/odpi/openmetadata/frameworks/integration/iterator/MemberElement.java index 0b89dd7718b..cc793fe0e43 100644 --- a/open-metadata-implementation/frameworks/open-integration-framework/src/main/java/org/odpi/openmetadata/frameworks/integration/iterator/MemberElement.java +++ b/open-metadata-implementation/frameworks/open-integration-framework/src/main/java/org/odpi/openmetadata/frameworks/integration/iterator/MemberElement.java @@ -69,6 +69,11 @@ public class MemberElement this.targetPermittedSynchronization = targetPermittedSynchronization; this.auditLog = auditLog; + if (element == null) + { + this.instanceSyncDirection = PermittedSynchronization.FROM_THIRD_PARTY; + } + if (externalIdentifiers != null) { for (MetadataCorrelationHeader externalIdentifier : externalIdentifiers) @@ -146,8 +151,8 @@ public Map getVendorProperties(String vendorIdentifier) * @param thirdPartyElementLastUpdateTime the time that the element was last updated in the third party * @return the action to take */ - public MemberAction getMemberAction(Date thirdPartyElementCreationTime, - Date thirdPartyElementLastUpdateTime) + public MemberAction getMemberAction(Date thirdPartyElementCreationTime, + Date thirdPartyElementLastUpdateTime) { final String methodName = "getMemberAction"; @@ -255,15 +260,7 @@ else if (dateComparison == DateComparison.THIRD_PARTY_COPY_NEWEST) } else /* No element in Egeria. */ { - if ((instanceSyncDirection == PermittedSynchronization.FROM_THIRD_PARTY) || - (instanceSyncDirection == PermittedSynchronization.BOTH_DIRECTIONS)) - { - return MemberAction.CREATE_INSTANCE_IN_OPEN_METADATA; - } - else if (instanceSyncDirection == PermittedSynchronization.TO_THIRD_PARTY) - { - return MemberAction.DELETE_INSTANCE_IN_THIRD_PARTY; - } + return MemberAction.CREATE_INSTANCE_IN_OPEN_METADATA; } /* @@ -320,7 +317,29 @@ private DateComparison compareAges(Date thirdPartyElementCreationTime, if (elementComparisonDate.after(thirdPartyComparisonDate)) { - return DateComparison.EGERIA_COPY_NEWEST; + /* + * Egeria has the newest copy - has it been updated since the last + * synchronization date? + */ + if (externalIdentifiers != null) + { + for (MetadataCorrelationHeader externalIdentifier : externalIdentifiers) + { + if (externalIdentifier != null) + { + if ((externalIdentifier.getLastSynchronized() != null) && (externalIdentifier.getLastSynchronized().before(elementComparisonDate))) + { + return DateComparison.EGERIA_COPY_NEWEST; + } + } + } + + return DateComparison.DATES_EQUAL; + } + else + { + return DateComparison.EGERIA_COPY_NEWEST; + } } return DateComparison.DATES_EQUAL; diff --git a/open-metadata-implementation/frameworks/open-metadata-framework/src/main/java/org/odpi/openmetadata/frameworks/openmetadata/enums/PermittedSynchronization.java b/open-metadata-implementation/frameworks/open-metadata-framework/src/main/java/org/odpi/openmetadata/frameworks/openmetadata/enums/PermittedSynchronization.java index 8ada8fb1de0..b13a490168d 100644 --- a/open-metadata-implementation/frameworks/open-metadata-framework/src/main/java/org/odpi/openmetadata/frameworks/openmetadata/enums/PermittedSynchronization.java +++ b/open-metadata-implementation/frameworks/open-metadata-framework/src/main/java/org/odpi/openmetadata/frameworks/openmetadata/enums/PermittedSynchronization.java @@ -199,6 +199,6 @@ public static String getOpenTypeDescriptionWiki() @Override public String toString() { - return "PermittedSynchronization{" + name + "= name }"; + return "PermittedSynchronization{ name=" + name + " }"; } } \ No newline at end of file