Merge pull request odpi#8503 from mandy-chessell/oak2024

Add cloud information model archive builder
dwolfson · Nov 22, 2024 · 7d66508 · 7d66508
2 parents c6103a5 + a64b446
commit 7d66508
Show file tree

Hide file tree

Showing 49 changed files with 53,753 additions and 73 deletions.
diff --git a/CloudInformationModelGUIDMap.json b/CloudInformationModelGUIDMap.json
diff --git a/EgeriaContentPacksGUIDMap.json b/EgeriaContentPacksGUIDMap.json
diff --git a/cloud-information-model.jsonld b/cloud-information-model.jsonld
diff --git a/content-packs/APIsContentPack.omarchive b/content-packs/APIsContentPack.omarchive
diff --git a/content-packs/ApacheAtlasContentPack.omarchive b/content-packs/ApacheAtlasContentPack.omarchive
diff --git a/content-packs/ApacheKafkaContentPack.omarchive b/content-packs/ApacheKafkaContentPack.omarchive
diff --git a/content-packs/CloudInformationModel.omarchive b/content-packs/CloudInformationModel.omarchive
diff --git a/content-packs/CocoBusinessSystemsArchive.omarchive b/content-packs/CocoBusinessSystemsArchive.omarchive
diff --git a/content-packs/CocoClinicalTrialsTemplatesArchive.omarchive b/content-packs/CocoClinicalTrialsTemplatesArchive.omarchive
diff --git a/content-packs/CocoComboArchive.omarchive b/content-packs/CocoComboArchive.omarchive
diff --git a/content-packs/CocoGovernanceEngineDefinitionsArchive.omarchive b/content-packs/CocoGovernanceEngineDefinitionsArchive.omarchive
diff --git a/content-packs/CocoGovernanceProgramArchive.omarchive b/content-packs/CocoGovernanceProgramArchive.omarchive
diff --git a/content-packs/CocoOrganizationArchive.omarchive b/content-packs/CocoOrganizationArchive.omarchive
diff --git a/content-packs/CocoSustainabilityArchive.omarchive b/content-packs/CocoSustainabilityArchive.omarchive
diff --git a/content-packs/CocoTypesArchive.omarchive b/content-packs/CocoTypesArchive.omarchive
diff --git a/content-packs/CoreContentPack.omarchive b/content-packs/CoreContentPack.omarchive
diff --git a/content-packs/ObservabilityContentPack.omarchive b/content-packs/ObservabilityContentPack.omarchive
diff --git a/content-packs/PostgresContentPack.omarchive b/content-packs/PostgresContentPack.omarchive
diff --git a/content-packs/UnityCatalogContentPack.omarchive b/content-packs/UnityCatalogContentPack.omarchive
diff --git a/...main/java/org/odpi/openmetadata/adapters/connectors/resource/jdbc/mappers/BaseMapper.java b/...main/java/org/odpi/openmetadata/adapters/connectors/resource/jdbc/mappers/BaseMapper.java
@@ -74,6 +74,15 @@ public String getStringPropertyFromColumn(String                     columnName,
     {
         final String methodName = "getStringPropertyFromColumn";
 
+        if (instanceTableRow == null)
+        {
+            throw new RepositoryErrorException(JDBCErrorCode.MISSING_DATABASE_VALUE.getMessageDefinition(columnName,
+                                                                                                         methodName,
+                                                                                                         this.getClass().getName()),
+                                               this.getClass().getName(),
+                                               methodName);
+        }
+
         JDBCDataValue jdbcDataValue = instanceTableRow.get(columnName);
         String        newValue = null;
 
@@ -110,6 +119,15 @@ public List<String> getStringArrayPropertyFromColumn(String
     {
         final String methodName = "getStringArrayPropertyFromColumn";
 
+        if (instanceTableRow == null)
+        {
+            throw new RepositoryErrorException(JDBCErrorCode.MISSING_DATABASE_VALUE.getMessageDefinition(columnName,
+                                                                                                         methodName,
+                                                                                                         this.getClass().getName()),
+                                               this.getClass().getName(),
+                                               methodName);
+        }
+
         JDBCDataValue jdbcDataValue = instanceTableRow.get(columnName);
         String        newValue = null;
 
@@ -151,6 +169,15 @@ public Map<String, Serializable> getSerializableMapPropertyFromColumn(String
     {
         final String methodName = "getSerializableMapPropertyFromColumn";
 
+        if (instanceTableRow == null)
+        {
+            throw new RepositoryErrorException(JDBCErrorCode.MISSING_DATABASE_VALUE.getMessageDefinition(columnName,
+                                                                                                         methodName,
+                                                                                                         this.getClass().getName()),
+                                               this.getClass().getName(),
+                                               methodName);
+        }
+
         JDBCDataValue jdbcDataValue = instanceTableRow.get(columnName);
         String        newValue = null;
 
@@ -210,6 +237,15 @@ public Date getDatePropertyFromColumn(String                     columnName,
     {
         final String methodName = "getDatePropertyFromColumn";
 
+        if (instanceTableRow == null)
+        {
+            throw new RepositoryErrorException(JDBCErrorCode.MISSING_DATABASE_VALUE.getMessageDefinition(columnName,
+                                                                                                         methodName,
+                                                                                                         this.getClass().getName()),
+                                               this.getClass().getName(),
+                                               methodName);
+        }
+
         JDBCDataValue jdbcDataValue = instanceTableRow.get(columnName);
         Date          newValue = null;
 
@@ -251,6 +287,15 @@ public long getLongPropertyFromColumn(String                     columnName,
     {
         final String methodName = "getLongPropertyFromColumn";
 
+        if (instanceTableRow == null)
+        {
+            throw new RepositoryErrorException(JDBCErrorCode.MISSING_DATABASE_VALUE.getMessageDefinition(columnName,
+                                                                                                         methodName,
+                                                                                                         this.getClass().getName()),
+                                               this.getClass().getName(),
+                                               methodName);
+        }
+
         JDBCDataValue jdbcDataValue = instanceTableRow.get(columnName);
 
         if ((jdbcDataValue != null) && (jdbcDataValue.getDataValue() instanceof Long longValue))
@@ -291,6 +336,15 @@ public boolean getBooleanPropertyFromColumn(String                     columnNam
     {
         final String methodName = "getBooleanPropertyFromColumn";
 
+        if (instanceTableRow == null)
+        {
+            throw new RepositoryErrorException(JDBCErrorCode.MISSING_DATABASE_VALUE.getMessageDefinition(columnName,
+                                                                                                         methodName,
+                                                                                                         this.getClass().getName()),
+                                               this.getClass().getName(),
+                                               methodName);
+        }
+
         JDBCDataValue jdbcDataValue = instanceTableRow.get(columnName);
 
         if ((jdbcDataValue != null) && (jdbcDataValue.getDataValue() instanceof Boolean booleanValue))

diff --git a/...pi/openmetadata/adapters/connectors/integration/openapis/properties/OpenAPIOperation.java b/...pi/openmetadata/adapters/connectors/integration/openapis/properties/OpenAPIOperation.java
@@ -29,7 +29,6 @@ public class OpenAPIOperation
     private OpenAPIRequestBody           requestBody  = null;
     private Map<String, OpenAPIResponse> responses    = null;
     private boolean                      deprecated   = false;
-    private Map<String, List<String>>    security     = null;
     private List<OpenAPIServer>          servers      = null;
     private Map<String, Object>          extensions   = null;
 
@@ -146,17 +145,6 @@ public void setDeprecated(boolean deprecated)
     }
 
 
-    public Map<String, List<String>> getSecurity()
-    {
-        return security;
-    }
-
-
-    public void setSecurity(Map<String, List<String>> security)
-    {
-        this.security = security;
-    }
-
 
     public List<OpenAPIServer> getServers()
     {
@@ -195,7 +183,6 @@ public String toString()
                        ", requestBody=" + requestBody +
                        ", responses=" + responses +
                        ", deprecated=" + deprecated +
-                       ", security=" + security +
                        ", servers=" + servers +
                        ", extensions=" + extensions +
                        '}';

diff --git a/...a/adapters/repositoryservices/postgres/repositoryconnector/PostgresOMRSMetadataStore.java b/...a/adapters/repositoryservices/postgres/repositoryconnector/PostgresOMRSMetadataStore.java
@@ -793,7 +793,7 @@ EntityDetail addEntityToStore(EntityDetail entityDetail) throws RepositoryErrorE
         }
         else if (entityDetail.getVersion() > storedEntity.getEntityDetail().getVersion())
         {
-            databaseStore.updatePreviousEntityVersionEndTime(storedEntity, this.getVersionEndDate(entityDetail.getUpdateTime()));
+            databaseStore.updatePreviousEntityVersionEndTime(storedEntity, databaseStore.getVersionEndDate(entityDetail.getUpdateTime()));
 
             databaseStore.addEntityToStore(new EntityMapper(entityDetail, repositoryHelper, repositoryName));
             return entityDetail;
@@ -803,32 +803,6 @@ else if (entityDetail.getVersion() > storedEntity.getEntityDetail().getVersion()
     }
 
 
-    /**
-     * Determine the end time for the previous version of an entity, relationship or classification.
-     *
-     * @param updateTime update time from next version
-     * @return date - 1 millisecond earlier than that the update time
-     * @throws RepositoryErrorException no update time in next version
-     */
-    private Date getVersionEndDate(Date updateTime) throws RepositoryErrorException
-    {
-        final String methodName = "getVersionEndDate";
-
-        if (updateTime != null)
-        {
-            long versionEndDate = updateTime.getTime() - 1;
-
-            return new Date(versionEndDate);
-        }
-
-        throw new RepositoryErrorException(PostgresErrorCode.MISSING_MAPPING_VALUE.getMessageDefinition("updateTime",
-                                                                                                        methodName,
-                                                                                                        this.getClass().getName()),
-                                           this.getClass().getName(),
-                                           methodName);
-    }
-
-
     /**
      * Save an entity proxy to the entity store.
      *
@@ -877,7 +851,7 @@ Relationship addRelationshipToStore(Relationship relationship) throws Repository
         }
         else if (relationship.getVersion() > storedRelationship.getRelationship().getVersion())
         {
-            databaseStore.updatePreviousRelationshipVersionEndTime(storedRelationship, this.getVersionEndDate(relationship.getUpdateTime()));
+            databaseStore.updatePreviousRelationshipVersionEndTime(storedRelationship, databaseStore.getVersionEndDate(relationship.getUpdateTime()));
             databaseStore.addRelationshipToStore(new RelationshipMapper(relationship, repositoryHelper, repositoryName));
             return relationship;
         }
@@ -913,7 +887,7 @@ void saveClassification(String          entityGUID,
         }
         else if (classification.getVersion() > storedClassification.getClassification().getVersion())
         {
-            databaseStore.updatePreviousClassificationVersionEndTime(storedClassification, this.getVersionEndDate(classification.getUpdateTime()));
+            databaseStore.updatePreviousClassificationVersionEndTime(storedClassification, databaseStore.getVersionEndDate(classification.getUpdateTime()));
             databaseStore.saveClassification(new ClassificationMapper(entityGUID, classification, repositoryHelper, repositoryName));
         }
     }

diff --git a/...data/adapters/repositoryservices/postgres/repositoryconnector/database/DatabaseStore.java b/...data/adapters/repositoryservices/postgres/repositoryconnector/database/DatabaseStore.java
@@ -1230,7 +1230,29 @@ public void addEntityToStore(EntityMapper entityMapper) throws RepositoryErrorEx
             jdbcResourceConnector.insertRowsIntoTable(RepositoryTable.ENTITY_ATTRIBUTE_VALUE.getTableName(),
                                                       entityMapper.getEntityPropertiesTableRows());
 
-            saveClassifications(entityMapper.getClassificationMappers());
+            /*
+             * Classifications are updated independently of entities.  Therefore, we only update them if the version of
+             * the classification supplied by the caller is new or has a version later than the stored value.
+             */
+            List<ClassificationMapper> classificationMappers = entityMapper.getClassificationMappers();
+
+            if (classificationMappers != null)
+            {
+                for (ClassificationMapper classificationMapper : classificationMappers)
+                {
+                    ClassificationMapper storedClassification = this.getClassificationForUpdate(classificationMapper.getEntityGUID(), classificationMapper.getClassification().getName());
+
+                    if (storedClassification == null)
+                    {
+                        this.saveClassification(classificationMapper);
+                    }
+                    else if (classificationMapper.getClassification().getVersion() > storedClassification.getClassification().getVersion())
+                    {
+                        this.updatePreviousClassificationVersionEndTime(storedClassification, this.getVersionEndDate(classificationMapper.getClassification().getUpdateTime()));
+                        this.saveClassification(classificationMapper);
+                    }
+                }
+            }
         }
         catch (PropertyServerException sqlException)
         {
@@ -1245,6 +1267,32 @@ public void addEntityToStore(EntityMapper entityMapper) throws RepositoryErrorEx
     }
 
 
+    /**
+     * Determine the end time for the previous version of an entity, relationship or classification.
+     *
+     * @param updateTime update time from next version
+     * @return date - 1 millisecond earlier than that the update time
+     * @throws RepositoryErrorException no update time in next version
+     */
+    public Date getVersionEndDate(Date updateTime) throws RepositoryErrorException
+    {
+        final String methodName = "getVersionEndDate";
+
+        if (updateTime != null)
+        {
+            long versionEndDate = updateTime.getTime() - 1;
+
+            return new Date(versionEndDate);
+        }
+
+        throw new RepositoryErrorException(PostgresErrorCode.MISSING_MAPPING_VALUE.getMessageDefinition("updateTime",
+                                                                                                        methodName,
+                                                                                                        this.getClass().getName()),
+                                           this.getClass().getName(),
+                                           methodName);
+    }
+
+
     /**
      * Add an entity proxy to the database only if the entity is not known.  Its header information (and effectivity dates) goes into the
      * entity table. The properties go into the entity attribute values table.

diff --git a/...penmetadata/adapters/connectors/egeriainfrastructure/servers/OMAGServerConnectorBase.java b/...penmetadata/adapters/connectors/egeriainfrastructure/servers/OMAGServerConnectorBase.java
@@ -14,7 +14,6 @@
 import org.odpi.openmetadata.frameworks.auditlog.AuditLog;
 import org.odpi.openmetadata.frameworks.auditlog.AuditLoggingComponent;
 import org.odpi.openmetadata.frameworks.auditlog.ComponentDescription;
-import org.odpi.openmetadata.frameworks.connectors.Connector;
 import org.odpi.openmetadata.frameworks.connectors.ConnectorBase;
 import org.odpi.openmetadata.frameworks.connectors.VirtualConnectorExtension;
 import org.odpi.openmetadata.frameworks.connectors.ffdc.ConnectorCheckedException;

diff --git a/.../main/java/org/odpi/openmetadata/samples/archiveutilities/SimpleCatalogArchiveHelper.java b/.../main/java/org/odpi/openmetadata/samples/archiveutilities/SimpleCatalogArchiveHelper.java
@@ -2770,12 +2770,12 @@ public String addDesignModel(String               typeName,
             elementTypeName = typeName;
         }
 
-        List<Classification> entityClassifications = null;
+        List<Classification> entityClassifications = new ArrayList<>();
+
+        entityClassifications.add(this.getAnchorClassification(null, elementTypeName, OpenMetadataType.DESIGN_MODEL_TYPE_NAME, methodName));
 
         if (classificationName != null)
         {
-            entityClassifications = new ArrayList<>();
-
             Classification classification = archiveHelper.getClassification(classificationName, null, InstanceStatus.ACTIVE);
 
             entityClassifications.add(classification);

diff --git a/...penmetadata/archiveutilities/openconnectors/apacheatlas/ApacheAtlasPackArchiveWriter.java b/...penmetadata/archiveutilities/openconnectors/apacheatlas/ApacheAtlasPackArchiveWriter.java
@@ -13,7 +13,6 @@
 import org.odpi.openmetadata.archiveutilities.openconnectors.core.CorePackArchiveWriter;
 import org.odpi.openmetadata.repositoryservices.connectors.stores.archivestore.properties.OpenMetadataArchive;
 
-import java.util.Date;
 
 /**
  * ApacheAtlasPackArchiveWriter creates an open metadata archive that includes the connector type
@@ -105,6 +104,12 @@ public void getArchiveContent()
                                                            RequestTypeDefinition.CATALOG_ATLAS_SERVER,
                                                            GovernanceEngineDefinition.ASSET_ONBOARDING_ENGINE);
 
+        this.deleteAsCatalogTargetGovernanceActionProcess("ApacheAtlasServer",
+                                                           AtlasDeployedImplementationType.APACHE_ATLAS_SERVER.getAssociatedTypeName(),
+                                                           AtlasDeployedImplementationType.APACHE_ATLAS_SERVER.getDeployedImplementationType(),
+                                                           RequestTypeDefinition.DELETE_ATLAS_SERVER,
+                                                           GovernanceEngineDefinition.ASSET_ONBOARDING_ENGINE);
+
         /*
          * Saving the GUIDs means tha the guids in the archive are stable between runs of the archive writer.
          */

diff --git a/...penmetadata/archiveutilities/openconnectors/apachekafka/ApacheKafkaPackArchiveWriter.java b/...penmetadata/archiveutilities/openconnectors/apachekafka/ApacheKafkaPackArchiveWriter.java
@@ -103,6 +103,11 @@ public void getArchiveContent()
                                                            GovernanceEngineDefinition.KAFKA_GOVERNANCE_ENGINE,
                                                            RequestTypeDefinition.CATALOG_KAFKA_SERVER,
                                                            GovernanceEngineDefinition.KAFKA_GOVERNANCE_ENGINE);
+        this.deleteAsCatalogTargetGovernanceActionProcess("ApacheKafkaServer",
+                                                          KafkaDeployedImplementationType.APACHE_KAFKA_SERVER.getAssociatedTypeName(),
+                                                          KafkaDeployedImplementationType.APACHE_KAFKA_SERVER.getDeployedImplementationType(),
+                                                          RequestTypeDefinition.DELETE_KAFKA_SERVER,
+                                                          GovernanceEngineDefinition.KAFKA_GOVERNANCE_ENGINE);
 
         /*
          * Saving the GUIDs means tha the guids in the archive are stable between runs of the archive writer.