From 1628a0d458e344f42b7506d80997cf1f8f80c765 Mon Sep 17 00:00:00 2001 From: Ljupcho Palashevski Date: Fri, 11 Aug 2023 13:31:21 +0200 Subject: [PATCH 01/13] Remove custom config not needed at this point. Signed-off-by: Ljupcho Palashevski --- .../AvailabilityProbesAutoConfiguration.java | 40 ------------------- 1 file changed, 40 deletions(-) delete mode 100644 open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/AvailabilityProbesAutoConfiguration.java diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/AvailabilityProbesAutoConfiguration.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/AvailabilityProbesAutoConfiguration.java deleted file mode 100644 index 0ec27300d03..00000000000 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/AvailabilityProbesAutoConfiguration.java +++ /dev/null @@ -1,40 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 */ -/* Copyright Contributors to the ODPi Egeria project. */ -package org.odpi.openmetadata.serverchassis.springboot.config; - -import org.springframework.boot.actuate.autoconfigure.health.ConditionalOnEnabledHealthIndicator; -import org.springframework.boot.actuate.availability.LivenessStateHealthIndicator; -import org.springframework.boot.actuate.availability.ReadinessStateHealthIndicator; -import org.springframework.boot.autoconfigure.AutoConfigureAfter; -import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.availability.ApplicationAvailability; -import org.springframework.context.annotation.Bean; - -import org.springframework.context.annotation.Configuration; - - -/** - * This class provides configuration for Application Availability support components. - */ -@Configuration(proxyBeanMethods = false) -@AutoConfigureAfter(ApplicationAvailabilityAutoConfiguration.class) -public class AvailabilityProbesAutoConfiguration { - - @Bean - @ConditionalOnEnabledHealthIndicator("livenessState") - @ConditionalOnMissingBean - public LivenessStateHealthIndicator livenessStateHealthIndicator(ApplicationAvailability applicationAvailability) { - return new LivenessStateHealthIndicator(applicationAvailability); - } - - @Bean - @ConditionalOnEnabledHealthIndicator("readinessState") - @ConditionalOnMissingBean - public ReadinessStateHealthIndicator readinessStateHealthIndicator( - ApplicationAvailability applicationAvailability) { - return new ReadinessStateHealthIndicator(applicationAvailability); - } - -} - From f064918a3e91c1fc790ed9ad4a95514dee154ac3 Mon Sep 17 00:00:00 2001 From: Ljupcho Palashevski Date: Fri, 11 Aug 2023 13:32:44 +0200 Subject: [PATCH 02/13] Setting zones to empty causes NullPointer, needs to be raised as bug. Signed-off-by: Ljupcho Palashevski --- .../src/main/resources/samples/metadata-access-server.json | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-access-server.json b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-access-server.json index 6a6373b0ebf..a68dd7b1c29 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-access-server.json +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-access-server.json @@ -5,7 +5,7 @@ "localServerName": "cocoMDS1", "localServerType": "Open Metadata and Governance Server", "localServerURL": "https://localhost:9443", - "localServerUserId": "OMAGServer", + "localServerUserId": "cocoMDS1npa", "maxPageSize": 1000, "accessServicesConfig": [ { @@ -20,8 +20,6 @@ "accessServiceWiki": "https://egeria-project.org/services/omas/asset-catalog/overview/", "accessServiceOperationalStatus": "ENABLED", "accessServiceOptions": { - "SupportedZones": [], - "DefaultZones": [], "SupportedTypesForSearch": [ "RelationalTable", "TabularColumn", @@ -49,8 +47,6 @@ "accessServiceWiki": "https://egeria-project.org/services/omas/asset-manager/overview/", "accessServiceOperationalStatus": "ENABLED", "accessServiceOptions": { - "SupportedZones": [], - "DefaultZones": [], "SupportedTypesForSearch": [ "RelationalTable", "TabularColumn", @@ -113,7 +109,6 @@ }, "configurationProperties": { "supportedSeverities": [ - "Information", "Startup", "Shutdown" ] From 8178a99ef602e11af2bbb01ff2d10ebad1f8930e Mon Sep 17 00:00:00 2001 From: Ljupcho Palashevski Date: Fri, 11 Aug 2023 13:33:38 +0200 Subject: [PATCH 03/13] work in progress on the new server configuration feature Signed-off-by: Ljupcho Palashevski --- .../server-chassis-spring/build.gradle | 4 + .../serverchassis/springboot/OMAGServer.java | 62 ++++------ .../springboot/config/OMAGConfigHelper.java | 106 ++++++++++++++++++ .../config/OMAGServerProperties.java | 65 ++++++++++- .../config/OMAGServicesConfiguration.java | 18 ++- .../main/resources/application-dev.properties | 5 +- .../src/main/resources/application-dev.yml | 23 ++++ .../resources/application-test.properties | 6 +- 8 files changed, 235 insertions(+), 54 deletions(-) create mode 100644 open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java create mode 100644 open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle b/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle index 0de2c37ad3b..5f3005f2f23 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle @@ -37,11 +37,15 @@ dependencies { compileOnly 'io.swagger.core.v3:swagger-annotations-jakarta' compileOnly project(':open-metadata-implementation:common-services:ffdc-services') + implementation project(':open-metadata-implementation:frameworks:open-connector-framework') implementation project(':open-metadata-implementation:adapters:authentication-plugins:http-helper') implementation project(':open-metadata-implementation:admin-services:admin-services-api') implementation project(':open-metadata-implementation:server-operations:server-operations-api') implementation project(':open-metadata-implementation:server-operations:server-operations-server') + implementation project(path: ':open-metadata-implementation:adapters:open-connectors:connector-configuration-factory') + implementation project(path: ':open-metadata-implementation:repository-services:repository-services-implementation') + /* IN DEVELOPMENT */ /* Pulling dependencies for some fo the sub-systems enabling 'Metadata Access Store' services */ diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/OMAGServer.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/OMAGServer.java index 683a57a2109..6c18e212104 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/OMAGServer.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/OMAGServer.java @@ -2,11 +2,10 @@ /* Copyright Contributors to the ODPi Egeria project. */ package org.odpi.openmetadata.serverchassis.springboot; -import com.fasterxml.jackson.databind.ObjectMapper; import org.odpi.openmetadata.adminservices.configuration.properties.OMAGServerConfig; -import org.odpi.openmetadata.serveroperations.rest.SuccessMessageResponse; -import org.odpi.openmetadata.serveroperations.server.OMAGServerOperationalServices; -import org.odpi.openmetadata.serverchassis.springboot.config.OMAGServerProperties; +import org.odpi.openmetadata.platformservices.rest.SuccessMessageResponse; +import org.odpi.openmetadata.platformservices.server.OMAGServerOperationalServices; +import org.odpi.openmetadata.serverchassis.springboot.config.OMAGConfigHelper; import org.odpi.openmetadata.serverchassis.springboot.exception.OMAGServerActivationError; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,17 +15,14 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.event.ContextClosedEvent; import org.springframework.context.event.EventListener; -import java.io.IOException; import java.util.List; /** * OMAGServer provides the main program for the OMAG Server spring-boot based starter application. */ -@EnableConfigurationProperties(OMAGServerProperties.class) @SpringBootApplication( scanBasePackages = {"org.odpi.openmetadata"} ) @@ -34,24 +30,21 @@ public class OMAGServer implements ApplicationRunner { private static final Logger LOG = LoggerFactory.getLogger(OMAGServer.class); private final ConfigurableApplicationContext context; - private final ObjectMapper objectMapper; private final OMAGServerOperationalServices operationalServices; - private final OMAGServerProperties serverProperties; - private OMAGServerConfig serverConfigDocument; - private String serverName; + private OMAGConfigHelper configHelper; /** * Constructor injecting the beans required. */ @Autowired - public OMAGServer(ConfigurableApplicationContext ctx, ObjectMapper objectMapper,OMAGServerProperties omagServerProperties, OMAGServerOperationalServices operationalServices) { + public OMAGServer(ConfigurableApplicationContext ctx, OMAGConfigHelper configHelper, OMAGServerOperationalServices omagServerOperationalServices) { this.context = ctx; - this.objectMapper = objectMapper; - this.serverProperties = omagServerProperties; - this.operationalServices = operationalServices; + this.operationalServices = omagServerOperationalServices; + this.configHelper = configHelper; } + /** * Main program, creating spring boot application instance. * @param args @@ -69,7 +62,7 @@ public static void main(String[] args) { public void run(ApplicationArguments args) { LOG.debug("Application runner executing run"); try { - activateOMAGServerUsingPlatformServices(); + activateOMAGServer(); } catch (OMAGServerActivationError e) { LOG.error("Server activation failed due to internal application error", e); /** @@ -86,9 +79,10 @@ public void run(ApplicationArguments args) { */ @EventListener(ContextClosedEvent.class) private void onContextClosedEvent() { - if (serverName != null) { - LOG.info("Application stopped, deactivating server: {}", serverName); - operationalServices.deactivateTemporarilyServerList(serverProperties.getServerUser(), List.of(serverName)); + //TODO: Improvement required: the name of the server should be taken for the configuration. + if (configHelper.getOmagServerConfig().getLocalServerName() != null) { + LOG.info("Application stopped, deactivating server: {}", configHelper.getOmagServerConfig().getLocalServerName()); + operationalServices.deactivateTemporarilyServerList(configHelper.getServerProperties().getDefaultUser(), List.of(configHelper.getOmagServerConfig().getLocalServerName())); } } @@ -97,37 +91,27 @@ private void onContextClosedEvent() { * @see OMAGServerOperationalServices * * The activation process requires OMAGServerConfig document. - * @see org.odpi.openmetadata.adminservices.configuration.properties.OMAGServerConfig + * @see OMAGServerConfig * * The OMAGServerConfig document location is provided as org.springframework.core.io.Resource and configured by application property `omag.server-config`. * * @throws OMAGServerActivationError */ - private void activateOMAGServerUsingPlatformServices() throws OMAGServerActivationError { + private void activateOMAGServer() throws OMAGServerActivationError { LOG.debug("Activation started"); - try { - LOG.info("Configuration {}", serverProperties.getServerConfig()); - serverConfigDocument = objectMapper.reader().readValue(serverProperties.getServerConfig().getInputStream(), OMAGServerConfig.class); - - if (serverConfigDocument == null) { - LOG.info("Activation failed, the cause is that the OMAGServerConfig document is null"); - throw new OMAGServerActivationError("Activation failed, the cause is that the OMAGServerConfig document is null"); - } - - serverName = serverConfigDocument.getLocalServerName(); - LOG.info("Configuration document for server: {} - loaded successfully", serverName); - } catch (IOException e) { - LOG.info("Configuration document cannot be loaded from the resource provided - check application configuration"); - throw new OMAGServerActivationError( - String.format("Configuration document cannot be loaded from the resource provided - check application configuration"),e); + configHelper.loadConfig(); + + if (configHelper.getOmagServerConfig() == null) { + LOG.info("Activation failed, the cause is that the OMAGServerConfig document is null"); + throw new OMAGServerActivationError("Activation failed, the cause is that the OMAGServerConfig document is null"); } - LOG.info("Sending activation request for server: {} and user: {}", serverName, serverProperties.getServerUser()); + LOG.info("Sending activation request for server: {} and user: {}", configHelper.getOmagServerConfig().getLocalServerName(), configHelper.getOmagServerConfig().getLocalServerUserId()); SuccessMessageResponse response = operationalServices - .activateWithSuppliedConfig(serverProperties.getServerUser().trim(), serverConfigDocument.getLocalServerName(), serverConfigDocument); + .activateWithSuppliedConfig(configHelper.getOmagServerConfig().getLocalServerUserId(), configHelper.getOmagServerConfig().getLocalServerName(), configHelper.getOmagServerConfig()); if (response == null) { LOG.info("Activation has failed. The cause is that response is null"); @@ -140,7 +124,7 @@ private void activateOMAGServerUsingPlatformServices() throws OMAGServerActivati } if (response.getRelatedHTTPCode() == 200) { - LOG.info("Activation succeeded for server: {}", serverConfigDocument.getLocalServerName()); + LOG.info("Activation succeeded for server: {}", configHelper.getOmagServerConfig().getLocalServerName()); } } diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java new file mode 100644 index 00000000000..45c0df5b254 --- /dev/null +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java @@ -0,0 +1,106 @@ +package org.odpi.openmetadata.serverchassis.springboot.config; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.Getter; +import org.odpi.openmetadata.adapters.repositoryservices.ConnectorConfigurationFactory; +import org.odpi.openmetadata.adminservices.configuration.properties.LocalRepositoryConfig; +import org.odpi.openmetadata.adminservices.configuration.properties.OMAGServerConfig; +import org.odpi.openmetadata.adminservices.configuration.properties.RepositoryServicesConfig; +import org.odpi.openmetadata.frameworks.connectors.properties.beans.Connection; +import org.odpi.openmetadata.frameworks.connectors.properties.beans.ConnectorType; +import org.odpi.openmetadata.repositoryservices.admin.OMRSConfigurationFactory; +import org.odpi.openmetadata.serverchassis.springboot.exception.OMAGServerActivationError; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + + +@EnableConfigurationProperties(OMAGServerProperties.class) +@Configuration +public class OMAGConfigHelper { + + @Getter final OMAGServerProperties serverProperties; + @Getter OMAGServerConfig omagServerConfig; + private static final Logger LOG = LoggerFactory.getLogger(OMAGConfigHelper.class); + private final ObjectMapper mapper; + ConnectorConfigurationFactory connectorConfigurationFactory; + OMRSConfigurationFactory omrsConfigurationFactory; + @Autowired + public OMAGConfigHelper(OMAGServerProperties properties, ObjectMapper objectMapper, ConnectorConfigurationFactory connectorConfigurationFactory, OMRSConfigurationFactory omrsConfigurationFactory) { + this.serverProperties = properties; + this.mapper = objectMapper; + this.connectorConfigurationFactory = connectorConfigurationFactory; + this.omrsConfigurationFactory = omrsConfigurationFactory; + } + + + public void loadConfig() throws OMAGServerActivationError { + + try { + + if (serverProperties.getJsonConfigFile() == null) { + + LOG.info("JSON configuration file not used. Configuring server using omag. application properties"); + + omagServerConfig = new OMAGServerConfig(); + + omagServerConfig.setLocalServerName(serverProperties.getServer().getName()); + omagServerConfig.setLocalServerUserId(serverProperties.getServer().getUser()); + omagServerConfig.setMaxPageSize(serverProperties.getServer().getMaxPageSize()); + + if(serverProperties.getServer().getRepositoryServices() != null) { + RepositoryServicesConfig repositoryServicesConfig = omrsConfigurationFactory.getDefaultRepositoryServicesConfig(); + if (serverProperties.getServer().getRepositoryServices().getAuditLogConnectors() != null) { + + List auditLogConnectors = serverProperties.getServer().getRepositoryServices().getAuditLogConnectors().stream().map(o -> { + Connection c = new Connection(); + ConnectorType ct = new ConnectorType(); + ct.setConnectorProviderClassName(o.getProviderClassName()); + c.setConnectorType(ct); + //TODO: Fix map/list binding problem. The type of supportedSeverities in configurationProperties should be resolved to List instead of Map!!! + c.setConfigurationProperties(o.getConfigurationProperties()); + return c; + }).collect(Collectors.toList()); + + repositoryServicesConfig.setAuditLogConnections(auditLogConnectors); + } + //TODO: Work in progress currently using default in-meme local repo. + LocalRepositoryConfig localRepositoryConfig = getLocalRepositoryConfig(); + repositoryServicesConfig.setLocalRepositoryConfig(localRepositoryConfig); + omagServerConfig.setRepositoryServicesConfig(repositoryServicesConfig); + } + + + } else { + + LOG.info("Configuration {}", serverProperties.getJsonConfigFile()); + omagServerConfig = mapper.reader().readValue(serverProperties.getJsonConfigFile().getInputStream(), OMAGServerConfig.class); + LOG.info("Configuration document for server: {} - loaded successfully", omagServerConfig.getLocalServerName()); + + } + + + } catch (IOException e) { + LOG.info("Configuration document cannot be loaded from the resource provided - check application configuration"); + throw new OMAGServerActivationError( + String.format("Configuration document cannot be loaded from the resource provided - check application configuration"), e); + } + + } + + private LocalRepositoryConfig getLocalRepositoryConfig() { + LocalRepositoryConfig localRepositoryConfig = new LocalRepositoryConfig(); + localRepositoryConfig.setEventsToSendRule(getServerProperties().getServer().getRepositoryServices().getLocalRepository().getEventsToSend()); + localRepositoryConfig.setEventsToSaveRule(getServerProperties().getServer().getRepositoryServices().getLocalRepository().getEventsToSave()); + localRepositoryConfig.setMetadataCollectionId(getServerProperties().getServer().getRepositoryServices().getLocalRepository().getMetadataCollectionId()); + localRepositoryConfig.setLocalRepositoryLocalConnection(connectorConfigurationFactory.getInMemoryLocalRepositoryLocalConnection()); + return localRepositoryConfig; + } + +} diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java index 1b455fa40a0..ffed1042aad 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java @@ -6,14 +6,19 @@ import lombok.Getter; import lombok.Setter; import org.odpi.openmetadata.adminservices.configuration.properties.OMAGServerConfig; +import org.odpi.openmetadata.adminservices.configuration.properties.OpenMetadataExchangeRule; +import org.odpi.openmetadata.adminservices.configuration.registration.ServerTypeClassification; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.core.io.Resource; import org.springframework.validation.annotation.Validated; +import java.util.List; +import java.util.Map; /** * This class provides validation support for OMAG specific application properties. */ -@ConfigurationProperties(prefix = "omag") +@ConfigurationProperties(prefix = "omag", ignoreUnknownFields=false) @Getter @Setter @Validated @@ -23,13 +28,61 @@ public class OMAGServerProperties { * Configures the location of the OMAGServerConfig json document defined as org.springframework.core.io.Resource * This property is required and cannot be null. */ - @NotNull - private Resource serverConfig; +// @NotNull + private Resource jsonConfigFile; /** - * Configures the username parameter used to activate the OMAG server instance using platform operational services. - * Default value is set to 'system', can be overwritted + * Configures the default user parameter used to activate the OMAG server instance using platform operational services. + * @see org.odpi.openmetadata.platformservices.server.OMAGServerOperationalServices#activateWithSuppliedConfig(String, String, OMAGServerConfig) */ - private String serverUser = "system"; + @Value("${omag.server-user:${omag.server.user:system}}") + private String defaultUser; + + + /** + * Server bean represents simplified OMAG server configuration model used for YAML based application configuration. + */ + private Server server; + + @Getter + @Setter + public static class Server { + private String name; + private ServerTypeClassification type; + private String user; + private int maxPageSize; + private RepositoryServices repositoryServices; + } + + @Getter + @Setter + public static class RepositoryServices { + private List auditLogConnectors; + private LocalRepository localRepository; + } + + @Getter + @Setter + public static class Connector { + private String providerClassName; + private Map configurationProperties; + private Endpoint endpoint; + } + + @Getter + @Setter + public static class LocalRepository { + String metadataCollectionId; + Connector localConnection; + Connector remoteConnection; + OpenMetadataExchangeRule eventsToSave; + OpenMetadataExchangeRule eventsToSend; + } + + @Getter + @Setter + public static class Endpoint { + private String address; + } } diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServicesConfiguration.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServicesConfiguration.java index 74dd34484ef..2bc19a4b107 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServicesConfiguration.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServicesConfiguration.java @@ -2,7 +2,9 @@ /* Copyright Contributors to the ODPi Egeria project. */ package org.odpi.openmetadata.serverchassis.springboot.config; -import org.odpi.openmetadata.serveroperations.server.OMAGServerOperationalServices; +import org.odpi.openmetadata.adapters.repositoryservices.ConnectorConfigurationFactory; +import org.odpi.openmetadata.platformservices.server.OMAGServerOperationalServices; +import org.odpi.openmetadata.repositoryservices.admin.OMRSConfigurationFactory; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; @@ -18,11 +20,19 @@ public class OMAGServicesConfiguration { * Provides singleton bean instance of OMAGServerOperationalServices * * @return OMAGServerOperationalServices instance - * @see OMAGServerOperationalServices + * @see org.odpi.openmetadata.platformservices.server.OMAGServerOperationalServices */ @Primary - @Bean(name = {"serverOperationalServices"}) - public OMAGServerOperationalServices serverOperationalServices(){ + @Bean(name = {"platformOperationalServices"}) + public OMAGServerOperationalServices platformOperationalServices(){ return new OMAGServerOperationalServices(); } + + @Bean(name = {"connectorConfigurationFactory"}) + public ConnectorConfigurationFactory connectorConfigurationFactory() { + return new ConnectorConfigurationFactory(); + } + + @Bean(name = {"omrsConfigurationFactory"}) + public OMRSConfigurationFactory omrsConfigurationFactory() {return new OMRSConfigurationFactory(); } } diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties index 784c900e6d9..bf65c02201d 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties @@ -15,6 +15,7 @@ management.endpoint.health.show-details=always management.endpoint.metrics.enabled=true management.endpoints.web.exposure.include=metrics,health -omag.server-config=classpath:samples/metadata-repository-server.json -omag.server-user=admin +#omag.json-config-file=classpath:samples/metadata-access-server.json +default.omag.server.user="omag-system" + diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml new file mode 100644 index 00000000000..5a7b8f77f26 --- /dev/null +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml @@ -0,0 +1,23 @@ +--- +omag: + server: + max-page-size: 1000 + name: cocoMDS1 + user: "${default.omag.server.user}" + type: OMAG_SERVER + repository-services: + audit-log-connectors: + - provider-class-name: "org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.console.ConsoleAuditLogStoreProvider" + configuration-properties: + supportedSeverities: ["Error"] + local-repository: + metadata-collection-id: "838005fc-e09f-4688-af89-58254a155cc6" + local-connection: + provider-class-name: "org.odpi.openmetadata.adapters.repositoryservices.inmemory.repositoryconnector.InMemoryOMRSRepositoryConnectorProvider" + remote-connection: + provider-class-name: "org.odpi.openmetadata.adapters.repositoryservices.rest.repositoryconnector.OMRSRESTRepositoryConnectorProvider" + endpoint: + address: "https://localhost:9443/servers/cocoMDS1" + events-to-save: ALL + events-to-send: ALL + diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/test/resources/application-test.properties b/open-metadata-implementation/server-chassis/server-chassis-spring/src/test/resources/application-test.properties index 2d730e29603..be116e30a99 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/test/resources/application-test.properties +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/test/resources/application-test.properties @@ -44,9 +44,9 @@ logging.level.org.odpi.openmetadata.repositoryservices=info logging.level.org.odpi.openmetadata.server=debug logging.level.org.springframework.web=info -logging.file.path=/var/logs -logging.file.name=egeria.log -logging.level.*=INFO +#logging.file.path=/var/logs +#logging.file.name=egeria.log +#logging.level.*=INFO logging.level.org.springframework=error #logging.level.org.odpi.openmetadata.commonservices=DEBUG From 98fb043c3ee85a3a02aa59ec7003dcb93ce6dbb7 Mon Sep 17 00:00:00 2001 From: Ljupcho Palashevski Date: Mon, 14 Aug 2023 17:00:30 +0200 Subject: [PATCH 04/13] WIP POC: added custom yaml configuration support using snakeyaml. Signed-off-by: Ljupcho Palashevski --- .../server-chassis-spring/build.gradle | 3 +- .../springboot/config/OMAGConfigHelper.java | 41 ++++++++++--------- .../config/OMAGServerProperties.java | 5 +-- .../main/resources/application-dev.properties | 5 ++- .../src/main/resources/application-dev.yml | 3 +- .../samples/metadata-repository-server.yml | 31 ++++++++++++++ 6 files changed, 60 insertions(+), 28 deletions(-) create mode 100644 open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yml diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle b/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle index 5f3005f2f23..173f6469830 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle @@ -20,8 +20,6 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-actuator' implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'org.springframework.boot:spring-boot-starter-validation' - implementation 'org.springframework.boot:spring-boot-actuator-autoconfigure' - implementation 'org.springframework.boot:spring-boot-actuator' implementation 'org.springframework.boot:spring-boot-starter-tomcat' implementation 'org.slf4j:slf4j-api' implementation 'com.fasterxml.jackson.core:jackson-databind' @@ -29,6 +27,7 @@ dependencies { implementation 'jakarta.servlet:jakarta.servlet-api' implementation 'jakarta.validation:jakarta.validation-api' implementation 'org.projectlombok:lombok' + implementation 'org.yaml:snakeyaml' annotationProcessor 'org.projectlombok:lombok' runtimeOnly 'io.micrometer:micrometer-registry-prometheus' diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java index 45c0df5b254..a528615460a 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* Copyright Contributors to the ODPi Egeria project. */ package org.odpi.openmetadata.serverchassis.springboot.config; import com.fasterxml.jackson.databind.ObjectMapper; @@ -15,9 +17,11 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Configuration; +import org.yaml.snakeyaml.Yaml; import java.io.IOException; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; @@ -43,50 +47,49 @@ public OMAGConfigHelper(OMAGServerProperties properties, ObjectMapper objectMapp public void loadConfig() throws OMAGServerActivationError { try { + if (serverProperties.getServerConfigFile() != null) { + + LOG.info("Configuration {}", serverProperties.getServerConfigFile()); + //TODO: This is POC implementation. Better way to identify content type may be required. + if (Objects.requireNonNull(serverProperties.getServerConfigFile().getFilename()).endsWith("json")) { + omagServerConfig = mapper.reader().readValue(serverProperties.getServerConfigFile().getInputStream(), OMAGServerConfig.class); + } else { + //TODO: This is POC implementation. We need better code to deal with yaml i.e. one option is to use Jackson and Yaml data format. + Yaml yaml = new Yaml(); + omagServerConfig = yaml.loadAs(serverProperties.getServerConfigFile().getInputStream(), OMAGServerConfig.class); + } + LOG.info("Configuration document for server: {} - loaded successfully", omagServerConfig.getLocalServerName()); - if (serverProperties.getJsonConfigFile() == null) { - - LOG.info("JSON configuration file not used. Configuring server using omag. application properties"); + } else if (serverProperties.getServer() != null) { + LOG.info("Configuring server using omag. application properties [EXPERIMENTAL]"); omagServerConfig = new OMAGServerConfig(); - omagServerConfig.setLocalServerName(serverProperties.getServer().getName()); omagServerConfig.setLocalServerUserId(serverProperties.getServer().getUser()); omagServerConfig.setMaxPageSize(serverProperties.getServer().getMaxPageSize()); - if(serverProperties.getServer().getRepositoryServices() != null) { + if (serverProperties.getServer().getRepositoryServices() != null) { RepositoryServicesConfig repositoryServicesConfig = omrsConfigurationFactory.getDefaultRepositoryServicesConfig(); if (serverProperties.getServer().getRepositoryServices().getAuditLogConnectors() != null) { - List auditLogConnectors = serverProperties.getServer().getRepositoryServices().getAuditLogConnectors().stream().map(o -> { Connection c = new Connection(); ConnectorType ct = new ConnectorType(); ct.setConnectorProviderClassName(o.getProviderClassName()); c.setConnectorType(ct); - //TODO: Fix map/list binding problem. The type of supportedSeverities in configurationProperties should be resolved to List instead of Map!!! + //TODO: Fix binding problem. The type of supportedSeverities in configurationProperties should be resolved to List instead of Map!!! c.setConfigurationProperties(o.getConfigurationProperties()); return c; }).collect(Collectors.toList()); - repositoryServicesConfig.setAuditLogConnections(auditLogConnectors); } - //TODO: Work in progress currently using default in-meme local repo. + //TODO: Work in progress currently using default in-mem local repo. LocalRepositoryConfig localRepositoryConfig = getLocalRepositoryConfig(); repositoryServicesConfig.setLocalRepositoryConfig(localRepositoryConfig); omagServerConfig.setRepositoryServicesConfig(repositoryServicesConfig); } - - - } else { - - LOG.info("Configuration {}", serverProperties.getJsonConfigFile()); - omagServerConfig = mapper.reader().readValue(serverProperties.getJsonConfigFile().getInputStream(), OMAGServerConfig.class); - LOG.info("Configuration document for server: {} - loaded successfully", omagServerConfig.getLocalServerName()); - } - - } catch (IOException e) { + } catch (Exception e) { LOG.info("Configuration document cannot be loaded from the resource provided - check application configuration"); throw new OMAGServerActivationError( String.format("Configuration document cannot be loaded from the resource provided - check application configuration"), e); diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java index ffed1042aad..177e0241edf 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java @@ -2,7 +2,6 @@ /* Copyright Contributors to the ODPi Egeria project. */ package org.odpi.openmetadata.serverchassis.springboot.config; -import jakarta.validation.constraints.NotNull; import lombok.Getter; import lombok.Setter; import org.odpi.openmetadata.adminservices.configuration.properties.OMAGServerConfig; @@ -29,7 +28,7 @@ public class OMAGServerProperties { * This property is required and cannot be null. */ // @NotNull - private Resource jsonConfigFile; + private Resource serverConfigFile; /** * Configures the default user parameter used to activate the OMAG server instance using platform operational services. @@ -65,7 +64,7 @@ public static class RepositoryServices { @Setter public static class Connector { private String providerClassName; - private Map configurationProperties; + private Map configurationProperties; private Endpoint endpoint; } diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties index bf65c02201d..6a0da195aee 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties @@ -15,7 +15,8 @@ management.endpoint.health.show-details=always management.endpoint.metrics.enabled=true management.endpoints.web.exposure.include=metrics,health -#omag.json-config-file=classpath:samples/metadata-access-server.json -default.omag.server.user="omag-system" +#omag.server-config-file=classpath:samples/metadata-access-server.json +omag.server-config-file=classpath:samples/metadata-repository-server.yml +default.omag.server.user=omag-system diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml index 5a7b8f77f26..450d63f0135 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml @@ -19,5 +19,4 @@ omag: endpoint: address: "https://localhost:9443/servers/cocoMDS1" events-to-save: ALL - events-to-send: ALL - + events-to-send: ALL \ No newline at end of file diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yml b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yml new file mode 100644 index 00000000000..95eb0e0f678 --- /dev/null +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yml @@ -0,0 +1,31 @@ +--- +localServerName: cocoMDS1 +localServerType: Open Metadata and Governance Server +localServerURL: https://localhost:9443 +localServerUserId: OMAGServer +maxPageSize: 1000 +repositoryServicesConfig: + auditLogConnections: + - connectorType: + connectorProviderClassName: >- + org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.console.ConsoleAuditLogStoreProvider + configurationProperties: + supportedSeverities: + - Error + localRepositoryConfig: + metadataCollectionId: 838005fc-e09f-4688-af89-58254a155cc6 + localRepositoryMode: OPEN_METADATA_NATIVE + localRepositoryLocalConnection: + connectorType: + connectorProviderClassName: >- + org.odpi.openmetadata.adapters.repositoryservices.inmemory.repositoryconnector.InMemoryOMRSRepositoryConnectorProvider + localRepositoryRemoteConnection: + headerVersion: 0 + displayName: Local Repository Remote Connection + connectorType: + connectorProviderClassName: >- + org.odpi.openmetadata.adapters.repositoryservices.rest.repositoryconnector.OMRSRESTRepositoryConnectorProvider + endpoint: + address: https://localhost:9443/servers/cocoMDS1 + eventsToSaveRule: ALL + eventsToSendRule: ALL \ No newline at end of file From 54fcb5843e16c3f8a11082e0639ba2f1d4e6efd4 Mon Sep 17 00:00:00 2001 From: Ljupcho Palashevski Date: Tue, 15 Aug 2023 13:54:40 +0200 Subject: [PATCH 05/13] Update Gradle build configuration Signed-off-by: Ljupcho Palashevski --- .../server-chassis/server-chassis-spring/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle b/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle index 173f6469830..70c65fe2359 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle @@ -25,7 +25,6 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.core:jackson-annotations' implementation 'jakarta.servlet:jakarta.servlet-api' - implementation 'jakarta.validation:jakarta.validation-api' implementation 'org.projectlombok:lombok' implementation 'org.yaml:snakeyaml' annotationProcessor 'org.projectlombok:lombok' @@ -34,6 +33,7 @@ dependencies { runtimeOnly 'ch.qos.logback:logback-classic' runtimeOnly 'org.hibernate:hibernate-validator' compileOnly 'io.swagger.core.v3:swagger-annotations-jakarta' + runtimeOnly 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' compileOnly project(':open-metadata-implementation:common-services:ffdc-services') implementation project(':open-metadata-implementation:frameworks:open-connector-framework') From c57cdeea437745c91e03df325e3906e62ef3f75e Mon Sep 17 00:00:00 2001 From: Ljupcho Palashevski Date: Wed, 16 Aug 2023 11:04:05 +0200 Subject: [PATCH 06/13] Server instance should only rely on OMAG config document Signed-off-by: Ljupcho Palashevski --- .../openmetadata/serverchassis/springboot/OMAGServer.java | 5 ++--- .../springboot/config/OMAGServerProperties.java | 8 -------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/OMAGServer.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/OMAGServer.java index 6c18e212104..59b369e247a 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/OMAGServer.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/OMAGServer.java @@ -79,10 +79,9 @@ public void run(ApplicationArguments args) { */ @EventListener(ContextClosedEvent.class) private void onContextClosedEvent() { - //TODO: Improvement required: the name of the server should be taken for the configuration. - if (configHelper.getOmagServerConfig().getLocalServerName() != null) { + if (configHelper.getOmagServerConfig() != null) { LOG.info("Application stopped, deactivating server: {}", configHelper.getOmagServerConfig().getLocalServerName()); - operationalServices.deactivateTemporarilyServerList(configHelper.getServerProperties().getDefaultUser(), List.of(configHelper.getOmagServerConfig().getLocalServerName())); + operationalServices.deactivateTemporarilyServerList(configHelper.getOmagServerConfig().getLocalServerUserId(), List.of(configHelper.getOmagServerConfig().getLocalServerName())); } } diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java index 177e0241edf..8f6cc8b59ea 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java @@ -30,14 +30,6 @@ public class OMAGServerProperties { // @NotNull private Resource serverConfigFile; - /** - * Configures the default user parameter used to activate the OMAG server instance using platform operational services. - * @see org.odpi.openmetadata.platformservices.server.OMAGServerOperationalServices#activateWithSuppliedConfig(String, String, OMAGServerConfig) - */ - @Value("${omag.server-user:${omag.server.user:system}}") - private String defaultUser; - - /** * Server bean represents simplified OMAG server configuration model used for YAML based application configuration. */ From 1235a7c63569fd5dc13d8059d55a7dd7f677c09b Mon Sep 17 00:00:00 2001 From: Ljupcho Palashevski Date: Fri, 8 Sep 2023 18:21:04 +0200 Subject: [PATCH 07/13] Marge conflicts while cherry-picking 24daa4fb02c6225ba114fec4093bf75b1779a349 Signed-off-by: Ljupcho Palashevski --- .../openmetadata/serverchassis/springboot/OMAGServer.java | 4 ++-- .../springboot/config/OMAGServicesConfiguration.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/OMAGServer.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/OMAGServer.java index 59b369e247a..c046e1d8e1b 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/OMAGServer.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/OMAGServer.java @@ -3,10 +3,10 @@ package org.odpi.openmetadata.serverchassis.springboot; import org.odpi.openmetadata.adminservices.configuration.properties.OMAGServerConfig; -import org.odpi.openmetadata.platformservices.rest.SuccessMessageResponse; -import org.odpi.openmetadata.platformservices.server.OMAGServerOperationalServices; import org.odpi.openmetadata.serverchassis.springboot.config.OMAGConfigHelper; import org.odpi.openmetadata.serverchassis.springboot.exception.OMAGServerActivationError; +import org.odpi.openmetadata.serveroperations.rest.SuccessMessageResponse; +import org.odpi.openmetadata.serveroperations.server.OMAGServerOperationalServices; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServicesConfiguration.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServicesConfiguration.java index 2bc19a4b107..e861b5cd965 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServicesConfiguration.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServicesConfiguration.java @@ -3,8 +3,8 @@ package org.odpi.openmetadata.serverchassis.springboot.config; import org.odpi.openmetadata.adapters.repositoryservices.ConnectorConfigurationFactory; -import org.odpi.openmetadata.platformservices.server.OMAGServerOperationalServices; import org.odpi.openmetadata.repositoryservices.admin.OMRSConfigurationFactory; +import org.odpi.openmetadata.serveroperations.server.OMAGServerOperationalServices; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; @@ -20,7 +20,7 @@ public class OMAGServicesConfiguration { * Provides singleton bean instance of OMAGServerOperationalServices * * @return OMAGServerOperationalServices instance - * @see org.odpi.openmetadata.platformservices.server.OMAGServerOperationalServices + * @see OMAGServerOperationalServices */ @Primary @Bean(name = {"platformOperationalServices"}) From bd35e6d96352e21aeeb8d554cbeb2fa2d4653620 Mon Sep 17 00:00:00 2001 From: Ljupcho Palashevski Date: Wed, 16 Aug 2023 11:07:22 +0200 Subject: [PATCH 08/13] Minimize json/yaml configuration files used, POC testing changes Signed-off-by: Ljupcho Palashevski --- .../springboot/config/OMAGConfigHelper.java | 7 +- .../main/resources/application-dev.properties | 3 +- .../src/main/resources/application-dev.yml | 11 ++- .../samples/metadata-repository-server.json | 67 +------------------ .../samples/metadata-repository-server.yml | 2 - 5 files changed, 17 insertions(+), 73 deletions(-) diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java index a528615460a..f8098a91a80 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java @@ -58,10 +58,11 @@ public void loadConfig() throws OMAGServerActivationError { Yaml yaml = new Yaml(); omagServerConfig = yaml.loadAs(serverProperties.getServerConfigFile().getInputStream(), OMAGServerConfig.class); } - LOG.info("Configuration document for server: {} - loaded successfully", omagServerConfig.getLocalServerName()); } else if (serverProperties.getServer() != null) { - LOG.info("Configuring server using omag. application properties [EXPERIMENTAL]"); + + LOG.info("Configuration document for server not provided"); + LOG.info("[EXPERIMENTAL] Configuring server using omag. application properties"); omagServerConfig = new OMAGServerConfig(); omagServerConfig.setLocalServerName(serverProperties.getServer().getName()); @@ -89,6 +90,8 @@ public void loadConfig() throws OMAGServerActivationError { } } + LOG.info("Configuration document for server: {} - loaded successfully", omagServerConfig.getLocalServerName()); + } catch (Exception e) { LOG.info("Configuration document cannot be loaded from the resource provided - check application configuration"); throw new OMAGServerActivationError( diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties index 6a0da195aee..6b4cfe49351 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties @@ -15,8 +15,7 @@ management.endpoint.health.show-details=always management.endpoint.metrics.enabled=true management.endpoints.web.exposure.include=metrics,health -#omag.server-config-file=classpath:samples/metadata-access-server.json +#omag.server-config-file=classpath:samples/metadata-repository-server.json omag.server-config-file=classpath:samples/metadata-repository-server.yml -default.omag.server.user=omag-system diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml index 450d63f0135..4592b9793d5 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml @@ -3,13 +3,15 @@ omag: server: max-page-size: 1000 name: cocoMDS1 - user: "${default.omag.server.user}" + user: system type: OMAG_SERVER repository-services: audit-log-connectors: - provider-class-name: "org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.console.ConsoleAuditLogStoreProvider" configuration-properties: - supportedSeverities: ["Error"] + supportedSeverities: + - Error + dummySecretProperty: "${app-secrets.dummyPassword}" local-repository: metadata-collection-id: "838005fc-e09f-4688-af89-58254a155cc6" local-connection: @@ -19,4 +21,7 @@ omag: endpoint: address: "https://localhost:9443/servers/cocoMDS1" events-to-save: ALL - events-to-send: ALL \ No newline at end of file + events-to-send: ALL +--- +app-secrets: + dummyPassword: top-secret \ No newline at end of file diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.json b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.json index 414b7c6df0c..52e968111dc 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.json +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.json @@ -1,10 +1,7 @@ { "class": "OMAGServerConfig", "versionId": "V2.0", - "localServerId": "996892b6-7299-4046-8928-b6687a7818ee", "localServerName": "cocoMDS1", - "localServerType": "Open Metadata and Governance Server", - "localServerURL": "https://localhost:9443", "localServerUserId": "OMAGServer", "maxPageSize": 1000, "repositoryServicesConfig": { @@ -17,27 +14,7 @@ "displayName": "Console", "connectorType": { "class": "ConnectorType", - "headerVersion": 0, - "type": { - "typeId": "954421eb-33a6-462d-a8ca-b5709a1bd0d4", - "typeName": "ConnectorType", - "typeVersion": 1, - "typeDescription": "A set of properties describing a type of connector." - }, - "guid": "4afac741-3dcc-4c60-a4ca-a6dede994e3f", - "qualifiedName": "Egeria:AuditLogDestinationConnector:Console", - "displayName": "Console Audit Log Destination Connector", - "description": "Connector supports logging of audit log messages to stdout.", - "connectorProviderClassName": "org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.console.ConsoleAuditLogStoreProvider", - "connectorFrameworkName": "Open Connector Framework (OCF)", - "connectorInterfaceLanguage": "Java", - "connectorInterfaces": [ - "org.odpi.openmetadata.frameworks.connectors.SecureConnectorExtension", - "org.odpi.openmetadata.repositoryservices.connectors.stores.auditlogstore.OMRSAuditLogStore" - ], - "recognizedConfigurationProperties": [ - "supportedSeverities" - ] + "connectorProviderClassName": "org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.console.ConsoleAuditLogStoreProvider" }, "configurationProperties": { "supportedSeverities": [ @@ -61,32 +38,13 @@ "localRepositoryConfig": { "class": "LocalRepositoryConfig", "metadataCollectionId": "838005fc-e09f-4688-af89-58254a155cc6", - "localRepositoryMode": "OPEN_METADATA_NATIVE", "localRepositoryLocalConnection": { "class": "Connection", "headerVersion": 0, "displayName": "In Memory Local Repository Connection", "connectorType": { "class": "ConnectorType", - "headerVersion": 0, - "type": { - "typeId": "954421eb-33a6-462d-a8ca-b5709a1bd0d4", - "typeName": "ConnectorType", - "typeVersion": 1, - "typeDescription": "A set of properties describing a type of connector." - }, - "guid": "65cc9091-757f-4bcd-b937-426160be8bc2", - "qualifiedName": "Egeria:OMRSRepositoryConnector:InMemory", - "displayName": "In Memory OMRS Repository Connector", - "description": "Native open metadata repository connector that maps open metadata calls to a set of in memory hash maps - demo use only.", - "connectorProviderClassName": "org.odpi.openmetadata.adapters.repositoryservices.inmemory.repositoryconnector.InMemoryOMRSRepositoryConnectorProvider", - "connectorFrameworkName": "Open Connector Framework (OCF)", - "connectorInterfaceLanguage": "Java", - "connectorInterfaces": [ - "org.odpi.openmetadata.frameworks.connectors.SecureConnectorExtension", - "org.odpi.openmetadata.frameworks.auditlog.AuditLoggingComponent", - "org.odpi.openmetadata.repositoryservices.connectors.stores.metadatacollectionstore.repositoryconnector.OMRSMetadataCollectionManager" - ] + "connectorProviderClassName": "org.odpi.openmetadata.adapters.repositoryservices.inmemory.repositoryconnector.InMemoryOMRSRepositoryConnectorProvider" } }, "localRepositoryRemoteConnection": { @@ -95,29 +53,10 @@ "displayName": "Local Repository Remote Connection", "connectorType": { "class": "ConnectorType", - "headerVersion": 0, - "type": { - "typeId": "954421eb-33a6-462d-a8ca-b5709a1bd0d4", - "typeName": "ConnectorType", - "typeVersion": 1, - "typeDescription": "A set of properties describing a type of connector." - }, - "guid": "75ea56d1-656c-43fb-bc0c-9d35c5553b9e", - "qualifiedName": "Egeria:OMRSRepositoryConnector:CohortMemberClient:REST", - "displayName": "REST Cohort Member Client Connector", - "description": "Cohort member client connector that provides access to open metadata located in a remote repository via REST calls.", - "connectorProviderClassName": "org.odpi.openmetadata.adapters.repositoryservices.rest.repositoryconnector.OMRSRESTRepositoryConnectorProvider", - "connectorFrameworkName": "Open Connector Framework (OCF)", - "connectorInterfaceLanguage": "Java", - "connectorInterfaces": [ - "org.odpi.openmetadata.frameworks.connectors.SecureConnectorExtension", - "org.odpi.openmetadata.frameworks.auditlog.AuditLoggingComponent", - "org.odpi.openmetadata.repositoryservices.connectors.stores.metadatacollectionstore.repositoryconnector.OMRSMetadataCollectionManager" - ] + "connectorProviderClassName": "org.odpi.openmetadata.adapters.repositoryservices.rest.repositoryconnector.OMRSRESTRepositoryConnectorProvider" }, "endpoint": { "class": "Endpoint", - "headerVersion": 0, "address": "https://localhost:9443/servers/cocoMDS1" } }, diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yml b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yml index 95eb0e0f678..eb4d74c0f81 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yml +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yml @@ -1,7 +1,6 @@ --- localServerName: cocoMDS1 localServerType: Open Metadata and Governance Server -localServerURL: https://localhost:9443 localServerUserId: OMAGServer maxPageSize: 1000 repositoryServicesConfig: @@ -14,7 +13,6 @@ repositoryServicesConfig: - Error localRepositoryConfig: metadataCollectionId: 838005fc-e09f-4688-af89-58254a155cc6 - localRepositoryMode: OPEN_METADATA_NATIVE localRepositoryLocalConnection: connectorType: connectorProviderClassName: >- From 601b10269370f0ff5076f83f4ee444b80c5c5d23 Mon Sep 17 00:00:00 2001 From: Ljupcho Palashevski Date: Wed, 16 Aug 2023 13:07:15 +0200 Subject: [PATCH 09/13] POC cont. Using Jackson/Yaml parsing Signed-off-by: Ljupcho Palashevski --- .../server-chassis-spring/build.gradle | 5 ++- .../springboot/config/OMAGConfigHelper.java | 7 +++- .../main/resources/application-dev.properties | 2 +- .../samples/metadata-repository-server.yml | 14 +++++-- .../samples/metadata-repository-server2.yml | 37 +++++++++++++++++++ 5 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server2.yml diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle b/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle index 70c65fe2359..cd0b0a2a3df 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle @@ -26,14 +26,15 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-annotations' implementation 'jakarta.servlet:jakarta.servlet-api' implementation 'org.projectlombok:lombok' - implementation 'org.yaml:snakeyaml' +// implementation 'org.yaml:snakeyaml' + implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' annotationProcessor 'org.projectlombok:lombok' runtimeOnly 'io.micrometer:micrometer-registry-prometheus' runtimeOnly 'ch.qos.logback:logback-classic' runtimeOnly 'org.hibernate:hibernate-validator' compileOnly 'io.swagger.core.v3:swagger-annotations-jakarta' - runtimeOnly 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' +// runtimeOnly 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' compileOnly project(':open-metadata-implementation:common-services:ffdc-services') implementation project(':open-metadata-implementation:frameworks:open-connector-framework') diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java index f8098a91a80..59cb552151d 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java @@ -3,6 +3,7 @@ package org.odpi.openmetadata.serverchassis.springboot.config; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import lombok.Getter; import org.odpi.openmetadata.adapters.repositoryservices.ConnectorConfigurationFactory; import org.odpi.openmetadata.adminservices.configuration.properties.LocalRepositoryConfig; @@ -55,8 +56,10 @@ public void loadConfig() throws OMAGServerActivationError { omagServerConfig = mapper.reader().readValue(serverProperties.getServerConfigFile().getInputStream(), OMAGServerConfig.class); } else { //TODO: This is POC implementation. We need better code to deal with yaml i.e. one option is to use Jackson and Yaml data format. - Yaml yaml = new Yaml(); - omagServerConfig = yaml.loadAs(serverProperties.getServerConfigFile().getInputStream(), OMAGServerConfig.class); +// Yaml yaml = new Yaml(); +// omagServerConfig = yaml.loadAs(serverProperties.getServerConfigFile().getInputStream(), OMAGServerConfig.class); + ObjectMapper yamlmapper = new ObjectMapper(new YAMLFactory()); + omagServerConfig = yamlmapper.reader().readValue(serverProperties.getServerConfigFile().getInputStream(), OMAGServerConfig.class); } } else if (serverProperties.getServer() != null) { diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties index 6b4cfe49351..8008c38b6d4 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties @@ -16,6 +16,6 @@ management.endpoint.metrics.enabled=true management.endpoints.web.exposure.include=metrics,health #omag.server-config-file=classpath:samples/metadata-repository-server.json -omag.server-config-file=classpath:samples/metadata-repository-server.yml +omag.server-config-file=classpath:samples/metadata-repository-server2.yml diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yml b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yml index eb4d74c0f81..06d6b3555af 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yml +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yml @@ -1,29 +1,37 @@ --- +class: OMAGServerConfig localServerName: cocoMDS1 localServerType: Open Metadata and Governance Server localServerUserId: OMAGServer maxPageSize: 1000 repositoryServicesConfig: + class: RepositoryServicesConfig auditLogConnections: - - connectorType: + - class: Connection + connectorType: + class: ConnectorType connectorProviderClassName: >- org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.console.ConsoleAuditLogStoreProvider configurationProperties: supportedSeverities: - Error localRepositoryConfig: + class: LocalRepositoryConfig metadataCollectionId: 838005fc-e09f-4688-af89-58254a155cc6 localRepositoryLocalConnection: + class: Connection connectorType: + class: ConnectorType connectorProviderClassName: >- org.odpi.openmetadata.adapters.repositoryservices.inmemory.repositoryconnector.InMemoryOMRSRepositoryConnectorProvider localRepositoryRemoteConnection: - headerVersion: 0 - displayName: Local Repository Remote Connection + class: Connection connectorType: + class: ConnectorType connectorProviderClassName: >- org.odpi.openmetadata.adapters.repositoryservices.rest.repositoryconnector.OMRSRESTRepositoryConnectorProvider endpoint: + class: Endpoint address: https://localhost:9443/servers/cocoMDS1 eventsToSaveRule: ALL eventsToSendRule: ALL \ No newline at end of file diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server2.yml b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server2.yml new file mode 100644 index 00000000000..06d6b3555af --- /dev/null +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server2.yml @@ -0,0 +1,37 @@ +--- +class: OMAGServerConfig +localServerName: cocoMDS1 +localServerType: Open Metadata and Governance Server +localServerUserId: OMAGServer +maxPageSize: 1000 +repositoryServicesConfig: + class: RepositoryServicesConfig + auditLogConnections: + - class: Connection + connectorType: + class: ConnectorType + connectorProviderClassName: >- + org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.console.ConsoleAuditLogStoreProvider + configurationProperties: + supportedSeverities: + - Error + localRepositoryConfig: + class: LocalRepositoryConfig + metadataCollectionId: 838005fc-e09f-4688-af89-58254a155cc6 + localRepositoryLocalConnection: + class: Connection + connectorType: + class: ConnectorType + connectorProviderClassName: >- + org.odpi.openmetadata.adapters.repositoryservices.inmemory.repositoryconnector.InMemoryOMRSRepositoryConnectorProvider + localRepositoryRemoteConnection: + class: Connection + connectorType: + class: ConnectorType + connectorProviderClassName: >- + org.odpi.openmetadata.adapters.repositoryservices.rest.repositoryconnector.OMRSRESTRepositoryConnectorProvider + endpoint: + class: Endpoint + address: https://localhost:9443/servers/cocoMDS1 + eventsToSaveRule: ALL + eventsToSendRule: ALL \ No newline at end of file From b7fef8b0ef4da69b9558bb187d3627a89f2d4eea Mon Sep 17 00:00:00 2001 From: tijanapavicic Date: Wed, 16 Aug 2023 18:18:09 +0200 Subject: [PATCH 10/13] POC revamping implementation squash me Signed-off-by: Ljupcho Palashevski --- bom/build.gradle | 2 - .../server-chassis-spring/build.gradle | 2 + .../springboot/config/OMAGConfigHelper.java | 157 +++++++++++++----- .../config/OMAGServerProperties.java | 2 - .../config/ObjectMapperConfiguration.java | 60 +++++-- .../springboot/constants/Extensions.java | 35 ++++ 6 files changed, 194 insertions(+), 64 deletions(-) create mode 100644 open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/Extensions.java diff --git a/bom/build.gradle b/bom/build.gradle index 33099041433..dd74b1a1c59 100644 --- a/bom/build.gradle +++ b/bom/build.gradle @@ -87,7 +87,6 @@ ext { servletVersion = '4.0.1' jakartaServletVersion = '6.0.0' sleepycatVersion = '18.3.12' - snakeyamlVersion = '2.0' slf4jVersion = '2.0.6' snappyVersion = '1.1.10.2' springbootVersion = '3.1.1' @@ -255,7 +254,6 @@ dependencies { // testng also used in our 'source' code to support unit tests api("org.testng:testng:${testngVersion}") api("joda-time:joda-time:${jodatimeVersion}") - api("org.yaml:snakeyaml:${snakeyamlVersion}") api("org.antlr:antlr-runtime:${antlrVersion}") api("org.antlr:ST4:${ST4Version}") api("org.apache.jena:jena-arq:${jenaVersion}") diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle b/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle index cd0b0a2a3df..6883bb1f934 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle @@ -28,6 +28,7 @@ dependencies { implementation 'org.projectlombok:lombok' // implementation 'org.yaml:snakeyaml' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' + implementation 'com.fasterxml.jackson.core:jackson-core' annotationProcessor 'org.projectlombok:lombok' runtimeOnly 'io.micrometer:micrometer-registry-prometheus' @@ -46,6 +47,7 @@ dependencies { implementation project(path: ':open-metadata-implementation:adapters:open-connectors:connector-configuration-factory') implementation project(path: ':open-metadata-implementation:repository-services:repository-services-implementation') + implementation 'com.google.guava:guava' /* IN DEVELOPMENT */ /* Pulling dependencies for some fo the sub-systems enabling 'Metadata Access Store' services */ diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java index 59cb552151d..3f75775dc8e 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java @@ -3,8 +3,9 @@ package org.odpi.openmetadata.serverchassis.springboot.config; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.google.common.io.Files; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.odpi.openmetadata.adapters.repositoryservices.ConnectorConfigurationFactory; import org.odpi.openmetadata.adminservices.configuration.properties.LocalRepositoryConfig; import org.odpi.openmetadata.adminservices.configuration.properties.OMAGServerConfig; @@ -12,13 +13,14 @@ import org.odpi.openmetadata.frameworks.connectors.properties.beans.Connection; import org.odpi.openmetadata.frameworks.connectors.properties.beans.ConnectorType; import org.odpi.openmetadata.repositoryservices.admin.OMRSConfigurationFactory; +import org.odpi.openmetadata.serverchassis.springboot.constants.Extensions; import org.odpi.openmetadata.serverchassis.springboot.exception.OMAGServerActivationError; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Configuration; -import org.yaml.snakeyaml.Yaml; +import org.springframework.core.io.Resource; import java.io.IOException; import java.util.List; @@ -28,63 +30,84 @@ @EnableConfigurationProperties(OMAGServerProperties.class) @Configuration +@Slf4j public class OMAGConfigHelper { - - @Getter final OMAGServerProperties serverProperties; - @Getter OMAGServerConfig omagServerConfig; - private static final Logger LOG = LoggerFactory.getLogger(OMAGConfigHelper.class); - private final ObjectMapper mapper; + public static final String WRONG_EXTENSION_MESSAGE = "Unallowed config file extension, " + + "allowed config file extensions are: " + Extensions.stream(); + @Getter + final OMAGServerProperties serverProperties; + private final ObjectMapper jsonObjectMapper; + private final ObjectMapper yamlObjectMapper; + @Getter + OMAGServerConfig omagServerConfig; ConnectorConfigurationFactory connectorConfigurationFactory; OMRSConfigurationFactory omrsConfigurationFactory; + @Autowired - public OMAGConfigHelper(OMAGServerProperties properties, ObjectMapper objectMapper, ConnectorConfigurationFactory connectorConfigurationFactory, OMRSConfigurationFactory omrsConfigurationFactory) { + public OMAGConfigHelper(OMAGServerProperties properties, + @Qualifier("jsonObjectMapper") ObjectMapper jsonObjectMapper, + @Qualifier("yamlObjectMapper") ObjectMapper yamlObjectMapper, + ConnectorConfigurationFactory connectorConfigurationFactory, + OMRSConfigurationFactory omrsConfigurationFactory) { this.serverProperties = properties; - this.mapper = objectMapper; + this.jsonObjectMapper = jsonObjectMapper; + this.yamlObjectMapper = yamlObjectMapper; this.connectorConfigurationFactory = connectorConfigurationFactory; this.omrsConfigurationFactory = omrsConfigurationFactory; } - + /** + * Method implementing logic for deciding proper configuration source + * and loading its content into configuration object + * @throws OMAGServerActivationError + */ public void loadConfig() throws OMAGServerActivationError { try { - if (serverProperties.getServerConfigFile() != null) { + if (isConfigurationFileProvided()) { + + Resource serverConfigFile = serverProperties.getServerConfigFile(); + + //TODO: This is POC implementation. + // Better way to identify content type may be required. + + if (!isJsonConfigurationFile(serverConfigFile) && !isYamlConfigurationFile(serverConfigFile)) { + log.error(WRONG_EXTENSION_MESSAGE); + throw new OMAGServerActivationError(WRONG_EXTENSION_MESSAGE); + } - LOG.info("Configuration {}", serverProperties.getServerConfigFile()); - //TODO: This is POC implementation. Better way to identify content type may be required. - if (Objects.requireNonNull(serverProperties.getServerConfigFile().getFilename()).endsWith("json")) { - omagServerConfig = mapper.reader().readValue(serverProperties.getServerConfigFile().getInputStream(), OMAGServerConfig.class); - } else { + log.info("{} based configuration with server-config-file {}", getFileExtension(serverConfigFile), serverConfigFile); + + if (isJsonConfigurationFile(serverConfigFile)) { + omagServerConfig = jsonObjectMapper.reader() + .readValue(serverConfigFile.getInputStream(), OMAGServerConfig.class); + + } else if (isYamlConfigurationFile(serverConfigFile)) { //TODO: This is POC implementation. We need better code to deal with yaml i.e. one option is to use Jackson and Yaml data format. // Yaml yaml = new Yaml(); -// omagServerConfig = yaml.loadAs(serverProperties.getServerConfigFile().getInputStream(), OMAGServerConfig.class); - ObjectMapper yamlmapper = new ObjectMapper(new YAMLFactory()); - omagServerConfig = yamlmapper.reader().readValue(serverProperties.getServerConfigFile().getInputStream(), OMAGServerConfig.class); +// omagServerConfig = yaml.loadAs(serverConfigFile.getInputStream(), OMAGServerConfig.class); + + omagServerConfig = yamlObjectMapper.reader() + .readValue(serverConfigFile.getInputStream(), OMAGServerConfig.class); } - } else if (serverProperties.getServer() != null) { + } else if (isPropertiesConfiguration()) { + + log.info("Configuring server using omag. application properties [EXPERIMENTAL]"); - LOG.info("Configuration document for server not provided"); - LOG.info("[EXPERIMENTAL] Configuring server using omag. application properties"); + OMAGServerProperties.Server omagServer = serverProperties.getServer(); omagServerConfig = new OMAGServerConfig(); - omagServerConfig.setLocalServerName(serverProperties.getServer().getName()); - omagServerConfig.setLocalServerUserId(serverProperties.getServer().getUser()); - omagServerConfig.setMaxPageSize(serverProperties.getServer().getMaxPageSize()); + omagServerConfig.setLocalServerName(omagServer.getName()); + omagServerConfig.setLocalServerUserId(omagServer.getUser()); + omagServerConfig.setMaxPageSize(omagServer.getMaxPageSize()); - if (serverProperties.getServer().getRepositoryServices() != null) { + if (omagServer.getRepositoryServices() != null) { RepositoryServicesConfig repositoryServicesConfig = omrsConfigurationFactory.getDefaultRepositoryServicesConfig(); - if (serverProperties.getServer().getRepositoryServices().getAuditLogConnectors() != null) { - List auditLogConnectors = serverProperties.getServer().getRepositoryServices().getAuditLogConnectors().stream().map(o -> { - Connection c = new Connection(); - ConnectorType ct = new ConnectorType(); - ct.setConnectorProviderClassName(o.getProviderClassName()); - c.setConnectorType(ct); - //TODO: Fix binding problem. The type of supportedSeverities in configurationProperties should be resolved to List instead of Map!!! - c.setConfigurationProperties(o.getConfigurationProperties()); - return c; - }).collect(Collectors.toList()); - repositoryServicesConfig.setAuditLogConnections(auditLogConnectors); + + if (omagServer.getRepositoryServices().getAuditLogConnectors() != null) { + + repositoryServicesConfig.setAuditLogConnections(getAuditLogConnectors()); } //TODO: Work in progress currently using default in-mem local repo. LocalRepositoryConfig localRepositoryConfig = getLocalRepositoryConfig(); @@ -93,21 +116,65 @@ public void loadConfig() throws OMAGServerActivationError { } } - LOG.info("Configuration document for server: {} - loaded successfully", omagServerConfig.getLocalServerName()); + log.info("Configuration document for server: {} - loaded successfully", omagServerConfig.getLocalServerName()); - } catch (Exception e) { - LOG.info("Configuration document cannot be loaded from the resource provided - check application configuration"); + } catch (IOException e) { + log.info("Configuration document cannot be loaded from the resource provided - check application configuration"); throw new OMAGServerActivationError( String.format("Configuration document cannot be loaded from the resource provided - check application configuration"), e); } + } + private String getFileExtension(Resource serverConfigFile) { + return Files.getFileExtension(serverConfigFile.getFilename()); + } + + private boolean isConfigurationFileProvided() { + return serverProperties.getServerConfigFile() != null; + } + + private boolean isJsonConfigurationFile(Resource serverConfigFile) { + return Extensions.JSON.name().equalsIgnoreCase(Objects.requireNonNull(getFileExtension(serverConfigFile))); + } + + private boolean isYamlConfigurationFile(Resource serverConfigFile) { + return Extensions.YAML.name().equalsIgnoreCase(Objects.requireNonNull(getFileExtension(serverConfigFile))) + || Extensions.YML.name().equalsIgnoreCase(Objects.requireNonNull(getFileExtension(serverConfigFile))); + } + + private boolean isPropertiesConfiguration() { + return serverProperties.getServer() != null; + } + + private List getAuditLogConnectors() { + + return serverProperties.getServer() + .getRepositoryServices() + .getAuditLogConnectors() + .stream() + .map(o -> getConnection(o)) + .collect(Collectors.toList()); + } + + private Connection getConnection(OMAGServerProperties.Connector o) { + Connection c = new Connection(); + ConnectorType ct = new ConnectorType(); + ct.setConnectorProviderClassName(o.getProviderClassName()); + c.setConnectorType(ct); + //TODO: Fix binding problem. The type of supportedSeverities in configurationProperties should be resolved to List instead of Map!!! + c.setConfigurationProperties(o.getConfigurationProperties()); + return c; } private LocalRepositoryConfig getLocalRepositoryConfig() { LocalRepositoryConfig localRepositoryConfig = new LocalRepositoryConfig(); - localRepositoryConfig.setEventsToSendRule(getServerProperties().getServer().getRepositoryServices().getLocalRepository().getEventsToSend()); - localRepositoryConfig.setEventsToSaveRule(getServerProperties().getServer().getRepositoryServices().getLocalRepository().getEventsToSave()); - localRepositoryConfig.setMetadataCollectionId(getServerProperties().getServer().getRepositoryServices().getLocalRepository().getMetadataCollectionId()); + + OMAGServerProperties.LocalRepository localRepository = + getServerProperties().getServer().getRepositoryServices().getLocalRepository(); + + localRepositoryConfig.setEventsToSendRule(localRepository.getEventsToSend()); + localRepositoryConfig.setEventsToSaveRule(localRepository.getEventsToSave()); + localRepositoryConfig.setMetadataCollectionId(localRepository.getMetadataCollectionId()); localRepositoryConfig.setLocalRepositoryLocalConnection(connectorConfigurationFactory.getInMemoryLocalRepositoryLocalConnection()); return localRepositoryConfig; } diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java index 8f6cc8b59ea..045ff8dc8b5 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java @@ -4,10 +4,8 @@ import lombok.Getter; import lombok.Setter; -import org.odpi.openmetadata.adminservices.configuration.properties.OMAGServerConfig; import org.odpi.openmetadata.adminservices.configuration.properties.OpenMetadataExchangeRule; import org.odpi.openmetadata.adminservices.configuration.registration.ServerTypeClassification; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.core.io.Resource; import org.springframework.validation.annotation.Validated; diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/ObjectMapperConfiguration.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/ObjectMapperConfiguration.java index b3856550b07..882d131424f 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/ObjectMapperConfiguration.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/ObjectMapperConfiguration.java @@ -2,31 +2,38 @@ /* Copyright Contributors to the ODPi Egeria project. */ package org.odpi.openmetadata.serverchassis.springboot.config; - +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; +import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; +import lombok.extern.slf4j.Slf4j; + +import org.odpi.openmetadata.serverchassis.springboot.constants.Extensions; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder; +import org.springframework.util.StringUtils; import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; /** - * This class provides configuration bean for customizing Jackson object mapper singleton instance configuration. + * This class provides configurations for customizing + * json bean Jackson object mapper singleton instance configuration + * and + * yaml bean Jackson object mapper singleton instance configuration */ +@Slf4j @Configuration public class ObjectMapperConfiguration { - public static final String PREFIX = "Object mapper configuration started."; - private final Logger log = LoggerFactory.getLogger(this.getClass()); + public static final String START_LOG_SUFIX = "ObjectMapper configuration started."; - public static ObjectMapper newObjectMapper(Jackson2ObjectMapperBuilder builder) { + private static ObjectMapper newObjectMapper(Jackson2ObjectMapperBuilder jackson2ObjectMapperBuilder) { - return builder + return jackson2ObjectMapperBuilder .serializationInclusion(NON_NULL) .failOnEmptyBeans(false) .failOnUnknownProperties(false) @@ -39,13 +46,36 @@ public static ObjectMapper newObjectMapper(Jackson2ObjectMapperBuilder builder) .build(); } - @Bean(name = {"objectMapper"}) - @Primary - ObjectMapper objectMapper(Jackson2ObjectMapperBuilder builder) { + /** + * @param jackson2ObjectMapperBuilder a builder used to create ObjectMapper instances with a fluent API. + * @return JsonMapper a JSON-format specific ObjectMapper implementation + */ + @Bean + @Qualifier("jsonObjectMapper") + ObjectMapper objectMapper(Jackson2ObjectMapperBuilder jackson2ObjectMapperBuilder) { - log.debug(PREFIX); + log.info("{}{}", StringUtils.capitalize(Extensions.JSON.name()), START_LOG_SUFIX); - return newObjectMapper(builder); + return newObjectMapper(jackson2ObjectMapperBuilder); } -} + /** + * @return YAMLMapper a convenience version of ObjectMapper which is configured with YAMLFactory + */ + @Bean + @Qualifier("yamlObjectMapper") + public ObjectMapper yamlMapper() { + + log.info("{}{}", StringUtils.capitalize(Extensions.YAML.name()), START_LOG_SUFIX); + + return YAMLMapper.builder() + .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) + .disable(YAMLGenerator.Feature.ALLOW_LONG_KEYS) + .serializationInclusion(JsonInclude.Include.NON_NULL) + .configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .enable(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY) + .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) + .build(); + } +} \ No newline at end of file diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/Extensions.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/Extensions.java new file mode 100644 index 00000000000..bc15e3352d1 --- /dev/null +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/Extensions.java @@ -0,0 +1,35 @@ +package org.odpi.openmetadata.serverchassis.springboot.constants; + +import java.io.Serializable; +import java.util.stream.Stream; + +/** + * Created by YM21WO on egeria + * on 17/08/2023 at 11:19 + */ +public enum Extensions implements Serializable { + + YAML(".yaml"), + YML(".yml"), + JSON(".json"); + + + private final String value; + + Extensions(String value) { + this.value = value; + } + + public static String stream() { + StringBuffer stringBuffer = new StringBuffer(); + Stream.of(Extensions.values()) + .forEach(e -> stringBuffer.append(e.getValue()).append(", ")); + + return stringBuffer.toString(); + } + + private String getValue() { + return value; + } + +} From 76169d7582a148660bd21da2edea4f006d7bfec4 Mon Sep 17 00:00:00 2001 From: Ljupcho Palashevski Date: Fri, 18 Aug 2023 13:53:52 +0200 Subject: [PATCH 11/13] POC cont. cleanup, non-functional changes. Signed-off-by: Ljupcho Palashevski --- .../server-chassis-spring/build.gradle | 4 +- .../springboot/config/OMAGConfigHelper.java | 66 +------------------ .../config/OMAGServerProperties.java | 52 +-------------- .../springboot/constants/Extensions.java | 7 +- .../main/resources/application-dev.properties | 2 +- .../src/main/resources/application-dev.yml | 42 ++++++------ ...=> metadata-repository-server_minimal.yml} | 13 +--- 7 files changed, 34 insertions(+), 152 deletions(-) rename open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/{metadata-repository-server2.yml => metadata-repository-server_minimal.yml} (74%) diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle b/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle index 6883bb1f934..39c70654306 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle @@ -28,7 +28,6 @@ dependencies { implementation 'org.projectlombok:lombok' // implementation 'org.yaml:snakeyaml' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' - implementation 'com.fasterxml.jackson.core:jackson-core' annotationProcessor 'org.projectlombok:lombok' runtimeOnly 'io.micrometer:micrometer-registry-prometheus' @@ -36,9 +35,10 @@ dependencies { runtimeOnly 'org.hibernate:hibernate-validator' compileOnly 'io.swagger.core.v3:swagger-annotations-jakarta' // runtimeOnly 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' + runtimeOnly 'com.fasterxml.jackson.core:jackson-core' compileOnly project(':open-metadata-implementation:common-services:ffdc-services') - implementation project(':open-metadata-implementation:frameworks:open-connector-framework') +// implementation project(':open-metadata-implementation:frameworks:open-connector-framework') implementation project(':open-metadata-implementation:adapters:authentication-plugins:http-helper') implementation project(':open-metadata-implementation:admin-services:admin-services-api') implementation project(':open-metadata-implementation:server-operations:server-operations-api') diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java index 3f75775dc8e..128b863da0f 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java @@ -7,15 +7,10 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.odpi.openmetadata.adapters.repositoryservices.ConnectorConfigurationFactory; -import org.odpi.openmetadata.adminservices.configuration.properties.LocalRepositoryConfig; import org.odpi.openmetadata.adminservices.configuration.properties.OMAGServerConfig; -import org.odpi.openmetadata.adminservices.configuration.properties.RepositoryServicesConfig; -import org.odpi.openmetadata.frameworks.connectors.properties.beans.Connection; -import org.odpi.openmetadata.frameworks.connectors.properties.beans.ConnectorType; import org.odpi.openmetadata.repositoryservices.admin.OMRSConfigurationFactory; import org.odpi.openmetadata.serverchassis.springboot.constants.Extensions; import org.odpi.openmetadata.serverchassis.springboot.exception.OMAGServerActivationError; - import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -23,9 +18,7 @@ import org.springframework.core.io.Resource; import java.io.IOException; -import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; @EnableConfigurationProperties(OMAGServerProperties.class) @@ -86,34 +79,14 @@ public void loadConfig() throws OMAGServerActivationError { //TODO: This is POC implementation. We need better code to deal with yaml i.e. one option is to use Jackson and Yaml data format. // Yaml yaml = new Yaml(); // omagServerConfig = yaml.loadAs(serverConfigFile.getInputStream(), OMAGServerConfig.class); - omagServerConfig = yamlObjectMapper.reader() .readValue(serverConfigFile.getInputStream(), OMAGServerConfig.class); } } else if (isPropertiesConfiguration()) { - log.info("Configuring server using omag. application properties [EXPERIMENTAL]"); - - OMAGServerProperties.Server omagServer = serverProperties.getServer(); - - omagServerConfig = new OMAGServerConfig(); - omagServerConfig.setLocalServerName(omagServer.getName()); - omagServerConfig.setLocalServerUserId(omagServer.getUser()); - omagServerConfig.setMaxPageSize(omagServer.getMaxPageSize()); - - if (omagServer.getRepositoryServices() != null) { - RepositoryServicesConfig repositoryServicesConfig = omrsConfigurationFactory.getDefaultRepositoryServicesConfig(); - - if (omagServer.getRepositoryServices().getAuditLogConnectors() != null) { - - repositoryServicesConfig.setAuditLogConnections(getAuditLogConnectors()); - } - //TODO: Work in progress currently using default in-mem local repo. - LocalRepositoryConfig localRepositoryConfig = getLocalRepositoryConfig(); - repositoryServicesConfig.setLocalRepositoryConfig(localRepositoryConfig); - omagServerConfig.setRepositoryServicesConfig(repositoryServicesConfig); - } + log.info("[EXPERIMENTAL] Configuring server using omag. application properties"); + omagServerConfig = serverProperties.getServerConfig(); } log.info("Configuration document for server: {} - loaded successfully", omagServerConfig.getLocalServerName()); @@ -143,40 +116,7 @@ private boolean isYamlConfigurationFile(Resource serverConfigFile) { } private boolean isPropertiesConfiguration() { - return serverProperties.getServer() != null; - } - - private List getAuditLogConnectors() { - - return serverProperties.getServer() - .getRepositoryServices() - .getAuditLogConnectors() - .stream() - .map(o -> getConnection(o)) - .collect(Collectors.toList()); - } - - private Connection getConnection(OMAGServerProperties.Connector o) { - Connection c = new Connection(); - ConnectorType ct = new ConnectorType(); - ct.setConnectorProviderClassName(o.getProviderClassName()); - c.setConnectorType(ct); - //TODO: Fix binding problem. The type of supportedSeverities in configurationProperties should be resolved to List instead of Map!!! - c.setConfigurationProperties(o.getConfigurationProperties()); - return c; - } - - private LocalRepositoryConfig getLocalRepositoryConfig() { - LocalRepositoryConfig localRepositoryConfig = new LocalRepositoryConfig(); - - OMAGServerProperties.LocalRepository localRepository = - getServerProperties().getServer().getRepositoryServices().getLocalRepository(); - - localRepositoryConfig.setEventsToSendRule(localRepository.getEventsToSend()); - localRepositoryConfig.setEventsToSaveRule(localRepository.getEventsToSave()); - localRepositoryConfig.setMetadataCollectionId(localRepository.getMetadataCollectionId()); - localRepositoryConfig.setLocalRepositoryLocalConnection(connectorConfigurationFactory.getInMemoryLocalRepositoryLocalConnection()); - return localRepositoryConfig; + return serverProperties.getServerConfig() != null; } } diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java index 045ff8dc8b5..3929a9fb1c8 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java @@ -4,13 +4,10 @@ import lombok.Getter; import lombok.Setter; -import org.odpi.openmetadata.adminservices.configuration.properties.OpenMetadataExchangeRule; -import org.odpi.openmetadata.adminservices.configuration.registration.ServerTypeClassification; +import org.odpi.openmetadata.adminservices.configuration.properties.OMAGServerConfig; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.core.io.Resource; import org.springframework.validation.annotation.Validated; -import java.util.List; -import java.util.Map; /** * This class provides validation support for OMAG specific application properties. @@ -27,51 +24,6 @@ public class OMAGServerProperties { */ // @NotNull private Resource serverConfigFile; - - /** - * Server bean represents simplified OMAG server configuration model used for YAML based application configuration. - */ - private Server server; - - @Getter - @Setter - public static class Server { - private String name; - private ServerTypeClassification type; - private String user; - private int maxPageSize; - private RepositoryServices repositoryServices; - } - - @Getter - @Setter - public static class RepositoryServices { - private List auditLogConnectors; - private LocalRepository localRepository; - } - - @Getter - @Setter - public static class Connector { - private String providerClassName; - private Map configurationProperties; - private Endpoint endpoint; - } - - @Getter - @Setter - public static class LocalRepository { - String metadataCollectionId; - Connector localConnection; - Connector remoteConnection; - OpenMetadataExchangeRule eventsToSave; - OpenMetadataExchangeRule eventsToSend; - } - - @Getter - @Setter - public static class Endpoint { - private String address; - } + private OMAGServerConfig serverConfig; } diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/Extensions.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/Extensions.java index bc15e3352d1..8bd5ff7e49e 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/Extensions.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/Extensions.java @@ -1,12 +1,9 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* Copyright Contributors to the ODPi Egeria project. */ package org.odpi.openmetadata.serverchassis.springboot.constants; import java.io.Serializable; import java.util.stream.Stream; - -/** - * Created by YM21WO on egeria - * on 17/08/2023 at 11:19 - */ public enum Extensions implements Serializable { YAML(".yaml"), diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties index 8008c38b6d4..6b4cfe49351 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties @@ -16,6 +16,6 @@ management.endpoint.metrics.enabled=true management.endpoints.web.exposure.include=metrics,health #omag.server-config-file=classpath:samples/metadata-repository-server.json -omag.server-config-file=classpath:samples/metadata-repository-server2.yml +omag.server-config-file=classpath:samples/metadata-repository-server.yml diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml index 4592b9793d5..323b6d34888 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml @@ -1,27 +1,31 @@ --- omag: - server: - max-page-size: 1000 - name: cocoMDS1 - user: system - type: OMAG_SERVER - repository-services: - audit-log-connectors: - - provider-class-name: "org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.console.ConsoleAuditLogStoreProvider" - configuration-properties: + server-config: + localServerName: cocoMDS1 + localServerUserId: OMAGServer + maxPageSize: 1000 + repositoryServicesConfig: + auditLogConnections: + - connectorType: + connectorProviderClassName: >- + org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.console.ConsoleAuditLogStoreProvider + configurationProperties: supportedSeverities: - Error - dummySecretProperty: "${app-secrets.dummyPassword}" - local-repository: - metadata-collection-id: "838005fc-e09f-4688-af89-58254a155cc6" - local-connection: - provider-class-name: "org.odpi.openmetadata.adapters.repositoryservices.inmemory.repositoryconnector.InMemoryOMRSRepositoryConnectorProvider" - remote-connection: - provider-class-name: "org.odpi.openmetadata.adapters.repositoryservices.rest.repositoryconnector.OMRSRESTRepositoryConnectorProvider" + localRepositoryConfig: + metadataCollectionId: 838005fc-e09f-4688-af89-58254a155cc6 + localRepositoryLocalConnection: + connectorType: + connectorProviderClassName: >- + org.odpi.openmetadata.adapters.repositoryservices.inmemory.repositoryconnector.InMemoryOMRSRepositoryConnectorProvider + localRepositoryRemoteConnection: + connectorType: + connectorProviderClassName: >- + org.odpi.openmetadata.adapters.repositoryservices.rest.repositoryconnector.OMRSRESTRepositoryConnectorProvider endpoint: - address: "https://localhost:9443/servers/cocoMDS1" - events-to-save: ALL - events-to-send: ALL + address: https://localhost:9443/servers/cocoMDS1 + eventsToSaveRule: ALL + eventsToSendRule: ALL --- app-secrets: dummyPassword: top-secret \ No newline at end of file diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server2.yml b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server_minimal.yml similarity index 74% rename from open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server2.yml rename to open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server_minimal.yml index 06d6b3555af..0f199e570b3 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server2.yml +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server_minimal.yml @@ -1,37 +1,26 @@ --- -class: OMAGServerConfig localServerName: cocoMDS1 -localServerType: Open Metadata and Governance Server localServerUserId: OMAGServer maxPageSize: 1000 repositoryServicesConfig: - class: RepositoryServicesConfig auditLogConnections: - - class: Connection - connectorType: - class: ConnectorType + - connectorType: connectorProviderClassName: >- org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.console.ConsoleAuditLogStoreProvider configurationProperties: supportedSeverities: - Error localRepositoryConfig: - class: LocalRepositoryConfig metadataCollectionId: 838005fc-e09f-4688-af89-58254a155cc6 localRepositoryLocalConnection: - class: Connection connectorType: - class: ConnectorType connectorProviderClassName: >- org.odpi.openmetadata.adapters.repositoryservices.inmemory.repositoryconnector.InMemoryOMRSRepositoryConnectorProvider localRepositoryRemoteConnection: - class: Connection connectorType: - class: ConnectorType connectorProviderClassName: >- org.odpi.openmetadata.adapters.repositoryservices.rest.repositoryconnector.OMRSRESTRepositoryConnectorProvider endpoint: - class: Endpoint address: https://localhost:9443/servers/cocoMDS1 eventsToSaveRule: ALL eventsToSendRule: ALL \ No newline at end of file From f2092cf47825d92f82e2c930089fee0111e498ea Mon Sep 17 00:00:00 2001 From: Ljupcho Palashevski Date: Tue, 5 Sep 2023 23:56:58 +0200 Subject: [PATCH 12/13] Cleanup, further code hardening Signed-off-by: Ljupcho Palashevski --- bom/build.gradle | 2 + .../server-chassis-spring/build.gradle | 7 +-- .../springboot/config/OMAGConfigHelper.java | 46 ++++++++----------- .../config/ObjectMapperConfiguration.java | 8 ---- .../springboot/constants/Extensions.java | 32 ------------- .../constants/SupportedConfigTypes.java | 11 +++++ .../src/main/resources/application-dev.yml | 7 ++- ...al.yml => metadata-repository-server.yaml} | 13 +++++- .../samples/metadata-repository-server.yml | 13 +----- 9 files changed, 52 insertions(+), 87 deletions(-) delete mode 100644 open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/Extensions.java create mode 100644 open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/SupportedConfigTypes.java rename open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/{metadata-repository-server_minimal.yml => metadata-repository-server.yaml} (74%) diff --git a/bom/build.gradle b/bom/build.gradle index dd74b1a1c59..0e7b9ffd1c8 100644 --- a/bom/build.gradle +++ b/bom/build.gradle @@ -113,6 +113,7 @@ ext { reactivestreamsVersion = '1.0.4' springdocStarterVersion = '2.1.0' jacocoVersion = '0.8.8' + snakeYamlVersion = '2.2' } dependencies { @@ -282,6 +283,7 @@ dependencies { api("org.slf4j:slf4j-simple:${slf4jVersion}") api("org.testng:testng:${testngVersion}") api("org.hamcrest:hamcrest:${hamcrestVersion}") + api("org.yaml:snakeyaml:${snakeYamlVersion}") // Explicitly enforced versions of transitive dependencies to mitigate potential CVEs reported by static security scans. //TODO: Remove dependency line below in case the new parent library is updated and pulls good version. diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle b/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle index 39c70654306..d7d4bf3779b 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle @@ -26,28 +26,25 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-annotations' implementation 'jakarta.servlet:jakarta.servlet-api' implementation 'org.projectlombok:lombok' -// implementation 'org.yaml:snakeyaml' + implementation 'org.yaml:snakeyaml' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' + implementation 'com.google.guava:guava' annotationProcessor 'org.projectlombok:lombok' runtimeOnly 'io.micrometer:micrometer-registry-prometheus' runtimeOnly 'ch.qos.logback:logback-classic' runtimeOnly 'org.hibernate:hibernate-validator' compileOnly 'io.swagger.core.v3:swagger-annotations-jakarta' -// runtimeOnly 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' runtimeOnly 'com.fasterxml.jackson.core:jackson-core' compileOnly project(':open-metadata-implementation:common-services:ffdc-services') -// implementation project(':open-metadata-implementation:frameworks:open-connector-framework') implementation project(':open-metadata-implementation:adapters:authentication-plugins:http-helper') implementation project(':open-metadata-implementation:admin-services:admin-services-api') implementation project(':open-metadata-implementation:server-operations:server-operations-api') implementation project(':open-metadata-implementation:server-operations:server-operations-server') - implementation project(path: ':open-metadata-implementation:adapters:open-connectors:connector-configuration-factory') implementation project(path: ':open-metadata-implementation:repository-services:repository-services-implementation') - implementation 'com.google.guava:guava' /* IN DEVELOPMENT */ /* Pulling dependencies for some fo the sub-systems enabling 'Metadata Access Store' services */ diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java index 128b863da0f..665ae92006c 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java @@ -9,13 +9,14 @@ import org.odpi.openmetadata.adapters.repositoryservices.ConnectorConfigurationFactory; import org.odpi.openmetadata.adminservices.configuration.properties.OMAGServerConfig; import org.odpi.openmetadata.repositoryservices.admin.OMRSConfigurationFactory; -import org.odpi.openmetadata.serverchassis.springboot.constants.Extensions; +import org.odpi.openmetadata.serverchassis.springboot.constants.SupportedConfigTypes; import org.odpi.openmetadata.serverchassis.springboot.exception.OMAGServerActivationError; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Configuration; import org.springframework.core.io.Resource; +import org.yaml.snakeyaml.Yaml; import java.io.IOException; import java.util.Objects; @@ -25,8 +26,7 @@ @Configuration @Slf4j public class OMAGConfigHelper { - public static final String WRONG_EXTENSION_MESSAGE = "Unallowed config file extension, " + - "allowed config file extensions are: " + Extensions.stream(); + @Getter final OMAGServerProperties serverProperties; private final ObjectMapper jsonObjectMapper; @@ -61,26 +61,17 @@ public void loadConfig() throws OMAGServerActivationError { Resource serverConfigFile = serverProperties.getServerConfigFile(); - //TODO: This is POC implementation. - // Better way to identify content type may be required. - - if (!isJsonConfigurationFile(serverConfigFile) && !isYamlConfigurationFile(serverConfigFile)) { - log.error(WRONG_EXTENSION_MESSAGE); - throw new OMAGServerActivationError(WRONG_EXTENSION_MESSAGE); - } - - log.info("{} based configuration with server-config-file {}", getFileExtension(serverConfigFile), serverConfigFile); + log.info("Configuration {}", serverConfigFile); if (isJsonConfigurationFile(serverConfigFile)) { - omagServerConfig = jsonObjectMapper.reader() - .readValue(serverConfigFile.getInputStream(), OMAGServerConfig.class); - + omagServerConfig = jsonObjectMapper.reader().readValue(serverConfigFile.getInputStream(), OMAGServerConfig.class); } else if (isYamlConfigurationFile(serverConfigFile)) { - //TODO: This is POC implementation. We need better code to deal with yaml i.e. one option is to use Jackson and Yaml data format. -// Yaml yaml = new Yaml(); -// omagServerConfig = yaml.loadAs(serverConfigFile.getInputStream(), OMAGServerConfig.class); - omagServerConfig = yamlObjectMapper.reader() - .readValue(serverConfigFile.getInputStream(), OMAGServerConfig.class); + omagServerConfig = yamlObjectMapper.reader().readValue(serverConfigFile.getInputStream(), OMAGServerConfig.class); + } else if (isYmlConfigurationFile(serverConfigFile)) { + Yaml yaml = new Yaml(); + omagServerConfig = yaml.loadAs(serverConfigFile.getInputStream(), OMAGServerConfig.class); + } else { + throw new OMAGServerActivationError("Configuration file is not supported"); } } else if (isPropertiesConfiguration()) { @@ -89,17 +80,15 @@ public void loadConfig() throws OMAGServerActivationError { omagServerConfig = serverProperties.getServerConfig(); } - log.info("Configuration document for server: {} - loaded successfully", omagServerConfig.getLocalServerName()); - } catch (IOException e) { log.info("Configuration document cannot be loaded from the resource provided - check application configuration"); throw new OMAGServerActivationError( - String.format("Configuration document cannot be loaded from the resource provided - check application configuration"), e); + "Configuration document cannot be loaded from the resource provided - check application configuration", e); } } private String getFileExtension(Resource serverConfigFile) { - return Files.getFileExtension(serverConfigFile.getFilename()); + return Files.getFileExtension(Objects.requireNonNull(serverConfigFile.getFilename())); } private boolean isConfigurationFileProvided() { @@ -107,12 +96,13 @@ private boolean isConfigurationFileProvided() { } private boolean isJsonConfigurationFile(Resource serverConfigFile) { - return Extensions.JSON.name().equalsIgnoreCase(Objects.requireNonNull(getFileExtension(serverConfigFile))); + return SupportedConfigTypes.JSON.name().equalsIgnoreCase(Objects.requireNonNull(getFileExtension(serverConfigFile))); } - private boolean isYamlConfigurationFile(Resource serverConfigFile) { - return Extensions.YAML.name().equalsIgnoreCase(Objects.requireNonNull(getFileExtension(serverConfigFile))) - || Extensions.YML.name().equalsIgnoreCase(Objects.requireNonNull(getFileExtension(serverConfigFile))); + return SupportedConfigTypes.YAML.name().equalsIgnoreCase(Objects.requireNonNull(getFileExtension(serverConfigFile))); + } + private boolean isYmlConfigurationFile(Resource serverConfigFile) { + return SupportedConfigTypes.YML.name().equalsIgnoreCase(Objects.requireNonNull(getFileExtension(serverConfigFile))); } private boolean isPropertiesConfiguration() { diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/ObjectMapperConfiguration.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/ObjectMapperConfiguration.java index 882d131424f..e0744704ac2 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/ObjectMapperConfiguration.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/ObjectMapperConfiguration.java @@ -11,12 +11,10 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import lombok.extern.slf4j.Slf4j; -import org.odpi.openmetadata.serverchassis.springboot.constants.Extensions; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder; -import org.springframework.util.StringUtils; import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; @@ -53,9 +51,6 @@ private static ObjectMapper newObjectMapper(Jackson2ObjectMapperBuilder jackson2 @Bean @Qualifier("jsonObjectMapper") ObjectMapper objectMapper(Jackson2ObjectMapperBuilder jackson2ObjectMapperBuilder) { - - log.info("{}{}", StringUtils.capitalize(Extensions.JSON.name()), START_LOG_SUFIX); - return newObjectMapper(jackson2ObjectMapperBuilder); } @@ -65,9 +60,6 @@ ObjectMapper objectMapper(Jackson2ObjectMapperBuilder jackson2ObjectMapperBuilde @Bean @Qualifier("yamlObjectMapper") public ObjectMapper yamlMapper() { - - log.info("{}{}", StringUtils.capitalize(Extensions.YAML.name()), START_LOG_SUFIX); - return YAMLMapper.builder() .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) .disable(YAMLGenerator.Feature.ALLOW_LONG_KEYS) diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/Extensions.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/Extensions.java deleted file mode 100644 index 8bd5ff7e49e..00000000000 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/Extensions.java +++ /dev/null @@ -1,32 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 */ -/* Copyright Contributors to the ODPi Egeria project. */ -package org.odpi.openmetadata.serverchassis.springboot.constants; - -import java.io.Serializable; -import java.util.stream.Stream; -public enum Extensions implements Serializable { - - YAML(".yaml"), - YML(".yml"), - JSON(".json"); - - - private final String value; - - Extensions(String value) { - this.value = value; - } - - public static String stream() { - StringBuffer stringBuffer = new StringBuffer(); - Stream.of(Extensions.values()) - .forEach(e -> stringBuffer.append(e.getValue()).append(", ")); - - return stringBuffer.toString(); - } - - private String getValue() { - return value; - } - -} diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/SupportedConfigTypes.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/SupportedConfigTypes.java new file mode 100644 index 00000000000..9d70bbbafa7 --- /dev/null +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/SupportedConfigTypes.java @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* Copyright Contributors to the ODPi Egeria project. */ +package org.odpi.openmetadata.serverchassis.springboot.constants; + +import java.io.Serializable; + +public enum SupportedConfigTypes implements Serializable { + YAML, + YML, + JSON; +} diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml index 323b6d34888..cfb636acc7d 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml @@ -1,5 +1,8 @@ --- omag: +# server-config-file: classpath:samples/metadata-repository-server.json +# server-config-file: classpath:samples/metadata-repository-server.yaml +# server-config-file: classpath:samples/metadata-repository-server.yml server-config: localServerName: cocoMDS1 localServerUserId: OMAGServer @@ -22,10 +25,12 @@ omag: connectorType: connectorProviderClassName: >- org.odpi.openmetadata.adapters.repositoryservices.rest.repositoryconnector.OMRSRESTRepositoryConnectorProvider + configurationProperties: + password: "${app-secrets.dummyPasswordDemo}" endpoint: address: https://localhost:9443/servers/cocoMDS1 eventsToSaveRule: ALL eventsToSendRule: ALL --- app-secrets: - dummyPassword: top-secret \ No newline at end of file + dummyPasswordDemo: top-secret \ No newline at end of file diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server_minimal.yml b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yaml similarity index 74% rename from open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server_minimal.yml rename to open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yaml index 0f199e570b3..06d6b3555af 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server_minimal.yml +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yaml @@ -1,26 +1,37 @@ --- +class: OMAGServerConfig localServerName: cocoMDS1 +localServerType: Open Metadata and Governance Server localServerUserId: OMAGServer maxPageSize: 1000 repositoryServicesConfig: + class: RepositoryServicesConfig auditLogConnections: - - connectorType: + - class: Connection + connectorType: + class: ConnectorType connectorProviderClassName: >- org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.console.ConsoleAuditLogStoreProvider configurationProperties: supportedSeverities: - Error localRepositoryConfig: + class: LocalRepositoryConfig metadataCollectionId: 838005fc-e09f-4688-af89-58254a155cc6 localRepositoryLocalConnection: + class: Connection connectorType: + class: ConnectorType connectorProviderClassName: >- org.odpi.openmetadata.adapters.repositoryservices.inmemory.repositoryconnector.InMemoryOMRSRepositoryConnectorProvider localRepositoryRemoteConnection: + class: Connection connectorType: + class: ConnectorType connectorProviderClassName: >- org.odpi.openmetadata.adapters.repositoryservices.rest.repositoryconnector.OMRSRESTRepositoryConnectorProvider endpoint: + class: Endpoint address: https://localhost:9443/servers/cocoMDS1 eventsToSaveRule: ALL eventsToSendRule: ALL \ No newline at end of file diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yml b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yml index 06d6b3555af..0f199e570b3 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yml +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/samples/metadata-repository-server.yml @@ -1,37 +1,26 @@ --- -class: OMAGServerConfig localServerName: cocoMDS1 -localServerType: Open Metadata and Governance Server localServerUserId: OMAGServer maxPageSize: 1000 repositoryServicesConfig: - class: RepositoryServicesConfig auditLogConnections: - - class: Connection - connectorType: - class: ConnectorType + - connectorType: connectorProviderClassName: >- org.odpi.openmetadata.adapters.repositoryservices.auditlogstore.console.ConsoleAuditLogStoreProvider configurationProperties: supportedSeverities: - Error localRepositoryConfig: - class: LocalRepositoryConfig metadataCollectionId: 838005fc-e09f-4688-af89-58254a155cc6 localRepositoryLocalConnection: - class: Connection connectorType: - class: ConnectorType connectorProviderClassName: >- org.odpi.openmetadata.adapters.repositoryservices.inmemory.repositoryconnector.InMemoryOMRSRepositoryConnectorProvider localRepositoryRemoteConnection: - class: Connection connectorType: - class: ConnectorType connectorProviderClassName: >- org.odpi.openmetadata.adapters.repositoryservices.rest.repositoryconnector.OMRSRESTRepositoryConnectorProvider endpoint: - class: Endpoint address: https://localhost:9443/servers/cocoMDS1 eventsToSaveRule: ALL eventsToSendRule: ALL \ No newline at end of file From 363d17c0cf2666b006c63d9431728d6b88861c71 Mon Sep 17 00:00:00 2001 From: Ljupcho Palashevski Date: Thu, 7 Sep 2023 10:24:59 +0200 Subject: [PATCH 13/13] Further enhancements, code cleanup and documentation improvements. Signed-off-by: Ljupcho Palashevski --- .../server-chassis-spring/README.md | 56 +++++++++++--- .../server-chassis-spring/build.gradle | 76 ++++++++++++++----- .../springboot/config/OMAGConfigHelper.java | 68 ++++++++++------- .../config/OMAGServerProperties.java | 6 +- .../config/OMAGServicesConfiguration.java | 16 +--- .../config/ObjectMapperConfiguration.java | 3 +- .../constants/SupportedConfigTypes.java | 11 --- .../main/resources/application-dev.properties | 5 +- .../src/main/resources/application-dev.yml | 2 +- .../resources/application-test.properties | 3 +- 10 files changed, 159 insertions(+), 87 deletions(-) delete mode 100644 open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/SupportedConfigTypes.java diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/README.md b/open-metadata-implementation/server-chassis/server-chassis-spring/README.md index 6d9d5bc5a46..3065f8500f9 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/README.md +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/README.md @@ -1,5 +1,5 @@ - + # OMAG Server Chassis Spring @@ -15,18 +15,54 @@ To build the boot application jar from the current module use: ### Starting the application locally -You can run the application locally from this module with java using following command: +You can run the application locally in the current module with java using following command: ```bash -java -jar build/libs/server-chassis-spring-*-SNAPSHOT.jar --omag.server-config=classpath:samples/metadata-repository-server.json --server.port=9080 --server.ssl.enabled=false +java -jar build/libs/server-chassis-spring-*-SNAPSHOT.jar --omag.server-config-file=classpath:samples/metadata-repository-server.yml --server.port=9080 --server.ssl.enabled=false ``` -The command will run the application using provided parameters. For demo purpose we turn ssl off and run the application on http port 9080. +``` + Project Egeria - Open Metadata and Governance + ____ __ ___ ___ ______ _____ + / __ \ / |/ // | / ____/ / ___/ ___ ____ _ __ ___ ____ + / / / // /|_/ // /| | / / __ \__ \ / _ \ / __/| | / // _ \ / __/ + / /_/ // / / // ___ |/ /_/ / ___/ // __// / | |/ // __// / + \____//_/ /_//_/ |_|\____/ /____/ \___//_/ |___/ \___//_/ + + :: Powered by Spring Boot (v3.1.1) :: + +2023-09-07T10:08:05.779+02:00 INFO 4334 --- [ main] o.o.o.s.springboot.OMAGServer : Starting OMAGServer using Java 17.0.8 with PID 4334 (/Developer/egeria/open-metadata-implementation/server-chassis/server-chassis-spring/build/libs/server-chassis-spring-4.4-SNAPSHOT.jar started by DEVELOPER in /Developer/egeria/open-metadata-implementation/server-chassis/server-chassis-spring) +2023-09-07T10:08:05.781+02:00 INFO 4334 --- [ main] o.o.o.s.springboot.OMAGServer : No active profile set, falling back to 1 default profile: "default" +2023-09-07T10:08:07.435+02:00 INFO 4334 --- [ main] o.s.b.w.embedded.tomcat.TomcatWebServer : Tomcat initialized with port(s): 9080 (http) +2023-09-07T10:08:07.444+02:00 INFO 4334 --- [ main] o.apache.catalina.core.StandardService : Starting service [Tomcat] +2023-09-07T10:08:07.444+02:00 INFO 4334 --- [ main] o.apache.catalina.core.StandardEngine : Starting Servlet engine: [Apache Tomcat/10.1.10] +2023-09-07T10:08:07.505+02:00 INFO 4334 --- [ main] o.a.c.c.C.[Tomcat].[localhost].[/] : Initializing Spring embedded WebApplicationContext +2023-09-07T10:08:07.506+02:00 INFO 4334 --- [ main] w.s.c.ServletWebServerApplicationContext : Root WebApplicationContext: initialization completed in 1681 ms +2023-09-07T10:08:08.228+02:00 INFO 4334 --- [ main] EnvironmentConfiguration$$SpringCGLIB$$0 : SSL configuration started working directory: /Developer/egeria/open-metadata-implementation/server-chassis/server-chassis-spring +2023-09-07T10:08:08.685+02:00 INFO 4334 --- [ main] o.s.b.a.e.web.EndpointLinksResolver : Exposing 2 endpoint(s) beneath base path '/actuator' +2023-09-07T10:08:08.749+02:00 INFO 4334 --- [ main] o.s.b.w.embedded.tomcat.TomcatWebServer : Tomcat started on port(s): 9080 (http) with context path '' +2023-09-07T10:08:08.763+02:00 INFO 4334 --- [ main] o.o.o.s.springboot.OMAGServer : Started OMAGServer in 3.384 seconds (process running for 3.851) +2023-09-07T10:08:08.786+02:00 INFO 4334 --- [ main] o.o.o.s.s.config.OMAGConfigHelper : Using configuration from class path resource [samples/metadata-repository-server.yml] +2023-09-07T10:08:09.024+02:00 INFO 4334 --- [ main] o.o.o.s.springboot.OMAGServer : Sending activation request for server: cocoMDS1 and user: OMAGServer +2023-09-07T10:08:09.208+02:00 INFO 4334 --- [ main] o.o.o.s.springboot.OMAGServer : Activation succeeded for server: cocoMDS1 + +``` + +The command will run the application using parameters provided. The OMAG server instance is created and activated using the configuration file supplied via application property `omag.server-config-file`. +
To demonstrate basic functionality, we turn ssl off `server.ssl.enabled=false` and run the application on http port 9080 `server.port=9080`. + +### Quick-start configuration properties + +| Property name | Environment variable | Description | +|-------------------------|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| omag.server-config-file | OMAG_SERVERCONFIGFILE | [REQUIRED] The [OMAGServerConfig document](https://egeria-project.org/concepts/configuration-document/) file location.
Note the value should be defined as spring Resource i.e. starting with `classpath:` or `file:`
Both JSON and YAML files are supported. See [samples](src%2Fmain%2Fresources%2Fsamples) for sample configuration files. | +| server.port | SERVER_PORT | Configures port used by the embedded Tomcat server. | +| server.ssl.enabled | SERVER_SSL_ENABLED | Configures if SSL should be enabled for the embedded Tomcat server. | -### Configuration properties +Application can be further customized by setting supported spring boot and application specific properties. +
The default configuration that is already packaged within the JAR distribution is [application.properties](src%2Fmain%2Fresources%2Fapplication.properties). +
Following Spring application [external configuration](https://docs.spring.io/spring-boot/docs/current/reference/html/features.html#features.external-config) feature, all the properties can be customized at deploy/run time as shown in the quick-start example above. -| Property name | Environment variable | | Description | -|--------------------|----------------------|:----|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| omag.server-config | OMAG_SERVER-CONFIG | | The [OMAGServerConfig document](https://egeria-project.org/concepts/configuration-document/) json file location **(Required)**. Note the value should be spring Resource i.e. starting with `classpath:` or `file:` | -| server.port | SERVER_PORT | | Configures port used by the embedded Tomcat server | -| server.ssl.enabled | SERVER_SSL_ENABLED | | Configures if SSL should be enabled for the embedded Tomcat server | \ No newline at end of file +---- +License: [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/), +Copyright Contributors to the ODPi Egeria project. \ No newline at end of file diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle b/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle index d7d4bf3779b..dc99453e82d 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/build.gradle @@ -42,24 +42,66 @@ dependencies { implementation project(':open-metadata-implementation:admin-services:admin-services-api') implementation project(':open-metadata-implementation:server-operations:server-operations-api') implementation project(':open-metadata-implementation:server-operations:server-operations-server') - implementation project(path: ':open-metadata-implementation:adapters:open-connectors:connector-configuration-factory') - implementation project(path: ':open-metadata-implementation:repository-services:repository-services-implementation') - /* IN DEVELOPMENT */ - - /* Pulling dependencies for some fo the sub-systems enabling 'Metadata Access Store' services */ - - runtimeOnly project(':open-metadata-implementation:repository-services:repository-services-spring') - runtimeOnly project(':open-metadata-implementation:access-services:asset-manager:asset-manager-spring') - runtimeOnly project(':open-metadata-implementation:access-services:asset-catalog:asset-catalog-spring') - runtimeOnly project(':open-metadata-implementation:access-services:data-manager:data-manager-spring') - - /* Pulling dependencies for some fo the sub-systems enabling 'Integration Daemon' services to test Database Integrator via JDBC integration connector */ - -// runtimeOnly project(':open-metadata-implementation:integration-services:database-integrator:database-integrator-server') -// runtimeOnly 'org.odpi.egeria:egeria-connector-resource-jdbc:1.1' -// runtimeOnly 'org.odpi.egeria:egeria-connector-integration-jdbc:1.1' -// runtimeOnly 'com.oracle.database.jdbc:ojdbc10:19.19.0.0' + if (!project.hasProperty("adminChassisOnly")) { + runtimeOnly project(':open-metadata-implementation:view-services:server-author-view:server-author-view-spring') + runtimeOnly project(':open-metadata-implementation:view-services:glossary-author-view:glossary-author-view-spring') + runtimeOnly project(':open-metadata-implementation:view-services:glossary-browser:glossary-browser-spring') + runtimeOnly project(':open-metadata-implementation:view-services:glossary-workflow:glossary-workflow-spring') + runtimeOnly project(':open-metadata-implementation:view-services:my-profile:my-profile-spring') + runtimeOnly project(':open-metadata-implementation:view-services:rex-view:rex-view-spring') + runtimeOnly project(':open-metadata-implementation:view-services:tex-view:tex-view-spring') + runtimeOnly project(':open-metadata-implementation:view-services:dino-view:dino-view-spring') + runtimeOnly project(':open-metadata-implementation:access-services:security-officer:security-officer-spring') + runtimeOnly project(':open-metadata-implementation:access-services:security-manager:security-manager-spring') + runtimeOnly project(':open-metadata-implementation:access-services:data-manager:data-manager-spring') + runtimeOnly project(':open-metadata-implementation:access-services:glossary-view:glossary-view-spring') + runtimeOnly project(':open-metadata-implementation:access-services:asset-lineage:asset-lineage-spring') + runtimeOnly project(':open-metadata-implementation:access-services:data-engine:data-engine-spring') + runtimeOnly project(':open-metadata-implementation:access-services:subject-area:subject-area-spring') + runtimeOnly project(':open-metadata-implementation:access-services:asset-catalog:asset-catalog-spring') + runtimeOnly project(':open-metadata-implementation:access-services:governance-program:governance-program-spring') + runtimeOnly project(':open-metadata-implementation:access-services:governance-engine:governance-engine-spring') + runtimeOnly project(':open-metadata-implementation:access-services:discovery-engine:discovery-engine-spring') + runtimeOnly project(':open-metadata-implementation:access-services:stewardship-action:stewardship-action-spring') + runtimeOnly project(':open-metadata-implementation:access-services:community-profile:community-profile-spring') + runtimeOnly project(':open-metadata-implementation:access-services:design-model:design-model-spring') + runtimeOnly project(':open-metadata-implementation:access-services:data-privacy:data-privacy-spring') + runtimeOnly project(':open-metadata-implementation:access-services:it-infrastructure:it-infrastructure-spring') + runtimeOnly project(':open-metadata-implementation:access-services:project-management:project-management-spring') + runtimeOnly project(':open-metadata-implementation:access-services:dev-ops:dev-ops-spring') + runtimeOnly project(':open-metadata-implementation:access-services:software-developer:software-developer-spring') + runtimeOnly project(':open-metadata-implementation:access-services:digital-architecture:digital-architecture-spring') + runtimeOnly project(':open-metadata-implementation:access-services:digital-service:digital-service-spring') + runtimeOnly project(':open-metadata-implementation:access-services:data-science:data-science-spring') + runtimeOnly project(':open-metadata-implementation:access-services:asset-consumer:asset-consumer-spring') + runtimeOnly project(':open-metadata-implementation:access-services:asset-manager:asset-manager-spring') + runtimeOnly project(':open-metadata-implementation:access-services:asset-owner:asset-owner-spring') + runtimeOnly project(':open-metadata-implementation:engine-services:asset-analysis:asset-analysis-spring') + runtimeOnly project(':open-metadata-implementation:engine-services:repository-governance:repository-governance-spring') + runtimeOnly project(':open-metadata-implementation:engine-services:governance-action:governance-action-spring') + runtimeOnly project(':open-metadata-implementation:integration-services:security-integrator:security-integrator-spring') + runtimeOnly project(':open-metadata-implementation:integration-services:organization-integrator:organization-integrator-spring') + runtimeOnly project(':open-metadata-implementation:integration-services:infrastructure-integrator:infrastructure-integrator-spring') + runtimeOnly project(':open-metadata-implementation:integration-services:lineage-integrator:lineage-integrator-spring') + runtimeOnly project(':open-metadata-implementation:integration-services:files-integrator:files-integrator-spring') + runtimeOnly project(':open-metadata-implementation:integration-services:display-integrator:display-integrator-spring') + runtimeOnly project(':open-metadata-implementation:integration-services:database-integrator:database-integrator-spring') + runtimeOnly project(':open-metadata-implementation:integration-services:analytics-integrator:analytics-integrator-spring') + runtimeOnly project(':open-metadata-implementation:integration-services:api-integrator:api-integrator-spring') + runtimeOnly project(':open-metadata-implementation:integration-services:catalog-integrator:catalog-integrator-spring') + runtimeOnly project(':open-metadata-implementation:integration-services:search-integrator:search-integrator-spring') + runtimeOnly project(':open-metadata-implementation:integration-services:topic-integrator:topic-integrator-spring') + runtimeOnly project(':open-metadata-implementation:repository-services:repository-services-spring') + runtimeOnly project(':open-metadata-conformance-suite:open-metadata-conformance-suite-spring') + runtimeOnly project(':open-metadata-implementation:framework-services:ocf-metadata-management:ocf-metadata-spring') + runtimeOnly project(':open-metadata-implementation:framework-services:gaf-metadata-management:gaf-metadata-spring') + runtimeOnly project(':open-metadata-implementation:framework-services:oif-metadata-management:oif-metadata-spring') + runtimeOnly project(':open-metadata-implementation:governance-servers:data-engine-proxy-services:data-engine-proxy-services-spring') + runtimeOnly project(':open-metadata-implementation:governance-servers:integration-daemon-services:integration-daemon-services-spring') + runtimeOnly project(':open-metadata-implementation:governance-servers:engine-host-services:engine-host-services-spring') + runtimeOnly project(':open-metadata-implementation:governance-servers:open-lineage-services:open-lineage-services-spring') + } testImplementation 'org.junit.jupiter:junit-jupiter-api' testImplementation 'org.springframework.boot:spring-boot-test' diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java index 665ae92006c..9658b9a4ba7 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGConfigHelper.java @@ -3,13 +3,11 @@ package org.odpi.openmetadata.serverchassis.springboot.config; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.exc.InvalidTypeIdException; import com.google.common.io.Files; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.odpi.openmetadata.adapters.repositoryservices.ConnectorConfigurationFactory; import org.odpi.openmetadata.adminservices.configuration.properties.OMAGServerConfig; -import org.odpi.openmetadata.repositoryservices.admin.OMRSConfigurationFactory; -import org.odpi.openmetadata.serverchassis.springboot.constants.SupportedConfigTypes; import org.odpi.openmetadata.serverchassis.springboot.exception.OMAGServerActivationError; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -18,10 +16,13 @@ import org.springframework.core.io.Resource; import org.yaml.snakeyaml.Yaml; -import java.io.IOException; import java.util.Objects; - +/** + * This class provides support for loading OMAGServerConfig document from different configuration options/styles. + * It supports JSON and YAML file based OMAG server configuration; + * Additionally provides EXPERIMENTAL feature: configure OMAG server using native spring configuration properties. + */ @EnableConfigurationProperties(OMAGServerProperties.class) @Configuration @Slf4j @@ -33,25 +34,30 @@ public class OMAGConfigHelper { private final ObjectMapper yamlObjectMapper; @Getter OMAGServerConfig omagServerConfig; - ConnectorConfigurationFactory connectorConfigurationFactory; - OMRSConfigurationFactory omrsConfigurationFactory; + static final String YAML = "yaml"; + static final String YML = "yml"; + static final String JSON = "json"; + /** + * Constructor that injects required beans such as omag application properties and jackson object mappers. + * + * @param properties application properties prefixed with 'omag.' + * @param jsonObjectMapper pre-configured object mapper bean for json processing + * @param yamlObjectMapper pre-configured object mapper bean for yaml processing + */ @Autowired public OMAGConfigHelper(OMAGServerProperties properties, @Qualifier("jsonObjectMapper") ObjectMapper jsonObjectMapper, - @Qualifier("yamlObjectMapper") ObjectMapper yamlObjectMapper, - ConnectorConfigurationFactory connectorConfigurationFactory, - OMRSConfigurationFactory omrsConfigurationFactory) { + @Qualifier("yamlObjectMapper") ObjectMapper yamlObjectMapper) { this.serverProperties = properties; this.jsonObjectMapper = jsonObjectMapper; this.yamlObjectMapper = yamlObjectMapper; - this.connectorConfigurationFactory = connectorConfigurationFactory; - this.omrsConfigurationFactory = omrsConfigurationFactory; } /** - * Method implementing logic for deciding proper configuration source - * and loading its content into configuration object + * Implements logic for deciding proper configuration source + * and loading its content into OMAGServerConfig configuration object + * * @throws OMAGServerActivationError */ public void loadConfig() throws OMAGServerActivationError { @@ -61,26 +67,38 @@ public void loadConfig() throws OMAGServerActivationError { Resource serverConfigFile = serverProperties.getServerConfigFile(); - log.info("Configuration {}", serverConfigFile); + log.info("Using configuration from {}", serverConfigFile); if (isJsonConfigurationFile(serverConfigFile)) { + // Read json based omag server configuration file + log.debug("JSON file detected, reading values using jsonObjectMapper..."); omagServerConfig = jsonObjectMapper.reader().readValue(serverConfigFile.getInputStream(), OMAGServerConfig.class); } else if (isYamlConfigurationFile(serverConfigFile)) { - omagServerConfig = yamlObjectMapper.reader().readValue(serverConfigFile.getInputStream(), OMAGServerConfig.class); - } else if (isYmlConfigurationFile(serverConfigFile)) { - Yaml yaml = new Yaml(); - omagServerConfig = yaml.loadAs(serverConfigFile.getInputStream(), OMAGServerConfig.class); + try { + // Read yaml based omag server configuration file + log.debug("YAML file detected, reading values using yamlObjectMapper..."); + omagServerConfig = yamlObjectMapper.reader().readValue(serverConfigFile.getInputStream(), OMAGServerConfig.class); + } catch (InvalidTypeIdException e) { + log.debug("yamlObjectMapper failed reading values, trying snakeYaml..."); + Yaml yaml = new Yaml(); + omagServerConfig = yaml.loadAs(serverConfigFile.getInputStream(), OMAGServerConfig.class); + } } else { throw new OMAGServerActivationError("Configuration file is not supported"); } } else if (isPropertiesConfiguration()) { - log.info("[EXPERIMENTAL] Configuring server using omag. application properties"); omagServerConfig = serverProperties.getServerConfig(); + //TODO: Continue investigation + // At this point we should have property mapped OMAGServerConfig object. + // However, due to internal spring yaml to java object marshaling logic, Connection beans get wrongly typed Map for configurationProperties: + // Our code expects Map but gets Map -- this leads to wrong connector configuration compromising the connector functionality. + // Reference to the problematic code https://github.com/spring-projects/spring-framework/blob/main/spring-beans/src/main/java/org/springframework/beans/factory/config/YamlProcessor.java#L330C11-L330C52 + // The same mapping problem is described in spring-boot issue https://github.com/spring-projects/spring-boot/issues/6180 } - } catch (IOException e) { + } catch (Exception e) { log.info("Configuration document cannot be loaded from the resource provided - check application configuration"); throw new OMAGServerActivationError( "Configuration document cannot be loaded from the resource provided - check application configuration", e); @@ -96,13 +114,11 @@ private boolean isConfigurationFileProvided() { } private boolean isJsonConfigurationFile(Resource serverConfigFile) { - return SupportedConfigTypes.JSON.name().equalsIgnoreCase(Objects.requireNonNull(getFileExtension(serverConfigFile))); + return JSON.equalsIgnoreCase(Objects.requireNonNull(getFileExtension(serverConfigFile))); } private boolean isYamlConfigurationFile(Resource serverConfigFile) { - return SupportedConfigTypes.YAML.name().equalsIgnoreCase(Objects.requireNonNull(getFileExtension(serverConfigFile))); - } - private boolean isYmlConfigurationFile(Resource serverConfigFile) { - return SupportedConfigTypes.YML.name().equalsIgnoreCase(Objects.requireNonNull(getFileExtension(serverConfigFile))); + String fileExtension = Objects.requireNonNull(getFileExtension(serverConfigFile)); + return YAML.equalsIgnoreCase(fileExtension) || YML.equalsIgnoreCase(fileExtension); } private boolean isPropertiesConfiguration() { diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java index 3929a9fb1c8..47c81712657 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServerProperties.java @@ -10,7 +10,7 @@ import org.springframework.validation.annotation.Validated; /** - * This class provides validation support for OMAG specific application properties. + * Provides validation support for OMAG specific application properties. */ @ConfigurationProperties(prefix = "omag", ignoreUnknownFields=false) @Getter @@ -24,6 +24,10 @@ public class OMAGServerProperties { */ // @NotNull private Resource serverConfigFile; + /** + * Application property that maps to OMAGServerConfig document directly. + * USED ONLY TO EXPERIMENT DUE TO UNDERLYING SPRING YAML CONFIGURATION PROCESSING BEHAVIOUR THAT COMPROMISES THE CURRENT FUNCTIONALITY. + */ private OMAGServerConfig serverConfig; } diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServicesConfiguration.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServicesConfiguration.java index e861b5cd965..dca69ad3630 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServicesConfiguration.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/OMAGServicesConfiguration.java @@ -2,8 +2,6 @@ /* Copyright Contributors to the ODPi Egeria project. */ package org.odpi.openmetadata.serverchassis.springboot.config; -import org.odpi.openmetadata.adapters.repositoryservices.ConnectorConfigurationFactory; -import org.odpi.openmetadata.repositoryservices.admin.OMRSConfigurationFactory; import org.odpi.openmetadata.serveroperations.server.OMAGServerOperationalServices; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -11,7 +9,7 @@ /** - * This class provides configuration supporting OMAG related components required by the application. + * Provides configuration supporting OMAG related components required by the application. */ @Configuration public class OMAGServicesConfiguration { @@ -23,16 +21,8 @@ public class OMAGServicesConfiguration { * @see OMAGServerOperationalServices */ @Primary - @Bean(name = {"platformOperationalServices"}) - public OMAGServerOperationalServices platformOperationalServices(){ + @Bean(name = {"serverOperationalServices"}) + public OMAGServerOperationalServices serverOperationalServices(){ return new OMAGServerOperationalServices(); } - - @Bean(name = {"connectorConfigurationFactory"}) - public ConnectorConfigurationFactory connectorConfigurationFactory() { - return new ConnectorConfigurationFactory(); - } - - @Bean(name = {"omrsConfigurationFactory"}) - public OMRSConfigurationFactory omrsConfigurationFactory() {return new OMRSConfigurationFactory(); } } diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/ObjectMapperConfiguration.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/ObjectMapperConfiguration.java index e0744704ac2..c944d263d01 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/ObjectMapperConfiguration.java +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/config/ObjectMapperConfiguration.java @@ -19,7 +19,7 @@ import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; /** - * This class provides configurations for customizing + * Provides configurations for customizing * json bean Jackson object mapper singleton instance configuration * and * yaml bean Jackson object mapper singleton instance configuration @@ -27,7 +27,6 @@ @Slf4j @Configuration public class ObjectMapperConfiguration { - public static final String START_LOG_SUFIX = "ObjectMapper configuration started."; private static ObjectMapper newObjectMapper(Jackson2ObjectMapperBuilder jackson2ObjectMapperBuilder) { diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/SupportedConfigTypes.java b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/SupportedConfigTypes.java deleted file mode 100644 index 9d70bbbafa7..00000000000 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/java/org/odpi/openmetadata/serverchassis/springboot/constants/SupportedConfigTypes.java +++ /dev/null @@ -1,11 +0,0 @@ -/* SPDX-License-Identifier: Apache-2.0 */ -/* Copyright Contributors to the ODPi Egeria project. */ -package org.odpi.openmetadata.serverchassis.springboot.constants; - -import java.io.Serializable; - -public enum SupportedConfigTypes implements Serializable { - YAML, - YML, - JSON; -} diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties index 6b4cfe49351..b177aaa4121 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.properties @@ -3,7 +3,7 @@ logging.level.org.odpi.openmetadata=error logging.level.org.odpi.openmetadata.frameworks.auditlog=info -logging.level.org.odpi.openmetadata.serverchassis.springboot=info +logging.level.org.odpi.openmetadata.serverchassis.springboot.config=debug management.health.defaults.enabled=false management.health.livenessstate.enabled=true @@ -15,7 +15,4 @@ management.endpoint.health.show-details=always management.endpoint.metrics.enabled=true management.endpoints.web.exposure.include=metrics,health -#omag.server-config-file=classpath:samples/metadata-repository-server.json -omag.server-config-file=classpath:samples/metadata-repository-server.yml - diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml index cfb636acc7d..8e800fe9750 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/main/resources/application-dev.yml @@ -2,7 +2,7 @@ omag: # server-config-file: classpath:samples/metadata-repository-server.json # server-config-file: classpath:samples/metadata-repository-server.yaml -# server-config-file: classpath:samples/metadata-repository-server.yml + server-config-file: classpath:samples/metadata-repository-server.yml server-config: localServerName: cocoMDS1 localServerUserId: OMAGServer diff --git a/open-metadata-implementation/server-chassis/server-chassis-spring/src/test/resources/application-test.properties b/open-metadata-implementation/server-chassis/server-chassis-spring/src/test/resources/application-test.properties index be116e30a99..3fb1a269448 100644 --- a/open-metadata-implementation/server-chassis/server-chassis-spring/src/test/resources/application-test.properties +++ b/open-metadata-implementation/server-chassis/server-chassis-spring/src/test/resources/application-test.properties @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright Contributors to the ODPi Egeria project. -omag.server-config=classpath:samples/metadata-repository-server.json -omag.server-user=system-npa +omag.server-config-file=classpath:samples/metadata-repository-server.json server.port=8001 server.servlet.context-path=/actuator