diff --git a/API.md b/API.md index e5b7a16..b7593ef 100644 --- a/API.md +++ b/API.md @@ -211,6 +211,10 @@ Response body is empty, with `204 No Content` response code. ## ## ### Secrets API + +`/secrets/**` are accessible for users with admin role only. +Please look at the README for `http` configuration to configure admin users. + __GET/secrets/{schema}__ __Returns:__ @@ -219,6 +223,12 @@ Set containing names of keys of custom secrets in specified schema. __Response body example:__ +__CURL example:__ + +`curl -u "admin:password" 'http://my-cluster:30000/editor/backend/secrets/my-schema'` + +`["cassandraPassword"]` + If there are two keys in custom secrets file of this schema ```json [ @@ -254,6 +264,12 @@ __Response body example:__ "key2" ] ``` + +__CURL example:__ + +`curl curl -X PUT 'http://my-cluster:30000/editor/backend/secrets/my-schema' + -u "admin:password" -H 'Content-Type: application/json' + -d '[{"key":"key1","data":"dXBkYXRlZHZhbHVl","key":"key2","data":"c29tZS1zZWNyZXQtdmFsdWU="}]'` ## __DELETE/secrets/{schema}__ @@ -275,4 +291,24 @@ __Response body example:__ "key1" ] ``` -## \ No newline at end of file + +## +## +### Namespace API + +`/namespace/**` endpoints are accessible for users with admin role only. +Please look at the README for `http` configuration to configure admin users. + +__DELETE/namespace/{schema}__ + +Deletes Kubernetes namespace related to `schema` if related git branch is deleted or spec.k8s-propagation: deny + +__Returns:__ + +Name of deleted Kubernetes namespace. + +__Response body example:__ +`th2-my-schema` + +__CURL example:__ +`curl -X DELETE 'http://my-cluster:30000/editor/backend/namespace/my-schema' -u "admin:password" ` \ No newline at end of file diff --git a/README.md b/README.md index 39ed0aa..04969ab 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# infra-mgr +# infra-mgr (2.4.0) infra-mgr is a component responsible for rolling out schemas from git repository to kubernetes. It watches for changes in the repositories and deploys changed components to kubernets. Depending on the schema configuration, it also monitors kubernetes and if it detects external manipulation on deployed component, redeploys them from latest repository version. @@ -89,8 +89,35 @@ infra-mgr configuration is given with *config.yml* file that should be on the cl rabbitmqManagement: rabbitmq-mng-params # individual ConfigMaps for components to be copied from infra-mgr namespace to schema namespace # this ConfigMaps will be populated with schema specific data before copying to target namespace + + behaviour: + permittedToRemoveNamespace: true + # Has infra-mgr got permission to remove Kubernetes namespace when + # branch is disabled (spec.k8s-propagation: deny) or removed. + # Infra-manager removes Kubernetes namespace when this option is true otherwise + # stops maintenance for the namespace without deleting any resources. + # Maintenance is continued when user enable or create the branch related to namespace: `` + # Default value is `true` + http: + adminAccounts: + jack: $2a$10$OvtVdHUf1/n1YL8lrf.69e3mCLA0HLWjUusHmSSxC6dVcEfIvJM6a + emily: $2a$10$Sj2H49Lav.3BsAoq660KAeoFPQFoat8DXlpuhTtID/jJUixzZDRB6 + # Map of username to encrypted by BCrypt (strength >= 10) password pairs. + # @see BCrypt + # This is required parameters because user must have admin role to + # access the `/secrets/**` and `/namespace/**` endpoints. + # example: `curl -u ":" 'http://localhost:8080/secrets/demo'` ``` ## ## For API documentation please refer to -[API Documentation](API.md) \ No newline at end of file +[API Documentation](API.md) + +## Changes: + +### 2.4.0 ++ Added `behaviour.permittedToRemoveNamespace` option ++ Added `http.adminAccounts` required option + + `/secrets/**` and `/namespace/**` are accessible for users with admin role only ++ `curl -X PUT 'http://localhost:8080/secrets/demo' -u ":" ...` endpoint + check secret value format when user uploads secrets via HTTP API \ No newline at end of file diff --git a/build.gradle b/build.gradle index afd7dab..7d27e72 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,7 @@ repositories { mavenCentral() } -configurations.all { +configurations.configureEach { exclude group: 'org.springframework.boot', module: 'spring-boot-starter-logging' exclude group: 'org.springframework.boot', module: 'logback-classic' } @@ -70,6 +70,8 @@ configurations.configureEach() { dependencies { implementation platform("org.springframework.boot:spring-boot-dependencies:${springboot_version}") + implementation "org.springframework.boot:spring-boot-starter-security:${springboot_version}" + implementation "org.springframework.boot:spring-boot-starter-web:${springboot_version}" implementation "com.exactpro.th2:infra-repo:${infra_repo_version}" @@ -125,16 +127,8 @@ test { useJUnitPlatform() } -compileKotlin { - kotlinOptions { - jvmTarget = "17" - } -} - -compileTestKotlin { - kotlinOptions { - jvmTarget = "17" - } +kotlin { + jvmToolchain(17) } jar { diff --git a/gradle.properties b/gradle.properties index 22c7faf..9bff325 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ # -# Copyright 2020-2022 Exactpro (Exactpro Systems Limited) +# Copyright 2020-2023 Exactpro (Exactpro Systems Limited) # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +14,9 @@ # limitations under the License. # -release_version = 2.3.7 +release_version = 2.4.0 springboot_version = 3.1.0 kotlin_version = 1.8.22 detekt_version = 1.22.0 -owaspVersion = 8.1.2 +owaspVersion = 8.1.2 \ No newline at end of file diff --git a/src/main/java/com/exactpro/th2/inframgr/BasicAuthConfiguration.java b/src/main/java/com/exactpro/th2/inframgr/BasicAuthConfiguration.java new file mode 100644 index 0000000..61e17ee --- /dev/null +++ b/src/main/java/com/exactpro/th2/inframgr/BasicAuthConfiguration.java @@ -0,0 +1,81 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.inframgr; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; +import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer; +import org.springframework.security.core.userdetails.User; +import org.springframework.security.core.userdetails.UserDetails; +import org.springframework.security.core.userdetails.UserDetailsService; +import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; +import org.springframework.security.crypto.password.PasswordEncoder; +import org.springframework.security.provisioning.InMemoryUserDetailsManager; +import org.springframework.security.web.SecurityFilterChain; + +import java.util.List; +import java.util.Map; + +import static org.springframework.security.config.Customizer.withDefaults; + +@Configuration +@EnableWebSecurity +public class BasicAuthConfiguration { + + private static final String ADMIN_ROLE = "ADMIN"; + + @Autowired + private Config config; + + @Bean + public SecurityFilterChain filterChain(HttpSecurity http) throws Exception { + http.authorizeHttpRequests((authorizeHttpRequests) -> + authorizeHttpRequests + .requestMatchers("/secrets/**").hasRole(ADMIN_ROLE) + .requestMatchers("/namespace/**").hasRole(ADMIN_ROLE) + .requestMatchers("/**").permitAll() + ).httpBasic(withDefaults()) + // CSRF is disabled because user uses curl only to call REST API + .csrf(AbstractHttpConfigurer::disable); + return http.build(); + } + + @Bean + public UserDetailsService userDetailsService() { + Map adminAccounts = config.getHttp().getAdminAccounts(); + if (adminAccounts.isEmpty()) { + throw new IllegalStateException("'http.adminAccounts' mustn't be empty"); + } + + List admins = adminAccounts.entrySet().stream() + .map(entry -> User.builder() + .username(entry.getKey()) + .password(entry.getValue()) + .roles(ADMIN_ROLE) + .build()) + .toList(); + return new InMemoryUserDetailsManager(admins); + } + + @Bean + public PasswordEncoder passwordEncoder() { + return new BCryptPasswordEncoder(); + } +} diff --git a/src/main/java/com/exactpro/th2/inframgr/Config.java b/src/main/java/com/exactpro/th2/inframgr/Config.java index 1a8a712..5318f5b 100644 --- a/src/main/java/com/exactpro/th2/inframgr/Config.java +++ b/src/main/java/com/exactpro/th2/inframgr/Config.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,13 @@ package com.exactpro.th2.inframgr; -import com.exactpro.th2.inframgr.util.cfg.*; +import com.exactpro.th2.inframgr.util.cfg.BehaviourCfg; +import com.exactpro.th2.inframgr.util.cfg.CassandraConfig; +import com.exactpro.th2.inframgr.util.cfg.GitCfg; +import com.exactpro.th2.inframgr.util.cfg.HttpCfg; +import com.exactpro.th2.inframgr.util.cfg.K8sConfig; +import com.exactpro.th2.inframgr.util.cfg.PrometheusConfig; +import com.exactpro.th2.inframgr.util.cfg.RabbitMQConfig; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; @@ -32,20 +38,26 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; +// TODO: instant this class as spring @Bean instead of singleton public class Config { + private static final Logger LOGGER = LoggerFactory.getLogger(Config.class); + private static final String CONFIG_FILE = "config.yml"; private static final String CONFIG_DIR_SYSTEM_PROPERTY = "inframgr.config.dir"; private static volatile Config instance; - private final Logger logger; - - private String configDir; + private static final Path CONFIG_DIR = Path.of(System.getProperty(CONFIG_DIR_SYSTEM_PROPERTY, ".")); // config fields + private BehaviourCfg behaviour; + + private HttpCfg http; + private GitCfg git; private RabbitMQConfig rabbitmq; @@ -56,6 +68,14 @@ public class Config { private K8sConfig kubernetes; + public BehaviourCfg getBehaviour() { + return behaviour; + } + + public HttpCfg getHttp() { + return http; + } + public GitCfg getGit() { return git; } @@ -77,6 +97,14 @@ public K8sConfig getKubernetes() { return kubernetes; } + public void setBehaviour(BehaviourCfg behaviour) { + this.behaviour = behaviour; + } + + public void setHttp(HttpCfg http) { + this.http = http; + } + public void setGit(GitCfg git) { this.git = git; } @@ -98,13 +126,14 @@ public void setKubernetes(K8sConfig kubernetes) { this.kubernetes = kubernetes; } - private Config() { - logger = LoggerFactory.getLogger(Config.class); - configDir = System.getProperty(CONFIG_DIR_SYSTEM_PROPERTY, "."); - configDir += "/"; + private Config() {} + + public static Config createInstance() throws IOException { + Path file = CONFIG_DIR.resolve(CONFIG_FILE); + return readConfiguration(file); } - private void parseFile(File file, ObjectMapper mapper, Object object) throws IOException { + private static void parseFile(File file, ObjectMapper mapper, Object object) throws IOException { String fileContent = new String(Files.readAllBytes(file.toPath())); StringSubstitutor stringSubstitutor = new StringSubstitutor( @@ -114,43 +143,39 @@ private void parseFile(File file, ObjectMapper mapper, Object object) throws IOE mapper.readerForUpdating(object).readValue(enrichedContent); } - private void readConfiguration() throws IOException { - + static Config readConfiguration(Path configFile) throws IOException { try { - File file = new File(configDir + CONFIG_FILE); - - parseFile(file, new ObjectMapper(new YAMLFactory()).enable( - JsonParser.Feature.STRICT_DUPLICATE_DETECTION). - registerModule(new KotlinModule.Builder().build()), this); + Config config = new Config(); + parseFile(configFile.toFile(), new ObjectMapper(new YAMLFactory()).enable( + JsonParser.Feature.STRICT_DUPLICATE_DETECTION). + registerModule(new KotlinModule.Builder().build()), config); - if (rabbitmq == null) { - rabbitmq = new RabbitMQConfig(); + if (config.getRabbitMQ() == null) { + config.setRabbitMQ(new RabbitMQConfig()); + } + if (config.getBehaviour() == null) { + config.setBehaviour(new BehaviourCfg()); + } + if (config.getCassandra() == null) { + config.setCassandra(new CassandraConfig()); + } + if (config.getHttp() == null) { + throw new IllegalStateException("'http' config can't be null"); + } + if (config.getHttp().getAdminAccounts() == null) { + throw new IllegalStateException("'http.adminAccounts' config can't be null"); } + return config; } catch (UnrecognizedPropertyException e) { - logger.error("Bad configuration: unknown property(\"{}\") specified in configuration file \"{}\"" + LOGGER.error("Bad configuration: unknown property(\"{}\") specified in configuration file \"{}\"" , e.getPropertyName() , CONFIG_FILE); throw new RuntimeException("Configuration exception", e); } catch (JsonParseException e) { - logger.error("Bad configuration: exception while parsing configuration file \"{}\"" + LOGGER.error("Bad configuration: exception while parsing configuration file \"{}\"" , CONFIG_FILE); throw new RuntimeException("Configuration exception", e); } } - - public static Config getInstance() throws IOException { - if (instance == null) { - synchronized (Config.class) { - if (instance == null) { - Config config = new Config(); - config.readConfiguration(); - - instance = config; - } - } - } - - return instance; - } } diff --git a/src/main/java/com/exactpro/th2/inframgr/DeploymentController.java b/src/main/java/com/exactpro/th2/inframgr/DeploymentController.java index a37736f..6997153 100644 --- a/src/main/java/com/exactpro/th2/inframgr/DeploymentController.java +++ b/src/main/java/com/exactpro/th2/inframgr/DeploymentController.java @@ -23,12 +23,12 @@ import com.exactpro.th2.inframgr.statuswatcher.ResourceCondition; import com.exactpro.th2.inframgr.statuswatcher.StatusCache; import com.fasterxml.jackson.annotation.JsonProperty; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.ResponseBody; import java.util.ArrayList; import java.util.List; @@ -36,8 +36,6 @@ @Controller public class DeploymentController { - private static final Logger logger = LoggerFactory.getLogger(DeploymentController.class); - public static final String UNKNOWN_ERROR = "UNKNOWN_ERROR"; public static final String BAD_RESOURCE_NAME = "BAD_RESOURCE_NAME"; @@ -64,12 +62,11 @@ public List getResourceDeploymentStatuses( } return response; - } catch (ServiceException e) { throw e; } catch (Exception e) { - logger.error("Exception retrieving schema {} from repository", schemaName, e); - throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, UNKNOWN_ERROR, e.getMessage()); + throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, UNKNOWN_ERROR, + "Exception retrieving schema " + schemaName + " from repository", e); } } diff --git a/src/main/java/com/exactpro/th2/inframgr/DescriptorController.java b/src/main/java/com/exactpro/th2/inframgr/DescriptorController.java index c1b294f..7cd2fa4 100644 --- a/src/main/java/com/exactpro/th2/inframgr/DescriptorController.java +++ b/src/main/java/com/exactpro/th2/inframgr/DescriptorController.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,9 +25,11 @@ import com.exactpro.th2.inframgr.errors.ServiceException; import com.exactpro.th2.inframgr.k8s.K8sCustomResource; import com.exactpro.th2.inframgr.k8s.Kubernetes; +import com.exactpro.th2.inframgr.k8s.KubernetesService; import com.exactpro.th2.infrarepo.ResourceType; import io.fabric8.kubernetes.client.ResourceNotFoundException; import jakarta.servlet.http.HttpServletResponse; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; @@ -49,6 +51,9 @@ public class DescriptorController { private static final String UNKNOWN_ERROR = "UNKNOWN_ERROR"; + @Autowired + private KubernetesService kubernetesService; + @GetMapping("/descriptor/{schema}/{kind}/{box}") @ResponseBody public Response getDescriptor(@PathVariable(name = "schema") String schemaName, @@ -69,20 +74,20 @@ public Response getDescriptor(@PathVariable(name = "schema") String schemaName, String descriptor; try { - Kubernetes kube = new Kubernetes(Config.getInstance().getKubernetes(), schemaName); - RegistryCredentialLookup secretMapper = new RegistryCredentialLookup(kube); + Kubernetes schemaKube = kubernetesService.getKubernetes(schemaName); + RegistryCredentialLookup secretMapper = new RegistryCredentialLookup(schemaKube); RegistryConnection registryConnection = new RegistryConnection(secretMapper.getCredentials()); - DescriptorExtractor descriptorExtractor = new DescriptorExtractor(registryConnection, kube); + DescriptorExtractor descriptorExtractor = new DescriptorExtractor(registryConnection, schemaKube); String resourceLabel = annotationFor(schemaName, kind, box); descriptor = descriptorExtractor.getImageDescriptor(resourceLabel, kind, box, PROTOBUF_DESCRIPTOR); } catch (ResourceNotFoundException e) { - throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, HttpStatus.NOT_FOUND.name(), e.getMessage()); + throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, HttpStatus.NOT_FOUND.name(), e); } catch (InvalidImageNameFormatException e) { - throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, FORMATTING_ERROR, e.getMessage()); + throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, FORMATTING_ERROR, e); } catch (RegistryRequestException e) { - throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, REGISTRY_ERROR, e.getMessage()); + throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, REGISTRY_ERROR, e); } catch (Exception e) { - throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, UNKNOWN_ERROR, e.getMessage()); + throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, UNKNOWN_ERROR, e); } if (descriptor != null) { return new Response(PROTOBUF_DESCRIPTOR, descriptor); @@ -91,23 +96,5 @@ public Response getDescriptor(@PathVariable(name = "schema") String schemaName, return null; } - private static class Response { - private final String descriptor; - - private final String content; - - public Response(String descriptor, String content) { - this.descriptor = descriptor; - this.content = content; - } - - public String getDescriptor() { - return descriptor; - } - - public String getContent() { - return content; - } - } - + public record Response(String descriptor, String content) { } } diff --git a/src/main/java/com/exactpro/th2/inframgr/InfraManagerApplication.java b/src/main/java/com/exactpro/th2/inframgr/InfraManagerApplication.java index bcfea0c..90ef50a 100644 --- a/src/main/java/com/exactpro/th2/inframgr/InfraManagerApplication.java +++ b/src/main/java/com/exactpro/th2/inframgr/InfraManagerApplication.java @@ -16,15 +16,17 @@ package com.exactpro.th2.inframgr; -import com.exactpro.th2.inframgr.metrics.PrometheusServer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.cassandra.CassandraAutoConfiguration; +import org.springframework.context.annotation.Bean; import org.springframework.scheduling.annotation.EnableScheduling; +import java.io.IOException; + @SpringBootApplication @EnableScheduling @EnableAutoConfiguration(exclude = {CassandraAutoConfiguration.class}) @@ -33,16 +35,16 @@ public class InfraManagerApplication { public static void main(String[] args) { try { - // preload configuration - Config.getInstance(); - - PrometheusServer.start(); SpringApplication application = new SpringApplication(InfraManagerApplication.class); application.run(args); - } catch (Exception e) { Logger logger = LoggerFactory.getLogger(InfraManagerApplication.class); logger.error("Exiting with exception", e); } } + + @Bean + public Config config() throws IOException { + return Config.createInstance(); + } } diff --git a/src/main/java/com/exactpro/th2/inframgr/JobController.java b/src/main/java/com/exactpro/th2/inframgr/JobController.java index 588b92e..965bf8f 100644 --- a/src/main/java/com/exactpro/th2/inframgr/JobController.java +++ b/src/main/java/com/exactpro/th2/inframgr/JobController.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import com.exactpro.th2.inframgr.errors.ServiceException; import com.exactpro.th2.inframgr.k8s.K8sCustomResource; import com.exactpro.th2.inframgr.k8s.Kubernetes; +import com.exactpro.th2.inframgr.k8s.KubernetesService; import com.exactpro.th2.infrarepo.*; import com.exactpro.th2.infrarepo.git.Gitter; import com.exactpro.th2.infrarepo.git.GitterContext; @@ -28,6 +29,7 @@ import org.eclipse.jgit.api.errors.GitAPIException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.*; @@ -38,20 +40,24 @@ @RestController public class JobController { - private static final String UNKNOWN_ERROR = "UNKNOWN_ERROR"; - private static final String GIT_ERROR = "GIT_ERROR"; private static final String CONFIG_ERROR = "GIT_ERROR"; private static final String BAD_RESOURCE_NAME = "BAD_RESOURCE_NAME"; - private static final Logger logger = LoggerFactory.getLogger(JobController.class); + private static final Logger LOGGER = LoggerFactory.getLogger(JobController.class); + + @Autowired + private Config config; + + @Autowired + private KubernetesService kubernetesService; @PutMapping("/jobs/{schemaName}/{jobName}") public void putSecrets(@PathVariable(name = "schemaName") String schemaName, @PathVariable(name = "jobName") String jobName) { - logger.debug("received request for job creation, job name: {}", jobName); + LOGGER.debug("received request for job creation, job name: {}", jobName); if (!K8sCustomResource.isSchemaNameValid(schemaName)) { throw new NotAcceptableException(BAD_RESOURCE_NAME, "Invalid schema name"); } @@ -59,27 +65,28 @@ public void putSecrets(@PathVariable(name = "schemaName") String schemaName, throw new NotAcceptableException(BAD_RESOURCE_NAME, "Invalid resource name"); } - try (Kubernetes kube = new Kubernetes(Config.getInstance().getKubernetes(), schemaName)) { + try { + Kubernetes schemaKube = kubernetesService.getKubernetes(schemaName); RepositoryResource resource; String resourceLabel; - GitterContext ctx = GitterContext.getContext(Config.getInstance().getGit()); + GitterContext ctx = GitterContext.getContext(config.getGit()); Gitter gitter = ctx.getGitter(schemaName); try { gitter.lock(); gitter.checkout(); resource = Repository.getResource(gitter, ResourceType.Th2Job.kind(), jobName); - resourceLabel = annotationFor(kube.getNamespaceName(), ResourceType.Th2Job.kind(), jobName); + resourceLabel = annotationFor(schemaKube.getNamespaceName(), ResourceType.Th2Job.kind(), jobName); } catch (GitAPIException e) { - throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, GIT_ERROR, e.getMessage()); + throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, GIT_ERROR, e); } finally { gitter.unlock(); } - kube.deleteCustomResource(resource); - logger.info("Delete resource : {}", resourceLabel); - kube.createCustomResource(resource); - logger.info("Created job with name : {}", resourceLabel); + schemaKube.deleteCustomResource(resource); + LOGGER.info("Delete resource : {}", resourceLabel); + schemaKube.createCustomResource(resource); + LOGGER.info("Created job with name : {}", resourceLabel); } catch (IOException e) { - throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, CONFIG_ERROR, e.getMessage()); + throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, CONFIG_ERROR, e); } } } diff --git a/src/main/java/com/exactpro/th2/inframgr/NamespaceController.java b/src/main/java/com/exactpro/th2/inframgr/NamespaceController.java new file mode 100644 index 0000000..0c7d41c --- /dev/null +++ b/src/main/java/com/exactpro/th2/inframgr/NamespaceController.java @@ -0,0 +1,111 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.inframgr; + +import com.exactpro.th2.inframgr.errors.NotAcceptableException; +import com.exactpro.th2.inframgr.errors.ServiceException; +import com.exactpro.th2.inframgr.k8s.K8sCustomResource; +import com.exactpro.th2.infrarepo.git.Gitter; +import com.exactpro.th2.infrarepo.git.GitterContext; +import com.exactpro.th2.infrarepo.repo.Repository; +import com.exactpro.th2.infrarepo.settings.RepositorySettingsResource; +import io.fabric8.kubernetes.api.model.Namespace; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClientBuilder; +import io.fabric8.kubernetes.client.dsl.Resource; +import jakarta.servlet.http.HttpServletRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.ResponseBody; + +import java.util.Map; + +@Controller +public class NamespaceController { + + private static final Logger LOGGER = LoggerFactory.getLogger(NamespaceController.class); + + public static final String UNKNOWN_ERROR = "UNKNOWN_ERROR"; + + public static final String NAMESPACE_DOES_NOT_EXIST = "NAMESPACE_DOES_NOT_EXIST"; + + public static final String GIT_BRANCH_IS_ACTIVE = "GIT_BRANCH_IS_ACTIVE"; + + public static final String BAD_RESOURCE_NAME = "BAD_RESOURCE_NAME"; + + @Autowired + private Config config; + + @DeleteMapping("/namespace/{schemaName}") + @ResponseBody + public String getResourceDeploymentStatuses(HttpServletRequest request, + @PathVariable(name = "schemaName") String schemaName) { + + try { + // check schema name against valid pattern + if (!K8sCustomResource.isSchemaNameValid(schemaName)) { + throw new NotAcceptableException(BAD_RESOURCE_NAME, "Invalid schema name"); + } + + String namespace = config.getKubernetes().getNamespacePrefix() + schemaName; + LOGGER.debug("Checking namespace \"{}\"", namespace); + try (KubernetesClient kubeClient = new KubernetesClientBuilder().build()) { + Resource namespaceResource = kubeClient.namespaces().withName(namespace); + if (namespaceResource.get() == null) { + throw new ServiceException(HttpStatus.GONE, NAMESPACE_DOES_NOT_EXIST, + "Kube doesn't contain namespace \"" + namespace + + "\" related to schema \"" + schemaName + "\""); + } + + LOGGER.debug("Checking branch \"{}\"", schemaName); + GitterContext ctx = GitterContext.getContext(config.getGit()); + Map commits = ctx.getAllBranchesCommits(); + + if (commits.containsKey(schemaName)) { + LOGGER.debug("Checking propagation for schema \"{}\"", schemaName); + Gitter gitter = ctx.getGitter(schemaName); + gitter.lock(); + try { + RepositorySettingsResource repositorySettings = Repository.getSettings(gitter); + if (repositorySettings != null && !repositorySettings.getSpec().isK8sPropagationDenied()) { + throw new NotAcceptableException(GIT_BRANCH_IS_ACTIVE, + "The schema \"" + schemaName + "\" is active, propagation \"" + + repositorySettings.getSpec().getK8sPropagation() + "\""); + } + } finally { + gitter.unlock(); + } + } + + namespaceResource.delete(); + LOGGER.info("Deleted namespace \"{}\" related to the schema \"{}\", user: \"{}\"", + namespace, schemaName, request.getUserPrincipal().getName()); + return namespace; + } + } catch (ServiceException e) { + throw e; + } catch (Exception e) { + throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, UNKNOWN_ERROR, + "Exception deleting namespace related to schema " + schemaName + " from kubernetes", e); + } + } +} diff --git a/src/main/java/com/exactpro/th2/inframgr/PodController.java b/src/main/java/com/exactpro/th2/inframgr/PodController.java index 92b2c85..c680eaa 100644 --- a/src/main/java/com/exactpro/th2/inframgr/PodController.java +++ b/src/main/java/com/exactpro/th2/inframgr/PodController.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import com.exactpro.th2.inframgr.errors.ServiceException; import com.exactpro.th2.inframgr.k8s.K8sCustomResource; import com.exactpro.th2.inframgr.k8s.Kubernetes; +import com.exactpro.th2.inframgr.k8s.KubernetesService; import com.exactpro.th2.inframgr.statuswatcher.StatusCache; import io.fabric8.kubernetes.client.KubernetesClientException; import org.slf4j.Logger; @@ -36,16 +37,21 @@ @Controller public class PodController { - - private static final Logger logger = LoggerFactory.getLogger(PodController.class); + private static final Logger LOGGER = LoggerFactory.getLogger(PodController.class); private static final String UNKNOWN_ERROR = "UNKNOWN_ERROR"; private static final String BAD_RESOURCE_NAME = "BAD_RESOURCE_NAME"; + @Autowired + private Config config; + @Autowired private StatusCache statusCache; + @Autowired + private KubernetesService kubernetesService; + @DeleteMapping("/pod/{schema}/{kind}/{resource}") public ResponseEntity deleteResourcePods( @PathVariable(name = "schema") String schemaName, @@ -59,24 +65,27 @@ public ResponseEntity deleteResourcePods( throw new NotAcceptableException(BAD_RESOURCE_NAME, "Invalid schema name"); } - Kubernetes kubernetes = new Kubernetes(Config.getInstance().getKubernetes(), schemaName); + Kubernetes schemaKube = kubernetesService.getKubernetes(schemaName); for (var resource : statusCache.getResourceDependencyStatuses(schemaName, kind, resourceName)) { if (resource.getKind().equals(Kubernetes.KIND_POD)) { + String annotation = annotationFor(schemaKube.getNamespaceName(), + Kubernetes.KIND_POD, resource.getName()); try { - kubernetes.deletePodWithName(resource.getName(), force); + schemaKube.deletePodWithName(resource.getName(), force); + LOGGER.info("Deleted pod \"{}\", schema name \"{}\"", annotation, schemaName); } catch (KubernetesClientException e) { - logger.error("Could not delete pod \"{}\"", - annotationFor(kubernetes.getNamespaceName(), Kubernetes.KIND_POD, resource.getName())); + LOGGER.error("Could not delete pod \"{}\"", annotation, e); } } } - + // TODO: return correct HTTP response return new ResponseEntity<>(HttpStatus.NO_CONTENT); } catch (ServiceException e) { throw e; } catch (Exception e) { - logger.error("Exception deleting pods for \"{}/{}\" in schema \"{}\"", kind, resourceName, schemaName, e); - throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, UNKNOWN_ERROR, e.getMessage()); + throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, UNKNOWN_ERROR, + "Exception deleting pods for \"" + kind + "/" + resourceName + + "\" in schema \"" + schemaName + "\"", e); } } } diff --git a/src/main/java/com/exactpro/th2/inframgr/SchemaController.java b/src/main/java/com/exactpro/th2/inframgr/SchemaController.java index 6b2f2b2..abbef3e 100644 --- a/src/main/java/com/exactpro/th2/inframgr/SchemaController.java +++ b/src/main/java/com/exactpro/th2/inframgr/SchemaController.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,6 @@ import com.exactpro.th2.inframgr.models.ResourceEntry; import com.exactpro.th2.inframgr.repository.RepositoryUpdateEvent; import com.exactpro.th2.inframgr.util.SchemaErrorPrinter; -import com.exactpro.th2.inframgr.util.cfg.GitCfg; import com.exactpro.th2.infrarepo.InconsistentRepositoryStateException; import com.exactpro.th2.infrarepo.SchemaUtils; import com.exactpro.th2.infrarepo.git.Gitter; @@ -37,7 +36,6 @@ import com.exactpro.th2.validator.SchemaValidator; import com.exactpro.th2.validator.ValidationReport; import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.module.kotlin.KotlinModule; @@ -45,15 +43,27 @@ import org.eclipse.jgit.api.errors.RefNotFoundException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.*; - -import java.util.*; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.ResponseBody; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; @Controller public class SchemaController { + private static final Logger LOGGER = LoggerFactory.getLogger(SchemaController.class); + public static final String SCHEMA_EXISTS = "SCHEMA_EXISTS"; private static final String REPOSITORY_ERROR = "REPOSITORY_ERROR"; @@ -62,41 +72,41 @@ public class SchemaController { public static final String SOURCE_BRANCH = "master"; - private static final Logger logger = LoggerFactory.getLogger(SchemaController.class); + @Autowired + private Config config; @GetMapping("/schemas") @ResponseBody public Set getAvailableSchemas() throws ServiceException { try { - GitterContext ctx = GitterContext.getContext(Config.getInstance().getGit()); + GitterContext ctx = GitterContext.getContext(config.getGit()); Set schemas = ctx.getBranches(); schemas.remove(SOURCE_BRANCH); return schemas; } catch (Exception e) { - throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, REPOSITORY_ERROR, e.getMessage()); + throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, REPOSITORY_ERROR, e); } } @GetMapping("/schema/{name}") @ResponseBody - public SchemaControllerResponse getSchemaFiles(@PathVariable(name = "name") String schemaName) throws Exception { + public SchemaControllerResponse getSchemaFiles(@PathVariable(name = "name") String schemaName) { if (schemaName.equals(SOURCE_BRANCH)) { throw new NotAcceptableException(REPOSITORY_ERROR, "Not Allowed"); } - GitCfg gitConfig = Config.getInstance().getGit(); - GitterContext ctx = GitterContext.getContext(gitConfig); + GitterContext ctx = GitterContext.getContext(config.getGit()); final Gitter gitter = ctx.getGitter(schemaName); try { gitter.lock(); return new SchemaControllerResponse(Repository.getSnapshot(gitter)); } catch (RefNotAdvertisedException | RefNotFoundException e) { - throw new ServiceException(HttpStatus.NOT_FOUND, HttpStatus.NOT_FOUND.name(), "schema does not exists"); + throw new ServiceException(HttpStatus.NOT_FOUND, HttpStatus.NOT_FOUND.name(), "schema does not exists", e); } catch (Exception e) { - logger.error("Exception retrieving schema {} from repository", schemaName, e); - throw new NotAcceptableException(REPOSITORY_ERROR, e.getMessage()); + LOGGER.error("Exception retrieving schema {} from repository", schemaName, e); + throw new NotAcceptableException(REPOSITORY_ERROR, e); } finally { gitter.unlock(); } @@ -104,7 +114,7 @@ public SchemaControllerResponse getSchemaFiles(@PathVariable(name = "name") Stri @PutMapping("/schema/{name}") @ResponseBody - public SchemaControllerResponse createSchema(@PathVariable(name = "name") String schemaName) throws Exception { + public SchemaControllerResponse createSchema(@PathVariable(name = "name") String schemaName) { if (schemaName.equals(SOURCE_BRANCH)) { throw new NotAcceptableException(REPOSITORY_ERROR, "Not Allowed"); @@ -114,9 +124,7 @@ public SchemaControllerResponse createSchema(@PathVariable(name = "name") String throw new NotAcceptableException(BAD_RESOURCE_NAME, "Invalid schema name"); } - Config config = Config.getInstance(); - GitCfg gitConfig = config.getGit(); - GitterContext ctx = GitterContext.getContext(gitConfig); + GitterContext ctx = GitterContext.getContext(config.getGit()); if (schemaAlreadyExists(schemaName, ctx)) { throw new NotAcceptableException(SCHEMA_EXISTS, "Error creating schema. schema already exists"); @@ -136,13 +144,14 @@ public SchemaControllerResponse createSchema(@PathVariable(name = "name") String issueRepoUpdateEvent(schemaName, snapshot); + LOGGER.info("Created schema \"{}\"", schemaName); return new SchemaControllerResponse(snapshot); } catch (ServiceException se) { throw se; } catch (Exception e) { - logger.error("Exception creating schema \"{}\"", schemaName, e); - throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, REPOSITORY_ERROR, e.getMessage()); + throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, REPOSITORY_ERROR, + "Exception creating schema \"" + schemaName + "\"", e); } } @@ -151,7 +160,7 @@ private boolean schemaAlreadyExists(String schemaName, GitterContext ctx) { try { branches = ctx.getBranches(); } catch (Exception e) { - throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, REPOSITORY_ERROR, e.getMessage()); + throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, REPOSITORY_ERROR, e); } return branches.contains(schemaName); } @@ -159,8 +168,7 @@ private boolean schemaAlreadyExists(String schemaName, GitterContext ctx) { @PostMapping("/schema/{name}") @ResponseBody public SchemaControllerResponse updateSchema(@PathVariable(name = "name") String schemaName, - @RequestBody String requestBody - ) throws Exception { + @RequestBody String requestBody) { if (schemaName.equals(SOURCE_BRANCH)) { throw new NotAcceptableException(REPOSITORY_ERROR, "Not Allowed"); @@ -175,17 +183,16 @@ public SchemaControllerResponse updateSchema(@PathVariable(name = "name") String operations = mapper.readValue(requestBody, new TypeReference<>() { }); } catch (Exception e) { - throw new BadRequestException(e.getMessage()); + throw new BadRequestException(e); } validateResourceNames(operations); - Config config = Config.getInstance(); - GitCfg gitConfig = config.getGit(); - GitterContext ctx = GitterContext.getContext(gitConfig); + GitterContext ctx = GitterContext.getContext(config.getGit()); if (!schemaAlreadyExists(schemaName, ctx)) { - throw new ServiceException(HttpStatus.NOT_FOUND, HttpStatus.NOT_FOUND.name(), "Schema does not exist"); + throw new ServiceException(HttpStatus.NOT_FOUND, + HttpStatus.NOT_FOUND.name(), "Schema does not exist", null); } //validate schema and apply updates if valid @@ -207,7 +214,7 @@ public SchemaControllerResponse updateSchema(@PathVariable(name = "name") String ); if (!validationContext.isValid()) { // do not update repository and kubernetes if requested changes contain errors. - logger.error("Schema \"{}\" contains errors, update request will be ignored", schemaName); + LOGGER.error("Schema \"{}\" contains errors, update request will be ignored", schemaName); ValidationReport report = validationContext.getReport(); SchemaErrorPrinter.printErrors(report, "editor"); return new SchemaControllerResponse(report); @@ -220,7 +227,7 @@ public SchemaControllerResponse updateSchema(@PathVariable(name = "name") String } if (commitRef == null) { - logger.info("Nothing changed, leaving"); + LOGGER.info("Nothing changed, leaving"); } else { issueRepoUpdateEvent(schemaName, snapshot); } @@ -228,13 +235,12 @@ public SchemaControllerResponse updateSchema(@PathVariable(name = "name") String } catch (ServiceException se) { throw se; } catch (Exception e) { - logger.error("Exception updating schema \"{}\" request", schemaName, e); - throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, REPOSITORY_ERROR, e.getMessage()); + throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, REPOSITORY_ERROR, + "Exception updating schema \"" + schemaName + "\" request", e); } } - private void issueRepoUpdateEvent(String schemaName, RepositorySnapshot snapshot) - throws JsonProcessingException { + private void issueRepoUpdateEvent(String schemaName, RepositorySnapshot snapshot) { SchemaEventRouter router = SchemaEventRouter.getInstance(); RepositoryUpdateEvent event = new RepositoryUpdateEvent(schemaName, snapshot.getCommitRef()); RepositorySettingsSpec rs = snapshot.getRepositorySettingsSpec(); @@ -262,26 +268,26 @@ private String updateRepository(Gitter gitter, List operations) th } return gitter.commitAndPush("schema update"); - } catch (InconsistentRepositoryStateException irse) { + } catch (InconsistentRepositoryStateException e) { // this exception is thrown when inconsistent state of git repository is expected // discard local cache and re-download repository - logger.error("Inconsistent repository state exception for branch \"{}\"", branchName, irse); + LOGGER.error("Inconsistent repository state exception for branch \"{}\"", branchName, e); - var se = new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, REPOSITORY_ERROR, irse.getMessage()); - se.addSuppressed(irse); + var se = new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, REPOSITORY_ERROR, e); + se.addSuppressed(e); try { gitter.recreateCache(); } catch (Exception re) { - logger.error("Exception recreating repository's local cache for branch \"{}\"", branchName, re); + LOGGER.error("Exception recreating repository's local cache for branch \"{}\"", branchName, re); se.addSuppressed(re); } throw se; } catch (Exception e) { - logger.error("Exception updating repository for branch \"{}\"", branchName, e); + LOGGER.error("Exception updating repository for branch \"{}\"", branchName, e); gitter.reset(); - throw new NotAcceptableException(REPOSITORY_ERROR, e.getMessage()); + throw new NotAcceptableException(REPOSITORY_ERROR, e); } } @@ -293,7 +299,7 @@ public static void validateResourceNames(List operations) { String resourceName = entry.getPayload().getName(); if (!K8sCustomResource.isNameValid(resourceName)) { - logger.error("Invalid resource name: \"{}\"", resourceName); + LOGGER.error("Invalid resource name: \"{}\"", resourceName); throw new NotAcceptableException(BAD_RESOURCE_NAME, String.format( "Invalid resource name : \"%s\" (%s)" , entry.getPayload().getName() @@ -302,7 +308,7 @@ public static void validateResourceNames(List operations) { } if (!names.add(resourceName)) { - logger.error("Multiple operations on the same resource: \"{}\"", resourceName); + LOGGER.error("Multiple operations on the same resource: \"{}\"", resourceName); throw new NotAcceptableException(REPOSITORY_ERROR, "Multiple operation on the resource"); } } diff --git a/src/main/java/com/exactpro/th2/inframgr/SchemaValidationController.java b/src/main/java/com/exactpro/th2/inframgr/SchemaValidationController.java index 6d1c734..d9e5f82 100644 --- a/src/main/java/com/exactpro/th2/inframgr/SchemaValidationController.java +++ b/src/main/java/com/exactpro/th2/inframgr/SchemaValidationController.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,6 +42,7 @@ import com.fasterxml.jackson.module.kotlin.KotlinModule; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.*; @@ -59,6 +60,9 @@ public class SchemaValidationController { private static final String REPOSITORY_ERROR = "REPOSITORY_ERROR"; + @Autowired + private Config config; + @PostMapping("/validation/{schemaName}") @ResponseBody public String validateRequestedSchema( @@ -70,7 +74,6 @@ public String validateRequestedSchema( throw new NotAcceptableException(REPOSITORY_ERROR, "Not Allowed"); } - Config config = Config.getInstance(); var fullRepositoryMap = toRepositoryMap(allResourcesStr); SchemaValidationContext validationContext = SchemaValidator.validate( schemaName, @@ -129,8 +132,7 @@ private Map> toRepositoryMap(String allR @GetMapping("/validation/{schemaName}") @ResponseBody public SchemaValidationContext validateSchema(@PathVariable(name = "schemaName") String schemaName, - @RequestBody String requestBody - ) throws Exception { + @RequestBody String requestBody) { if (schemaName.equals(SOURCE_BRANCH)) { throw new NotAcceptableException(REPOSITORY_ERROR, "Not Allowed"); } @@ -145,15 +147,13 @@ public SchemaValidationContext validateSchema(@PathVariable(name = "schemaName") request = mapper.readValue(requestBody, new TypeReference<>() { }); } catch (Exception e) { - throw new BadRequestException(e.getMessage()); + throw new BadRequestException(e); } List operations = request.operations; validateResourceNames(operations); - Config config = Config.getInstance(); - var fullRepositoryMap = toRepositoryMap(operations); if (request.fullSchema) { @@ -174,7 +174,7 @@ public SchemaValidationContext validateSchema(@PathVariable(name = "schemaName") try { branches = ctx.getBranches(); } catch (Exception e) { - throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, REPOSITORY_ERROR, e.getMessage()); + throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, REPOSITORY_ERROR, e); } if (!branches.contains(schemaName)) { throw new ServiceException(HttpStatus.NOT_FOUND, HttpStatus.NOT_FOUND.name(), "schema does not exists"); @@ -201,7 +201,7 @@ public SchemaValidationContext validateSchema(@PathVariable(name = "schemaName") return validationContext; } catch (Exception e) { logger.error("Exception updating schema \"{}\" request", schemaName, e); - throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, REPOSITORY_ERROR, e.getMessage()); + throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, REPOSITORY_ERROR, e); } } @@ -217,7 +217,7 @@ private Map> toRepositoryMap(List getSecrets(@PathVariable(name = "schemaName") String schemaName) throws ServiceException { @@ -48,17 +65,19 @@ public Set getSecrets(@PathVariable(name = "schemaName") String schemaNa throw new NotAcceptableException(BAD_RESOURCE_NAME, "Invalid schema name"); } try { - SecretsManager secretsManager = new SecretsManager(); + SecretsManager secretsManager = new SecretsManager(config.getKubernetes().getNamespacePrefix()); Map secretData = secretsManager.getCustomSecret(schemaName).getData(); return secretData != null ? secretData.keySet() : Collections.emptySet(); } catch (Exception e) { - throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, UNKNOWN_ERROR, e.getMessage()); + throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, UNKNOWN_ERROR, + "get secrets failure, schema name: \"" + schemaName + "\"", e); } } @PutMapping("/secrets/{schemaName}") @ResponseBody - public Set putSecrets(@PathVariable(name = "schemaName") String schemaName, + public Set putSecrets(HttpServletRequest request, + @PathVariable(name = "schemaName") String schemaName, @RequestBody String requestBody) { if (!K8sCustomResource.isSchemaNameValid(schemaName)) { throw new NotAcceptableException(BAD_RESOURCE_NAME, "Invalid schema name"); @@ -68,23 +87,39 @@ public Set putSecrets(@PathVariable(name = "schemaName") String schemaNa ObjectMapper mapper = new ObjectMapper() .enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION) .registerModule(new KotlinModule.Builder().build()); - secretEntries = mapper.readValue(requestBody, new TypeReference<>() { - }); + secretEntries = mapper.readValue(requestBody, new TypeReference<>() { }); } catch (Exception e) { - throw new BadRequestException(e.getMessage()); + throw new BadRequestException("Parsing secret body failure, schema name: \"" + schemaName + "\"", e); + } + + Set secretKeys = new HashSet<>(); + for (SecretsRequestEntry secretEntry : secretEntries) { + try { + DECODER.decode(secretEntry.data); + } catch (IllegalArgumentException e) { + secretKeys.add(secretEntry.key); + } + } + if (!secretKeys.isEmpty()) { + throw new BadRequestException("Values for secrets " + secretKeys + " haven't got base 64 format"); } try { - SecretsManager secretsManager = new SecretsManager(); - return secretsManager.createOrReplaceSecrets(schemaName, secretEntries); + SecretsManager secretsManager = new SecretsManager(config.getKubernetes().getNamespacePrefix()); + Set secrets = secretsManager.createOrReplaceSecrets(schemaName, + secretEntries, request.getUserPrincipal().getName()); + LOGGER.info("Updated secrets \"{}\", schema name: \"{}\"", secrets, schemaName); + return secrets; } catch (Exception e) { - throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, UNKNOWN_ERROR, e.getMessage()); + throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, UNKNOWN_ERROR, + "Create or replace secrets failure, schema name: \"" + schemaName + "\"", e); } } @DeleteMapping("/secrets/{schemaName}") @ResponseBody - public Set deleteSecrets(@PathVariable(name = "schemaName") String schemaName, + public Set deleteSecrets(HttpServletRequest request, + @PathVariable(name = "schemaName") String schemaName, @RequestBody String requestBody) { if (!K8sCustomResource.isSchemaNameValid(schemaName)) { throw new NotAcceptableException(BAD_RESOURCE_NAME, "Invalid schema name"); @@ -97,14 +132,20 @@ public Set deleteSecrets(@PathVariable(name = "schemaName") String schem secretsNames = mapper.readValue(requestBody, new TypeReference<>() { }); } catch (Exception e) { - throw new BadRequestException(e.getMessage()); + throw new BadRequestException( + "Parsing secret body failure, schema name: \"" + schemaName + "\", body: \"" + requestBody + "\"", + e); } try { - SecretsManager secretsManager = new SecretsManager(); - return secretsManager.deleteSecrets(schemaName, secretsNames); + SecretsManager secretsManager = new SecretsManager(config.getKubernetes().getNamespacePrefix()); + Set secrets = secretsManager.deleteSecrets(schemaName, secretsNames); + LOGGER.info("Deleted secrets \"{}\", schema name: \"{}\", user: \"{}\"", + secrets, schemaName, request.getUserPrincipal().getName()); + return secrets; } catch (Exception e) { - throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, UNKNOWN_ERROR, e.getMessage()); + throw new ServiceException(HttpStatus.INTERNAL_SERVER_ERROR, UNKNOWN_ERROR, + "Delete secrets failure, schema name: \"" + schemaName + "\"", e); } } diff --git a/src/main/java/com/exactpro/th2/inframgr/controllers/backup/NamespaceBackupController.java b/src/main/java/com/exactpro/th2/inframgr/controllers/backup/NamespaceBackupController.java index 196d792..7e5a271 100644 --- a/src/main/java/com/exactpro/th2/inframgr/controllers/backup/NamespaceBackupController.java +++ b/src/main/java/com/exactpro/th2/inframgr/controllers/backup/NamespaceBackupController.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ package com.exactpro.th2.inframgr.controllers.backup; +import com.exactpro.th2.inframgr.Config; import com.exactpro.th2.inframgr.errors.NotAcceptableException; import com.exactpro.th2.inframgr.k8s.K8sCustomResource; import com.exactpro.th2.inframgr.k8s.SecretsManager; @@ -32,6 +33,7 @@ import org.apache.commons.text.RandomStringGenerator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.*; @@ -53,6 +55,9 @@ public class NamespaceBackupController { private static final Logger logger = LoggerFactory.getLogger(NamespaceBackupController.class); + @Autowired + private Config config; + ObjectMapper mapper = new ObjectMapper() .enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION) .registerModule(new KotlinModule.Builder().build()); @@ -77,7 +82,7 @@ public BackupObject getBackupZip(@PathVariable(name = "schemaName") String schem zipParameters.setAesKeyStrength(AesKeyStrength.KEY_STRENGTH_256); // add custom-secrets-config as zip content - SecretsManager secretsManager = new SecretsManager(); + SecretsManager secretsManager = new SecretsManager(config.getKubernetes().getNamespacePrefix()); var secret = secretsManager.getCustomSecret(schemaName); if (secret == null || secret.getData() == null) { logger.info("There are no custom secrets present in schema: \"{}\"", schemaName); @@ -125,7 +130,7 @@ public BackupResponse putBackupZip(@PathVariable(name = "schemaName") String sch if (entryName.contains(CUSTOM_SECRETS_SUFFIX)) { String content = reader.readLine(); if (content != null) { - SecretsManager secretsManager = new SecretsManager(); + SecretsManager secretsManager = new SecretsManager(config.getKubernetes().getNamespacePrefix()); Map secretEntries = mapper.readValue(content, new TypeReference<>() { }); backupResponse.addCustomSecrets(secretsManager.createOrReplaceSecrets(schemaName, secretEntries)); diff --git a/src/main/java/com/exactpro/th2/inframgr/docker/monitoring/DynamicResourceProcessor.java b/src/main/java/com/exactpro/th2/inframgr/docker/monitoring/DynamicResourceProcessor.java index c928557..fb3f15a 100644 --- a/src/main/java/com/exactpro/th2/inframgr/docker/monitoring/DynamicResourceProcessor.java +++ b/src/main/java/com/exactpro/th2/inframgr/docker/monitoring/DynamicResourceProcessor.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,16 +23,18 @@ import com.exactpro.th2.inframgr.docker.util.SpecUtils; import com.exactpro.th2.inframgr.docker.util.VersionNumberUtils; import com.exactpro.th2.inframgr.k8s.Kubernetes; +import com.exactpro.th2.inframgr.k8s.KubernetesService; import com.exactpro.th2.inframgr.statuswatcher.ResourcePath; import com.exactpro.th2.infrarepo.ResourceType; import com.exactpro.th2.infrarepo.repo.RepositoryResource; import jakarta.annotation.PostConstruct; import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.io.IOException; import java.util.List; @Component @@ -44,6 +46,12 @@ public class DynamicResourceProcessor { private static final long REGISTRY_CHECK_INITIAL_DELAY_SECONDS = 30; + @Autowired + private Config config; + + @Autowired + private KubernetesService kubernetesService; + private static final List monitoredKinds = List.of( ResourceType.Th2Box.kind(), ResourceType.Th2CoreBox.kind(), @@ -131,17 +139,10 @@ private static void updateTrackedResources(String schema, } @PostConstruct - public void start() throws IOException { + public void start() { try { - Kubernetes kube = new Kubernetes(Config.getInstance().getKubernetes(), null); - RegistryCredentialLookup secretMapper = new RegistryCredentialLookup(kube); - RegistryConnection registryConnection = new RegistryConnection(secretMapper.getCredentials()); - RegistryWatcher registryWatcher = new RegistryWatcher( - REGISTRY_CHECK_INITIAL_DELAY_SECONDS, - REGISTRY_CHECK_PERIOD_SECONDS, - registryConnection - ); - registryWatcher.startWatchingRegistry(); + getRegistryWatcher() + .startWatchingRegistry(); logger.info("DynamicResourceProcessor has been started"); } catch (Exception e) { logger.error("Exception while starting DynamicResourceProcessor. " + @@ -149,4 +150,17 @@ public void start() throws IOException { throw e; } } + + @NotNull + private RegistryWatcher getRegistryWatcher() { + Kubernetes anonKube = kubernetesService.getKubernetes(); + RegistryCredentialLookup secretMapper = new RegistryCredentialLookup(anonKube); + RegistryConnection registryConnection = new RegistryConnection(secretMapper.getCredentials()); + return new RegistryWatcher( + config, + REGISTRY_CHECK_INITIAL_DELAY_SECONDS, + REGISTRY_CHECK_PERIOD_SECONDS, + registryConnection + ); + } } diff --git a/src/main/java/com/exactpro/th2/inframgr/docker/monitoring/watcher/RegistryWatcher.java b/src/main/java/com/exactpro/th2/inframgr/docker/monitoring/watcher/RegistryWatcher.java index 3596f03..50be7d1 100644 --- a/src/main/java/com/exactpro/th2/inframgr/docker/monitoring/watcher/RegistryWatcher.java +++ b/src/main/java/com/exactpro/th2/inframgr/docker/monitoring/watcher/RegistryWatcher.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,7 +22,6 @@ import com.exactpro.th2.inframgr.util.cfg.GitCfg; import com.exactpro.th2.infrarepo.git.GitterContext; -import java.io.IOException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -41,17 +40,19 @@ public class RegistryWatcher implements Runnable { private final RegistryConnection connection; + private final Config config; + private GitterContext ctx; - public RegistryWatcher(long initialDelay, long repeatPeriod, RegistryConnection connection) { + public RegistryWatcher(Config config, long initialDelay, long repeatPeriod, RegistryConnection connection) { + this.config = config; this.taskScheduler = new ScheduledThreadPoolExecutor(THREAD_POOL_SIZE_SCHEDULER); this.initialDelay = initialDelay; this.repeatPeriod = repeatPeriod; this.connection = connection; } - public void startWatchingRegistry() throws IOException { - Config config = Config.getInstance(); + public void startWatchingRegistry() { GitCfg gitConfig = config.getGit(); ctx = GitterContext.getContext(gitConfig); diff --git a/src/main/java/com/exactpro/th2/inframgr/errors/BadRequestException.java b/src/main/java/com/exactpro/th2/inframgr/errors/BadRequestException.java index 8f558f2..e9d1997 100644 --- a/src/main/java/com/exactpro/th2/inframgr/errors/BadRequestException.java +++ b/src/main/java/com/exactpro/th2/inframgr/errors/BadRequestException.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,16 @@ import org.springframework.http.HttpStatus; public class BadRequestException extends ServiceException { + + public BadRequestException(String message, Exception e) { + super(HttpStatus.BAD_REQUEST, HttpStatus.BAD_REQUEST.name(), message, e); + } + public BadRequestException(String message) { - super(HttpStatus.BAD_REQUEST, HttpStatus.BAD_REQUEST.name(), message); + this(message, null); + } + + public BadRequestException(Exception e) { + this(e.getMessage(), e); } } diff --git a/src/main/java/com/exactpro/th2/inframgr/errors/ErrorResponse.java b/src/main/java/com/exactpro/th2/inframgr/errors/ErrorResponse.java index ebb089b..a32f48b 100644 --- a/src/main/java/com/exactpro/th2/inframgr/errors/ErrorResponse.java +++ b/src/main/java/com/exactpro/th2/inframgr/errors/ErrorResponse.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,10 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.springframework.http.HttpStatus; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + public class ErrorResponse { public static final String STATUS_CODE = "status_code"; @@ -28,20 +32,29 @@ public class ErrorResponse { public static final String MESSAGE = "message"; - private HttpStatus httpStatus; + public static final String CAUSES = "causes"; - private String errorCode; + private final HttpStatus httpStatus; - private String message; + private final String errorCode; - public ErrorResponse(HttpStatus httpStatus, String errorCode) { - this(httpStatus, errorCode, null); - } + private final String message; - public ErrorResponse(HttpStatus httpStatus, String errorCode, String message) { + private final List causes; + + public ErrorResponse(HttpStatus httpStatus, String errorCode, String message, Exception e) { this.httpStatus = httpStatus; this.errorCode = errorCode; this.message = message; + this.causes = collectCauses(e); + } + + public ErrorResponse(HttpStatus httpStatus, String errorCode, Exception e) { + this(httpStatus, errorCode, e.getMessage(), e); + } + + public ErrorResponse(HttpStatus httpStatus, String errorCode, String message) { + this(httpStatus, errorCode, message, null); } @JsonIgnore @@ -63,4 +76,23 @@ public String getErrorCode() { public String getMessage() { return message; } + + @JsonProperty(CAUSES) + public List getCauses() { + return causes; + } + + private List collectCauses(Exception e) { + if (e == null || e.getCause() == null) { + return Collections.emptyList(); + } + + Throwable cause = e.getCause(); + List causes = new ArrayList<>(); + while (cause != null) { + causes.add(cause.getMessage()); + cause = cause.getCause(); + } + return causes; + } } diff --git a/src/main/java/com/exactpro/th2/inframgr/errors/NotAcceptableException.java b/src/main/java/com/exactpro/th2/inframgr/errors/NotAcceptableException.java index b26e7f5..df018d1 100644 --- a/src/main/java/com/exactpro/th2/inframgr/errors/NotAcceptableException.java +++ b/src/main/java/com/exactpro/th2/inframgr/errors/NotAcceptableException.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,15 @@ import org.springframework.http.HttpStatus; public class NotAcceptableException extends ServiceException { + public NotAcceptableException(String errorCode, String message, Exception e) { + super(HttpStatus.NOT_ACCEPTABLE, errorCode, message, e); + } + public NotAcceptableException(String errorCode, String message) { - super(HttpStatus.NOT_ACCEPTABLE, errorCode, message); + this(errorCode, message, null); + } + + public NotAcceptableException(String errorCode, Exception e) { + this(errorCode, e.getMessage(), e); } } diff --git a/src/main/java/com/exactpro/th2/inframgr/errors/ServiceException.java b/src/main/java/com/exactpro/th2/inframgr/errors/ServiceException.java index 6b71586..f6e9d0f 100644 --- a/src/main/java/com/exactpro/th2/inframgr/errors/ServiceException.java +++ b/src/main/java/com/exactpro/th2/inframgr/errors/ServiceException.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,14 +20,25 @@ public class ServiceException extends RuntimeException { - public ErrorResponse getErrorResponse() { - return errorResponse; - } + private final ErrorResponse errorResponse; - private ErrorResponse errorResponse; + public ServiceException(HttpStatus statusCode, String errorCode, String message, Exception e) { + super(message, e); + errorResponse = new ErrorResponse(statusCode, errorCode, message, e); + } public ServiceException(HttpStatus statusCode, String errorCode, String message) { + super(message); errorResponse = new ErrorResponse(statusCode, errorCode, message); } + public ServiceException(HttpStatus statusCode, String errorCode, Exception e) { + super(e); + errorResponse = new ErrorResponse(statusCode, errorCode, e); + } + + public ErrorResponse getErrorResponse() { + return errorResponse; + } + } diff --git a/src/main/java/com/exactpro/th2/inframgr/errors/ServiceExceptionHandler.java b/src/main/java/com/exactpro/th2/inframgr/errors/ServiceExceptionHandler.java index 23390e4..5351f1e 100644 --- a/src/main/java/com/exactpro/th2/inframgr/errors/ServiceExceptionHandler.java +++ b/src/main/java/com/exactpro/th2/inframgr/errors/ServiceExceptionHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,18 +16,21 @@ package com.exactpro.th2.inframgr.errors; +import jakarta.servlet.http.HttpServletRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.ControllerAdvice; import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.context.request.WebRequest; import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; @ControllerAdvice public class ServiceExceptionHandler extends ResponseEntityExceptionHandler { + private static final Logger LOGGER = LoggerFactory.getLogger(ServiceExceptionHandler.class); @ExceptionHandler(ServiceException.class) - public ResponseEntity handleServiceException(ServiceException e, WebRequest request) { - + public ResponseEntity handleServiceException(HttpServletRequest req, ServiceException e) { + LOGGER.error("Request: {} failure", req.getRequestURL(), e); ErrorResponse response = e.getErrorResponse(); return new ResponseEntity<>(response, response.getHttpStatus()); } diff --git a/src/main/java/com/exactpro/th2/inframgr/initializer/LoggingConfigMap.java b/src/main/java/com/exactpro/th2/inframgr/initializer/LoggingConfigMap.java index 75368c7..bb2f410 100644 --- a/src/main/java/com/exactpro/th2/inframgr/initializer/LoggingConfigMap.java +++ b/src/main/java/com/exactpro/th2/inframgr/initializer/LoggingConfigMap.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,14 +16,13 @@ package com.exactpro.th2.inframgr.initializer; -import com.exactpro.th2.inframgr.Config; import com.exactpro.th2.inframgr.k8s.Kubernetes; +import com.exactpro.th2.inframgr.util.cfg.K8sConfig; import com.exactpro.th2.infrarepo.repo.RepositoryResource; import io.fabric8.kubernetes.api.model.ConfigMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.Map; import static com.exactpro.th2.inframgr.statuswatcher.ResourcePath.annotationFor; @@ -76,29 +75,33 @@ private LoggingConfigMap() { "OFF", "FATAL" ); - public static void checkLoggingConfigMap(RepositoryResource resource, + public static void checkLoggingConfigMap(K8sConfig kubeConfig, + RepositoryResource resource, String logLevelRoot, String logLevelTh2, String fullCommitRef, - Kubernetes kube) throws IOException { - if (resource.getMetadata() != null && resource.getMetadata().getName().equals(getLoggingConfigMapName())) { - copyLoggingConfigMap(logLevelRoot, logLevelTh2, kube, fullCommitRef, true); + Kubernetes kube) { + if (resource.getMetadata() != null + && resource.getMetadata().getName().equals(getLoggingConfigMapName(kubeConfig))) { + copyLoggingConfigMap(kubeConfig, logLevelRoot, logLevelTh2, kube, fullCommitRef, true); } } - public static void copyLoggingConfigMap(String logLevelRoot, + public static void copyLoggingConfigMap(K8sConfig kubeConfig, + String logLevelRoot, String logLevelTh2, String fullCommitRef, - Kubernetes kube) throws IOException { - copyLoggingConfigMap(logLevelRoot, logLevelTh2, kube, fullCommitRef, false); + Kubernetes kube) { + copyLoggingConfigMap(kubeConfig, logLevelRoot, logLevelTh2, kube, fullCommitRef, false); } - public static void copyLoggingConfigMap(String logLevelRoot, + public static void copyLoggingConfigMap(K8sConfig kubeConfig, + String logLevelRoot, String logLevelTh2, Kubernetes kube, String fullCommitRef, - boolean forceUpdate) throws IOException { - String configMapName = getLoggingConfigMapName(); + boolean forceUpdate) { + String configMapName = getLoggingConfigMapName(kubeConfig); if (configMapName == null || configMapName.isEmpty()) { return; } @@ -168,7 +171,7 @@ public static void copyLoggingConfigMap(String logLevelRoot, } } - private static String getLoggingConfigMapName() throws IOException { - return Config.getInstance().getKubernetes().getConfigMaps().get(LOGGING_CONFIGMAP_PARAM); + private static String getLoggingConfigMapName(K8sConfig kubeConfig) { + return kubeConfig.getConfigMaps().get(LOGGING_CONFIGMAP_PARAM); } } diff --git a/src/main/java/com/exactpro/th2/inframgr/initializer/SchemaInitializer.java b/src/main/java/com/exactpro/th2/inframgr/initializer/SchemaInitializer.java index ef82c69..23092a3 100644 --- a/src/main/java/com/exactpro/th2/inframgr/initializer/SchemaInitializer.java +++ b/src/main/java/com/exactpro/th2/inframgr/initializer/SchemaInitializer.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,12 +41,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.Base64; import java.util.HashMap; import java.util.Map; +import java.util.Objects; -import static com.exactpro.th2.inframgr.k8s.Kubernetes.*; +import static com.exactpro.th2.inframgr.k8s.Kubernetes.KIND_SERVICE_MONITOR; +import static com.exactpro.th2.inframgr.k8s.Kubernetes.createMetaDataWithNewAnnotations; +import static com.exactpro.th2.inframgr.k8s.Kubernetes.createMetadataWithPreviousAnnotations; import static com.exactpro.th2.inframgr.statuswatcher.ResourcePath.annotationFor; import static com.exactpro.th2.inframgr.util.AnnotationUtils.setSourceHash; @@ -110,81 +112,76 @@ public enum SchemaSyncMode { FORCE } - public static void ensureSchema(String schemaName, Kubernetes kube) throws Exception { - ensureSchema(schemaName, kube, Config.getInstance().getKubernetes().getSchemaSyncMode()); - K8sResourceCache.INSTANCE.addNamespace(kube.formatNamespaceName(schemaName)); - } - - public static void ensureSchema(String schemaName, Kubernetes kube, SchemaSyncMode syncMode) throws Exception { - switch (syncMode) { + public static void ensureSchema(Config config, Kubernetes schemaKube) { + Objects.requireNonNull(schemaKube.getSchemaName(), "Kubernetes client is anonymous"); + switch (config.getKubernetes().getSchemaSyncMode()) { case CHECK_NAMESPACE: - if (kube.existsNamespace()) { - if (!kube.namespaceActive()) { - retryTaskQueue.add(new SchemaRecoveryTask(schemaName, NAMESPACE_RETRY_DELAY), true); + if (schemaKube.existsNamespace()) { + if (!schemaKube.namespaceActive()) { + retryTaskQueue.add(new SchemaRecoveryTask(schemaKube, NAMESPACE_RETRY_DELAY), true); throw new IllegalStateException( String.format( "Cannot synchronize branch \"%s\" as corresponding namespace in the wrong state" - , schemaName + , schemaKube.getSchemaName() ) ); } return; } - ensureNameSpace(schemaName, kube, false); + ensureNameSpace(config, schemaKube, false); break; case CHECK_RESOURCES: - ensureNameSpace(schemaName, kube, false); + ensureNameSpace(config, schemaKube, false); break; case FORCE: - ensureNameSpace(schemaName, kube, true); + ensureNameSpace(config, schemaKube, true); break; } + K8sResourceCache.INSTANCE.addNamespace(schemaKube.getNamespaceName()); } - private static void ensureNameSpace(String schemaName, Kubernetes kube, boolean forceUpdate) throws IOException { - - Config config = Config.getInstance(); - - if (!kube.existsNamespace()) { + private static void ensureNameSpace(Config config, Kubernetes schemaKube, boolean forceUpdate) { + if (!schemaKube.existsNamespace()) { // namespace not found, create it - logger.info("Creating namespace \"{}\"", kube.getNamespaceName()); - kube.createNamespace(); + logger.info("Creating namespace \"{}\"", schemaKube.getNamespaceName()); + schemaKube.createNamespace(); } // copy Th2BoxConfigurations config maps - copyConfigMap(kube, MQ_ROUTER_CM_NAME, forceUpdate); - copyConfigMap(kube, GRPC_ROUTER_CM_NAME, forceUpdate); - copyConfigMap(kube, CRADLE_MANAGER_CM_NAME, forceUpdate); - copyConfigMap(kube, BOOK_CONFIG_CM_NAME, forceUpdate); + copyConfigMap(schemaKube, MQ_ROUTER_CM_NAME, forceUpdate); + copyConfigMap(schemaKube, GRPC_ROUTER_CM_NAME, forceUpdate); + copyConfigMap(schemaKube, CRADLE_MANAGER_CM_NAME, forceUpdate); + copyConfigMap(schemaKube, BOOK_CONFIG_CM_NAME, forceUpdate); // ensure rabbitMq resources - ensureRabbitMQResources(config, schemaName, kube, forceUpdate); + ensureRabbitMQResources(config, schemaKube, forceUpdate); //ensure cassandra resources - copyCassandraSecret(config, kube, forceUpdate); - ensureCradleConfig(config, schemaName, kube, forceUpdate); + copyCassandraSecret(config, schemaKube, forceUpdate); + ensureCradleConfig(config, schemaKube, forceUpdate); // copy Service Monitor - copyServiceMonitor(config, kube, forceUpdate); + copyServiceMonitor(config, schemaKube, forceUpdate); // copy required secrets - copySecrets(config, kube, forceUpdate); + copySecrets(config, schemaKube, forceUpdate); // create custom-secrets resource - ensureCustomSecrets(kube, forceUpdate); + ensureCustomSecrets(schemaKube, forceUpdate); } - private static void copyConfigMap(Kubernetes kube, String configMapName, boolean forceUpdate) { - String newResourceLabel = annotationFor(kube.getNamespaceName(), Kubernetes.KIND_CONFIGMAP, configMapName); - ConfigMap originalConfigMap = kube.currentNamespace().getConfigMap(configMapName); + private static void copyConfigMap(Kubernetes schemaKube, String configMapName, boolean forceUpdate) { + String newResourceLabel = annotationFor(schemaKube.getNamespaceName(), + Kubernetes.KIND_CONFIGMAP, configMapName); + ConfigMap originalConfigMap = schemaKube.currentNamespace().getConfigMap(configMapName); if (originalConfigMap == null || originalConfigMap.getData() == null) { logger.error("Failed to load ConfigMap \"{}\" from default namespace", configMapName); return; } - if (kube.getConfigMap(configMapName) != null && !forceUpdate) { + if (schemaKube.getConfigMap(configMapName) != null && !forceUpdate) { logger.info("\"{}\" already exists, skipping", newResourceLabel); return; } @@ -199,28 +196,29 @@ private static void copyConfigMap(Kubernetes kube, String configMapName, boolean newConfigMap.setData(originalConfigMap.getData()); setSourceHash(newConfigMap); - kube.createOrReplaceConfigMap(newConfigMap); + schemaKube.createOrReplaceConfigMap(newConfigMap); logger.info("Created \"{}\" based on \"{}\" from default namespace", newResourceLabel, configMapName); } - static void ensureRabbitMQResources(Config config, String schemaName, Kubernetes kube, boolean forceUpdate) { + static void ensureRabbitMQResources(Config config, Kubernetes schemaKube, boolean forceUpdate) { + Objects.requireNonNull(schemaKube.getSchemaName(), "Kubernetes client is anonymous"); Map configMaps = config.getKubernetes().getConfigMaps(); String prefix = config.getKubernetes().getNamespacePrefix(); RabbitMQConfig rabbitMQConfig = config.getRabbitMQ(); String vHostName = rabbitMQConfig.getVhostName(); - String username = prefix + schemaName; - String exchange = prefix + schemaName; + String username = prefix + schemaKube.getSchemaName(); + String exchange = prefix + schemaKube.getSchemaName(); // copy config map with updated vHost value to namespace try { - createRabbitMQSecret(config, kube, username, forceUpdate); + createRabbitMQSecret(config, schemaKube, username, forceUpdate); copyRabbitMQConfigMap( - configMaps.get(RABBITMQ_CONFIGMAP_PARAM), vHostName, username, exchange, kube, forceUpdate + configMaps.get(RABBITMQ_CONFIGMAP_PARAM), vHostName, username, exchange, schemaKube, forceUpdate ); copyRabbitMQConfigMap( - configMaps.get(RABBITMQ_EXTERNAL_CM_PARAM), vHostName, username, exchange, kube, forceUpdate + configMaps.get(RABBITMQ_EXTERNAL_CM_PARAM), vHostName, username, exchange, schemaKube, forceUpdate ); } catch (Exception e) { logger.error("Exception writing RabbitMQ configuration resources", e); @@ -319,19 +317,19 @@ private static ConfigMap configMapWithNewData(String jsonKey, Object content) return newConfigMap; } - static void copyCassandraSecret(Config config, Kubernetes kube, boolean forceUpdate) { - + static void copyCassandraSecret(Config config, Kubernetes schemaKube, boolean forceUpdate) { + Objects.requireNonNull(schemaKube.getSchemaName(), "Kubernetes client is anonymous"); CassandraConfig cassandraConfig = config.getCassandra(); String secretName = cassandraConfig.getSecret(); - String newResourceLabel = annotationFor(kube.getNamespaceName(), + String newResourceLabel = annotationFor(schemaKube.getNamespaceName(), Kubernetes.KIND_SECRET, CASSANDRA_SECRET_NAME_FOR_NAMESPACES); - if (kube.getSecret(CASSANDRA_SECRET_NAME_FOR_NAMESPACES) != null && !forceUpdate) { + if (schemaKube.getSecret(CASSANDRA_SECRET_NAME_FOR_NAMESPACES) != null && !forceUpdate) { logger.info("\"{}\" already exists, skipping", newResourceLabel); return; } - Secret secret = kube.currentNamespace().getSecrets().get(secretName); + Secret secret = schemaKube.currentNamespace().getSecrets().get(secretName); if (secret == null || secret.getData() == null) { logger.error("Failed to load Secret \"{}\" from default namespace", secretName); return; @@ -343,30 +341,30 @@ static void copyCassandraSecret(Config config, Kubernetes kube, boolean forceUpd newResourceLabel, secret.getMetadata().getAnnotations()) ); - kube.createOrReplaceSecret(newResource); + schemaKube.createOrReplaceSecret(newResource); logger.info("Created \"{}\" based on \"{}\" from default namespace", newResourceLabel, secretName); } catch (Exception e) { logger.error("Exception creating \"{}\"", newResourceLabel, e); } } - static void ensureCradleConfig(Config config, String schemaName, Kubernetes kube, boolean forceUpdate) { - + static void ensureCradleConfig(Config config, Kubernetes schemaKube, boolean forceUpdate) { + Objects.requireNonNull(schemaKube.getSchemaName(), "Kubernetes client is anonymous"); try { GitterContext ctx = GitterContext.getContext(config.getGit()); - Gitter gitter = ctx.getGitter(schemaName); + Gitter gitter = ctx.getGitter(schemaKube.getSchemaName()); try { gitter.lock(); CradleConfig cradle = Repository.getSettings(gitter).getSpec().getCradle(); Map configMaps = config.getKubernetes().getConfigMaps(); - copyCradleConfigMap(configMaps.get(CASSANDRA_CONFIGMAP_PARAM), cradle, kube, forceUpdate); - copyCradleConfigMap(configMaps.get(CASSANDRA_EXT_CONFIGMAP_PARAM), cradle, kube, forceUpdate); + copyCradleConfigMap(configMaps.get(CASSANDRA_CONFIGMAP_PARAM), cradle, schemaKube, forceUpdate); + copyCradleConfigMap(configMaps.get(CASSANDRA_EXT_CONFIGMAP_PARAM), cradle, schemaKube, forceUpdate); } finally { gitter.unlock(); } } catch (Exception e) { - logger.error("Exception extracting keyspace for \"{}\"", schemaName, e); + logger.error("Exception extracting keyspace for \"{}\"", schemaKube.getSchemaName(), e); } } @@ -414,17 +412,18 @@ private static void copyCradleConfigMap(String configMapName, } } - private static void copyServiceMonitor(Config config, Kubernetes kube, boolean forceUpdate) { + private static void copyServiceMonitor(Config config, Kubernetes schemaKube, boolean forceUpdate) { String serviceMonitorName = config.getKubernetes().getServiceMonitor(); - ServiceMonitor.Type originalServiceMonitor = kube.currentNamespace().loadServiceMonitor(serviceMonitorName); + ServiceMonitor.Type originalServiceMonitor = schemaKube.currentNamespace() + .loadServiceMonitor(serviceMonitorName); if (originalServiceMonitor == null) { logger.error("Failed to load ServiceMonitor \"{}\" from default namespace", serviceMonitorName); return; } - String namespace = kube.getNamespaceName(); + String namespace = schemaKube.getNamespaceName(); String newResourceLabel = annotationFor(namespace, KIND_SERVICE_MONITOR, serviceMonitorName); try { - if (kube.loadServiceMonitor(namespace, serviceMonitorName) != null && !forceUpdate) { + if (schemaKube.loadServiceMonitor(namespace, serviceMonitorName) != null && !forceUpdate) { logger.info("\"{}\" already exists, skipping", newResourceLabel); return; } @@ -434,7 +433,7 @@ private static void copyServiceMonitor(Config config, Kubernetes kube, boolean f processInstanceLabel(newServiceMonitor.getMetadata(), namespace); newServiceMonitor.setKind(KIND_SERVICE_MONITOR); newServiceMonitor.setSpec(originalServiceMonitor.getSpec()); - kube.createServiceMonitor(newServiceMonitor); + schemaKube.createServiceMonitor(newServiceMonitor); logger.info("Created \"{}\" based on \"{}\" from default namespace", newResourceLabel, serviceMonitorName); } catch (Exception e) { logger.error("Exception creating ServiceMonitor \"{}\"", newResourceLabel, e); @@ -448,10 +447,10 @@ private static void processInstanceLabel(ObjectMeta metadata, String namespace) } } - private static void copySecrets(Config config, Kubernetes kube, boolean forceUpdate) { + private static void copySecrets(Config config, Kubernetes schemaKube, boolean forceUpdate) { - Map workingNamespaceSecrets = kube.currentNamespace().getSecrets(); - Map targetNamespaceSecrets = kube.getSecrets(); + Map workingNamespaceSecrets = schemaKube.currentNamespace().getSecrets(); + Map targetNamespaceSecrets = schemaKube.getSecrets(); String rmqSecretName = config.getRabbitMQ().getSecret(); String cassandraSecretName = config.getCassandra().getSecret(); @@ -462,7 +461,7 @@ private static void copySecrets(Config config, Kubernetes kube, boolean forceUpd } Secret originalSecret = workingNamespaceSecrets.get(secretName); - String newResourceLabel = annotationFor(kube.getNamespaceName(), Kubernetes.KIND_SECRET, secretName); + String newResourceLabel = annotationFor(schemaKube.getNamespaceName(), Kubernetes.KIND_SECRET, secretName); if (originalSecret == null || originalSecret.getData() == null) { logger.error("Failed to load Secret \"{}\" from default namespace", secretName); continue; @@ -478,7 +477,7 @@ private static void copySecrets(Config config, Kubernetes kube, boolean forceUpd newResourceLabel, originalSecret.getMetadata().getAnnotations()) ); - kube.createOrReplaceSecret(newResource); + schemaKube.createOrReplaceSecret(newResource); logger.info("Created \"{}\" based on \"{}\" from default namespace", newResourceLabel, secretName); } catch (Exception e) { logger.error("Exception creating \"{}\"", newResourceLabel, e); @@ -497,11 +496,11 @@ private static Secret makeSecretCopy(Secret secret) { return secretCopy; } - private static void ensureCustomSecrets(Kubernetes kube, boolean forceUpdate) { + private static void ensureCustomSecrets(Kubernetes schemaKube, boolean forceUpdate) { String secretName = SecretsManager.DEFAULT_SECRET_NAME; - String newResourceLabel = annotationFor(kube.getNamespaceName(), Kubernetes.KIND_SECRET, secretName); + String newResourceLabel = annotationFor(schemaKube.getNamespaceName(), Kubernetes.KIND_SECRET, secretName); - if (kube.getSecret(secretName) != null && !forceUpdate) { + if (schemaKube.getSecret(secretName) != null && !forceUpdate) { logger.info("\"{}\" already exists, skipping", newResourceLabel); return; } @@ -516,7 +515,7 @@ private static void ensureCustomSecrets(Kubernetes kube, boolean forceUpdate) { }}); try { - kube.createOrReplaceSecret(newSecret); + schemaKube.createOrReplaceSecret(newSecret); logger.info("Created \"{}\"", newResourceLabel); } catch (Exception e) { logger.error("Exception creating \"{}\"", newResourceLabel, e); diff --git a/src/main/java/com/exactpro/th2/inframgr/k8s/K8sOperator.java b/src/main/java/com/exactpro/th2/inframgr/k8s/K8sOperator.java index 01f8eef..1428379 100644 --- a/src/main/java/com/exactpro/th2/inframgr/k8s/K8sOperator.java +++ b/src/main/java/com/exactpro/th2/inframgr/k8s/K8sOperator.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,9 +37,9 @@ import jakarta.annotation.PreDestroy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.io.IOException; import java.util.Objects; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -52,17 +52,24 @@ public class K8sOperator { private static final int RECOVERY_THREAD_POOL_SIZE = 3; + @Autowired private Config config; + @Autowired + private RepositoryWatcherService repositoryWatcherService; + + @Autowired + private KubernetesService kubernetesService; + private K8sResourceCache cache; private RetryableTaskQueue taskQueue; private void startInformers() { // wait for startup synchronization to complete - logger.info("Operator is waiting for kubernetes startup synchronization to complete"); + logger.info("Operator is waiting for anon Kube startup synchronization to complete"); while (!(Thread.currentThread().isInterrupted() - || RepositoryWatcherService.isStartupSynchronizationComplete())) { + || repositoryWatcherService.isStartupSynchronizationComplete())) { try { Thread.sleep(1000); } catch (InterruptedException e) { @@ -72,33 +79,33 @@ private void startInformers() { } logger.info("Creating informers"); - Kubernetes kube = new Kubernetes(config.getKubernetes(), null); + Kubernetes anonKube = kubernetesService.getKubernetes(); cache = K8sResourceCache.INSTANCE; - kube.registerCustomResourceSharedInformers(new ResourceEventHandler() { + anonKube.registerCustomResourceSharedInformers(new ResourceEventHandler() { @Override public void onAdd(K8sCustomResource obj) { - processEvent(Watcher.Action.ADDED, obj, kube); + processEvent(Watcher.Action.ADDED, obj, anonKube); } @Override public void onUpdate(K8sCustomResource oldObj, K8sCustomResource newObj) { - processEvent(Watcher.Action.MODIFIED, newObj, kube); + processEvent(Watcher.Action.MODIFIED, newObj, anonKube); } @Override public void onDelete(K8sCustomResource obj, boolean deletedFinalStateUnknown) { - processEvent(Watcher.Action.DELETED, obj, kube); + processEvent(Watcher.Action.DELETED, obj, anonKube); } }); - kube.startInformers(); + anonKube.startInformers(); logger.info("Informers has been started"); } - private void processEvent(Watcher.Action action, K8sCustomResource res, Kubernetes kube) { + private void processEvent(Watcher.Action action, K8sCustomResource res, Kubernetes anonKube) { try { ObjectMeta meta = res.getMetadata(); @@ -136,7 +143,8 @@ private void processEvent(Watcher.Action action, K8sCustomResource res, Kubernet // action is needed as optimistic check did not draw enough conclusions GitterContext ctx = GitterContext.getContext(config.getGit()); - Gitter gitter = ctx.getGitter(kube.extractSchemaName(namespace)); + String schemaName = anonKube.extractSchemaName(namespace); + Gitter gitter = ctx.getGitter(schemaName); RepositoryResource resource = null; try { @@ -202,18 +210,18 @@ private void processEvent(Watcher.Action action, K8sCustomResource res, Kubernet logger.info("Detected external manipulation on {}, recreating resource {}", resourceLabel, hashTag); // check current status of namespace - Namespace n = kube.getNamespace(namespace); + Namespace n = anonKube.getNamespace(namespace); if (n == null || !n.getStatus().getPhase().equals(Kubernetes.PHASE_ACTIVE)) { logger.warn("Cannot recreate resource {} as namespace is in \"{}\" state. " + "Scheduled full schema synchronization" , resourceLabel, (n == null ? "Deleted" : n.getStatus().getPhase())); - taskQueue.add(new SchemaRecoveryTask(kube.extractSchemaName(namespace)), true); + taskQueue.add(new SchemaRecoveryTask(kubernetesService.getKubernetes(schemaName)), true); } else { - kube.createOrReplaceCustomResource(resource, namespace); + anonKube.createOrReplaceCustomResource(resource, namespace); } } else if (actionDelete) { logger.info("Detected external manipulation on {}, deleting resource {}", resourceLabel, hashTag); - kube.deleteCustomResource(resource, namespace); + anonKube.deleteCustomResource(resource, namespace); } } finally { @@ -226,9 +234,7 @@ private void processEvent(Watcher.Action action, K8sCustomResource res, Kubernet } @PostConstruct - public void start() throws IOException { - - config = Config.getInstance(); + public void start() { taskQueue = new RetryableTaskQueue(RECOVERY_THREAD_POOL_SIZE); // start repository event listener thread diff --git a/src/main/java/com/exactpro/th2/inframgr/k8s/K8sSynchronization.java b/src/main/java/com/exactpro/th2/inframgr/k8s/K8sSynchronization.java index 22e8eee..94a7e2c 100644 --- a/src/main/java/com/exactpro/th2/inframgr/k8s/K8sSynchronization.java +++ b/src/main/java/com/exactpro/th2/inframgr/k8s/K8sSynchronization.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,10 +19,10 @@ import com.exactpro.th2.inframgr.Config; import com.exactpro.th2.inframgr.SchemaEventRouter; import com.exactpro.th2.inframgr.docker.monitoring.DynamicResourceProcessor; -import com.exactpro.th2.inframgr.initializer.LoggingConfigMap; import com.exactpro.th2.inframgr.initializer.BookConfiguration; -import com.exactpro.th2.inframgr.initializer.Th2BoxConfigurations; +import com.exactpro.th2.inframgr.initializer.LoggingConfigMap; import com.exactpro.th2.inframgr.initializer.SchemaInitializer; +import com.exactpro.th2.inframgr.initializer.Th2BoxConfigurations; import com.exactpro.th2.inframgr.metrics.ManagerMetrics; import com.exactpro.th2.inframgr.repository.RepositoryUpdateEvent; import com.exactpro.th2.inframgr.util.SchemaErrorPrinter; @@ -46,10 +46,10 @@ import jakarta.annotation.PostConstruct; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import rx.schedulers.Schedulers; -import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -65,17 +65,22 @@ public class K8sSynchronization { private static final Logger logger = LoggerFactory.getLogger(K8sSynchronization.class); + @Autowired private Config config; + @Autowired + private KubernetesService kubernetesService; + private final K8sSynchronizationJobQueue jobQueue = new K8sSynchronizationJobQueue(); private void deleteNamespace(String schemaName) { - try (Kubernetes kube = new Kubernetes(config.getKubernetes(), schemaName)) { - if (kube.existsNamespace()) { + try { + Kubernetes schemaKube = kubernetesService.getKubernetes(schemaName); + if (schemaKube.existsNamespace()) { logger.info("Removing schema \"{}\" from kubernetes", schemaName); DynamicResourceProcessor.deleteSchema(schemaName); - K8sResourceCache.INSTANCE.removeNamespace(kube.formatNamespaceName(schemaName)); - kube.deleteNamespace(); + K8sResourceCache.INSTANCE.removeNamespace(schemaKube.getNamespaceName()); + schemaKube.deleteNamespace(); } } catch (Exception e) { logger.error("Exception removing schema \"{}\" from kubernetes", schemaName, e); @@ -89,17 +94,19 @@ private void synchronizeNamespace(String schemaName, Histogram.Timer timer = ManagerMetrics.getCommitTimer(); String shortCommitRef = getShortCommitRef(fullCommitRef); - try (Kubernetes kube = new Kubernetes(config.getKubernetes(), schemaName)) { - SchemaInitializer.ensureSchema(schemaName, kube); + try { + Kubernetes schemaKube = kubernetesService.getKubernetes(schemaName); + SchemaInitializer.ensureSchema(config, schemaKube); validateSchema(schemaName, repositoryResources, repositorySettings, shortCommitRef); RepositorySettingsSpec settingsSpec = repositorySettings.getSpec(); try { LoggingConfigMap.copyLoggingConfigMap( + config.getKubernetes(), settingsSpec.getLogLevelRoot(), settingsSpec.getLogLevelTh2(), fullCommitRef, - kube + schemaKube ); } catch (Exception e) { logger.error("Exception copying logging config map to schema \"{}\"", schemaName, e); @@ -107,7 +114,7 @@ private void synchronizeNamespace(String schemaName, BookConfiguration.synchronizeBookConfig( settingsSpec.getBookConfig(), - kube, + schemaKube, fullCommitRef ); @@ -116,10 +123,10 @@ private void synchronizeNamespace(String schemaName, settingsSpec.getGrpcRouter(), settingsSpec.getCradleManager(), fullCommitRef, - kube + schemaKube ); - syncCustomResourcesWithK8s(schemaName, repositoryResources, kube, shortCommitRef); + syncCustomResourcesWithK8s(schemaName, repositoryResources, schemaKube, shortCommitRef); } finally { timer.observeDuration(); @@ -162,10 +169,10 @@ private void validateSchema(String schemaName, private void syncCustomResourcesWithK8s(String schemaName, Map> repositoryResources, - Kubernetes kube, String shortCommitRef) { - String namespace = kube.formatNamespaceName(schemaName); + Kubernetes schemaKube, String shortCommitRef) { + String namespace = schemaKube.getNamespaceName(); K8sResourceCache cache = K8sResourceCache.INSTANCE; - Map> k8sResources = loadCustomResources(kube); + Map> k8sResources = loadCustomResources(schemaKube); // synchronize by resource type for (ResourceType type : ResourceType.values()) { if (type.isMangedResource() && !type.equals(ResourceType.Th2Job)) { @@ -187,7 +194,7 @@ private void syncCustomResourcesWithK8s(String schemaName, logger.info("Creating resource {} {}. [commit: {}]", resourceLabel, hashTag, shortCommitRef); try { - kube.createCustomResource(resource); + schemaKube.createCustomResource(resource); } catch (Exception e) { logger.error("Exception creating resource {} {}. [commit: {}]", resourceLabel, hashTag, shortCommitRef, e); @@ -202,7 +209,7 @@ private void syncCustomResourcesWithK8s(String schemaName, logger.info("Updating resource {} {}. [commit: {}]", resourceLabel, hashTag, shortCommitRef); try { - kube.replaceCustomResource(resource); + schemaKube.replaceCustomResource(resource); } catch (Exception e) { logger.error("Exception updating resource {} {}. [commit: {}]", resourceLabel, hashTag, shortCommitRef, e); @@ -223,7 +230,7 @@ private void syncCustomResourcesWithK8s(String schemaName, meta.setName(resourceName); resource.setMetadata(meta); DynamicResourceProcessor.checkResource(resource, schemaName, true); - kube.deleteCustomResource(resource); + schemaKube.deleteCustomResource(resource); } catch (Exception e) { logger.error("Exception deleting resource {}. [commit: {}]", resourceLabel, shortCommitRef, e); @@ -310,12 +317,6 @@ public void synchronizeBranch(String branch, long detectionTime) { public void start() { logger.info("Starting Kubernetes synchronization phase"); subscribeToRepositoryEvents(); - try { - config = Config.getInstance(); - } catch (IOException e) { - logger.error("Error loading config"); - throw new RuntimeException("Failed to start Kubernetes synchronization component"); - } } private void subscribeToRepositoryEvents() { diff --git a/src/main/java/com/exactpro/th2/inframgr/k8s/Kubernetes.java b/src/main/java/com/exactpro/th2/inframgr/k8s/Kubernetes.java index 79183b3..b3335ab 100644 --- a/src/main/java/com/exactpro/th2/inframgr/k8s/Kubernetes.java +++ b/src/main/java/com/exactpro/th2/inframgr/k8s/Kubernetes.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package com.exactpro.th2.inframgr.k8s; import com.exactpro.th2.inframgr.k8s.cr.*; +import com.exactpro.th2.inframgr.util.cfg.BehaviourCfg; import com.exactpro.th2.inframgr.util.cfg.K8sConfig; import com.exactpro.th2.infrarepo.ResourceType; import com.exactpro.th2.infrarepo.repo.RepositoryResource; @@ -29,6 +30,9 @@ import io.fabric8.kubernetes.client.informers.ResourceEventHandler; import io.fabric8.kubernetes.client.informers.SharedIndexInformer; import io.fabric8.kubernetes.client.informers.SharedInformerFactory; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.Closeable; import java.util.*; @@ -38,6 +42,7 @@ import static io.fabric8.kubernetes.internal.KubernetesDeserializer.registerCustomKind; public class Kubernetes implements Closeable { + private static final Logger LOGGER = LoggerFactory.getLogger(Kubernetes.class); public static final String KIND_SECRET = "Secret"; @@ -57,6 +62,8 @@ public class Kubernetes implements Closeable { private static final String ANTECEDENT_ANNOTATION_KEY = "th2.exactpro.com/antecedent"; + private final boolean permittedToRemoveNamespace; + private final String namespacePrefix; public String formatNamespaceName(String schemaName) { @@ -102,7 +109,7 @@ public String extractSchemaName(String namespaceName) { throw new IllegalArgumentException("Malformed namespace name"); } String schemaName = namespaceName.substring(namespacePrefix.length()); - if (schemaName.equals("")) { + if (StringUtils.isEmpty(schemaName)) { throw new IllegalArgumentException("Malformed namespace name"); } return schemaName; @@ -127,7 +134,7 @@ public > List customResources = customResourceList.getItems(); boolean resourceUpdated = false; - if (customResources.size() > 0) { + if (!customResources.isEmpty()) { for (T k8sResource : customResources) { if (k8sResource.getMetadata().getName().equals(repoResource.getMetadata().getName())) { k8sResource.getMetadata().setAnnotations(repoResource.getMetadata().getAnnotations()); @@ -211,7 +218,7 @@ public > var customResourceList = operation.inNamespace(namespace).list(); List customResources = customResourceList.getItems(); - if (customResources.size() > 0) { + if (!customResources.isEmpty()) { for (T k8sResource : customResources) { if (k8sResource.getMetadata().getName().equals(repoResource.getMetadata().getName())) { @@ -314,8 +321,13 @@ public Namespace getNamespace(String namespace) { return client.namespaces().withName(namespace).get(); } - public List deleteNamespace() { - return client.namespaces().withName(namespace).delete(); + public void deleteNamespace() { + if (permittedToRemoveNamespace) { + Collection statusDetails = client.namespaces().withName(namespace).delete(); + LOGGER.info("Deleted '{}' namespace, status details: {}", namespace, statusDetails); + } else { + LOGGER.warn("Stopping namespace \"{}\" maintenance", namespace); + } } public void createNamespace() { @@ -420,18 +432,18 @@ public void registerSharedInformersAll(ResourceEventHandler eventHandler) { registerCustomResourceSharedInformers(eventHandler); SharedInformerFactory factory = getInformerFactory(); - var filteringEventHanled = new FilteringResourceEventHandler().wrap(eventHandler); + var filteringEventHandled = new FilteringResourceEventHandler().wrap(eventHandler); factory.sharedIndexInformerFor(Deployment.class, 0) - .addEventHandler(filteringEventHanled); + .addEventHandler(filteringEventHandled); factory.sharedIndexInformerFor(Pod.class, 0) - .addEventHandler(filteringEventHanled); + .addEventHandler(filteringEventHandled); factory.sharedIndexInformerFor(Service.class, 0) - .addEventHandler(filteringEventHanled); + .addEventHandler(filteringEventHandled); factory.sharedIndexInformerFor(ConfigMap.class, 0) - .addEventHandler(filteringEventHanled); + .addEventHandler(filteringEventHandled); } private Map mapOf(List secrets) { @@ -456,16 +468,19 @@ public void startInformers() { informerFactory.startAllRegisteredInformers(); } - private KubernetesClient client; + private final KubernetesClient client; private Map operations; - private String namespace; + private final String schemaName; - public Kubernetes(K8sConfig config, String schemaName) { + private final String namespace; - // if we are not using custom configutation, let fabric8 handle initialization + public Kubernetes(BehaviourCfg behaviour, K8sConfig config, String schemaName) { + this.permittedToRemoveNamespace = behaviour.isPermittedToRemoveNamespace(); + // if we are not using custom configuration, let fabric8 handle initialization this.namespacePrefix = config.getNamespacePrefix(); + this.schemaName = schemaName; this.namespace = formatNamespaceName(schemaName); this.currentNamespace = new CurrentNamespace(); @@ -484,7 +499,7 @@ public Kubernetes(K8sConfig config, String schemaName) { .withTrustCerts(config.ignoreInsecureHosts()); // prioritize key & certificate data over files - if (config.getClientCertificate() != null && config.getClientCertificate().length() > 0) { + if (config.getClientCertificate() != null && !config.getClientCertificate().isEmpty()) { configBuilder.withClientCertData(new String( Base64.getEncoder().encode(config.getClientCertificate().getBytes())) ); @@ -492,7 +507,7 @@ public Kubernetes(K8sConfig config, String schemaName) { configBuilder.withClientCertFile(config.getClientCertificateFile()); } - if (config.getClientKey() != null && config.getClientKey().length() > 0) { + if (config.getClientKey() != null && !config.getClientKey().isEmpty()) { configBuilder.withClientKeyData(new String( Base64.getEncoder().encode(config.getClientKey().getBytes())) ); @@ -524,6 +539,10 @@ public Secret createOrReplaceSecret(Secret secret) { return client.resource(secret).inNamespace(namespace).createOrReplace(); } + public String getSchemaName() { + return schemaName; + } + public String getNamespaceName() { return namespace; } @@ -533,7 +552,7 @@ public void close() { client.close(); } - private CurrentNamespace currentNamespace; + private final CurrentNamespace currentNamespace; public CurrentNamespace currentNamespace() { return currentNamespace; diff --git a/src/main/java/com/exactpro/th2/inframgr/k8s/KubernetesService.java b/src/main/java/com/exactpro/th2/inframgr/k8s/KubernetesService.java new file mode 100644 index 0000000..8a5f4f1 --- /dev/null +++ b/src/main/java/com/exactpro/th2/inframgr/k8s/KubernetesService.java @@ -0,0 +1,84 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.inframgr.k8s; + +import com.exactpro.th2.inframgr.Config; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@Service +public class KubernetesService { + + private static final Logger LOGGER = LoggerFactory.getLogger(KubernetesService.class); + + private final ConcurrentMap clients = new ConcurrentHashMap<>(); + + private Kubernetes defaultClient; + + @Autowired + private Config config; + + public Kubernetes getKubernetes(String schema) { + if (schema == null) { + return defaultClient; + } + return clients.computeIfAbsent(schema, key -> { + LOGGER.info("Creating kubernetes client for the schema '{}'", key); + return new Kubernetes(config.getBehaviour(), config.getKubernetes(), key); + }); + } + + public Kubernetes getKubernetes() { + return getKubernetes(null); + } + + @PostConstruct + private void postConstruct() { + LOGGER.info("Initialising kubernetes controller"); + defaultClient = new Kubernetes(config.getBehaviour(), config.getKubernetes(), null); + } + + @PreDestroy + private void preDestroy() { + LOGGER.info("Closing kubernetes clients"); + try { + defaultClient.close(); + } catch (Exception e) { + LOGGER.error("Closing default kubernetes client failure", e); + } + + for (String schema : clients.keySet()) { + try { + Kubernetes client = clients.remove(schema); + if (client == null) { + LOGGER.warn("kubernetes client for schema '{}' is already removed", schema); + } else { + client.close(); + } + } catch (Exception e) { + LOGGER.error("Closing kubernetes client for schema '{}' failure", schema, e); + } + } + } +} diff --git a/src/main/java/com/exactpro/th2/inframgr/k8s/SchemaRecoveryTask.java b/src/main/java/com/exactpro/th2/inframgr/k8s/SchemaRecoveryTask.java index 89b484c..c3b40ee 100644 --- a/src/main/java/com/exactpro/th2/inframgr/k8s/SchemaRecoveryTask.java +++ b/src/main/java/com/exactpro/th2/inframgr/k8s/SchemaRecoveryTask.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,6 @@ package com.exactpro.th2.inframgr.k8s; -import com.exactpro.th2.inframgr.Config; import com.exactpro.th2.inframgr.SchemaEvent; import com.exactpro.th2.inframgr.SchemaEventRouter; import com.exactpro.th2.inframgr.util.RetryableTask; @@ -24,29 +23,31 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Objects; + public class SchemaRecoveryTask implements RetryableTask { private static final Logger logger = LoggerFactory.getLogger(SchemaRecoveryTask.class); private static final int RETRY_DELAY_SEC = 60; - private final String schema; + private final Kubernetes schemaKube; private final int retryDelay; - public SchemaRecoveryTask(String schema) { - this.schema = schema; - this.retryDelay = RETRY_DELAY_SEC; + public SchemaRecoveryTask(Kubernetes schemaKube, int retryDelay) { + Objects.requireNonNull(schemaKube.getSchemaName(), "Kubernetes client is anonymous"); + this.schemaKube = schemaKube; + this.retryDelay = retryDelay; } - public SchemaRecoveryTask(String schema, int retryDelay) { - this.schema = schema; - this.retryDelay = retryDelay; + public SchemaRecoveryTask(Kubernetes schemaKube) { + this(schemaKube, RETRY_DELAY_SEC); } @Override public String getUniqueKey() { - return SchemaRecoveryTask.class.getName() + ":" + schema; + return SchemaRecoveryTask.class.getName() + ":" + schemaKube.getSchemaName(); } @Override @@ -56,11 +57,9 @@ public long getRetryDelay() { @Override public void run() { + // check actual state of the namespace try { - // check actual state of the namespace - Config config = Config.getInstance(); - Kubernetes kube = new Kubernetes(config.getKubernetes(), schema); - Namespace namespace = kube.getNamespace(kube.getNamespaceName()); + Namespace namespace = schemaKube.getNamespace(schemaKube.getNamespaceName()); if (namespace != null && !namespace.getStatus().getPhase().equals(Kubernetes.PHASE_ACTIVE)) { // namespace is still unavailable for operations @@ -77,12 +76,13 @@ public void run() { // namespace not found or is marked as active // send synchronization request SchemaEventRouter router = SchemaEventRouter.getInstance(); - SchemaEvent event = new SynchronizationRequestEvent(schema); - router.addEvent(schema, event); + SchemaEvent event = new SynchronizationRequestEvent(schemaKube.getSchemaName()); + router.addEvent(schemaKube.getSchemaName(), event); } catch (Exception e) { // rethrow exception to re-execute this task in the future - logger.error("Exception recovering schema \"{}\", will be rescheduled for retry", schema, e); + logger.error("Exception recovering schema \"{}\", will be rescheduled for retry", + schemaKube.getSchemaName(), e); throw new RuntimeException(e); } } diff --git a/src/main/java/com/exactpro/th2/inframgr/k8s/SecretsManager.java b/src/main/java/com/exactpro/th2/inframgr/k8s/SecretsManager.java index b1bac2e..c90fbf0 100644 --- a/src/main/java/com/exactpro/th2/inframgr/k8s/SecretsManager.java +++ b/src/main/java/com/exactpro/th2/inframgr/k8s/SecretsManager.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,6 @@ package com.exactpro.th2.inframgr.k8s; -import com.exactpro.th2.inframgr.Config; import com.exactpro.th2.inframgr.SecretsController; import com.exactpro.th2.inframgr.statuswatcher.ResourcePath; import io.fabric8.kubernetes.api.model.Secret; @@ -25,8 +24,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.*; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; public class SecretsManager { @@ -38,8 +40,8 @@ public class SecretsManager { private final String prefix; - public SecretsManager() throws IOException { - this.prefix = Config.getInstance().getKubernetes().getNamespacePrefix(); + public SecretsManager(String prefix) { + this.prefix = prefix; } public Secret getCustomSecret(String schemaName) { @@ -55,7 +57,8 @@ public Secret getCustomSecret(String schemaName) { } public Set createOrReplaceSecrets(String schemaName, - List secretEntries) { + List secretEntries, + String user) { String namespace = prefix + schemaName; String resourceLabel = ResourcePath.annotationFor(namespace, Kubernetes.KIND_SECRET, DEFAULT_SECRET_NAME); Set updatedEntries = new HashSet<>(); @@ -70,8 +73,8 @@ public Set createOrReplaceSecrets(String schemaName, } secret.setData(data); try { - kubernetesClient.resource(secret).inNamespace(namespace).createOrReplace(); - logger.info("Updated \"{}\"", resourceLabel); + kubernetesClient.resource(secret).inNamespace(namespace).serverSideApply(); + logger.info("Updated \"{}\" by user \"{}\"", resourceLabel, user); return updatedEntries; } catch (Exception e) { logger.error("Exception while updating \"{}\"", resourceLabel, e); @@ -91,7 +94,7 @@ public Set createOrReplaceSecrets(String schemaName, data.putAll(secretEntries); secret.setData(data); try { - kubernetesClient.resource(secret).inNamespace(namespace).createOrReplace(); + kubernetesClient.resource(secret).inNamespace(namespace).serverSideApply(); logger.info("Updated \"{}\"", resourceLabel); return secretEntries.keySet(); } catch (Exception e) { @@ -113,7 +116,7 @@ public Set deleteSecrets(String schemaName, Set secretEntries) { } secret.setData(data); try { - kubernetesClient.resource(secret).inNamespace(namespace).createOrReplace(); + kubernetesClient.resource(secret).inNamespace(namespace).serverSideApply(); logger.info("Removed entries from \"{}\"", resourceLabel); return secretEntries; } catch (Exception e) { diff --git a/src/main/java/com/exactpro/th2/inframgr/metrics/PrometheusServer.java b/src/main/java/com/exactpro/th2/inframgr/metrics/PrometheusServer.java deleted file mode 100644 index 85fdcbf..0000000 --- a/src/main/java/com/exactpro/th2/inframgr/metrics/PrometheusServer.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.exactpro.th2.inframgr.metrics; - -import com.exactpro.th2.inframgr.Config; -import com.exactpro.th2.inframgr.util.cfg.PrometheusConfig; -import io.prometheus.client.exporter.HTTPServer; -import io.prometheus.client.hotspot.DefaultExports; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.concurrent.atomic.AtomicReference; - -public class PrometheusServer { - - private static final Logger logger = LoggerFactory.getLogger(PrometheusServer.class); - - private static final AtomicReference prometheusExporter = new AtomicReference<>(); - - public static void start() throws IOException { - DefaultExports.initialize(); - PrometheusConfig prometheusConfiguration = Config.getInstance().getPrometheusConfiguration(); - - String host = prometheusConfiguration.getHost(); - int port = prometheusConfiguration.getPort(); - boolean enabled = prometheusConfiguration.getEnabled(); - - prometheusExporter.updateAndGet(server -> { - if (server == null && enabled) { - try { - server = new HTTPServer(host, port); - logger.info("Started prometheus server on: \"{}:{}\"", host, port); - return server; - } catch (IOException e) { - throw new RuntimeException("Failed to create Prometheus exporter", e); - } - } - return server; - }); - } -} diff --git a/src/main/java/com/exactpro/th2/inframgr/metrics/PrometheusService.java b/src/main/java/com/exactpro/th2/inframgr/metrics/PrometheusService.java new file mode 100644 index 0000000..6d2b570 --- /dev/null +++ b/src/main/java/com/exactpro/th2/inframgr/metrics/PrometheusService.java @@ -0,0 +1,68 @@ +/* + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.inframgr.metrics; + +import com.exactpro.th2.inframgr.Config; +import com.exactpro.th2.inframgr.util.cfg.PrometheusConfig; +import io.prometheus.client.exporter.HTTPServer; +import io.prometheus.client.hotspot.DefaultExports; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicReference; + +@Component +// TODO: replace to spring implementation +public class PrometheusService { + + private static final Logger logger = LoggerFactory.getLogger(PrometheusService.class); + + private static final AtomicReference prometheusExporter = new AtomicReference<>(); + + @Autowired + private Config config; + + @PostConstruct + public void start() { + DefaultExports.initialize(); + PrometheusConfig prometheusConfiguration = config.getPrometheusConfiguration(); + + if (prometheusConfiguration.getEnabled()) { + try { + String host = prometheusConfiguration.getHost(); + int port = prometheusConfiguration.getPort(); + prometheusExporter.set(new HTTPServer(host, port)); + logger.info("Started prometheus server on: \"{}:{}\"", host, port); + } catch (IOException e) { + throw new RuntimeException("Failed to create Prometheus exporter", e); + } + } + } + + @PreDestroy + public void stop() { + HTTPServer httpServer = prometheusExporter.get(); + if (httpServer != null) { + httpServer.close(); + } + } +} diff --git a/src/main/java/com/exactpro/th2/inframgr/repository/RepositoryWatcherService.java b/src/main/java/com/exactpro/th2/inframgr/repository/RepositoryWatcherService.java index f81bb88..dd76dc1 100644 --- a/src/main/java/com/exactpro/th2/inframgr/repository/RepositoryWatcherService.java +++ b/src/main/java/com/exactpro/th2/inframgr/repository/RepositoryWatcherService.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,127 +20,157 @@ import com.exactpro.th2.inframgr.SchemaEventRouter; import com.exactpro.th2.inframgr.docker.monitoring.DynamicResourceProcessor; import com.exactpro.th2.inframgr.k8s.K8sResourceCache; -import com.exactpro.th2.inframgr.util.cfg.GitCfg; import com.exactpro.th2.infrarepo.git.GitterContext; import io.fabric8.kubernetes.api.model.Namespace; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientBuilder; import io.fabric8.kubernetes.client.dsl.Resource; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; + +import static com.exactpro.th2.inframgr.SchemaController.SOURCE_BRANCH; @Component public class RepositoryWatcherService { - private final Map commitHistory; + private static final Logger LOGGER = LoggerFactory.getLogger(RepositoryWatcherService.class); - private final GitCfg config; + private static volatile boolean startupSynchronizationComplete; - private final SchemaEventRouter eventRouter; + private final Map commitHistory; - private final Logger logger; + private final SchemaEventRouter eventRouter; private final KubernetesClient kubeClient = new KubernetesClientBuilder().build(); - private int prevBranchCount; - - private final String namespacePrefix; + private Set prevBranches = Collections.emptySet(); - private static volatile boolean startupSynchronizationComplete; + @Autowired + private Config config; - public RepositoryWatcherService() throws Exception { - var fullConfig = Config.getInstance(); + public RepositoryWatcherService() { commitHistory = new HashMap<>(); - config = fullConfig.getGit(); - namespacePrefix = fullConfig.getKubernetes().getNamespacePrefix(); eventRouter = SchemaEventRouter.getInstance(); - logger = LoggerFactory.getLogger(RepositoryWatcherService.class); } @Scheduled(fixedDelayString = "${GIT_FETCH_INTERVAL:14000}") private void scheduledJob() { try { - logger.debug("fetching changes from git"); - GitterContext ctx = GitterContext.getContext(config); + LOGGER.debug("fetching changes from git"); + GitterContext ctx = GitterContext.getContext(config.getGit()); Map commits = ctx.getAllBranchesCommits(); - if (prevBranchCount > commits.size()) { + if (!prevBranches.equals(commits.keySet())) { + LOGGER.info("Fetched branches: {}, previous branches: {}", commits.keySet(), prevBranches); removeExtinctedNamespaces(commits.keySet()); + } else { + notifyAboutExtinctedNamespaces(commits.keySet()); } - prevBranchCount = commits.size(); + prevBranches = commits.keySet(); if (commitHistory.isEmpty()) { doInitialSynchronization(commits); } commits.forEach((branch, commitRef) -> { - if (!(branch.equals("master") + if (!(SOURCE_BRANCH.equals(branch) || commitHistory.isEmpty() || commitHistory.getOrDefault(branch, "").equals(commitRef))) { - logger.info("New commit \"{}\" detected for branch \"{}\"", commitRef, branch); + LOGGER.info("New commit \"{}\" detected for branch \"{}\"", commitRef, branch); RepositoryUpdateEvent event = new RepositoryUpdateEvent(branch, commitRef); boolean sent = eventRouter.addEventIfNotCached(branch, event); if (!sent) { - logger.info("Event is recently processed, ignoring"); + LOGGER.info("Event is recently processed, ignoring"); } } }); commitHistory.putAll(commits); } catch (Exception e) { - logger.error("Error fetching repository", e); + LOGGER.error("Error fetching repository", e); } } private void doInitialSynchronization(Map commits) { - logger.info("Starting Kubernetes synchronization phase"); + LOGGER.info("Starting Kubernetes synchronization phase"); commits.forEach((branch, commitRef) -> { - if (!branch.equals("master")) { + if (!SOURCE_BRANCH.equals(branch)) { RepositoryUpdateEvent event = new RepositoryUpdateEvent(branch, commitRef); boolean sent = eventRouter.addEventIfNotCached(branch, event); if (!sent) { - logger.info("Event is recently processed, ignoring"); + LOGGER.info("Event is recently processed, ignoring"); } } }); startupSynchronizationComplete = true; - logger.info("Kubernetes synchronization phase complete"); + LOGGER.info("Kubernetes synchronization phase complete"); } private void removeExtinctedNamespaces(Set existingBranches) { - List extinctNamespaces = kubeClient.namespaces() - .list() - .getItems() - .stream() - .map(item -> item.getMetadata().getName()) - .filter(namespace -> namespace.startsWith(namespacePrefix) - && !existingBranches.contains(namespace.substring(namespacePrefix.length()))) - .collect(Collectors.toList()); + List extinctNamespaces = getExtinctNamespaces(existingBranches); for (String extinctNamespace : extinctNamespaces) { - String schemaName = extinctNamespace.substring(namespacePrefix.length()); + String schemaName = extinctNamespace.substring(config.getKubernetes().getNamespacePrefix().length()); DynamicResourceProcessor.deleteSchema(schemaName); K8sResourceCache.INSTANCE.removeNamespace(extinctNamespace); Resource namespaceResource = kubeClient.namespaces().withName(extinctNamespace); if (namespaceResource != null) { - String branchName = extinctNamespace.substring(namespacePrefix.length()); - eventRouter.removeEventsForSchema(branchName); - logger.info("branch \"{}\" was removed from remote repository, deleting corresponding namespace \"{}\"", - branchName, existingBranches); - namespaceResource.delete(); - commitHistory.remove(branchName); + eventRouter.removeEventsForSchema(schemaName); + if (config.getBehaviour().isPermittedToRemoveNamespace()) { + LOGGER.info( + "branch \"{}\" was removed from remote repository, deleting corresponding namespace \"{}\"", + schemaName, + extinctNamespace + ); + namespaceResource.delete(); + } else { + LOGGER.warn( + "branch \"{}\" was removed from remote repository, stopping namespace \"{}\" maintenance", + schemaName, + extinctNamespace + ); + } + commitHistory.remove(schemaName); } } } - public static boolean isStartupSynchronizationComplete() { + private void notifyAboutExtinctedNamespaces(Set existingBranches) { + if (!config.getBehaviour().isPermittedToRemoveNamespace() && LOGGER.isWarnEnabled()) { + List extinctNamespaces = getExtinctNamespaces(existingBranches); + if (!extinctNamespaces.isEmpty()) { + LOGGER.warn( + "Maintenance for namespaces \"{}\" are stopped, existed branches: \"{}\"", + extinctNamespaces, + existingBranches + ); + } + } + } + + @NotNull + private List getExtinctNamespaces(Set existingBranches) { + String namespacePrefix = config.getKubernetes().getNamespacePrefix(); + return kubeClient.namespaces() + .list() + .getItems() + .stream() + .map(item -> item.getMetadata().getName()) + .filter(namespace -> namespace.startsWith(namespacePrefix) + && !existingBranches.contains(namespace.substring(namespacePrefix.length()))) + .toList(); + } + + public boolean isStartupSynchronizationComplete() { return startupSynchronizationComplete; } } diff --git a/src/main/java/com/exactpro/th2/inframgr/statuswatcher/StatusCache.java b/src/main/java/com/exactpro/th2/inframgr/statuswatcher/StatusCache.java index eb9c2fe..9024495 100644 --- a/src/main/java/com/exactpro/th2/inframgr/statuswatcher/StatusCache.java +++ b/src/main/java/com/exactpro/th2/inframgr/statuswatcher/StatusCache.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 Exactpro (Exactpro Systems Limited) + * Copyright 2020-2023 Exactpro (Exactpro Systems Limited) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,30 +19,37 @@ import com.exactpro.th2.inframgr.Config; import com.exactpro.th2.inframgr.SchemaEventRouter; import com.exactpro.th2.inframgr.k8s.Kubernetes; +import com.exactpro.th2.inframgr.k8s.KubernetesService; import com.exactpro.th2.infrarepo.ResourceType; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.client.informers.ResourceEventHandler; import jakarta.annotation.PostConstruct; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.util.*; @Component public class StatusCache { + private static final Logger LOGGER = LoggerFactory.getLogger(StatusCache.class); - private NamespaceResources resources; + private final NamespaceResources resources; - private Map> dependencies; + private final Map> dependencies; - private Map dependents; + private final Map dependents; - private SchemaEventRouter eventRouter; + private final SchemaEventRouter eventRouter; - private Kubernetes kube; + private Kubernetes anonKube; - private static final Logger logger = LoggerFactory.getLogger(StatusCache.class); + @Autowired + private Config config; + + @Autowired + private KubernetesService kubernetesService; public StatusCache() { resources = new NamespaceResources(); @@ -57,10 +64,7 @@ private synchronized void update(ResourceCondition resource, String schema, Acti ResourcePath path = ResourcePath.fromMetadata(resource); ResourceType type = ResourceType.forKind(path.getKind()); - boolean isSchemaElement = false; - if (type != null && type.isMangedResource()) { - isSchemaElement = true; - } + boolean isSchemaElement = type != null && type.isMangedResource(); ResourcePath annotationPath = null; switch (action) { @@ -96,7 +100,7 @@ public synchronized List getStatuses(String schema) { List events = new ArrayList<>(); - String namespace = kube.formatNamespaceName(schema); + String namespace = anonKube.formatNamespaceName(schema); List schemaElements = resources.getSchemaElements(namespace); if (schemaElements == null) { return null; @@ -116,9 +120,8 @@ public synchronized List getResourceDependencyStatuses(String String kind, String resourceName) { - String namespace = kube.formatNamespaceName(schema); - List elements = resources.getResourceElements(namespace, kind, resourceName); - return elements; + String namespace = anonKube.formatNamespaceName(schema); + return resources.getResourceElements(namespace, kind, resourceName); } private ResourceCondition.Status calculateStatus(ResourceCondition resource) { @@ -135,7 +138,7 @@ private ResourceCondition.Status calculateStatus(ResourceCondition resource) { } // find all pods for this component - // and compute lowest status as a common status + // and compute the lowest status as a common status ResourceCondition.Status podsStatus = null; for (ResourcePath p : components) { if (p.getKind().equals("Pod")) { @@ -196,18 +199,17 @@ private void unindex(ResourcePath resourcePath) { } @PostConstruct - public void start() throws Exception { - logger.info("Starting resource status monitoring"); + public void start() { + LOGGER.info("Starting resource status monitoring"); - Config config = Config.getInstance(); - kube = new Kubernetes(config.getKubernetes(), null); + anonKube = kubernetesService.getKubernetes(); - kube.registerSharedInformersAll(new ResourceEventHandler() { + anonKube.registerSharedInformersAll(new ResourceEventHandler() { @Override public void onAdd(HasMetadata obj) { ResourceCondition resource = ResourceCondition.extractFrom(obj); - String schema = kube.extractSchemaName(resource.getNamespace()); + String schema = anonKube.extractSchemaName(resource.getNamespace()); update(resource, schema, StatusCache.Action.ADD); } @@ -215,7 +217,7 @@ public void onAdd(HasMetadata obj) { @Override public void onUpdate(HasMetadata oldObj, HasMetadata newObj) { ResourceCondition resource = ResourceCondition.extractFrom(newObj); - String schema = kube.extractSchemaName(resource.getNamespace()); + String schema = anonKube.extractSchemaName(resource.getNamespace()); update(resource, schema, StatusCache.Action.ADD); } @@ -223,13 +225,13 @@ public void onUpdate(HasMetadata oldObj, HasMetadata newObj) { @Override public void onDelete(HasMetadata obj, boolean deletedFinalStateUnknown) { ResourceCondition resource = ResourceCondition.extractFrom(obj); - String schema = kube.extractSchemaName(resource.getNamespace()); + String schema = anonKube.extractSchemaName(resource.getNamespace()); update(resource, schema, StatusCache.Action.REMOVE); } }); - kube.startInformers(); + anonKube.startInformers(); } private enum Action { diff --git a/src/main/java/com/exactpro/th2/inframgr/util/cfg/BehaviourCfg.java b/src/main/java/com/exactpro/th2/inframgr/util/cfg/BehaviourCfg.java new file mode 100644 index 0000000..42b560f --- /dev/null +++ b/src/main/java/com/exactpro/th2/inframgr/util/cfg/BehaviourCfg.java @@ -0,0 +1,36 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.inframgr.util.cfg; + +public class BehaviourCfg { + /** + * Has infra-mgr got permission to remove Kubernetes namespace when + * branch is disabled (spec.k8s-propagation: deny) or removed. + * Infra-manager removes Kubernetes namespace when this option is true otherwise + * stops maintenance for the namespace without deleting any resources. + * Maintenance is continued when user enable or create the branch related to namespace: `` + */ + private boolean permittedToRemoveNamespace = true; + + public boolean isPermittedToRemoveNamespace() { + return permittedToRemoveNamespace; + } + + public void setPermittedToRemoveNamespace(boolean permittedToRemoveNamespace) { + this.permittedToRemoveNamespace = permittedToRemoveNamespace; + } +} diff --git a/src/main/java/com/exactpro/th2/inframgr/util/cfg/HttpCfg.java b/src/main/java/com/exactpro/th2/inframgr/util/cfg/HttpCfg.java new file mode 100644 index 0000000..9baa5be --- /dev/null +++ b/src/main/java/com/exactpro/th2/inframgr/util/cfg/HttpCfg.java @@ -0,0 +1,36 @@ +/* + * Copyright 2023 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.inframgr.util.cfg; + +import java.util.Map; + +public class HttpCfg { + + /** + * Map of username to encrypted by BCrypt (strength >= 10) password pairs. + * @see BCrypt + */ + private Map adminAccounts; + + public Map getAdminAccounts() { + return adminAccounts; + } + + public void setAdminAccounts(Map adminAccounts) { + this.adminAccounts = adminAccounts; + } +}