diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 119b9c06..c66b63c8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,8 @@ variables: APP_NAME: "th2-infra-mgr" MAJOR_VERSION: "0" - MINOR_VERSION: "8" - MAINTENANCE_VERSION: "6" + MINOR_VERSION: "9" + MAINTENANCE_VERSION: "0" DOCKER_PUBLISH_ENABLED: "true" include: diff --git a/gradle.properties b/gradle.properties index 00d93e71..15f6af75 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -release_version = 0.8.6 \ No newline at end of file +release_version = 0.9.0 \ No newline at end of file diff --git a/src/main/java/com/exactpro/th2/inframgr/SchemaController.java b/src/main/java/com/exactpro/th2/inframgr/SchemaController.java index 687c6a80..1d79967e 100644 --- a/src/main/java/com/exactpro/th2/inframgr/SchemaController.java +++ b/src/main/java/com/exactpro/th2/inframgr/SchemaController.java @@ -22,15 +22,10 @@ import com.exactpro.th2.inframgr.initializer.SchemaInitializer; import com.exactpro.th2.inframgr.k8s.K8sCustomResource; import com.exactpro.th2.inframgr.k8s.Kubernetes; -import com.exactpro.th2.inframgr.models.RepositoryResource; -import com.exactpro.th2.inframgr.models.RepositorySettings; -import com.exactpro.th2.inframgr.models.RepositorySnapshot; import com.exactpro.th2.inframgr.models.RequestEntry; -import com.exactpro.th2.inframgr.repository.Gitter; -import com.exactpro.th2.inframgr.repository.InconsistentRepositoryStateException; -import com.exactpro.th2.inframgr.repository.Repository; import com.exactpro.th2.inframgr.repository.RepositoryUpdateEvent; import com.exactpro.th2.inframgr.util.Stringifier; +import com.exactpro.th2.infrarepo.*; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import org.eclipse.jgit.api.errors.RefNotAdvertisedException; @@ -69,7 +64,7 @@ public Set getAvailableSchemas() throws ServiceException { @GetMapping("/schema/{name}") @ResponseBody - public RepositorySnapshot getSchemaFiles(@PathVariable(name="name") String schemaName) throws Exception { + public SchemaControllerResponse getSchemaFiles(@PathVariable(name="name") String schemaName) throws Exception { if (schemaName.equals(SOURCE_BRANCH)) throw new NotAcceptableException(REPOSITORY_ERROR, "Not Allowed"); @@ -78,7 +73,7 @@ public RepositorySnapshot getSchemaFiles(@PathVariable(name="name") String schem final Gitter gitter = Gitter.getBranch(gitConfig, schemaName); try { gitter.lock(); - return Repository.getSnapshot(gitter); + return new SchemaControllerResponse(Repository.getSnapshot(gitter)); } catch (RefNotAdvertisedException | RefNotFoundException e) { throw new ServiceException(HttpStatus.NOT_FOUND, HttpStatus.NOT_FOUND.name(), "schema does not exists"); } catch (Exception e) { @@ -91,7 +86,7 @@ public RepositorySnapshot getSchemaFiles(@PathVariable(name="name") String schem @PutMapping("/schema/{name}") @ResponseBody - public RepositorySnapshot createSchema(@PathVariable(name="name") String schemaName) throws Exception { + public SchemaControllerResponse createSchema(@PathVariable(name="name") String schemaName) throws Exception { if (schemaName.equals(SOURCE_BRANCH)) throw new NotAcceptableException(REPOSITORY_ERROR, "Not Allowed"); @@ -131,7 +126,7 @@ public RepositorySnapshot createSchema(@PathVariable(name="name") String schemaN event.setSyncingK8s(!(rs != null && (rs.isK8sPropagationDenied() || rs.isK8sSynchronizationRequired()))); router.addEvent(event); - return snapshot; + return new SchemaControllerResponse(snapshot); } catch (ServiceException se) { throw se; @@ -144,7 +139,7 @@ public RepositorySnapshot createSchema(@PathVariable(name="name") String schemaN @PostMapping("/schema/{name}") @ResponseBody - public RepositorySnapshot updateSchema(@PathVariable(name="name") String schemaName, @RequestBody String requestBody) + public SchemaControllerResponse updateSchema(@PathVariable(name="name") String schemaName, @RequestBody String requestBody) throws Exception { if (schemaName.equals(SOURCE_BRANCH)) @@ -208,7 +203,7 @@ public RepositorySnapshot updateSchema(@PathVariable(name="name") String schemaN // delegate this job to K8sSynchronization event.setSyncingK8s(false); router.addEvent(event); - return snapshot; + return new SchemaControllerResponse(snapshot); } event.setSyncingK8s(true); @@ -218,7 +213,7 @@ public RepositorySnapshot updateSchema(@PathVariable(name="name") String schemaN synchronizeWithK8s(config.getKubernetes(), operations, schemaName); } - return snapshot; + return new SchemaControllerResponse(snapshot); } catch (ServiceException se) { throw se; } catch (Exception e) { @@ -240,7 +235,7 @@ private void synchronizeWithK8s(Config.K8sConfig k8sConfig, List o if (entry.getPayload().getKind().isK8sResource()) { try { Stringifier.stringify(entry.getPayload().getSpec()); - RepositoryResource resource = new RepositoryResource(entry.getPayload()); + RepositoryResource resource = entry.getPayload().toRepositoryResource(); switch (entry.getOperation()) { case add: kube.createCustomResource(resource); @@ -281,13 +276,13 @@ private String updateRepository(Gitter gitter, List operations) th for (RequestEntry entry : operations) switch (entry.getOperation()) { case add: - Repository.add(gitter.getConfig(), branchName, entry.getPayload()); + Repository.add(gitter.getConfig(), branchName, entry.getPayload().toRepositoryResource()); break; case update: - Repository.update(gitter.getConfig(), branchName, entry.getPayload()); + Repository.update(gitter.getConfig(), branchName, entry.getPayload().toRepositoryResource()); break; case remove: - Repository.remove(gitter.getConfig(), branchName, entry.getPayload()); + Repository.remove(gitter.getConfig(), branchName, entry.getPayload().toRepositoryResource()); break; } return gitter.commitAndPush("schema update"); diff --git a/src/main/java/com/exactpro/th2/inframgr/SchemaControllerResponse.java b/src/main/java/com/exactpro/th2/inframgr/SchemaControllerResponse.java new file mode 100644 index 00000000..f6cef169 --- /dev/null +++ b/src/main/java/com/exactpro/th2/inframgr/SchemaControllerResponse.java @@ -0,0 +1,42 @@ +/* + * Copyright 2020-2020 Exactpro (Exactpro Systems Limited) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exactpro.th2.inframgr; + +import com.exactpro.th2.inframgr.models.ResourceEntry; +import com.exactpro.th2.infrarepo.RepositorySnapshot; + +import java.util.HashSet; +import java.util.Set; + +public class SchemaControllerResponse { + private String commitRef; + private Set resources; + + public SchemaControllerResponse(RepositorySnapshot snapshot) { + this.commitRef = snapshot.getCommitRef(); + resources = new HashSet<>(); + snapshot.getResources().forEach(e -> resources.add(new ResourceEntry(e))); + } + + public String getCommitRef() { + return commitRef; + } + + public Set getResources() { + return resources; + } +} diff --git a/src/main/java/com/exactpro/th2/inframgr/initializer/SchemaInitializer.java b/src/main/java/com/exactpro/th2/inframgr/initializer/SchemaInitializer.java index f9da4b88..6a0dce96 100644 --- a/src/main/java/com/exactpro/th2/inframgr/initializer/SchemaInitializer.java +++ b/src/main/java/com/exactpro/th2/inframgr/initializer/SchemaInitializer.java @@ -19,8 +19,8 @@ import com.exactpro.th2.inframgr.Config; import com.exactpro.th2.inframgr.k8s.K8sCustomResource; import com.exactpro.th2.inframgr.k8s.Kubernetes; -import com.exactpro.th2.inframgr.models.RepositoryResource; -import com.exactpro.th2.inframgr.models.ResourceType; +import com.exactpro.th2.infrarepo.RepositoryResource; +import com.exactpro.th2.infrarepo.ResourceType; import com.exactpro.th2.inframgr.statuswatcher.ResourcePath; import com.fasterxml.jackson.databind.ObjectMapper; import io.fabric8.kubernetes.api.model.ConfigMap; @@ -210,8 +210,7 @@ private static void copyIngress(Config config, Kubernetes kube) { logger.info("Creating \"{}\"", ResourcePath.annotationFor(namespace, "Ingress", ingressName)); K8sCustomResource ingress = kube.currentNamespace().loadCustomResource(ResourceType.HelmRelease, ingressName); - RepositoryResource.Metadata meta = new RepositoryResource.Metadata(); - meta.setName(ingressName); + RepositoryResource.Metadata meta = new RepositoryResource.Metadata(ingressName); RepositoryResource resource = new RepositoryResource(ResourceType.HelmRelease); resource.setSpec(ingress.getSpec()); diff --git a/src/main/java/com/exactpro/th2/inframgr/k8s/K8sOperator.java b/src/main/java/com/exactpro/th2/inframgr/k8s/K8sOperator.java index 675e3819..5fbdc607 100644 --- a/src/main/java/com/exactpro/th2/inframgr/k8s/K8sOperator.java +++ b/src/main/java/com/exactpro/th2/inframgr/k8s/K8sOperator.java @@ -17,12 +17,10 @@ package com.exactpro.th2.inframgr.k8s; import com.exactpro.th2.inframgr.Config; -import com.exactpro.th2.inframgr.models.*; -import com.exactpro.th2.inframgr.repository.Gitter; -import com.exactpro.th2.inframgr.repository.Repository; import com.exactpro.th2.inframgr.statuswatcher.ResourcePath; import com.exactpro.th2.inframgr.util.RetryableTaskQueue; import com.exactpro.th2.inframgr.util.Stringifier; +import com.exactpro.th2.infrarepo.*; import io.fabric8.kubernetes.api.model.Namespace; import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.client.KubernetesClientException; @@ -111,7 +109,7 @@ private void processEvent(Watcher.Action action, K8sCustomResource res, Kubernet String hash = res.getSourceHash(); String resourceLabel = "\"" + ResourcePath.annotationFor(namespace, kind, name) + "\""; - String hashTag = "[" + (hash == null ? "no-hash" : hash) + "]"; + String hashTag = "[" + (res.getSourceHash() == null ? "no-hash" : res.getSourceHash()) + "]"; logger.debug("Received {} event on resource {} {}", action.name(), resourceLabel, hashTag); Lock lock = cache.lockFor(namespace, kind, name); @@ -123,14 +121,14 @@ private void processEvent(Watcher.Action action, K8sCustomResource res, Kubernet String cachedHash = cacheEntry == null ? null : cacheEntry.getHash(); if (action.equals(Watcher.Action.DELETED) && cacheEntry != null && cacheEntry.isMarkedAsDeleted() && Objects.equals(cachedHash, hash)) { - logger.debug("No action needed for resource {}", resourceLabel); + logger.debug("No action needed for resource {} {}", resourceLabel, hashTag); return; } if (!action.equals(Watcher.Action.DELETED) && cacheEntry != null && !cacheEntry.isMarkedAsDeleted() && Objects.equals(cachedHash, hash)) { - logger.debug("No action needed for resource {}", resourceLabel); + logger.debug("No action needed for resource {} {}", resourceLabel, hashTag); return; } @@ -139,7 +137,7 @@ private void processEvent(Watcher.Action action, K8sCustomResource res, Kubernet Gitter gitter = Gitter.getBranch(config.getGit(), kube.extractSchemaName(namespace)); logger.info("Checking out branch \"{}\" from repository", gitter.getBranch()) ; - ResourceEntry resourceEntry = null; + RepositoryResource resource = null; try { gitter.lock(); RepositorySnapshot snapshot = Repository.getSnapshot(gitter); @@ -150,20 +148,16 @@ private void processEvent(Watcher.Action action, K8sCustomResource res, Kubernet return; // refresh cache for this namespace - for (ResourceEntry e :snapshot.getResources()) { - cache.add(namespace, e); - if (e.getKind().kind().equals(kind) && e.getName().equals(name)) - resourceEntry = e; + for (RepositoryResource r :snapshot.getResources()) { + cache.add(namespace, r); + if (r.getKind().equals(kind) && r.getMetadata().getName().equals(name)) + resource = r; } } finally { gitter.unlock(); } - hash = null; - if (resourceEntry != null) - hash = resourceEntry.getSourceHash(); - hashTag = "[" + (hash == null ? "no-hash" : hash) + "]"; // recheck item cacheEntry = cache.get(namespace, kind, name); @@ -183,16 +177,20 @@ private void processEvent(Watcher.Action action, K8sCustomResource res, Kubernet else { actionDelete = true; - resourceEntry = new ResourceEntry(); - resourceEntry.setKind(ResourceType.forKind(kind)); - resourceEntry.setName(name); + resource = new RepositoryResource(); + resource.setKind(kind); + resource.setMetadata(new RepositoryResource.Metadata(name)); } } - Stringifier.stringify(resourceEntry.getSpec()); - RepositoryResource resource = new RepositoryResource(resourceEntry); + hash = null; + if (resource != null) + hash = resource.getSourceHash(); + hashTag = "[" + (hash == null ? "no-hash" : hash) + "]"; + + Stringifier.stringify(resource.getSpec()); if (actionReplace) { - logger.info("Detected external manipulation on {} {}, recreating resource", resourceLabel,hashTag) ; + logger.info("Detected external manipulation on {} {}, recreating resource", resourceLabel, hashTag) ; // check current status of namespace Namespace n = kube.getNamespace(namespace); diff --git a/src/main/java/com/exactpro/th2/inframgr/k8s/K8sResourceCache.java b/src/main/java/com/exactpro/th2/inframgr/k8s/K8sResourceCache.java index 4c0628e7..c6b2cd0a 100644 --- a/src/main/java/com/exactpro/th2/inframgr/k8s/K8sResourceCache.java +++ b/src/main/java/com/exactpro/th2/inframgr/k8s/K8sResourceCache.java @@ -16,8 +16,7 @@ package com.exactpro.th2.inframgr.k8s; -import com.exactpro.th2.inframgr.models.ResourceEntry; -import com.exactpro.th2.inframgr.models.ResourceType; +import com.exactpro.th2.infrarepo.RepositoryResource; import java.util.HashMap; import java.util.Map; @@ -27,7 +26,7 @@ public enum K8sResourceCache { INSTANCE; - public class CacheEntry { + public static class CacheEntry { private boolean markedDeleted; private String hash; @@ -48,16 +47,16 @@ private void setHash(String hash) { } } - private Map cache = new HashMap<>(); - private Map locks = new HashMap<>(); + private final Map cache = new HashMap<>(); + private final Map locks = new HashMap<>(); - private String keyFor(String namespace, ResourceType type, String resourceName) { - return String.format("%s.%s.%s", namespace, type.kind(), resourceName); + private String keyFor(String namespace, String type, String resourceName) { + return String.format("%s:%s/%s", namespace, type, resourceName); } public synchronized void add(String namespace, K8sCustomResource resource) { - String key = keyFor(namespace, ResourceType.forKind(resource.getKind()), resource.getMetadata().getName()); + String key = keyFor(namespace, resource.getKind(), resource.getMetadata().getName()); CacheEntry entry = new CacheEntry(); entry.setHash(resource.getSourceHash()); @@ -65,9 +64,9 @@ public synchronized void add(String namespace, K8sCustomResource resource) { cache.put(key, entry); } - public synchronized void add(String namespace, ResourceEntry resource) { + public synchronized void add(String namespace, RepositoryResource resource) { - String key = keyFor(namespace, resource.getKind(), resource.getName()); + String key = keyFor(namespace, resource.getKind(), resource.getMetadata().getName()); CacheEntry entry = new CacheEntry(); entry.setHash(resource.getSourceHash()); @@ -76,7 +75,7 @@ public synchronized void add(String namespace, ResourceEntry resource) { public synchronized CacheEntry get(String namespace, String resourceType, String resourceName) { - String key = keyFor(namespace, ResourceType.forKind(resourceType), resourceName); + String key = keyFor(namespace, resourceType, resourceName); return cache.get(key); } @@ -87,7 +86,6 @@ public synchronized CacheEntry get(String namespace, K8sCustomResource resource) public synchronized void remove(String namespace, String resourceType, String resourceName) { - String key = keyFor(namespace, ResourceType.forKind(resourceType), resourceName); CacheEntry entry = get(namespace, resourceType, resourceName); if (entry != null) entry.markAsDeleted(); @@ -95,7 +93,7 @@ public synchronized void remove(String namespace, String resourceType, String re public synchronized Lock lockFor(String namespace, String resourceType, String resourceName) { - String key = keyFor(namespace, ResourceType.forKind(resourceType), resourceName); + String key = keyFor(namespace, resourceType, resourceName); return locks.computeIfAbsent(key, v -> new ReentrantLock()); } diff --git a/src/main/java/com/exactpro/th2/inframgr/k8s/K8sSynchronization.java b/src/main/java/com/exactpro/th2/inframgr/k8s/K8sSynchronization.java index 69e58fd3..10a48c0d 100644 --- a/src/main/java/com/exactpro/th2/inframgr/k8s/K8sSynchronization.java +++ b/src/main/java/com/exactpro/th2/inframgr/k8s/K8sSynchronization.java @@ -18,12 +18,10 @@ import com.exactpro.th2.inframgr.Config; import com.exactpro.th2.inframgr.SchemaEventRouter; import com.exactpro.th2.inframgr.initializer.SchemaInitializer; -import com.exactpro.th2.inframgr.models.*; -import com.exactpro.th2.inframgr.repository.Gitter; -import com.exactpro.th2.inframgr.repository.Repository; import com.exactpro.th2.inframgr.repository.RepositoryUpdateEvent; import com.exactpro.th2.inframgr.statuswatcher.ResourcePath; import com.exactpro.th2.inframgr.util.Stringifier; +import com.exactpro.th2.infrarepo.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -56,7 +54,7 @@ private void deleteNamespace(String schemaName) { } } - private void synchronizeNamespace(String schemaName, Map> repositoryEntries) throws Exception { + private void synchronizeNamespace(String schemaName, Map> repositoryResources) throws Exception { try (Kubernetes kube = new Kubernetes(config.getKubernetes(), schemaName)) { @@ -65,29 +63,28 @@ private void synchronizeNamespace(String schemaName, Map> k8sEntries = new HashMap<>(); + Map> k8sResources = new HashMap<>(); for (ResourceType t : ResourceType.values()) if (t.isK8sResource() && !t.equals(ResourceType.HelmRelease)) - k8sEntries.put(t, kube.loadCustomResources(t)); + k8sResources.put(t.kind(), kube.loadCustomResources(t)); // synchronize by resource type - for (ResourceType resourceType : ResourceType.values()) - if (resourceType.isK8sResource() && !resourceType.equals(ResourceType.HelmRelease)) { - Map entries = repositoryEntries.get(resourceType); - Map customResources = k8sEntries.get(resourceType); - - for (ResourceEntry entry: entries.values()) { - String resourceName = entry.getName(); - String resourceLabel = "\"" + ResourcePath.annotationFor(namespace, resourceType.kind(), resourceName) + "\""; - String hashTag = "[" + (entry.getSourceHash() == null ? "no-hash" : entry.getSourceHash()) + "]"; + for (ResourceType type : ResourceType.values()) + if (type.isK8sResource() && !type.equals(ResourceType.HelmRelease)) { + Map resources = repositoryResources.get(type.kind()); + Map customResources = k8sResources.get(type.kind()); + + for (RepositoryResource resource: resources.values()) { + String resourceName = resource.getMetadata().getName(); + String resourceLabel = "\"" + ResourcePath.annotationFor(namespace, type.kind(), resourceName) + "\""; + String hashTag = "[" + (resource.getSourceHash() == null ? "no-hash" : resource.getSourceHash()) + "]"; // add resource to cache - cache.add(namespace, entry); + cache.add(namespace, resource); // check repository items against k8s if (!customResources.containsKey(resourceName)) { // create custom resources that do not exist in k8s logger.info("Creating resource {} {}", resourceLabel, hashTag); - RepositoryResource resource = new RepositoryResource(entry); try { Stringifier.stringify(resource.getSpec()); kube.createCustomResource(resource); @@ -98,10 +95,9 @@ private void synchronizeNamespace(String schemaName, Map repositoryEntries = snapshot.getResources(); + Set repositoryResources = snapshot.getResources(); RepositorySettings repositorySettings = snapshot.getRepositorySettings(); if (repositorySettings != null && repositorySettings.isK8sPropagationDenied()) { @@ -161,16 +157,15 @@ public void synchronizeBranch(String branch) { logger.info("Proceeding with schema \"{}\"", branch); // convert to map - Map> repositoryMap = new HashMap<>(); + Map> repositoryMap = new HashMap<>(); for (ResourceType t : ResourceType.values()) if (t.isK8sResource()) - repositoryMap.put(t, new HashMap<>()); + repositoryMap.put(t.kind(), new HashMap<>()); - for (ResourceEntry entry : repositoryEntries) - if (entry.getKind().isK8sResource()) { - Map typeMap = repositoryMap.get(entry.getKind()); - repositoryMap.putIfAbsent(entry.getKind(), typeMap); - typeMap.put(entry.getName(), entry); + for (RepositoryResource resource : repositoryResources) + if (ResourceType.forKind(resource.getKind()).isK8sResource()) { + Map typeMap = repositoryMap.get(resource.getKind()); + typeMap.put(resource.getMetadata().getName(), resource); } // synchronize entries diff --git a/src/main/java/com/exactpro/th2/inframgr/k8s/Kubernetes.java b/src/main/java/com/exactpro/th2/inframgr/k8s/Kubernetes.java index a9ff5681..f87edf8a 100644 --- a/src/main/java/com/exactpro/th2/inframgr/k8s/Kubernetes.java +++ b/src/main/java/com/exactpro/th2/inframgr/k8s/Kubernetes.java @@ -16,8 +16,8 @@ package com.exactpro.th2.inframgr.k8s; import com.exactpro.th2.inframgr.Config; -import com.exactpro.th2.inframgr.models.RepositoryResource; -import com.exactpro.th2.inframgr.models.ResourceType; +import com.exactpro.th2.infrarepo.RepositoryResource; +import com.exactpro.th2.infrarepo.ResourceType; import io.fabric8.kubernetes.api.model.*; import io.fabric8.kubernetes.api.model.apps.Deployment; import io.fabric8.kubernetes.client.ConfigBuilder; diff --git a/src/main/java/com/exactpro/th2/inframgr/models/ResourceEntry.java b/src/main/java/com/exactpro/th2/inframgr/models/ResourceEntry.java index 9df5eb9a..e1912a6f 100644 --- a/src/main/java/com/exactpro/th2/inframgr/models/ResourceEntry.java +++ b/src/main/java/com/exactpro/th2/inframgr/models/ResourceEntry.java @@ -15,6 +15,9 @@ */ package com.exactpro.th2.inframgr.models; +import com.exactpro.th2.infrarepo.RepositoryResource; +import com.exactpro.th2.infrarepo.ResourceType; + public class ResourceEntry { private ResourceType kind; @@ -22,6 +25,14 @@ public class ResourceEntry { private Object spec; private String hash; + public ResourceEntry() {} + public ResourceEntry(RepositoryResource resource) { + this.kind = ResourceType.forKind(resource.getKind()); + this.name = resource.getMetadata().getName(); + this.spec = resource.getSpec(); + this.hash = resource.getSourceHash(); + } + public ResourceType getKind() { return kind; } @@ -53,4 +64,15 @@ public String getSourceHash() { public void setSourceHash(String hash) { this.hash = hash; } + + public RepositoryResource toRepositoryResource() { + RepositoryResource resource = new RepositoryResource(); + resource.setApiVersion(this.getKind().k8sApiVersion()); + resource.setKind(this.getKind().kind()); + resource.setSpec(this.getSpec()); + resource.setSourceHash(this.getSourceHash()); + resource.setMetadata(new RepositoryResource.Metadata(this.getName())); + return resource; + } + } diff --git a/src/main/java/com/exactpro/th2/inframgr/repository/RepositoryWatcherService.java b/src/main/java/com/exactpro/th2/inframgr/repository/RepositoryWatcherService.java index 977ca6bc..e0834fe7 100644 --- a/src/main/java/com/exactpro/th2/inframgr/repository/RepositoryWatcherService.java +++ b/src/main/java/com/exactpro/th2/inframgr/repository/RepositoryWatcherService.java @@ -17,6 +17,7 @@ import com.exactpro.th2.inframgr.Config; import com.exactpro.th2.inframgr.SchemaEventRouter; +import com.exactpro.th2.infrarepo.Gitter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Scheduled; diff --git a/src/main/java/com/exactpro/th2/inframgr/statuswatcher/NamespaceResources.java b/src/main/java/com/exactpro/th2/inframgr/statuswatcher/NamespaceResources.java index a051c536..affd845c 100644 --- a/src/main/java/com/exactpro/th2/inframgr/statuswatcher/NamespaceResources.java +++ b/src/main/java/com/exactpro/th2/inframgr/statuswatcher/NamespaceResources.java @@ -16,7 +16,7 @@ package com.exactpro.th2.inframgr.statuswatcher; -import com.exactpro.th2.inframgr.models.ResourceType; +import com.exactpro.th2.infrarepo.ResourceType; import java.util.ArrayList; import java.util.HashMap; diff --git a/src/main/java/com/exactpro/th2/inframgr/statuswatcher/ResourceCondition.java b/src/main/java/com/exactpro/th2/inframgr/statuswatcher/ResourceCondition.java index dd6113c3..ab2fd000 100644 --- a/src/main/java/com/exactpro/th2/inframgr/statuswatcher/ResourceCondition.java +++ b/src/main/java/com/exactpro/th2/inframgr/statuswatcher/ResourceCondition.java @@ -17,7 +17,7 @@ package com.exactpro.th2.inframgr.statuswatcher; import com.exactpro.th2.inframgr.k8s.K8sCustomResource; -import com.exactpro.th2.inframgr.models.ResourceType; +import com.exactpro.th2.infrarepo.ResourceType; import io.fabric8.kubernetes.api.model.*; import io.fabric8.kubernetes.api.model.apps.Deployment; import io.fabric8.kubernetes.api.model.apps.DeploymentCondition; diff --git a/src/main/java/com/exactpro/th2/inframgr/statuswatcher/StatusCache.java b/src/main/java/com/exactpro/th2/inframgr/statuswatcher/StatusCache.java index ac4a39a2..c59c72f4 100644 --- a/src/main/java/com/exactpro/th2/inframgr/statuswatcher/StatusCache.java +++ b/src/main/java/com/exactpro/th2/inframgr/statuswatcher/StatusCache.java @@ -19,7 +19,7 @@ import com.exactpro.th2.inframgr.Config; import com.exactpro.th2.inframgr.SchemaEventRouter; import com.exactpro.th2.inframgr.k8s.Kubernetes; -import com.exactpro.th2.inframgr.models.ResourceType; +import com.exactpro.th2.infrarepo.ResourceType; import io.fabric8.kubernetes.api.model.HasMetadata; import io.fabric8.kubernetes.client.KubernetesClientException; import org.slf4j.Logger; diff --git a/src/main/java/com/exactpro/th2/inframgr/util/cfg/_GitConfig.java b/src/main/java/com/exactpro/th2/inframgr/util/cfg/_GitConfig.java index 04f17830..f0725386 100644 --- a/src/main/java/com/exactpro/th2/inframgr/util/cfg/_GitConfig.java +++ b/src/main/java/com/exactpro/th2/inframgr/util/cfg/_GitConfig.java @@ -15,8 +15,9 @@ */ package com.exactpro.th2.inframgr.util.cfg; +import com.exactpro.th2.infrarepo.GitConfig; -public class _GitConfig { +public class _GitConfig implements GitConfig { private String remoteRepository; private boolean ignoreInsecureHosts; @@ -27,6 +28,7 @@ public class _GitConfig { private String privateKey; private byte[] privateKeyBytes; + @Override public String getRemoteRepository() { return remoteRepository; } @@ -35,6 +37,7 @@ public void setRemoteRepository(String remoteRepository) { this.remoteRepository = remoteRepository; } + @Override public boolean ignoreInsecureHosts() { return ignoreInsecureHosts; } @@ -43,6 +46,7 @@ public void setIgnoreInsecureHosts(boolean ignoreInsecureHosts) { this.ignoreInsecureHosts = ignoreInsecureHosts; } + @Override public String getLocalRepositoryRoot() { return localRepositoryRoot; } @@ -51,6 +55,7 @@ public void setLocalRepositoryRoot(String localRepositoryRoot) { this.localRepositoryRoot = localRepositoryRoot; } + @Override public String getPrivateKeyFile() { return privateKeyFile; } @@ -59,6 +64,7 @@ public void setPrivateKeyFile(String privateKeyFile) { this.privateKeyFile = privateKeyFile; } + @Override public byte[] getPrivateKey() { return privateKeyBytes; } diff --git a/src/main/java/com/exactpro/th2/inframgr/util/Hash.java b/src/main/java/com/exactpro/th2/infrarepo/GitConfig.java similarity index 50% rename from src/main/java/com/exactpro/th2/inframgr/util/Hash.java rename to src/main/java/com/exactpro/th2/infrarepo/GitConfig.java index c7e60afd..6a48fb12 100644 --- a/src/main/java/com/exactpro/th2/inframgr/util/Hash.java +++ b/src/main/java/com/exactpro/th2/infrarepo/GitConfig.java @@ -13,25 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.exactpro.th2.inframgr.util; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; +package com.exactpro.th2.infrarepo; -public class Hash { - public static String digest(String data) { - try { - MessageDigest md = MessageDigest.getInstance("SHA-256"); - byte[] digest = md.digest(data.getBytes()); - StringBuilder sb = new StringBuilder(); - for (byte b : digest) - sb.append(String.format("%02x", b)); +public interface GitConfig { + String getRemoteRepository(); - return sb.toString(); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } - } -} + boolean ignoreInsecureHosts(); + + String getLocalRepositoryRoot(); + String getPrivateKeyFile(); + byte[] getPrivateKey(); +} diff --git a/src/main/java/com/exactpro/th2/inframgr/repository/Gitter.java b/src/main/java/com/exactpro/th2/infrarepo/Gitter.java similarity index 93% rename from src/main/java/com/exactpro/th2/inframgr/repository/Gitter.java rename to src/main/java/com/exactpro/th2/infrarepo/Gitter.java index 411bba26..44095ba6 100644 --- a/src/main/java/com/exactpro/th2/inframgr/repository/Gitter.java +++ b/src/main/java/com/exactpro/th2/infrarepo/Gitter.java @@ -13,9 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.exactpro.th2.inframgr.repository; +package com.exactpro.th2.infrarepo; -import com.exactpro.th2.inframgr.Config; import com.jcraft.jsch.JSch; import com.jcraft.jsch.JSchException; import org.eclipse.jgit.api.Git; @@ -44,7 +43,7 @@ public class Gitter { public static final String REFS_HEADS = "refs/heads/"; private static volatile Map instance = new ConcurrentHashMap<>(); - private Config.GitConfig config; + private GitConfig config; private String branch; private Lock lock; private TransportConfigCallback callback; @@ -52,7 +51,7 @@ public class Gitter { private final String repositoryDir; - private Gitter(Config.GitConfig config, String branch) { + private Gitter(GitConfig config, String branch) { this.config = config; this.branch = branch; this.localCacheRoot = config.getLocalRepositoryRoot() + "/" + branch; @@ -61,7 +60,7 @@ private Gitter(Config.GitConfig config, String branch) { this.lock = new ReentrantLock(); } - public Config.GitConfig getConfig() { + public GitConfig getConfig() { return config; } @@ -77,12 +76,12 @@ public void unlock() { lock.unlock(); } - public static Gitter getBranch(Config.GitConfig config, String branch) { + public static Gitter getBranch(GitConfig config, String branch) { return instance.computeIfAbsent(branch, k -> new Gitter(config, k)); } - private static TransportConfigCallback transportConfigCallback(Config.GitConfig config) { + private static TransportConfigCallback transportConfigCallback(GitConfig config) { if (config.ignoreInsecureHosts()) JSch.setConfig("StrictHostKeyChecking", "no"); @@ -109,7 +108,7 @@ protected JSch createDefaultJSch(FS fs) throws JSchException { } - public static Map getAllBranchesCommits(Config.GitConfig config) throws Exception { + public static Map getAllBranchesCommits(GitConfig config) throws Exception { // retrieve all remote branches Collection allBranches = Git.lsRemoteRepository() @@ -127,13 +126,13 @@ public static Map getAllBranchesCommits(Config.GitConfig config) return result; } - public static Set getBranches(Config.GitConfig config) throws Exception { + public static Set getBranches(GitConfig config) throws Exception { Map commits = getAllBranchesCommits(config); return commits.keySet(); } - private String checkout(Config.GitConfig config, String branch, String targetDir) throws IOException, GitAPIException { + private String checkout(GitConfig config, String branch, String targetDir) throws IOException, GitAPIException { // create branch directory if it does not exist File dir = new File(targetDir); diff --git a/src/main/java/com/exactpro/th2/inframgr/repository/InconsistentRepositoryStateException.java b/src/main/java/com/exactpro/th2/infrarepo/InconsistentRepositoryStateException.java similarity index 95% rename from src/main/java/com/exactpro/th2/inframgr/repository/InconsistentRepositoryStateException.java rename to src/main/java/com/exactpro/th2/infrarepo/InconsistentRepositoryStateException.java index 12df8142..25a94ee2 100644 --- a/src/main/java/com/exactpro/th2/inframgr/repository/InconsistentRepositoryStateException.java +++ b/src/main/java/com/exactpro/th2/infrarepo/InconsistentRepositoryStateException.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.exactpro.th2.inframgr.repository; +package com.exactpro.th2.infrarepo; public class InconsistentRepositoryStateException extends IllegalStateException { public InconsistentRepositoryStateException(String message) { diff --git a/src/main/java/com/exactpro/th2/inframgr/repository/Repository.java b/src/main/java/com/exactpro/th2/infrarepo/Repository.java similarity index 52% rename from src/main/java/com/exactpro/th2/inframgr/repository/Repository.java rename to src/main/java/com/exactpro/th2/infrarepo/Repository.java index 9a4e91fc..a9f05b04 100644 --- a/src/main/java/com/exactpro/th2/inframgr/repository/Repository.java +++ b/src/main/java/com/exactpro/th2/infrarepo/Repository.java @@ -13,14 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.exactpro.th2.inframgr.repository; - -import com.exactpro.th2.inframgr.Config; -import com.exactpro.th2.inframgr.models.RepositoryResource; -import com.exactpro.th2.inframgr.models.RepositorySnapshot; -import com.exactpro.th2.inframgr.models.ResourceEntry; -import com.exactpro.th2.inframgr.models.ResourceType; -import com.exactpro.th2.inframgr.util.Hash; +package com.exactpro.th2.infrarepo; + import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; @@ -31,82 +25,81 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.HashSet; import java.util.Set; public class Repository { - private static ResourceEntry loadYMLFile(File file) throws IOException { + private static RepositoryResource loadYAMLFile(File file) throws IOException { String contents = Files.readString(file.toPath()); ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); RepositoryResource resource = mapper.readValue(contents, RepositoryResource.class); + resource.setSourceHash(Repository.digest(contents)); - ResourceEntry entry = new ResourceEntry(); - entry.setKind(ResourceType.forKind(resource.getKind())); - entry.setName(resource.getMetadata().getName()); - entry.setSpec(resource.getSpec()); - - entry.setSourceHash(Hash.digest(contents)); - - return entry; + return resource; } - private static Set loadBranchYMLFiles(File repositoryRoot) throws IOException { + + private static Set loadBranchYAMLFiles(File repositoryRoot) throws IOException { Logger logger = LoggerFactory.getLogger(Repository.class); - Set resources = new HashSet<>(); - Set keys = new HashSet<>(); + Set resources = new HashSet<>(); + Set keySet = new HashSet<>(); + for (ResourceType t : ResourceType.values()) if (t.isRepositoryResource()) { File dir = new File(repositoryRoot.getAbsolutePath() + "/" + t.path()); if (dir.exists()) { if (!dir.isDirectory()) { - logger.warn("entry expected to be a directory: \"{}\"", dir.getAbsoluteFile()); + logger.error("entry expected to be a directory: \"{}\"", dir.getAbsoluteFile()); continue; } File[] files = dir.listFiles(); - for (File f : files) { - if (f.isFile() && (f.getAbsolutePath().endsWith(".yml") || f.getAbsolutePath().endsWith(".yaml"))) { - ResourceEntry resourceEntry = Repository.loadYMLFile(f); - - if (!extractName(f.getName()).equals(resourceEntry.getName())) { - logger.warn("skipping \"{}\" | resource name does not match filename", f.getAbsolutePath()); - continue; - } - - if (!resourceEntry.getKind().path().equals(t.path())) { - logger.warn("skipping \"{}\" | resource is located in wrong directory. kind: {}, dir: {}" - , f.getAbsolutePath(), resourceEntry.getKind().kind(), t.path()); - continue; + if (files != null) + for (File f : files) { + if (f.isFile() && (f.getAbsolutePath().endsWith(".yml") || f.getAbsolutePath().endsWith(".yaml"))) { + RepositoryResource resource = Repository.loadYAMLFile(f); + RepositoryResource.Metadata meta = resource.getMetadata(); + + if (meta == null || !extractName(f.getName()).equals(meta.getName())) { + logger.warn("skipping \"{}\" | resource name does not match filename", f.getAbsolutePath()); + continue; + } + + if (!ResourceType.forKind(resource.getKind()).path().equals(t.path())) { + logger.error("skipping \"{}\" | resource is located in wrong directory. kind: {}, dir: {}" + , f.getAbsolutePath(), resource.getKind(), t.path()); + continue; + } + + String key = resource.getKind() + "/" + meta.getName(); + if (keySet.contains(key)) + continue; + + resources.add(resource); + keySet.add(key); } - - String key = resourceEntry.getKind() + "/" + resourceEntry.getName(); - if (keys.contains(key)) - continue; - - resources.add(resourceEntry); - keys.add(key); } - } } } return resources; } - private static File getFile(Config.GitConfig config, String branch, ResourceEntry entry) { + private static File fileFor(GitConfig config, String branch, RepositoryResource resource) { - File file = new File ( + return new File ( config.getLocalRepositoryRoot() + "/" + branch - + "/" + entry.getKind().path() - + "/" + entry.getName() + + "/" + ResourceType.forKind(resource.getKind()).path() + + "/" + resource.getMetadata().getName() + ".yml"); - return file; } @@ -129,7 +122,7 @@ public static RepositorySnapshot getSnapshot(Gitter gitter) throws IOException, String path = gitter.getConfig().getLocalRepositoryRoot() + "/" + gitter.getBranch(); String commitRef = gitter.checkout(); - Set resources = Repository.loadBranchYMLFiles(new File(path)); + Set resources = Repository.loadBranchYAMLFiles(new File(path)); RepositorySnapshot snapshot = new RepositorySnapshot(commitRef); snapshot.setResources(resources); @@ -137,41 +130,37 @@ public static RepositorySnapshot getSnapshot(Gitter gitter) throws IOException, } - private static void saveYMLFile(File file, RepositoryResource resource) throws IOException { + private static void saveYAMLFile(File file, RepositoryResource resource) throws IOException { file.getParentFile().mkdir(); ObjectMapper mapper = new ObjectMapper((new YAMLFactory()) .disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER) .enable(YAMLGenerator.Feature.MINIMIZE_QUOTES)); String contents = mapper.writeValueAsString(resource); - resource.setSourceHash(Hash.digest(contents)); + resource.setSourceHash(Repository.digest(contents)); Files.writeString(file.toPath(), contents); } - public static void add(Config.GitConfig config, String branch, ResourceEntry entry) throws IOException { + public static void add(GitConfig config, String branch, RepositoryResource resource) throws IOException { - File file = getFile(config, branch, entry); + File file = fileFor(config, branch, resource); if (file.exists()) throw new IllegalArgumentException("resource already exist"); - RepositoryResource resource = new RepositoryResource(entry); - Repository.saveYMLFile(file, resource); - entry.setSourceHash(resource.getSourceHash()); + Repository.saveYAMLFile(file, resource); } - public static void update(Config.GitConfig config, String branch, ResourceEntry entry) throws IOException { + public static void update(GitConfig config, String branch, RepositoryResource resource) throws IOException { - File file = getFile(config, branch, entry); + File file = fileFor(config, branch, resource); if (!file.exists() || !file.isFile()) throw new IllegalArgumentException("resource does not exist"); - RepositoryResource resource = new RepositoryResource(entry); - Repository.saveYMLFile(file, resource); - entry.setSourceHash(resource.getSourceHash()); + Repository.saveYAMLFile(file, resource); } - public static void remove(Config.GitConfig config, String branch, ResourceEntry entry){ + public static void remove(GitConfig config, String branch, RepositoryResource resource){ - File file = getFile(config, branch, entry); + File file = fileFor(config, branch, resource); if (!file.exists() || !file.isFile()) throw new IllegalArgumentException("resource does not exist"); file.delete(); @@ -186,4 +175,20 @@ private static String extractName(String fileName) { else return fileName.substring(0, index); } + + + public static String digest(String data) { + try { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + byte[] digest = md.digest(data.getBytes()); + StringBuilder sb = new StringBuilder(); + for (byte b : digest) + sb.append(String.format("%02x", b)); + + return sb.toString(); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + } diff --git a/src/main/java/com/exactpro/th2/inframgr/models/RepositoryResource.java b/src/main/java/com/exactpro/th2/infrarepo/RepositoryResource.java similarity index 87% rename from src/main/java/com/exactpro/th2/inframgr/models/RepositoryResource.java rename to src/main/java/com/exactpro/th2/infrarepo/RepositoryResource.java index 8833d8dc..5030aab4 100644 --- a/src/main/java/com/exactpro/th2/inframgr/models/RepositoryResource.java +++ b/src/main/java/com/exactpro/th2/infrarepo/RepositoryResource.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.exactpro.th2.inframgr.models; +package com.exactpro.th2.infrarepo; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -28,6 +28,8 @@ public String getName() { public void setName(String name) { this.name = name; } + public Metadata() {} + public Metadata(String name) {this.name = name;} } private String apiVersion; @@ -44,16 +46,6 @@ public RepositoryResource(ResourceType type) { this.kind = type.name(); } - public RepositoryResource(ResourceEntry data) { - setApiVersion(data.getKind().k8sApiVersion()); - setKind(data.getKind().kind()); - setSpec(data.getSpec()); - setSourceHash(data.getSourceHash()); - - setMetadata(new RepositoryResource.Metadata()); - getMetadata().setName(data.getName()); - } - public String getApiVersion() { return apiVersion; } diff --git a/src/main/java/com/exactpro/th2/inframgr/models/RepositorySettings.java b/src/main/java/com/exactpro/th2/infrarepo/RepositorySettings.java similarity index 98% rename from src/main/java/com/exactpro/th2/inframgr/models/RepositorySettings.java rename to src/main/java/com/exactpro/th2/infrarepo/RepositorySettings.java index 53f2b30d..b5e9e233 100644 --- a/src/main/java/com/exactpro/th2/inframgr/models/RepositorySettings.java +++ b/src/main/java/com/exactpro/th2/infrarepo/RepositorySettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.exactpro.th2.inframgr.models; +package com.exactpro.th2.infrarepo; import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonIgnore; diff --git a/src/main/java/com/exactpro/th2/inframgr/models/RepositorySnapshot.java b/src/main/java/com/exactpro/th2/infrarepo/RepositorySnapshot.java similarity index 73% rename from src/main/java/com/exactpro/th2/inframgr/models/RepositorySnapshot.java rename to src/main/java/com/exactpro/th2/infrarepo/RepositorySnapshot.java index cc201b79..da42fae0 100644 --- a/src/main/java/com/exactpro/th2/inframgr/models/RepositorySnapshot.java +++ b/src/main/java/com/exactpro/th2/infrarepo/RepositorySnapshot.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.exactpro.th2.inframgr.models; +package com.exactpro.th2.infrarepo; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.core.JsonProcessingException; @@ -23,7 +23,7 @@ public class RepositorySnapshot { private String commitRef; - private Set resources; + private Set resources; public RepositorySnapshot(String commitRef) { @@ -34,23 +34,22 @@ public String getCommitRef() { return commitRef; } - public Set getResources() { + public Set getResources() { return resources; } - public void setResources(Set resources) { + public void setResources(Set resources) { this.resources = resources; } @JsonIgnore public RepositorySettings getRepositorySettings() throws JsonProcessingException { - for (ResourceEntry entry : resources) - if (entry.getKind() == ResourceType.SettingsFile) { + + for (RepositoryResource resource : resources) + if (resource.getKind().equals(ResourceType.SettingsFile.kind())) { ObjectMapper mapper = new ObjectMapper(); - RepositorySettings s = mapper.readValue( - mapper.writeValueAsString(entry.getSpec()), - RepositorySettings.class - ); + RepositorySettings s = + mapper.readValue(mapper.writeValueAsString(resource.getSpec()), RepositorySettings.class); return s; } return null; diff --git a/src/main/java/com/exactpro/th2/inframgr/models/ResourceType.java b/src/main/java/com/exactpro/th2/infrarepo/ResourceType.java similarity index 98% rename from src/main/java/com/exactpro/th2/inframgr/models/ResourceType.java rename to src/main/java/com/exactpro/th2/infrarepo/ResourceType.java index 536e18e5..3558659a 100644 --- a/src/main/java/com/exactpro/th2/inframgr/models/ResourceType.java +++ b/src/main/java/com/exactpro/th2/infrarepo/ResourceType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.exactpro.th2.inframgr.models; +package com.exactpro.th2.infrarepo; import java.util.HashMap; import java.util.Map;