From a326e990bd3a3e82da441a29f27434b66266e4e5 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Wed, 18 Sep 2024 15:07:43 +0200 Subject: [PATCH 01/36] Enable compliance tests to use plugins for cluster provisioning Signed-off-by: Toni Finger --- Tests/kaas/plugin/README.md | 39 ++++++ Tests/kaas/plugin/interface.py | 188 +++++++++++++++++++++++++++++ Tests/kaas/plugin/plugin_kind.py | 19 +++ Tests/kaas/plugin/requirements.in | 3 + Tests/kaas/plugin/requirements.txt | 62 ++++++++++ Tests/kaas/plugin/run.py | 14 +++ 6 files changed, 325 insertions(+) create mode 100644 Tests/kaas/plugin/README.md create mode 100644 Tests/kaas/plugin/interface.py create mode 100644 Tests/kaas/plugin/plugin_kind.py create mode 100644 Tests/kaas/plugin/requirements.in create mode 100644 Tests/kaas/plugin/requirements.txt create mode 100644 Tests/kaas/plugin/run.py diff --git a/Tests/kaas/plugin/README.md b/Tests/kaas/plugin/README.md new file mode 100644 index 000000000..28cfa83ce --- /dev/null +++ b/Tests/kaas/plugin/README.md @@ -0,0 +1,39 @@ +# Plugin for provisioning k8s clusters and performing conformance tests on these clusters + +## Development environment + +### requirements + +* [docker](https://docs.docker.com/engine/install/) +* [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) +* [sonobuoy](https://sonobuoy.io/docs/v0.57.1/#installation) + +### setup for development + +1. Generate python 3.10 env + + ```bash + sudo apt-get install python3.10-dev + virtualenv -p /usr/bin/python3.10 venv + echo "*" >> venv/.gitignore + source venv/bin/activate + (venv) curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 + (venv) python3.10 -m pip install --upgrade pip + (venv) python3.10 -m pip --version + + ``` + +2. Install dependencies: + + ```bash + (venv) pip install pip-tools + (venv) pip-compile requirements.in + (venv) pip-sync requirements.txt + ``` + +3. Set environment variables and launch the process: + + ```bash + (venv) export CLUSTER_PROVIDER="kind" + (venv) python run.py + ``` diff --git a/Tests/kaas/plugin/interface.py b/Tests/kaas/plugin/interface.py new file mode 100644 index 000000000..cd37de939 --- /dev/null +++ b/Tests/kaas/plugin/interface.py @@ -0,0 +1,188 @@ +from abc import ABC, abstractmethod +from typing import final +from kubernetes import client, config +import os +import logging +from junitparser import JUnitXml + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger("interface") + + +def setup_k8s_client(kubeconfigfile=None): + + if kubeconfigfile: + logger.debug(f"loading kubeconfig file '{kubeconfigfile}'") + config.load_kube_config(kubeconfigfile) + logger.info("kubeconfigfile loaded successfully") + else: + logger.error("no kubeconfig file provided") + return None + + k8s_api_client = client.CoreV1Api() + + return k8s_api_client + + +class KubernetesClusterPlugin(ABC): + """ + An abstract base class for custom Kubernetes cluster provider plugins. + It represents an interface class from which the api provider-specific + plugins must be derived as child classes + """ + + kubeconfig_path = None + cluster_name = None + k8s_api_client = None + working_directory = None + plugin_result_directory = "plugins/e2e/results/global" + + @final + def __init__(self, config=None): + logger.debug(config) + self.working_directory = os.getcwd() + logger.debug(f"Working from {self.working_directory}") + + @final + def _preflight_check(self): + """ + Prefligth test to ensure that everything is set up correctly for execution + :param: None + :return: None + """ + logger.info("check kubeconfig") + self.k8s_api_client = setup_k8s_client(self.kubeconfig_path) + + for api in client.ApisApi().get_api_versions().groups: + versions = [] + for v in api.versions: + name = "" + if v.version == api.preferred_version.version and len(api.versions) > 1: + name += "*" + name += v.version + versions.append(name) + logger.debug(f"[supported api]: {api.name:<40} {','.join(versions)}") + + logger.debug("checks if sonobuoy is availabe") + return_value = os.system(f"sonobuoy version --kubeconfig='{self.kubeconfig_path}'") + if return_value != 0: + raise Exception("sonobuoy is not installed") + + @final + def _test_k8s_cncf_conformance(self): + """ + This method invokes the conformance tests with sononbuoy + :param: None + :return: None + """ + logger.info(f" invoke cncf conformance test") + # ~ os.system(f"sonobuoy run --wait --mode=certified-conformance --kubeconfig='{self.kubeconfig_path}'") + # TODO:!!! switch to the real test on the final merge !!! + # Only one test is currently being carried out for development purposes + os.system( + f"sonobuoy run --wait --plugin-env e2e.E2E_FOCUS=pods --plugin-env e2e.E2E_DRYRUN=true --kubeconfig='{self.kubeconfig_path}'" + ) + + @final + def _test_scs_kaas_conformance(self): + """ + This method invokes SCS's very own conformance tests by using sononbuoy + :param: None + :return: None + """ + raise NotImplementedError + + @final + def _cleanup_sonobuoy_resources(self): + """ + This method deletes all resources that sonobuoy has created in a k8s cluster for a test + :param: None + :return: None + """ + logger.info(f"removing sonobuoy tests from cluster") + os.system(f"sonobuoy delete --wait --kubeconfig='{self.kubeconfig_path}'") + + @final + def _retrieve_result(self, result_file_name): + """ + This method invokes sonobouy to store the results in a subdirectory of + the working directory. The Junit results file contained in it is then + analyzed in order to interpret the relevant information it containes + :param: result_file_name: + :return: None + """ + logger.debug(f"retrieving results to {result_file_name}") + result_directory = self.working_directory + "/" + result_file_name + if os.path.exists(result_directory): + os.system(f"rm -rf {result_directory}/*") + else: + os.mkdir(result_directory) + os.chdir(result_directory) + os.system( + f"sonobuoy retrieve -x --filename='{result_file_name}' --kubeconfig='{self.kubeconfig_path}'" + ) + + logger.debug( + f"parsing JUnit result from {result_directory + '/' + self.plugin_result_directory + '/junit_01.xml' } " + ) + xml = JUnitXml.fromfile( + result_directory + "/" + self.plugin_result_directory + "/junit_01.xml" + ) + failed_test_cases = 0 + passed_test_cases = 0 + skipped_test_cases = 0 + for suite in xml: + for case in suite: + if case.is_passed: + passed_test_cases += 1 + else: + failed_test_cases += 1 + if case.is_skipped: + skipped_test_cases += 1 + + logger.info( + f" {passed_test_cases} passed, {failed_test_cases} failed of which {skipped_test_cases} were skipped" + ) + os.chdir(self.working_directory) + + @abstractmethod + def _create_cluster(self, cluster_name) -> (str, int): + """ + Create a Kubernetes cluster to test aggainst. + :param: cluster_name: + :return: kubeconfig: kubeconfig of the cluster used for testing + """ + pass + + @abstractmethod + def _delete_cluster(self, cluster_name) -> (str, int): + """ + Delete the Kubernetes cluster. + :param: cluster_name: + :return: None + """ + pass + + @final + def run(self): + """ + This method is to be called to run the plugin + """ + + try: + self._create_cluster() + self._preflight_check() + self._test_k8s_cncf_conformance() + self._retrieve_result("cncf_result") + self._cleanup_sonobuoy_resources() + # self._test_scs_kaas_conformance() + # self._retrieve_result("scs_kaas_result") + except Exception as e: + logging.error(e) + + try: + self._cleanup_sonobuoy_resources() + except Exception as e: + logging.error(e) + finally: + self._delete_cluster() diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py new file mode 100644 index 000000000..210e17123 --- /dev/null +++ b/Tests/kaas/plugin/plugin_kind.py @@ -0,0 +1,19 @@ +from interface import KubernetesClusterPlugin +from pytest_kind import KindCluster +from pathlib import Path + +class PluginKind(KubernetesClusterPlugin): + """ + Plugin to handle the provisioning of kubernetes cluster for + conformance testing purpose with the use of Kind + """ + + def _create_cluster(self): + self.cluster_name= "scs-cluster" + self.cluster = KindCluster(self.cluster_name) + self.cluster.create() + self.kubeconfig_path=str(self.cluster.kubeconfig_path.resolve()) + + def _delete_cluster(self): + self.cluster.delete() + diff --git a/Tests/kaas/plugin/requirements.in b/Tests/kaas/plugin/requirements.in new file mode 100644 index 000000000..640831e54 --- /dev/null +++ b/Tests/kaas/plugin/requirements.in @@ -0,0 +1,3 @@ +pytest-kind +kubernetes +junitparser diff --git a/Tests/kaas/plugin/requirements.txt b/Tests/kaas/plugin/requirements.txt new file mode 100644 index 000000000..e99ac2285 --- /dev/null +++ b/Tests/kaas/plugin/requirements.txt @@ -0,0 +1,62 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile requirements.in +# +cachetools==5.5.0 + # via google-auth +certifi==2024.8.30 + # via + # kubernetes + # requests +charset-normalizer==3.3.2 + # via requests +google-auth==2.34.0 + # via kubernetes +idna==3.8 + # via requests +junitparser==3.2.0 + # via -r requirements.in +kubernetes==30.1.0 + # via -r requirements.in +oauthlib==3.2.2 + # via + # kubernetes + # requests-oauthlib +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.0 + # via google-auth +pykube-ng==23.6.0 + # via pytest-kind +pytest-kind==22.11.1 + # via -r requirements.in +python-dateutil==2.9.0.post0 + # via kubernetes +pyyaml==6.0.2 + # via + # kubernetes + # pykube-ng +requests==2.32.3 + # via + # kubernetes + # pykube-ng + # requests-oauthlib +requests-oauthlib==2.0.0 + # via kubernetes +rsa==4.9 + # via google-auth +six==1.16.0 + # via + # kubernetes + # python-dateutil +urllib3==2.2.2 + # via + # kubernetes + # pykube-ng + # requests +websocket-client==1.8.0 + # via kubernetes diff --git a/Tests/kaas/plugin/run.py b/Tests/kaas/plugin/run.py new file mode 100644 index 000000000..80f6c3abd --- /dev/null +++ b/Tests/kaas/plugin/run.py @@ -0,0 +1,14 @@ +from plugin_kind import PluginKind +import os + +if __name__ == "__main__": + + match os.environ['CLUSTER_PROVIDER']: + case "static": + plugin = PluginStatic() + case "kind": + plugin = PluginKind() + case _: + raise NotImplementedError(f"{os.environ['CLUSTER_PROVIDER']} is not valid ") + + plugin.run() From ab0803b3e78fac934433d699a4d6256c8e489582 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Wed, 18 Sep 2024 19:50:35 +0200 Subject: [PATCH 02/36] Fix python style And apply static plugin Signed-off-by: Toni Finger --- Tests/kaas/plugin/interface.py | 4 ++-- Tests/kaas/plugin/plugin_kind.py | 11 +++++------ Tests/kaas/plugin/plugin_static.py | 14 ++++++++++++++ Tests/kaas/plugin/run.py | 1 + 4 files changed, 22 insertions(+), 8 deletions(-) create mode 100644 Tests/kaas/plugin/plugin_static.py diff --git a/Tests/kaas/plugin/interface.py b/Tests/kaas/plugin/interface.py index cd37de939..7f7079d33 100644 --- a/Tests/kaas/plugin/interface.py +++ b/Tests/kaas/plugin/interface.py @@ -75,7 +75,7 @@ def _test_k8s_cncf_conformance(self): :param: None :return: None """ - logger.info(f" invoke cncf conformance test") + logger.info(" invoke cncf conformance test") # ~ os.system(f"sonobuoy run --wait --mode=certified-conformance --kubeconfig='{self.kubeconfig_path}'") # TODO:!!! switch to the real test on the final merge !!! # Only one test is currently being carried out for development purposes @@ -99,7 +99,7 @@ def _cleanup_sonobuoy_resources(self): :param: None :return: None """ - logger.info(f"removing sonobuoy tests from cluster") + logger.info("removing sonobuoy tests from cluster") os.system(f"sonobuoy delete --wait --kubeconfig='{self.kubeconfig_path}'") @final diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py index 210e17123..36265e9b5 100644 --- a/Tests/kaas/plugin/plugin_kind.py +++ b/Tests/kaas/plugin/plugin_kind.py @@ -1,19 +1,18 @@ from interface import KubernetesClusterPlugin from pytest_kind import KindCluster -from pathlib import Path + class PluginKind(KubernetesClusterPlugin): """ - Plugin to handle the provisioning of kubernetes cluster for + Plugin to handle the provisioning of kubernetes cluster for conformance testing purpose with the use of Kind """ - + def _create_cluster(self): - self.cluster_name= "scs-cluster" + self.cluster_name = "scs-cluster" self.cluster = KindCluster(self.cluster_name) self.cluster.create() - self.kubeconfig_path=str(self.cluster.kubeconfig_path.resolve()) + self.kubeconfig_path = str(self.cluster.kubeconfig_path.resolve()) def _delete_cluster(self): self.cluster.delete() - diff --git a/Tests/kaas/plugin/plugin_static.py b/Tests/kaas/plugin/plugin_static.py new file mode 100644 index 000000000..2d094396d --- /dev/null +++ b/Tests/kaas/plugin/plugin_static.py @@ -0,0 +1,14 @@ +from interface import KubernetesClusterPlugin +import os + + +class PluginStatic(KubernetesClusterPlugin): + """ + Plugin to handle the provisioning of kubernetes + using a kubeconfig file + """ + def _create_cluster(self): + self.kubeconfig_path = os.environ["KUBECONFIG"] + + def _delete_cluster(self): + pass diff --git a/Tests/kaas/plugin/run.py b/Tests/kaas/plugin/run.py index 80f6c3abd..88051e86e 100644 --- a/Tests/kaas/plugin/run.py +++ b/Tests/kaas/plugin/run.py @@ -1,4 +1,5 @@ from plugin_kind import PluginKind +from plugin_static import PluginStatic import os if __name__ == "__main__": From 199fdf2b08df6694f106dbef0a800b267b3e3f32 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Fri, 20 Sep 2024 14:05:16 +0200 Subject: [PATCH 03/36] Fix path to junit file Signed-off-by: Toni Finger --- Tests/kaas/plugin/interface.py | 22 +++++++++------------- Tests/kaas/plugin/plugin_kind.py | 2 ++ 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/Tests/kaas/plugin/interface.py b/Tests/kaas/plugin/interface.py index 7f7079d33..812c43118 100644 --- a/Tests/kaas/plugin/interface.py +++ b/Tests/kaas/plugin/interface.py @@ -35,7 +35,6 @@ class KubernetesClusterPlugin(ABC): cluster_name = None k8s_api_client = None working_directory = None - plugin_result_directory = "plugins/e2e/results/global" @final def __init__(self, config=None): @@ -103,7 +102,7 @@ def _cleanup_sonobuoy_resources(self): os.system(f"sonobuoy delete --wait --kubeconfig='{self.kubeconfig_path}'") @final - def _retrieve_result(self, result_file_name): + def _retrieve_result(self, result_dir_name): """ This method invokes sonobouy to store the results in a subdirectory of the working directory. The Junit results file contained in it is then @@ -111,22 +110,20 @@ def _retrieve_result(self, result_file_name): :param: result_file_name: :return: None """ - logger.debug(f"retrieving results to {result_file_name}") - result_directory = self.working_directory + "/" + result_file_name - if os.path.exists(result_directory): - os.system(f"rm -rf {result_directory}/*") + logger.debug(f"retrieving results to {result_dir_name}") + result_dir = self.working_directory + "/" + result_dir_name + if os.path.exists(result_dir): + os.system(f"rm -rf {result_dir}/*") else: - os.mkdir(result_directory) - os.chdir(result_directory) + os.mkdir(result_dir) os.system( - f"sonobuoy retrieve -x --filename='{result_file_name}' --kubeconfig='{self.kubeconfig_path}'" + f"sonobuoy retrieve {result_dir} -x --filename='{result_dir_name}' --kubeconfig='{self.kubeconfig_path}'" ) - logger.debug( - f"parsing JUnit result from {result_directory + '/' + self.plugin_result_directory + '/junit_01.xml' } " + f"parsing JUnit result from {result_dir + '/plugins/e2e/results/global/junit_01.xml' } " ) xml = JUnitXml.fromfile( - result_directory + "/" + self.plugin_result_directory + "/junit_01.xml" + result_dir + "/plugins/e2e/results/global/junit_01.xml" ) failed_test_cases = 0 passed_test_cases = 0 @@ -143,7 +140,6 @@ def _retrieve_result(self, result_file_name): logger.info( f" {passed_test_cases} passed, {failed_test_cases} failed of which {skipped_test_cases} were skipped" ) - os.chdir(self.working_directory) @abstractmethod def _create_cluster(self, cluster_name) -> (str, int): diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py index 36265e9b5..8abe46e18 100644 --- a/Tests/kaas/plugin/plugin_kind.py +++ b/Tests/kaas/plugin/plugin_kind.py @@ -1,5 +1,6 @@ from interface import KubernetesClusterPlugin from pytest_kind import KindCluster +import time class PluginKind(KubernetesClusterPlugin): @@ -12,6 +13,7 @@ def _create_cluster(self): self.cluster_name = "scs-cluster" self.cluster = KindCluster(self.cluster_name) self.cluster.create() + time.sleep(30) self.kubeconfig_path = str(self.cluster.kubeconfig_path.resolve()) def _delete_cluster(self): From f994cde8da7cb4142d5557c80e87a35a81ab6dc7 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Mon, 7 Oct 2024 10:16:10 +0200 Subject: [PATCH 04/36] Inital integration of the plugin into the 'scs-test-runner.py' Signed-off-by: Toni Finger --- Tests/config-kaas-example.toml | 92 ++++++++++++ Tests/kaas/plugin/interface.py | 155 ++++---------------- Tests/kaas/plugin/plugin_kind.py | 8 +- Tests/kaas/plugin/run.py | 10 ++ Tests/kaas/plugin/run_sonobuoy_executor.py | 34 +++++ Tests/kaas/plugin/sonobuoy_executor.py | 158 +++++++++++++++++++++ Tests/scs-compatible-kaas.yaml | 3 + Tests/scs-test-runner.py | 48 ++++++- 8 files changed, 374 insertions(+), 134 deletions(-) create mode 100644 Tests/config-kaas-example.toml mode change 100644 => 100755 Tests/kaas/plugin/run.py create mode 100755 Tests/kaas/plugin/run_sonobuoy_executor.py create mode 100644 Tests/kaas/plugin/sonobuoy_executor.py diff --git a/Tests/config-kaas-example.toml b/Tests/config-kaas-example.toml new file mode 100644 index 000000000..1d7c05c10 --- /dev/null +++ b/Tests/config-kaas-example.toml @@ -0,0 +1,92 @@ +[secrets] +keyfile = ".secret/keyfile" +tokenfile = ".secret/tokenfile" + + +[presets.default] +scopes = [ + "scs-compatible-iaas", +] +subjects = [ + "gx-scs", +] + + +[presets.all] +scopes = [ + "scs-compatible-iaas", +] +subjects = [ + "gx-scs", + "artcodix", + "pco-prod1", + "pco-prod2", + "pco-prod3", + "pco-prod4", + "poc-kdo", + "poc-wgcloud", + "regio-a", + "syseleven-dus2", + "syseleven-ham1", + "wavestack", +] +workers = 4 + + +############################################################################### + +[presets.all-iaas-kaas] +scopes = [ + "scs-compatible-kaas", + "scs-compatible-iaas", +] +subjects = [ + "cloud-ik-1", + "cloud-ik-2", + #"cloud-ik-3", +] +workers = 4 + + +[presets.all-kaas] +scopes = [ + "scs-compatible-kaas", +] +subjects = [ + "cloud-k-4", + "cloud-k-5", + #"cloud-k-6", +] +workers = 4 + + + +[scopes.scs-compatible-iaas] +spec = "./scs-compatible-iaas.yaml" + + +[scopes.scs-compatible-kaas] +spec = "./scs-compatible-kaas.yaml" + + +# default subject (not a real subject, but used to declare a default mapping) +# (this is the only mapping declaration that supports using Python string interpolation) +[subjects._.mapping] +os_cloud = "{subject}" + + +[subjects._.kubernetes_setup] +kube_plugin = "kind" +kube_versions = [ + "v1.12", + #"v1.13", + #"v1.14", +] + + +[subjects.cloud-ik-1.kubernetes_setup] +kube_plugin = "none" + +[subjects.cloud-ik-1.mapping] +os_cloud = "none" + diff --git a/Tests/kaas/plugin/interface.py b/Tests/kaas/plugin/interface.py index 812c43118..04bd1737e 100644 --- a/Tests/kaas/plugin/interface.py +++ b/Tests/kaas/plugin/interface.py @@ -2,28 +2,13 @@ from typing import final from kubernetes import client, config import os +import shutil import logging from junitparser import JUnitXml -logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger("interface") -def setup_k8s_client(kubeconfigfile=None): - - if kubeconfigfile: - logger.debug(f"loading kubeconfig file '{kubeconfigfile}'") - config.load_kube_config(kubeconfigfile) - logger.info("kubeconfigfile loaded successfully") - else: - logger.error("no kubeconfig file provided") - return None - - k8s_api_client = client.CoreV1Api() - - return k8s_api_client - - class KubernetesClusterPlugin(ABC): """ An abstract base class for custom Kubernetes cluster provider plugins. @@ -31,115 +16,18 @@ class KubernetesClusterPlugin(ABC): plugins must be derived as child classes """ - kubeconfig_path = None + kubeconfig = None cluster_name = None k8s_api_client = None working_directory = None @final def __init__(self, config=None): + logger.info(f"Inital provider plug in of type:{type(self)}") logger.debug(config) self.working_directory = os.getcwd() logger.debug(f"Working from {self.working_directory}") - @final - def _preflight_check(self): - """ - Prefligth test to ensure that everything is set up correctly for execution - :param: None - :return: None - """ - logger.info("check kubeconfig") - self.k8s_api_client = setup_k8s_client(self.kubeconfig_path) - - for api in client.ApisApi().get_api_versions().groups: - versions = [] - for v in api.versions: - name = "" - if v.version == api.preferred_version.version and len(api.versions) > 1: - name += "*" - name += v.version - versions.append(name) - logger.debug(f"[supported api]: {api.name:<40} {','.join(versions)}") - - logger.debug("checks if sonobuoy is availabe") - return_value = os.system(f"sonobuoy version --kubeconfig='{self.kubeconfig_path}'") - if return_value != 0: - raise Exception("sonobuoy is not installed") - - @final - def _test_k8s_cncf_conformance(self): - """ - This method invokes the conformance tests with sononbuoy - :param: None - :return: None - """ - logger.info(" invoke cncf conformance test") - # ~ os.system(f"sonobuoy run --wait --mode=certified-conformance --kubeconfig='{self.kubeconfig_path}'") - # TODO:!!! switch to the real test on the final merge !!! - # Only one test is currently being carried out for development purposes - os.system( - f"sonobuoy run --wait --plugin-env e2e.E2E_FOCUS=pods --plugin-env e2e.E2E_DRYRUN=true --kubeconfig='{self.kubeconfig_path}'" - ) - - @final - def _test_scs_kaas_conformance(self): - """ - This method invokes SCS's very own conformance tests by using sononbuoy - :param: None - :return: None - """ - raise NotImplementedError - - @final - def _cleanup_sonobuoy_resources(self): - """ - This method deletes all resources that sonobuoy has created in a k8s cluster for a test - :param: None - :return: None - """ - logger.info("removing sonobuoy tests from cluster") - os.system(f"sonobuoy delete --wait --kubeconfig='{self.kubeconfig_path}'") - - @final - def _retrieve_result(self, result_dir_name): - """ - This method invokes sonobouy to store the results in a subdirectory of - the working directory. The Junit results file contained in it is then - analyzed in order to interpret the relevant information it containes - :param: result_file_name: - :return: None - """ - logger.debug(f"retrieving results to {result_dir_name}") - result_dir = self.working_directory + "/" + result_dir_name - if os.path.exists(result_dir): - os.system(f"rm -rf {result_dir}/*") - else: - os.mkdir(result_dir) - os.system( - f"sonobuoy retrieve {result_dir} -x --filename='{result_dir_name}' --kubeconfig='{self.kubeconfig_path}'" - ) - logger.debug( - f"parsing JUnit result from {result_dir + '/plugins/e2e/results/global/junit_01.xml' } " - ) - xml = JUnitXml.fromfile( - result_dir + "/plugins/e2e/results/global/junit_01.xml" - ) - failed_test_cases = 0 - passed_test_cases = 0 - skipped_test_cases = 0 - for suite in xml: - for case in suite: - if case.is_passed: - passed_test_cases += 1 - else: - failed_test_cases += 1 - if case.is_skipped: - skipped_test_cases += 1 - - logger.info( - f" {passed_test_cases} passed, {failed_test_cases} failed of which {skipped_test_cases} were skipped" - ) @abstractmethod def _create_cluster(self, cluster_name) -> (str, int): @@ -160,25 +48,38 @@ def _delete_cluster(self, cluster_name) -> (str, int): pass @final - def run(self): + def create(self, name="scs-cluster", version=None, kubeconfig_filepath=None,): """ - This method is to be called to run the plugin + This method is to be called to create a k8s cluster + :param: kubernetes_version: + :return: uuid """ - + self.cluster_name=name + self.cluster_version=version try: self._create_cluster() - self._preflight_check() - self._test_k8s_cncf_conformance() - self._retrieve_result("cncf_result") - self._cleanup_sonobuoy_resources() - # self._test_scs_kaas_conformance() - # self._retrieve_result("scs_kaas_result") + except Exception as e: logging.error(e) + self._delete_cluster() + + if kubeconfig_filepath: + generated_file=self.kubeconfig + self.kubeconfig = shutil.move(generated_file, kubeconfig_filepath) + return self.kubeconfig + + @final + def destroy(self, cluster_uuid=None): + """ + This method is to be called unprovision a cluster + :param: cluster_uuid: + """ try: - self._cleanup_sonobuoy_resources() + self._delete_cluster() except Exception as e: logging.error(e) - finally: - self._delete_cluster() + + + + diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py index 8abe46e18..91eb93e30 100644 --- a/Tests/kaas/plugin/plugin_kind.py +++ b/Tests/kaas/plugin/plugin_kind.py @@ -1,4 +1,4 @@ -from interface import KubernetesClusterPlugin +from .interface import KubernetesClusterPlugin from pytest_kind import KindCluster import time @@ -10,11 +10,11 @@ class PluginKind(KubernetesClusterPlugin): """ def _create_cluster(self): - self.cluster_name = "scs-cluster" + print(self.cluster_version) self.cluster = KindCluster(self.cluster_name) self.cluster.create() - time.sleep(30) - self.kubeconfig_path = str(self.cluster.kubeconfig_path.resolve()) + time.sleep(30)#TODO:!!! make this deterministic + self.kubeconfig = str(self.cluster.kubeconfig_path.resolve()) def _delete_cluster(self): self.cluster.delete() diff --git a/Tests/kaas/plugin/run.py b/Tests/kaas/plugin/run.py old mode 100644 new mode 100755 index 88051e86e..024723e87 --- a/Tests/kaas/plugin/run.py +++ b/Tests/kaas/plugin/run.py @@ -1,15 +1,25 @@ +#!/usr/bin/env python3 +# vim: set ts=4 sw=4 et: +# + from plugin_kind import PluginKind from plugin_static import PluginStatic import os +import logging +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger("interface") if __name__ == "__main__": + logger.info("Select provider plugin") match os.environ['CLUSTER_PROVIDER']: case "static": plugin = PluginStatic() case "kind": plugin = PluginKind() case _: + logger.error("provider plugin not implemented") raise NotImplementedError(f"{os.environ['CLUSTER_PROVIDER']} is not valid ") + logger.info("Run plugin") plugin.run() diff --git a/Tests/kaas/plugin/run_sonobuoy_executor.py b/Tests/kaas/plugin/run_sonobuoy_executor.py new file mode 100755 index 000000000..7ef6666c7 --- /dev/null +++ b/Tests/kaas/plugin/run_sonobuoy_executor.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# vim: set ts=4 sw=4 et: +# + +from sonobuoy_executor import SonobuoyExecutor +import logging +import getopt +import sys +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger("sonobuoy_executor") + +#TODO:!!! move this to class file +def parse_arguments(argv): + """Parse cli arguments from the script call""" + try: + opts, args = getopt.gnu_getopt(argv, "k:t:h", ["kubeconfig=", "test=", "help"]) + except getopt.GetoptError: + raise ConfigException + + for opt in opts: + if opt[0] == "-h" or opt[0] == "--help": + raise HelpException + if opt[0] == "-k" or opt[0] == "--kubeconfig": + kubeconfig = opt[1] + + return kubeconfig + + +if __name__ == "__main__": + logger.info("read in kubeconfig") + kubeconfig_path = parse_arguments(sys.argv) + logger.info("Run sonobuoy_executor") + sonobuoy_executor = SonobuoyExecutor(kubeconfig_path) + sonobuoy_executor.run() diff --git a/Tests/kaas/plugin/sonobuoy_executor.py b/Tests/kaas/plugin/sonobuoy_executor.py new file mode 100644 index 000000000..05c6f68ae --- /dev/null +++ b/Tests/kaas/plugin/sonobuoy_executor.py @@ -0,0 +1,158 @@ +from typing import final +from kubernetes import client, config +import os +import logging +from junitparser import JUnitXml + +logger = logging.getLogger("sonobuoy_executor") + + +def setup_k8s_client(kubeconfigfile=None): + + if kubeconfigfile: + logger.debug(f"loading kubeconfig file '{kubeconfigfile}'") + config.load_kube_config(kubeconfigfile) + logger.info("kubeconfigfile loaded successfully") + else: + logger.error("no kubeconfig file provided") + return None + + k8s_api_client = client.CoreV1Api() + + return k8s_api_client + + +class SonobuoyExecutor: + """ + An abstract base class for custom Kubernetes cluster provider plugins. + It represents an interface class from which the api provider-specific + plugins must be derived as child classes + """ + + kubeconfig_path = None + + working_directory = None + + @final + def __init__(self, kubeconfig=None): + logger.info("Inital SonobuoyExecutor") + logger.debug(kubeconfig) + if kubeconfig is None: + raise Exception("No kubeconfig provided") + # TODO:!!! check if kubeconfig is valid + self.working_directory = os.getcwd() + logger.debug(f"Working from {self.working_directory}") + + @final + def _preflight_check(self): + """ + Prefligth test to ensure that everything is set up correctly for execution + :param: None + :return: None + """ + logger.info("check kubeconfig") + self.k8s_api_client = setup_k8s_client(self.kubeconfig_path) + + for api in client.ApisApi().get_api_versions().groups: + versions = [] + for v in api.versions: + name = "" + if v.version == api.preferred_version.version and len(api.versions) > 1: + name += "*" + name += v.version + versions.append(name) + logger.debug(f"[supported api]: {api.name:<40} {','.join(versions)}") + + logger.debug("checks if sonobuoy is availabe") + return_value = os.system(f"sonobuoy version --kubeconfig='{self.kubeconfig_path}'") + if return_value != 0: + raise Exception("sonobuoy is not installed") + + @final + def _test_k8s_cncf_conformance(self): + """ + This method invokes the conformance tests with sononbuoy + :param: None + :return: None + """ + logger.info(" invoke cncf conformance test") + # ~ os.system(f"sonobuoy run --wait --mode=certified-conformance --kubeconfig='{self.kubeconfig_path}'") + # TODO:!!! switch to the real test on the final merge !!! + # Only one test is currently being carried out for development purposes + os.system( + f"sonobuoy run --wait --plugin-env e2e.E2E_FOCUS=pods --plugin-env e2e.E2E_DRYRUN=true --kubeconfig='{self.kubeconfig_path}'" + ) + + @final + def _test_scs_kaas_conformance(self): + """ + This method invokes SCS's very own conformance tests by using sononbuoy + :param: None + :return: None + """ + raise NotImplementedError + + @final + def _cleanup_sonobuoy_resources(self): + """ + This method deletes all resources that sonobuoy has created in a k8s cluster for a test + :param: None + :return: None + """ + logger.info("removing sonobuoy tests from cluster") + os.system(f"sonobuoy delete --wait --kubeconfig='{self.kubeconfig_path}'") + + @final + def _retrieve_result(self, result_dir_name): + """ + This method invokes sonobouy to store the results in a subdirectory of + the working directory. The Junit results file contained in it is then + analyzed in order to interpret the relevant information it containes + :param: result_file_name: + :return: None + """ + logger.debug(f"retrieving results to {result_dir_name}") + result_dir = self.working_directory + "/" + result_dir_name + if os.path.exists(result_dir): + os.system(f"rm -rf {result_dir}/*") + else: + os.mkdir(result_dir) + os.system( + f"sonobuoy retrieve {result_dir} -x --filename='{result_dir_name}' --kubeconfig='{self.kubeconfig_path}'" + ) + logger.debug( + f"parsing JUnit result from {result_dir + '/plugins/e2e/results/global/junit_01.xml'} " + ) + xml = JUnitXml.fromfile( + result_dir + "/plugins/e2e/results/global/junit_01.xml" + ) + failed_test_cases = 0 + passed_test_cases = 0 + skipped_test_cases = 0 + for suite in xml: + for case in suite: + if case.is_passed: + passed_test_cases += 1 + else: + failed_test_cases += 1 + if case.is_skipped: + skipped_test_cases += 1 + + logger.info( + f" {passed_test_cases} passed, {failed_test_cases} failed of which {skipped_test_cases} were skipped" + ) + + @final + def run(self): + """ + This method is to be called to run the plugin + """ + try: + self._preflight_check() + self._test_k8s_cncf_conformance() + self._retrieve_result("cncf_result") + self._cleanup_sonobuoy_resources() + except Exception as e: + logging.error(e) + finally: + self._cleanup_sonobuoy_resources() diff --git a/Tests/scs-compatible-kaas.yaml b/Tests/scs-compatible-kaas.yaml index 4aa540999..33af1bf41 100644 --- a/Tests/scs-compatible-kaas.yaml +++ b/Tests/scs-compatible-kaas.yaml @@ -7,6 +7,9 @@ modules: - id: cncf-k8s-conformance name: CNCF Kubernetes conformance url: https://github.com/cncf/k8s-conformance/tree/master + run: + - executable: ./kaas/plugin/run_sonobuoy_executor.py + args: -k {kubeconfig} testcases: - id: cncf-k8s-conformance tags: [mandatory] diff --git a/Tests/scs-test-runner.py b/Tests/scs-test-runner.py index de7152428..5818e96eb 100755 --- a/Tests/scs-test-runner.py +++ b/Tests/scs-test-runner.py @@ -17,6 +17,9 @@ import click import tomli +# TODO:!!! provide plugin chosing at interface side +from kaas.plugin.plugin_kind import PluginKind +cluster_plugin = PluginKind() logger = logging.getLogger(__name__) MONITOR_URL = "https://compliance.sovereignit.cloud/" @@ -58,10 +61,43 @@ def get_subject_mapping(self, subject): mapping.update(self.subjects.get(subject, {}).get('mapping', {})) return mapping + def generate_compliance_check_jobs(self, subjects, scopes): + jobs = [] + for subject in subjects: + for scope in scopes: + if scope == "scs-compatible-iaas": + jobs.append([scope, subject]) + if scope == "scs-compatible-kaas": + k8s_setup = self.get_kubernetes_setup(subject) + for k8s_version in k8s_setup["kube_versions"]: + jobs.append([scope, subject, k8s_version, k8s_setup["kube_plugin"]]) + return jobs + + def get_kubernetes_setup(self, subject): + default_kubernetes_setup = self.subjects.get('_', {}).get('kubernetes_setup', {}) + kubernetes_setup = {"kube_plugin": default_kubernetes_setup['kube_plugin'], "kube_versions": default_kubernetes_setup['kube_versions']} + kubernetes_setup.update(self.subjects.get(subject, {}).get('kubernetes_setup', {})) + return kubernetes_setup + + def build_clusters_for_jobs_sequence(self, jobs): + for i in range(len(jobs)): + if jobs[i][0] == "scs-compatible-kaas": + cluster_id = f"{jobs[i][1]}-{jobs[i][2]}" + logger.debug(f"Provide cluster '{cluster_id}'") + kubeconfig_path = os.getcwd() + "/k8s_test_kubeconfigs/" + if not os.path.exists(kubeconfig_path): + os.mkdir(kubeconfig_path) + kubeconfig_file = f"{cluster_id}.yaml" + kubeconfig = cluster_plugin.create(cluster_id, jobs[i][2], (kubeconfig_path + kubeconfig_file)) + jobs[i].append(kubeconfig) + return jobs + def abspath(self, path): return os.path.join(self.cwd, path) - def build_check_command(self, scope, subject, output): + def build_check_command(self, job, output): + scope = job[0] + subject = job[1] # TODO figure out when to supply --debug here (but keep separated from our --debug) cmd = [ sys.executable, self.scs_compliance_check, self.abspath(self.scopes[scope]['spec']), @@ -69,6 +105,10 @@ def build_check_command(self, scope, subject, output): ] for key, value in self.get_subject_mapping(subject).items(): cmd.extend(['-a', f'{key}={value}']) + if len(job) == 5 and scope == "scs-compatible-kaas": + cmd.extend(['-a', f'kubeconfig={job[4]}']) + elif len(job) != 5 and scope == "scs-compatible-kaas": + logger.Error(f"Scope is '{job[0]}' but no kubeconfig was provided") return cmd def build_cleanup_command(self, subject): @@ -175,9 +215,11 @@ def run(cfg, scopes, subjects, preset, num_workers, monitor_url, report_yaml): logger.debug(f'monitor url: {monitor_url}, num_workers: {num_workers}, output: {report_yaml}') with tempfile.TemporaryDirectory(dir=cfg.cwd) as tdirname: report_yaml_tmp = os.path.join(tdirname, 'report.yaml') - jobs = [(scope, subject) for scope in scopes for subject in subjects] + jobs = cfg.generate_compliance_check_jobs(subjects, scopes) + logger.debug("Create clusters and provide kubeconfig") + jobs = cfg.build_clusters_for_jobs_sequence(jobs) outputs = [os.path.join(tdirname, f'report-{idx}.yaml') for idx in range(len(jobs))] - commands = [cfg.build_check_command(job[0], job[1], output) for job, output in zip(jobs, outputs)] + commands = [cfg.build_check_command(job, output) for job, output in zip(jobs, outputs)] _run_commands(commands, num_workers=num_workers) _concat_files(outputs, report_yaml_tmp) subprocess.run(cfg.build_sign_command(report_yaml_tmp)) From 4a51c0dabfc1b95650f30d454794f4ed4447d504 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Thu, 10 Oct 2024 13:04:49 +0200 Subject: [PATCH 05/36] Adjust 'scs-test-runner.py' to handle kaas tests Splitting the functionality into the handling of sonobuoys and the provision of K8s clusters. Signed-off-by: Toni Finger --- Tests/config-kaas-example.toml | 40 ++++----------- Tests/kaas/plugin/interface.py | 27 ++++------ Tests/kaas/plugin/plugin_kind.py | 7 +-- Tests/kaas/plugin/plugin_static.py | 6 +-- Tests/kaas/plugin/run.py | 25 ---------- Tests/kaas/plugin/run_plugin.py | 27 ++++++++++ Tests/kaas/plugin/run_sonobuoy_executor.py | 58 ++++++++++++++-------- Tests/kaas/plugin/sonobuoy_executor.py | 47 +++++++++++------- Tests/scs-compatible-kaas.yaml | 6 +-- Tests/scs-test-runner.py | 36 +++++++++----- 10 files changed, 146 insertions(+), 133 deletions(-) delete mode 100755 Tests/kaas/plugin/run.py create mode 100644 Tests/kaas/plugin/run_plugin.py diff --git a/Tests/config-kaas-example.toml b/Tests/config-kaas-example.toml index 1d7c05c10..43557a518 100644 --- a/Tests/config-kaas-example.toml +++ b/Tests/config-kaas-example.toml @@ -12,29 +12,6 @@ subjects = [ ] -[presets.all] -scopes = [ - "scs-compatible-iaas", -] -subjects = [ - "gx-scs", - "artcodix", - "pco-prod1", - "pco-prod2", - "pco-prod3", - "pco-prod4", - "poc-kdo", - "poc-wgcloud", - "regio-a", - "syseleven-dus2", - "syseleven-ham1", - "wavestack", -] -workers = 4 - - -############################################################################### - [presets.all-iaas-kaas] scopes = [ "scs-compatible-kaas", @@ -53,9 +30,8 @@ scopes = [ "scs-compatible-kaas", ] subjects = [ - "cloud-k-4", - "cloud-k-5", - #"cloud-k-6", + "cloud-kaas-a", + "cloud-kaas-b", ] workers = 4 @@ -78,15 +54,17 @@ os_cloud = "{subject}" [subjects._.kubernetes_setup] kube_plugin = "kind" kube_versions = [ - "v1.12", - #"v1.13", - #"v1.14", + "v1.29", + #"v1.30", + #"v1.31", ] -[subjects.cloud-ik-1.kubernetes_setup] -kube_plugin = "none" +[subjects.cloud-kaas-b.kubernetes_setup] +kube_plugin = "static" [subjects.cloud-ik-1.mapping] os_cloud = "none" + + diff --git a/Tests/kaas/plugin/interface.py b/Tests/kaas/plugin/interface.py index 04bd1737e..34d39fc97 100644 --- a/Tests/kaas/plugin/interface.py +++ b/Tests/kaas/plugin/interface.py @@ -1,10 +1,8 @@ from abc import ABC, abstractmethod from typing import final -from kubernetes import client, config import os import shutil import logging -from junitparser import JUnitXml logger = logging.getLogger("interface") @@ -24,11 +22,11 @@ class KubernetesClusterPlugin(ABC): @final def __init__(self, config=None): logger.info(f"Inital provider plug in of type:{type(self)}") - logger.debug(config) + self.config = config + logger.debug(self.config) self.working_directory = os.getcwd() logger.debug(f"Working from {self.working_directory}") - @abstractmethod def _create_cluster(self, cluster_name) -> (str, int): """ @@ -48,14 +46,14 @@ def _delete_cluster(self, cluster_name) -> (str, int): pass @final - def create(self, name="scs-cluster", version=None, kubeconfig_filepath=None,): + def create(self, name="scs-cluster", version=None, kubeconfig_filepath=None): """ This method is to be called to create a k8s cluster :param: kubernetes_version: :return: uuid """ - self.cluster_name=name - self.cluster_version=version + self.cluster_name = name + self.cluster_version = version try: self._create_cluster() @@ -64,22 +62,19 @@ def create(self, name="scs-cluster", version=None, kubeconfig_filepath=None,): self._delete_cluster() if kubeconfig_filepath: - generated_file=self.kubeconfig - self.kubeconfig = shutil.move(generated_file, kubeconfig_filepath) + generated_file = self.kubeconfig + self.kubeconfig = shutil.move(generated_file, kubeconfig_filepath) - return self.kubeconfig + return self.kubeconfig @final - def destroy(self, cluster_uuid=None): + def delete(self, cluster_name=None): """ - This method is to be called unprovision a cluster + This method is to be called in order to unprovision a cluster :param: cluster_uuid: """ + self.cluster_name = cluster_name try: self._delete_cluster() except Exception as e: logging.error(e) - - - - diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py index 91eb93e30..b5f490d17 100644 --- a/Tests/kaas/plugin/plugin_kind.py +++ b/Tests/kaas/plugin/plugin_kind.py @@ -1,6 +1,5 @@ from .interface import KubernetesClusterPlugin from pytest_kind import KindCluster -import time class PluginKind(KubernetesClusterPlugin): @@ -11,10 +10,12 @@ class PluginKind(KubernetesClusterPlugin): def _create_cluster(self): print(self.cluster_version) - self.cluster = KindCluster(self.cluster_name) + self.cluster = KindCluster( + self.cluster_name + ) # TODO: enrich with config and version self.cluster.create() - time.sleep(30)#TODO:!!! make this deterministic self.kubeconfig = str(self.cluster.kubeconfig_path.resolve()) def _delete_cluster(self): + self.cluster = KindCluster(self.cluster_name) self.cluster.delete() diff --git a/Tests/kaas/plugin/plugin_static.py b/Tests/kaas/plugin/plugin_static.py index 2d094396d..55ed75a29 100644 --- a/Tests/kaas/plugin/plugin_static.py +++ b/Tests/kaas/plugin/plugin_static.py @@ -1,5 +1,4 @@ -from interface import KubernetesClusterPlugin -import os +from .interface import KubernetesClusterPlugin class PluginStatic(KubernetesClusterPlugin): @@ -7,8 +6,9 @@ class PluginStatic(KubernetesClusterPlugin): Plugin to handle the provisioning of kubernetes using a kubeconfig file """ + def _create_cluster(self): - self.kubeconfig_path = os.environ["KUBECONFIG"] + self.kubeconfig = self.kubeconfig def _delete_cluster(self): pass diff --git a/Tests/kaas/plugin/run.py b/Tests/kaas/plugin/run.py deleted file mode 100755 index 024723e87..000000000 --- a/Tests/kaas/plugin/run.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 -# vim: set ts=4 sw=4 et: -# - -from plugin_kind import PluginKind -from plugin_static import PluginStatic -import os -import logging -logging.basicConfig(level=logging.DEBUG) -logger = logging.getLogger("interface") - -if __name__ == "__main__": - - logger.info("Select provider plugin") - match os.environ['CLUSTER_PROVIDER']: - case "static": - plugin = PluginStatic() - case "kind": - plugin = PluginKind() - case _: - logger.error("provider plugin not implemented") - raise NotImplementedError(f"{os.environ['CLUSTER_PROVIDER']} is not valid ") - - logger.info("Run plugin") - plugin.run() diff --git a/Tests/kaas/plugin/run_plugin.py b/Tests/kaas/plugin/run_plugin.py new file mode 100644 index 000000000..8fbf96ca5 --- /dev/null +++ b/Tests/kaas/plugin/run_plugin.py @@ -0,0 +1,27 @@ +from .plugin_kind import PluginKind +from .plugin_static import PluginStatic + + +def init_plugin(plugin_kind, config=None): + if plugin_kind == "static": + plugin = PluginStatic(config) + + elif plugin_kind == "kind": + plugin = PluginKind(config) + + else: + raise ValueError(f"unknown plugin '{plugin_kind}'") + + return plugin + + +def run_plugin_create(plugin_kind, cluster_id, k8s_version, kubeconfig_filepath): + plugin = init_plugin(plugin_kind) + kubeconfig = plugin.create(cluster_id, k8s_version, kubeconfig_filepath) + return kubeconfig + + +def run_plugin_delete(plugin_kind, cluster_id): + plugin = init_plugin(plugin_kind) + kubeconfig = plugin.delete(cluster_id) + return kubeconfig diff --git a/Tests/kaas/plugin/run_sonobuoy_executor.py b/Tests/kaas/plugin/run_sonobuoy_executor.py index 7ef6666c7..162ae7545 100755 --- a/Tests/kaas/plugin/run_sonobuoy_executor.py +++ b/Tests/kaas/plugin/run_sonobuoy_executor.py @@ -3,32 +3,46 @@ # from sonobuoy_executor import SonobuoyExecutor -import logging -import getopt import sys -logging.basicConfig(level=logging.DEBUG) -logger = logging.getLogger("sonobuoy_executor") +import click +import logging -#TODO:!!! move this to class file -def parse_arguments(argv): - """Parse cli arguments from the script call""" - try: - opts, args = getopt.gnu_getopt(argv, "k:t:h", ["kubeconfig=", "test=", "help"]) - except getopt.GetoptError: - raise ConfigException +logging_config = { + "level": "INFO", + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "sonobuoy_logs": { + "format": "%(levelname)s: %(message)s" + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "sonobuoy_logs", + "stream": "ext://sys.stdout" + } + }, + "root": { + "handlers": ["console"] + } +} - for opt in opts: - if opt[0] == "-h" or opt[0] == "--help": - raise HelpException - if opt[0] == "-k" or opt[0] == "--kubeconfig": - kubeconfig = opt[1] - return kubeconfig +logger = logging.getLogger(__name__) -if __name__ == "__main__": - logger.info("read in kubeconfig") - kubeconfig_path = parse_arguments(sys.argv) +@click.command() +@click.option('-k', '--kubeconfig', 'kubeconfig', type=click.Path(exists=False), default=None, help='path/to/kubeconfig_file.yaml') +@click.option('-r', '--result_dir_name', 'result_dir_name', type=str, default="sonobuoy_results", help='directory name to store results at') +@click.option('-c', '--check', 'check_name', type=str, default="sonobuoy_executor", help='name of the check to p') # TODO:!!! this should be eighter 'scs-kaas-tests' or 'cncf-conformance' +@click.option('--debug/--no-debug', default=False) # TODO: Not Yet Implemented +def sonobuoy_run(kubeconfig, result_dir_name, check_name, debug): logger.info("Run sonobuoy_executor") - sonobuoy_executor = SonobuoyExecutor(kubeconfig_path) - sonobuoy_executor.run() + sonobuoy_executor = SonobuoyExecutor(check_name, kubeconfig, result_dir_name) + return_code = sonobuoy_executor.run() + sys.exit(return_code) + + +if __name__ == "__main__": + sonobuoy_run() diff --git a/Tests/kaas/plugin/sonobuoy_executor.py b/Tests/kaas/plugin/sonobuoy_executor.py index 05c6f68ae..f76a4b3b4 100644 --- a/Tests/kaas/plugin/sonobuoy_executor.py +++ b/Tests/kaas/plugin/sonobuoy_executor.py @@ -30,18 +30,20 @@ class SonobuoyExecutor: """ kubeconfig_path = None - working_directory = None @final - def __init__(self, kubeconfig=None): - logger.info("Inital SonobuoyExecutor") - logger.debug(kubeconfig) + def __init__(self, check_name="sonobuoy_executor", kubeconfig=None, result_dir_name="sonobuoy_results"): + self.check_name = check_name + logger.info(f"Inital SonobuoyExecutor for {self.check_name}") + logger.debug(f"kubeconfig: {kubeconfig} ") if kubeconfig is None: raise Exception("No kubeconfig provided") - # TODO:!!! check if kubeconfig is valid + else: + self.kubeconfig_path = kubeconfig self.working_directory = os.getcwd() - logger.debug(f"Working from {self.working_directory}") + self.result_dir_name = result_dir_name + logger.debug(f"Working from {self.working_directory} placing results at {self.result_dir_name}") @final def _preflight_check(self): @@ -103,7 +105,7 @@ def _cleanup_sonobuoy_resources(self): os.system(f"sonobuoy delete --wait --kubeconfig='{self.kubeconfig_path}'") @final - def _retrieve_result(self, result_dir_name): + def _retrieve_result(self): """ This method invokes sonobouy to store the results in a subdirectory of the working directory. The Junit results file contained in it is then @@ -111,14 +113,14 @@ def _retrieve_result(self, result_dir_name): :param: result_file_name: :return: None """ - logger.debug(f"retrieving results to {result_dir_name}") - result_dir = self.working_directory + "/" + result_dir_name + logger.debug(f"retrieving results to {self.result_dir_name}") + result_dir = self.working_directory + "/" + self.result_dir_name if os.path.exists(result_dir): os.system(f"rm -rf {result_dir}/*") else: os.mkdir(result_dir) os.system( - f"sonobuoy retrieve {result_dir} -x --filename='{result_dir_name}' --kubeconfig='{self.kubeconfig_path}'" + f"sonobuoy retrieve {result_dir} -x --filename='{result_dir}' --kubeconfig='{self.kubeconfig_path}'" ) logger.debug( f"parsing JUnit result from {result_dir + '/plugins/e2e/results/global/junit_01.xml'} " @@ -131,28 +133,37 @@ def _retrieve_result(self, result_dir_name): skipped_test_cases = 0 for suite in xml: for case in suite: - if case.is_passed: + if case.is_passed is True: passed_test_cases += 1 + elif case.is_skipped is True: + skipped_test_cases += 1 + # ~ logger.warning(f"SKIPPED:{case.name}") # TODO:!!! decide if skipped is error or warning only ? else: failed_test_cases += 1 - if case.is_skipped: - skipped_test_cases += 1 + logger.error(f"ERROR: {case.name}") - logger.info( - f" {passed_test_cases} passed, {failed_test_cases} failed of which {skipped_test_cases} were skipped" - ) + result_message = f" {passed_test_cases} passed, {failed_test_cases} failed, {skipped_test_cases} skipped" + if failed_test_cases == 0 and skipped_test_cases == 0: + logger.info(result_message) + self.return_code = 0 + else: + logger.error("ERROR:" + result_message) + self.return_code = 3 @final def run(self): """ This method is to be called to run the plugin """ + self.return_code = 11 try: self._preflight_check() self._test_k8s_cncf_conformance() - self._retrieve_result("cncf_result") - self._cleanup_sonobuoy_resources() + self._retrieve_result() except Exception as e: logging.error(e) + self.return_code = 1 finally: self._cleanup_sonobuoy_resources() + print(self.check_name + ": " + ('PASS', 'FAIL')[min(1, self.return_code)]) + return self.return_code diff --git a/Tests/scs-compatible-kaas.yaml b/Tests/scs-compatible-kaas.yaml index 33af1bf41..185f3d41a 100644 --- a/Tests/scs-compatible-kaas.yaml +++ b/Tests/scs-compatible-kaas.yaml @@ -6,12 +6,12 @@ variables: modules: - id: cncf-k8s-conformance name: CNCF Kubernetes conformance - url: https://github.com/cncf/k8s-conformance/tree/master + url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0210-v2-k8s-version-policy.md # TODO:!!! this standard mentions cncf but is is the right reference? run: - executable: ./kaas/plugin/run_sonobuoy_executor.py - args: -k {kubeconfig} + args: -k {kubeconfig} -r {result_dir_name} -c 'cncf-conformance' testcases: - - id: cncf-k8s-conformance + - id: cncf-conformance tags: [mandatory] - id: scs-0210-v2 name: Kubernetes version policy diff --git a/Tests/scs-test-runner.py b/Tests/scs-test-runner.py index 5818e96eb..2eeaa71ae 100755 --- a/Tests/scs-test-runner.py +++ b/Tests/scs-test-runner.py @@ -17,9 +17,7 @@ import click import tomli -# TODO:!!! provide plugin chosing at interface side -from kaas.plugin.plugin_kind import PluginKind -cluster_plugin = PluginKind() +from kaas.plugin.run_plugin import run_plugin_create, run_plugin_delete logger = logging.getLogger(__name__) MONITOR_URL = "https://compliance.sovereignit.cloud/" @@ -70,7 +68,8 @@ def generate_compliance_check_jobs(self, subjects, scopes): if scope == "scs-compatible-kaas": k8s_setup = self.get_kubernetes_setup(subject) for k8s_version in k8s_setup["kube_versions"]: - jobs.append([scope, subject, k8s_version, k8s_setup["kube_plugin"]]) + kaas_subject = f"{subject}.{k8s_version}" + jobs.append([scope, kaas_subject, k8s_version, k8s_setup["kube_plugin"]]) return jobs def get_kubernetes_setup(self, subject): @@ -79,19 +78,26 @@ def get_kubernetes_setup(self, subject): kubernetes_setup.update(self.subjects.get(subject, {}).get('kubernetes_setup', {})) return kubernetes_setup - def build_clusters_for_jobs_sequence(self, jobs): + def build_clusters_for_jobs_in_sequence(self, jobs): for i in range(len(jobs)): if jobs[i][0] == "scs-compatible-kaas": - cluster_id = f"{jobs[i][1]}-{jobs[i][2]}" + cluster_id = f"{jobs[i][1]}" logger.debug(f"Provide cluster '{cluster_id}'") kubeconfig_path = os.getcwd() + "/k8s_test_kubeconfigs/" if not os.path.exists(kubeconfig_path): os.mkdir(kubeconfig_path) - kubeconfig_file = f"{cluster_id}.yaml" - kubeconfig = cluster_plugin.create(cluster_id, jobs[i][2], (kubeconfig_path + kubeconfig_file)) + kubeconfig_filepath = kubeconfig_path + f"{cluster_id}.yaml" + kubeconfig = run_plugin_create(jobs[i][3], cluster_id, jobs[i][2], (kubeconfig_filepath)) jobs[i].append(kubeconfig) return jobs + def delete_clusters_for_jobs_in_sequence(self, jobs): + for i in range(len(jobs)): + if jobs[i][0] == "scs-compatible-kaas": + cluster_id = f"{jobs[i][1]}" + logger.debug(f"Delete cluster '{cluster_id}'") + run_plugin_delete(jobs[i][3], cluster_id) + def abspath(self, path): return os.path.join(self.cwd, path) @@ -107,6 +113,7 @@ def build_check_command(self, job, output): cmd.extend(['-a', f'{key}={value}']) if len(job) == 5 and scope == "scs-compatible-kaas": cmd.extend(['-a', f'kubeconfig={job[4]}']) + cmd.extend(['-a', f'result_dir_name={job[1]}-{job[2]}']) elif len(job) != 5 and scope == "scs-compatible-kaas": logger.Error(f"Scope is '{job[0]}' but no kubeconfig was provided") return cmd @@ -192,8 +199,9 @@ def _move_file(source_path, target_path): @click.option('--num-workers', 'num_workers', type=int, default=5) @click.option('--monitor-url', 'monitor_url', type=str, default=MONITOR_URL) @click.option('-o', '--output', 'report_yaml', type=click.Path(exists=False), default=None) +@click.option('--upload/--no-upload', default=True) @click.pass_obj -def run(cfg, scopes, subjects, preset, num_workers, monitor_url, report_yaml): +def run(cfg, scopes, subjects, preset, num_workers, monitor_url, report_yaml, upload): """ run compliance tests and upload results to compliance monitor """ @@ -217,15 +225,19 @@ def run(cfg, scopes, subjects, preset, num_workers, monitor_url, report_yaml): report_yaml_tmp = os.path.join(tdirname, 'report.yaml') jobs = cfg.generate_compliance_check_jobs(subjects, scopes) logger.debug("Create clusters and provide kubeconfig") - jobs = cfg.build_clusters_for_jobs_sequence(jobs) + jobs = cfg.build_clusters_for_jobs_in_sequence(jobs) outputs = [os.path.join(tdirname, f'report-{idx}.yaml') for idx in range(len(jobs))] commands = [cfg.build_check_command(job, output) for job, output in zip(jobs, outputs)] _run_commands(commands, num_workers=num_workers) _concat_files(outputs, report_yaml_tmp) - subprocess.run(cfg.build_sign_command(report_yaml_tmp)) - subprocess.run(cfg.build_upload_command(report_yaml_tmp, monitor_url)) + if upload: + logging.debug("uploading results") + subprocess.run(cfg.build_sign_command(report_yaml_tmp)) + subprocess.run(cfg.build_upload_command(report_yaml_tmp, monitor_url)) if report_yaml is not None: _move_file(report_yaml_tmp, report_yaml) + logger.debug("delete clusters") + cfg.delete_clusters_for_jobs_in_sequence(jobs) return 0 From 928ea830d81a80215f567d681a1efec1e003296c Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Mon, 14 Oct 2024 12:34:41 +0200 Subject: [PATCH 06/36] Add ability to deploy clusters for different K8s versions Signed-off-by: Toni Finger --- Tests/config-kaas-example.toml | 9 +++++---- Tests/kaas/plugin/plugin_kind.py | 9 ++++----- Tests/kaas/plugin/run_plugin.py | 3 +-- Tests/kaas/plugin/sonobuoy_executor.py | 21 ++++++++++++++------- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/Tests/config-kaas-example.toml b/Tests/config-kaas-example.toml index 43557a518..e9a12bf6f 100644 --- a/Tests/config-kaas-example.toml +++ b/Tests/config-kaas-example.toml @@ -1,6 +1,7 @@ [secrets] keyfile = ".secret/keyfile" tokenfile = ".secret/tokenfile" +k8s_config = ".secret/k8s_config.yml" [presets.default] @@ -31,7 +32,7 @@ scopes = [ ] subjects = [ "cloud-kaas-a", - "cloud-kaas-b", + #"cloud-kaas-b", ] workers = 4 @@ -54,9 +55,9 @@ os_cloud = "{subject}" [subjects._.kubernetes_setup] kube_plugin = "kind" kube_versions = [ - "v1.29", - #"v1.30", - #"v1.31", + "v1.29.0", + "v1.30.0", + #"v1.31.0", ] diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py index b5f490d17..f0bd77a5f 100644 --- a/Tests/kaas/plugin/plugin_kind.py +++ b/Tests/kaas/plugin/plugin_kind.py @@ -9,11 +9,10 @@ class PluginKind(KubernetesClusterPlugin): """ def _create_cluster(self): - print(self.cluster_version) - self.cluster = KindCluster( - self.cluster_name - ) # TODO: enrich with config and version - self.cluster.create() + config_file = None + cluster_image = f"kindest/node:{self.cluster_version}" + self.cluster = KindCluster(self.cluster_name, self.kubeconfig, cluster_image) + self.cluster.create(config_file) self.kubeconfig = str(self.cluster.kubeconfig_path.resolve()) def _delete_cluster(self): diff --git a/Tests/kaas/plugin/run_plugin.py b/Tests/kaas/plugin/run_plugin.py index 8fbf96ca5..65bf7d830 100644 --- a/Tests/kaas/plugin/run_plugin.py +++ b/Tests/kaas/plugin/run_plugin.py @@ -23,5 +23,4 @@ def run_plugin_create(plugin_kind, cluster_id, k8s_version, kubeconfig_filepath) def run_plugin_delete(plugin_kind, cluster_id): plugin = init_plugin(plugin_kind) - kubeconfig = plugin.delete(cluster_id) - return kubeconfig + plugin.delete(cluster_id) diff --git a/Tests/kaas/plugin/sonobuoy_executor.py b/Tests/kaas/plugin/sonobuoy_executor.py index f76a4b3b4..28e10ab54 100644 --- a/Tests/kaas/plugin/sonobuoy_executor.py +++ b/Tests/kaas/plugin/sonobuoy_executor.py @@ -33,7 +33,12 @@ class SonobuoyExecutor: working_directory = None @final - def __init__(self, check_name="sonobuoy_executor", kubeconfig=None, result_dir_name="sonobuoy_results"): + def __init__( + self, + check_name="sonobuoy_executor", + kubeconfig=None, + result_dir_name="sonobuoy_results", + ): self.check_name = check_name logger.info(f"Inital SonobuoyExecutor for {self.check_name}") logger.debug(f"kubeconfig: {kubeconfig} ") @@ -43,7 +48,9 @@ def __init__(self, check_name="sonobuoy_executor", kubeconfig=None, result_dir_n self.kubeconfig_path = kubeconfig self.working_directory = os.getcwd() self.result_dir_name = result_dir_name - logger.debug(f"Working from {self.working_directory} placing results at {self.result_dir_name}") + logger.debug( + f"Working from {self.working_directory} placing results at {self.result_dir_name}" + ) @final def _preflight_check(self): @@ -66,7 +73,9 @@ def _preflight_check(self): logger.debug(f"[supported api]: {api.name:<40} {','.join(versions)}") logger.debug("checks if sonobuoy is availabe") - return_value = os.system(f"sonobuoy version --kubeconfig='{self.kubeconfig_path}'") + return_value = os.system( + f"sonobuoy version --kubeconfig='{self.kubeconfig_path}'" + ) if return_value != 0: raise Exception("sonobuoy is not installed") @@ -125,9 +134,7 @@ def _retrieve_result(self): logger.debug( f"parsing JUnit result from {result_dir + '/plugins/e2e/results/global/junit_01.xml'} " ) - xml = JUnitXml.fromfile( - result_dir + "/plugins/e2e/results/global/junit_01.xml" - ) + xml = JUnitXml.fromfile(result_dir + "/plugins/e2e/results/global/junit_01.xml") failed_test_cases = 0 passed_test_cases = 0 skipped_test_cases = 0 @@ -165,5 +172,5 @@ def run(self): self.return_code = 1 finally: self._cleanup_sonobuoy_resources() - print(self.check_name + ": " + ('PASS', 'FAIL')[min(1, self.return_code)]) + print(self.check_name + ": " + ("PASS", "FAIL")[min(1, self.return_code)]) return self.return_code From acc451dc04c2c26f45814a399237c7a137349c29 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Fri, 18 Oct 2024 11:36:37 +0200 Subject: [PATCH 07/36] Update `scs-test-runner.py` Signed-off-by: Toni Finger --- Tests/scs-test-runner.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Tests/scs-test-runner.py b/Tests/scs-test-runner.py index 2eeaa71ae..a1446decb 100755 --- a/Tests/scs-test-runner.py +++ b/Tests/scs-test-runner.py @@ -230,10 +230,6 @@ def run(cfg, scopes, subjects, preset, num_workers, monitor_url, report_yaml, up commands = [cfg.build_check_command(job, output) for job, output in zip(jobs, outputs)] _run_commands(commands, num_workers=num_workers) _concat_files(outputs, report_yaml_tmp) - if upload: - logging.debug("uploading results") - subprocess.run(cfg.build_sign_command(report_yaml_tmp)) - subprocess.run(cfg.build_upload_command(report_yaml_tmp, monitor_url)) if report_yaml is not None: _move_file(report_yaml_tmp, report_yaml) logger.debug("delete clusters") From e4d6c3cad282fc6aa46aa10c332a0fd82fb7ecf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Wed, 23 Oct 2024 22:17:28 +0200 Subject: [PATCH 08/36] still draft: a few steps towards my vision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/config-kaas-example.toml | 71 ------------- Tests/config.toml | 17 ++++ Tests/kaas/plugin/interface.py | 13 +-- Tests/kaas/plugin/plugin_kind.py | 13 ++- Tests/kaas/plugin/plugin_static.py | 2 +- Tests/kaas/plugin/run_plugin.py | 69 ++++++++++--- Tests/scs-compatible-kaas.yaml | 41 +++++--- Tests/scs-test-runner.py | 158 ++++++++++++++++------------- 8 files changed, 204 insertions(+), 180 deletions(-) delete mode 100644 Tests/config-kaas-example.toml mode change 100644 => 100755 Tests/kaas/plugin/run_plugin.py diff --git a/Tests/config-kaas-example.toml b/Tests/config-kaas-example.toml deleted file mode 100644 index e9a12bf6f..000000000 --- a/Tests/config-kaas-example.toml +++ /dev/null @@ -1,71 +0,0 @@ -[secrets] -keyfile = ".secret/keyfile" -tokenfile = ".secret/tokenfile" -k8s_config = ".secret/k8s_config.yml" - - -[presets.default] -scopes = [ - "scs-compatible-iaas", -] -subjects = [ - "gx-scs", -] - - -[presets.all-iaas-kaas] -scopes = [ - "scs-compatible-kaas", - "scs-compatible-iaas", -] -subjects = [ - "cloud-ik-1", - "cloud-ik-2", - #"cloud-ik-3", -] -workers = 4 - - -[presets.all-kaas] -scopes = [ - "scs-compatible-kaas", -] -subjects = [ - "cloud-kaas-a", - #"cloud-kaas-b", -] -workers = 4 - - - -[scopes.scs-compatible-iaas] -spec = "./scs-compatible-iaas.yaml" - - -[scopes.scs-compatible-kaas] -spec = "./scs-compatible-kaas.yaml" - - -# default subject (not a real subject, but used to declare a default mapping) -# (this is the only mapping declaration that supports using Python string interpolation) -[subjects._.mapping] -os_cloud = "{subject}" - - -[subjects._.kubernetes_setup] -kube_plugin = "kind" -kube_versions = [ - "v1.29.0", - "v1.30.0", - #"v1.31.0", -] - - -[subjects.cloud-kaas-b.kubernetes_setup] -kube_plugin = "static" - -[subjects.cloud-ik-1.mapping] -os_cloud = "none" - - - diff --git a/Tests/config.toml b/Tests/config.toml index 0f270cd4b..1c913210a 100644 --- a/Tests/config.toml +++ b/Tests/config.toml @@ -33,6 +33,17 @@ subjects = [ workers = 4 +[presets.all-kaas] +scopes = [ + "scs-compatible-kaas", +] +subjects = [ + "cloud-kaas-a", + #"cloud-kaas-b", +] +workers = 4 + + [scopes.scs-compatible-iaas] spec = "./scs-compatible-iaas.yaml" @@ -41,3 +52,9 @@ spec = "./scs-compatible-iaas.yaml" # (this is the only mapping declaration that supports using Python string interpolation) [subjects._.mapping] os_cloud = "{subject}" +subject_root = "{subject}" + + +[subjects._.kubernetes_setup] +kube_plugin = "kind" +clusterspec = "kaas/clusterspec.yaml" diff --git a/Tests/kaas/plugin/interface.py b/Tests/kaas/plugin/interface.py index 34d39fc97..7148e5e24 100644 --- a/Tests/kaas/plugin/interface.py +++ b/Tests/kaas/plugin/interface.py @@ -21,7 +21,7 @@ class KubernetesClusterPlugin(ABC): @final def __init__(self, config=None): - logger.info(f"Inital provider plug in of type:{type(self)}") + logger.info(f"Init provider plug-in of type {self.__class__.__name__}") self.config = config logger.debug(self.config) self.working_directory = os.getcwd() @@ -58,14 +58,15 @@ def create(self, name="scs-cluster", version=None, kubeconfig_filepath=None): self._create_cluster() except Exception as e: - logging.error(e) + logging.exception(e) self._delete_cluster() if kubeconfig_filepath: - generated_file = self.kubeconfig - self.kubeconfig = shutil.move(generated_file, kubeconfig_filepath) + shutil.move(self.kubeconfig, kubeconfig_filepath) + else: + kubeconfig_filepath = str(self.kubeconfig) - return self.kubeconfig + return kubeconfig_filepath @final def delete(self, cluster_name=None): @@ -77,4 +78,4 @@ def delete(self, cluster_name=None): try: self._delete_cluster() except Exception as e: - logging.error(e) + logging.exception(e) diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py index f0bd77a5f..840a38138 100644 --- a/Tests/kaas/plugin/plugin_kind.py +++ b/Tests/kaas/plugin/plugin_kind.py @@ -1,4 +1,4 @@ -from .interface import KubernetesClusterPlugin +from interface import KubernetesClusterPlugin from pytest_kind import KindCluster @@ -10,10 +10,17 @@ class PluginKind(KubernetesClusterPlugin): def _create_cluster(self): config_file = None - cluster_image = f"kindest/node:{self.cluster_version}" + cluster_version = self.cluster_version + if cluster_version == '1.29': + cluster_version = 'v1.29.8' + elif cluster_version == '1.30': + cluster_version = 'v1.30.4' + elif cluster_version == '1.31' or cluster_version == 'default': + cluster_version = 'v1.31.1' + cluster_image = f"kindest/node:{cluster_version}" self.cluster = KindCluster(self.cluster_name, self.kubeconfig, cluster_image) self.cluster.create(config_file) - self.kubeconfig = str(self.cluster.kubeconfig_path.resolve()) + self.kubeconfig = self.cluster.kubeconfig_path.resolve() def _delete_cluster(self): self.cluster = KindCluster(self.cluster_name) diff --git a/Tests/kaas/plugin/plugin_static.py b/Tests/kaas/plugin/plugin_static.py index 55ed75a29..f28f148fd 100644 --- a/Tests/kaas/plugin/plugin_static.py +++ b/Tests/kaas/plugin/plugin_static.py @@ -1,4 +1,4 @@ -from .interface import KubernetesClusterPlugin +from interface import KubernetesClusterPlugin class PluginStatic(KubernetesClusterPlugin): diff --git a/Tests/kaas/plugin/run_plugin.py b/Tests/kaas/plugin/run_plugin.py old mode 100644 new mode 100755 index 65bf7d830..26b9a8194 --- a/Tests/kaas/plugin/run_plugin.py +++ b/Tests/kaas/plugin/run_plugin.py @@ -1,26 +1,65 @@ -from .plugin_kind import PluginKind -from .plugin_static import PluginStatic +#!/usr/bin/env python3 +import logging +import os.path +import click +import yaml -def init_plugin(plugin_kind, config=None): - if plugin_kind == "static": - plugin = PluginStatic(config) +from plugin_kind import PluginKind +from plugin_static import PluginStatic - elif plugin_kind == "kind": - plugin = PluginKind(config) - else: - raise ValueError(f"unknown plugin '{plugin_kind}'") +PLUGIN_LOOKUP = { + "kind": PluginKind, + "static": PluginStatic, +} - return plugin + +def init_plugin(plugin_kind, config=None): + plugin_maker = PLUGIN_LOOKUP.get(plugin_kind) + if plugin_maker is None: + raise ValueError(f"unknown plugin '{plugin_kind}'") + return plugin_maker(config) -def run_plugin_create(plugin_kind, cluster_id, k8s_version, kubeconfig_filepath): +def run_plugin_create(plugin_kind, clusterspec): plugin = init_plugin(plugin_kind) - kubeconfig = plugin.create(cluster_id, k8s_version, kubeconfig_filepath) - return kubeconfig + for cluster_id, cluster_info in clusterspec.items(): + plugin.create(cluster_id, cluster_info['branch'], os.path.abspath(cluster_info['kubeconfig'])) -def run_plugin_delete(plugin_kind, cluster_id): +def run_plugin_delete(plugin_kind, clusterspec): plugin = init_plugin(plugin_kind) - plugin.delete(cluster_id) + for cluster_id in clusterspec: + plugin.delete(cluster_id) + + +def load_spec(clusterspec_path): + with open(clusterspec_path, "rb") as fileobj: + return yaml.load(fileobj, Loader=yaml.SafeLoader) + + +@click.group() +def cli(): + pass + + +@cli.command() +@click.argument('plugin_kind', type=click.Choice(list(PLUGIN_LOOKUP), case_sensitive=False)) +@click.argument('clusterspec_path', type=click.Path(exists=True, dir_okay=False)) +def create(plugin_kind, clusterspec_path): + clusterspec = load_spec(clusterspec_path)['clusters'] + run_plugin_create(plugin_kind, clusterspec) + + +@cli.command() +@click.argument('plugin_kind', type=click.Choice(list(PLUGIN_LOOKUP), case_sensitive=False)) +@click.argument('clusterspec_path', type=click.Path(exists=True, dir_okay=False)) +def delete(plugin_kind, clusterspec_path): + clusterspec = load_spec(clusterspec_path)['clusters'] + run_plugin_delete(plugin_kind, clusterspec) + + +if __name__ == '__main__': + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) + cli() diff --git a/Tests/scs-compatible-kaas.yaml b/Tests/scs-compatible-kaas.yaml index 185f3d41a..942121b80 100644 --- a/Tests/scs-compatible-kaas.yaml +++ b/Tests/scs-compatible-kaas.yaml @@ -2,32 +2,38 @@ name: SCS-compatible KaaS uuid: 1fffebe6-fd4b-44d3-a36c-fc58b4bb0180 url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Tests/scs-compatible-kaas.yaml variables: - - kubeconfig + - subject_root # directory containing the kubeconfig files for the subject under test modules: - id: cncf-k8s-conformance name: CNCF Kubernetes conformance url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0210-v2-k8s-version-policy.md # TODO:!!! this standard mentions cncf but is is the right reference? + parameters: + kubeconfig: relative path to kubeconfig file run: - executable: ./kaas/plugin/run_sonobuoy_executor.py - args: -k {kubeconfig} -r {result_dir_name} -c 'cncf-conformance' + args: -k {subject_root}/{kubeconfig} -r {result_dir_name} -c 'cncf-conformance' testcases: - - id: cncf-conformance + - id: cncf-k8s-conformance tags: [mandatory] - id: scs-0210-v2 name: Kubernetes version policy url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0210-v2-k8s-version-policy.md + parameters: + kubeconfig: relative path to kubeconfig file run: - executable: ./kaas/k8s-version-policy/k8s_version_policy.py - args: -k {kubeconfig} + args: -k {subject_root}/{kubeconfig} testcases: - id: version-policy-check tags: [mandatory] - id: scs-0214-v2 name: Kubernetes node distribution and availability url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0214-v1-k8s-node-distribution.md + parameters: + kubeconfig: relative path to kubeconfig file run: - executable: ./kaas/k8s-node-distribution/k8s_node_distribution_check.py - args: -k {kubeconfig} + args: -k {subject_root}/{kubeconfig} testcases: - id: node-distribution-check tags: [mandatory] @@ -36,18 +42,23 @@ timeline: - date: 2024-02-28 versions: v1: draft - v2: draft versions: - - version: v2 - include: - - cncf-k8s-conformance - - scs-0210-v2 - - scs-0214-v2 - targets: - main: mandatory - version: v1 include: - - scs-0210-v2 - - scs-0214-v2 + - ref: cncf-k8s-conformance + parameters: + kubeconfig: kubeconfig.yaml + - ref: scs-0210-v2 + parameters: + kubeconfig: kubeconfig.yaml + - ref: scs-0210-v2 + parameters: + kubeconfig: kubecfg129.yaml + - ref: scs-0210-v2 + parameters: + kubeconfig: kubecfg130.yaml + - ref: scs-0214-v2 + parameters: + kubeconfig: kubeconfig.yaml targets: main: mandatory diff --git a/Tests/scs-test-runner.py b/Tests/scs-test-runner.py index a1446decb..3ec6beaaa 100755 --- a/Tests/scs-test-runner.py +++ b/Tests/scs-test-runner.py @@ -17,17 +17,23 @@ import click import tomli -from kaas.plugin.run_plugin import run_plugin_create, run_plugin_delete - logger = logging.getLogger(__name__) MONITOR_URL = "https://compliance.sovereignit.cloud/" +def ensure_dir(path): + try: + os.makedirs(path) + except FileExistsError: + pass + + class Config: def __init__(self): self.cwd = os.path.abspath(os.path.dirname(sys.argv[0]) or os.getcwd()) self.scs_compliance_check = os.path.join(self.cwd, 'scs-compliance-check.py') self.cleanup_py = os.path.join(self.cwd, 'cleanup.py') + self.run_plugin_py = os.path.join(self.cwd, 'kaas', 'plugin', 'run_plugin.py') self.ssh_keygen = shutil.which('ssh-keygen') self.curl = shutil.which('curl') self.secrets = {} @@ -59,88 +65,76 @@ def get_subject_mapping(self, subject): mapping.update(self.subjects.get(subject, {}).get('mapping', {})) return mapping - def generate_compliance_check_jobs(self, subjects, scopes): - jobs = [] - for subject in subjects: - for scope in scopes: - if scope == "scs-compatible-iaas": - jobs.append([scope, subject]) - if scope == "scs-compatible-kaas": - k8s_setup = self.get_kubernetes_setup(subject) - for k8s_version in k8s_setup["kube_versions"]: - kaas_subject = f"{subject}.{k8s_version}" - jobs.append([scope, kaas_subject, k8s_version, k8s_setup["kube_plugin"]]) - return jobs - def get_kubernetes_setup(self, subject): default_kubernetes_setup = self.subjects.get('_', {}).get('kubernetes_setup', {}) - kubernetes_setup = {"kube_plugin": default_kubernetes_setup['kube_plugin'], "kube_versions": default_kubernetes_setup['kube_versions']} + kubernetes_setup = dict(default_kubernetes_setup) kubernetes_setup.update(self.subjects.get(subject, {}).get('kubernetes_setup', {})) return kubernetes_setup - def build_clusters_for_jobs_in_sequence(self, jobs): - for i in range(len(jobs)): - if jobs[i][0] == "scs-compatible-kaas": - cluster_id = f"{jobs[i][1]}" - logger.debug(f"Provide cluster '{cluster_id}'") - kubeconfig_path = os.getcwd() + "/k8s_test_kubeconfigs/" - if not os.path.exists(kubeconfig_path): - os.mkdir(kubeconfig_path) - kubeconfig_filepath = kubeconfig_path + f"{cluster_id}.yaml" - kubeconfig = run_plugin_create(jobs[i][3], cluster_id, jobs[i][2], (kubeconfig_filepath)) - jobs[i].append(kubeconfig) - return jobs - - def delete_clusters_for_jobs_in_sequence(self, jobs): - for i in range(len(jobs)): - if jobs[i][0] == "scs-compatible-kaas": - cluster_id = f"{jobs[i][1]}" - logger.debug(f"Delete cluster '{cluster_id}'") - run_plugin_delete(jobs[i][3], cluster_id) - def abspath(self, path): return os.path.join(self.cwd, path) - def build_check_command(self, job, output): - scope = job[0] - subject = job[1] + def build_check_command(self, scope, subject, output): # TODO figure out when to supply --debug here (but keep separated from our --debug) - cmd = [ + args = [ sys.executable, self.scs_compliance_check, self.abspath(self.scopes[scope]['spec']), '--debug', '-C', '-o', output, '-s', subject, ] for key, value in self.get_subject_mapping(subject).items(): - cmd.extend(['-a', f'{key}={value}']) - if len(job) == 5 and scope == "scs-compatible-kaas": - cmd.extend(['-a', f'kubeconfig={job[4]}']) - cmd.extend(['-a', f'result_dir_name={job[1]}-{job[2]}']) - elif len(job) != 5 and scope == "scs-compatible-kaas": - logger.Error(f"Scope is '{job[0]}' but no kubeconfig was provided") - return cmd + args.extend(['-a', f'{key}={value}']) + return {'args': args} + + def build_provision_command(self, subject): + kubernetes_setup = self.get_kubernetes_setup(subject) + subject_root = self.abspath(self.get_subject_mapping(subject).get('subject_root') or '.') + ensure_dir(subject_root) + return { + 'args': [ + sys.executable, self.run_plugin_py, + 'create', + kubernetes_setup['kube_plugin'], + self.abspath(kubernetes_setup['clusterspec']), + ], + 'cwd': subject_root, + } + + def build_unprovision_command(self, subject): + kubernetes_setup = self.get_kubernetes_setup(subject) + subject_root = self.abspath(self.get_subject_mapping(subject).get('subject_root') or '.') + ensure_dir(subject_root) + return { + 'args': [ + sys.executable, self.run_plugin_py, + 'delete', + kubernetes_setup['kube_plugin'], + self.abspath(kubernetes_setup['clusterspec']), + ], + 'cwd': subject_root, + } def build_cleanup_command(self, subject): # TODO figure out when to supply --debug here (but keep separated from our --debug) - return [ + return {'args': [ sys.executable, self.cleanup_py, '-c', self.get_subject_mapping(subject)['os_cloud'], '--prefix', '_scs-', '--ipaddr', '10.1.0.', '--debug', - ] + ]} def build_sign_command(self, target_path): - return [ + return {'args': [ self.ssh_keygen, '-Y', 'sign', '-f', self.abspath(self.secrets['keyfile']), '-n', 'report', target_path, - ] + ]} def build_upload_command(self, target_path, monitor_url): if not monitor_url.endswith('/'): monitor_url += '/' - return [ + return {'args': [ self.curl, '--fail-with-body', '--data-binary', f'@{target_path}.sig', @@ -148,7 +142,7 @@ def build_upload_command(self, target_path, monitor_url): '-H', 'Content-Type: application/x-signed-yaml', '-H', f'Authorization: Basic {self.auth_token}', f'{monitor_url}reports', - ] + ]} @click.group() @@ -170,7 +164,7 @@ def _run_commands(commands, num_workers=5): processes = [] while commands or processes: while commands and len(processes) < num_workers: - processes.append(subprocess.Popen(commands.pop())) + processes.append(subprocess.Popen(**commands.pop())) processes[:] = [p for p in processes if p.poll() is None] time.sleep(0.5) @@ -223,29 +217,18 @@ def run(cfg, scopes, subjects, preset, num_workers, monitor_url, report_yaml, up logger.debug(f'monitor url: {monitor_url}, num_workers: {num_workers}, output: {report_yaml}') with tempfile.TemporaryDirectory(dir=cfg.cwd) as tdirname: report_yaml_tmp = os.path.join(tdirname, 'report.yaml') - jobs = cfg.generate_compliance_check_jobs(subjects, scopes) + jobs = [(scope, subject) for scope in scopes for subject in subjects] logger.debug("Create clusters and provide kubeconfig") - jobs = cfg.build_clusters_for_jobs_in_sequence(jobs) outputs = [os.path.join(tdirname, f'report-{idx}.yaml') for idx in range(len(jobs))] commands = [cfg.build_check_command(job, output) for job, output in zip(jobs, outputs)] _run_commands(commands, num_workers=num_workers) _concat_files(outputs, report_yaml_tmp) if report_yaml is not None: _move_file(report_yaml_tmp, report_yaml) - logger.debug("delete clusters") - cfg.delete_clusters_for_jobs_in_sequence(jobs) return 0 -@cli.command() -@click.option('--subject', 'subjects', type=str) -@click.option('--preset', 'preset', type=str) -@click.option('--num-workers', 'num_workers', type=int, default=5) -@click.pass_obj -def cleanup(cfg, subjects, preset, num_workers): - """ - clean up any lingering resources - """ +def _run_command_for_subjects(cfg, subjects, preset, num_workers, command): if not subjects and not preset: preset = 'default' if preset: @@ -258,12 +241,49 @@ def cleanup(cfg, subjects, preset, num_workers): subjects = [subject.strip() for subject in subjects.split(',')] if subjects else [] if not subjects: raise click.UsageError('subject(s) must be non-empty') - logger.debug(f'cleaning up for subject(s) {", ".join(subjects)}, num_workers: {num_workers}') - commands = [cfg.build_cleanup_command(subject) for subject in subjects] + logger.debug(f'running {command} for subject(s) {", ".join(subjects)}, num_workers: {num_workers}') + m = getattr(cfg, f'build_{command}_command') + commands = [m(subject) for subject in subjects] _run_commands(commands, num_workers=num_workers) return 0 +@cli.command() +@click.option('--subject', 'subjects', type=str) +@click.option('--preset', 'preset', type=str) +@click.option('--num-workers', 'num_workers', type=int, default=5) +@click.pass_obj +def cleanup(cfg, subjects, preset, num_workers): + """ + clean up any lingering IaaS resources + """ + return _run_command_for_subjects(cfg, subjects, preset, num_workers, "cleanup") + + +@cli.command() +@click.option('--subject', 'subjects', type=str) +@click.option('--preset', 'preset', type=str) +@click.option('--num-workers', 'num_workers', type=int, default=5) +@click.pass_obj +def provision(cfg, subjects, preset, num_workers): + """ + create k8s clusters + """ + return _run_command_for_subjects(cfg, subjects, preset, num_workers, "provision") + + +@cli.command() +@click.option('--subject', 'subjects', type=str) +@click.option('--preset', 'preset', type=str) +@click.option('--num-workers', 'num_workers', type=int, default=5) +@click.pass_obj +def unprovision(cfg, subjects, preset, num_workers): + """ + clean up k8s clusters + """ + return _run_command_for_subjects(cfg, subjects, preset, num_workers, "unprovision") + + if __name__ == '__main__': logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) cli(obj=Config()) From 1dd220cc75d9e8d6f0e34d2814764730f6c23c95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20B=C3=BCchse?= Date: Wed, 23 Oct 2024 22:19:23 +0200 Subject: [PATCH 09/36] add missing file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matthias Büchse --- Tests/kaas/clusterspec.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 Tests/kaas/clusterspec.yaml diff --git a/Tests/kaas/clusterspec.yaml b/Tests/kaas/clusterspec.yaml new file mode 100644 index 000000000..b802613cc --- /dev/null +++ b/Tests/kaas/clusterspec.yaml @@ -0,0 +1,11 @@ +# this file specifies all clusters that have to be provisioned for the tests to run +clusters: + default: + branch: default + kubeconfig: kubeconfig.yaml + v1.29: + branch: "1.29" + kubeconfig: kubecfg129.yaml + v1.30: + branch: "1.30" + kubeconfig: kubecfg130.yaml From 7c063d84184e4072540bee44f25c80aab0abcec2 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Wed, 23 Oct 2024 23:18:00 +0200 Subject: [PATCH 10/36] Fixup! Rework rebase conflicts Signed-off-by: Toni Finger --- Tests/kaas/plugin/plugin_kind.py | 25 +++++++------ Tests/run_plugin.py | 63 ++++++++++++++++++++++++++++++++ Tests/scs-test-runner.py | 26 +++++++++++-- 3 files changed, 100 insertions(+), 14 deletions(-) create mode 100755 Tests/run_plugin.py diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py index 840a38138..3146db3b1 100644 --- a/Tests/kaas/plugin/plugin_kind.py +++ b/Tests/kaas/plugin/plugin_kind.py @@ -9,18 +9,21 @@ class PluginKind(KubernetesClusterPlugin): """ def _create_cluster(self): - config_file = None - cluster_version = self.cluster_version - if cluster_version == '1.29': - cluster_version = 'v1.29.8' - elif cluster_version == '1.30': - cluster_version = 'v1.30.4' - elif cluster_version == '1.31' or cluster_version == 'default': - cluster_version = 'v1.31.1' - cluster_image = f"kindest/node:{cluster_version}" + # ~ cluster_version = self.cluster_version #TODO:!!! rebase artefact + # ~ if cluster_version == '1.29': #TODO:!!! rebase artefact + # ~ cluster_version = 'v1.29.8' #TODO:!!! rebase artefact + # ~ elif cluster_version == '1.30': #TODO:!!! rebase artefact + # ~ cluster_version = 'v1.30.4' #TODO:!!! rebase artefact + # ~ elif cluster_version == '1.31' or cluster_version == 'default': #TODO:!!! rebase artefact + # ~ cluster_version = 'v1.31.1' #TODO:!!! rebase artefact + + cluster_image = f"kindest/node:{self.cluster_version}" self.cluster = KindCluster(self.cluster_name, self.kubeconfig, cluster_image) - self.cluster.create(config_file) - self.kubeconfig = self.cluster.kubeconfig_path.resolve() + if self.config is None: + self.cluster.create() + else: + self.cluster.create(self.config) + self.kubeconfig = str(self.cluster.kubeconfig_path.resolve()) def _delete_cluster(self): self.cluster = KindCluster(self.cluster_name) diff --git a/Tests/run_plugin.py b/Tests/run_plugin.py new file mode 100755 index 000000000..d0e5f4d85 --- /dev/null +++ b/Tests/run_plugin.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# vim: set ts=4 sw=4 et: +# +# scs-test-runner.py +# +# (c) Toni Finger +# SPDX-License-Identifier: Apache-2.0 + +from kaas.plugin.plugin_kind import PluginKind +from kaas.plugin.plugin_static import PluginStatic +import click +import sys + + +def init_plugin(plugin_kind, config=None): + if plugin_kind == "static": + plugin = PluginStatic(config) + + elif plugin_kind == "kind": + plugin = PluginKind(config) + + else: + raise ValueError(f"unknown plugin '{plugin_kind}'") + + return plugin + + +@click.group() +@click.pass_obj +def cli(cfg): + pass + + +@cli.result_callback() +def process_pipeline(rc, *args, **kwargs): + sys.exit(rc) + + +@cli.command() +@click.option('--plugin', 'plugin', type=str, default="kind", help='type of the plugin to be used') +@click.option('--clusterid', 'cluster_id', type=str, default=None, help='Id or name of the cluster to be identified for the deletion process') +@click.option('--version', 'k8s_version', type=str, default=None, help='Kubernetes version of the cluster to create') +@click.option('--kubeconfig', 'kubeconfig_filepath', type=click.Path(exists=False), default="./kubeconfig.yaml", help='path/to/kubeconfig_file.yaml') +@click.option('--config', 'config', type=str, default=None, help='k8s cluster provisioner specific configuration') +@click.pass_obj +def create(cfg, plugin, cluster_id, k8s_version, kubeconfig_filepath, config): + plugin = init_plugin(plugin, config) + plugin.create(cluster_id, k8s_version, kubeconfig_filepath) + return 0 + + +@cli.command() +@click.option('--plugin', 'plugin', type=str, default="kind", help='type of the plugin to be used') +@click.option('--clusterid', 'cluster_id', type=str, default="", help='Id or name of the cluster to be identified for the deletion process') +@click.pass_obj +def delete(cfg, plugin, cluster_id): + plugin = init_plugin(plugin) + plugin.delete(cluster_id) + return 0 + + +if __name__ == '__main__': + cli() diff --git a/Tests/scs-test-runner.py b/Tests/scs-test-runner.py index 3ec6beaaa..d59155e4a 100755 --- a/Tests/scs-test-runner.py +++ b/Tests/scs-test-runner.py @@ -33,7 +33,8 @@ def __init__(self): self.cwd = os.path.abspath(os.path.dirname(sys.argv[0]) or os.getcwd()) self.scs_compliance_check = os.path.join(self.cwd, 'scs-compliance-check.py') self.cleanup_py = os.path.join(self.cwd, 'cleanup.py') - self.run_plugin_py = os.path.join(self.cwd, 'kaas', 'plugin', 'run_plugin.py') + # self.run_plugin_py = os.path.join(self.cwd, 'kaas', 'plugin', 'run_plugin.py') #TODO:!!! rebase artefact + self.run_plugin = os.path.join(self.cwd, 'run_plugin.py') self.ssh_keygen = shutil.which('ssh-keygen') self.curl = shutil.which('curl') self.secrets = {} @@ -71,6 +72,24 @@ def get_kubernetes_setup(self, subject): kubernetes_setup.update(self.subjects.get(subject, {}).get('kubernetes_setup', {})) return kubernetes_setup + #TODO:!!! rebase artefact --> merge with functions `build_provision_command` and `build_unprovision_command` + #def build_create_cluster_command(self, plugin_type, cluster_id, k8s_version, kubeconfig_path): + # return [ + # self.run_plugin, + # "create", + # '--plugin', plugin_type, + # '--clusterid', cluster_id, + # '--version', k8s_version, + # '--kubeconfig', kubeconfig_path + # ] + #def build_delete_cluster_command(self, plugin_type, cluster_id): + # return [ + # self.run_plugin, + # "delete", + # '--plugin', plugin_type, + # '--clusterid', cluster_id + # ] + def abspath(self, path): return os.path.join(self.cwd, path) @@ -193,9 +212,8 @@ def _move_file(source_path, target_path): @click.option('--num-workers', 'num_workers', type=int, default=5) @click.option('--monitor-url', 'monitor_url', type=str, default=MONITOR_URL) @click.option('-o', '--output', 'report_yaml', type=click.Path(exists=False), default=None) -@click.option('--upload/--no-upload', default=True) @click.pass_obj -def run(cfg, scopes, subjects, preset, num_workers, monitor_url, report_yaml, upload): +def run(cfg, scopes, subjects, preset, num_workers, monitor_url, report_yaml): """ run compliance tests and upload results to compliance monitor """ @@ -223,6 +241,8 @@ def run(cfg, scopes, subjects, preset, num_workers, monitor_url, report_yaml, up commands = [cfg.build_check_command(job, output) for job, output in zip(jobs, outputs)] _run_commands(commands, num_workers=num_workers) _concat_files(outputs, report_yaml_tmp) + subprocess.run(cfg.build_sign_command(report_yaml_tmp)) + subprocess.run(cfg.build_upload_command(report_yaml_tmp, monitor_url)) if report_yaml is not None: _move_file(report_yaml_tmp, report_yaml) return 0 From 5b116418211c7f4c744fc455aafba5eb47ecb11f Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Thu, 24 Oct 2024 20:23:40 +0200 Subject: [PATCH 11/36] Fixup plugin kubeconfig file generation Signed-off-by: Toni Finger --- Tests/kaas/plugin/interface.py | 12 +++--- Tests/kaas/plugin/plugin_kind.py | 19 +++++----- Tests/run_plugin.py | 63 -------------------------------- Tests/scs-test-runner.py | 24 +----------- 4 files changed, 18 insertions(+), 100 deletions(-) delete mode 100755 Tests/run_plugin.py diff --git a/Tests/kaas/plugin/interface.py b/Tests/kaas/plugin/interface.py index 7148e5e24..dc53592a7 100644 --- a/Tests/kaas/plugin/interface.py +++ b/Tests/kaas/plugin/interface.py @@ -1,6 +1,7 @@ from abc import ABC, abstractmethod from typing import final import os +import os.path import shutil import logging @@ -54,12 +55,13 @@ def create(self, name="scs-cluster", version=None, kubeconfig_filepath=None): """ self.cluster_name = name self.cluster_version = version - try: - self._create_cluster() - except Exception as e: - logging.exception(e) - self._delete_cluster() + self._create_cluster() # TODO: maybe we do not need to use try exept here? + # try: + # self._create_cluster() + # except Exception as e: + # logging.exception(e) + # self._delete_cluster() if kubeconfig_filepath: shutil.move(self.kubeconfig, kubeconfig_filepath) diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py index 3146db3b1..e9279ed0a 100644 --- a/Tests/kaas/plugin/plugin_kind.py +++ b/Tests/kaas/plugin/plugin_kind.py @@ -9,16 +9,15 @@ class PluginKind(KubernetesClusterPlugin): """ def _create_cluster(self): - # ~ cluster_version = self.cluster_version #TODO:!!! rebase artefact - # ~ if cluster_version == '1.29': #TODO:!!! rebase artefact - # ~ cluster_version = 'v1.29.8' #TODO:!!! rebase artefact - # ~ elif cluster_version == '1.30': #TODO:!!! rebase artefact - # ~ cluster_version = 'v1.30.4' #TODO:!!! rebase artefact - # ~ elif cluster_version == '1.31' or cluster_version == 'default': #TODO:!!! rebase artefact - # ~ cluster_version = 'v1.31.1' #TODO:!!! rebase artefact - - cluster_image = f"kindest/node:{self.cluster_version}" - self.cluster = KindCluster(self.cluster_name, self.kubeconfig, cluster_image) + cluster_version = self.cluster_version + if cluster_version == '1.29': + cluster_version = 'v1.29.8' + elif cluster_version == '1.30': + cluster_version = 'v1.30.4' + elif cluster_version == '1.31' or cluster_version == 'default': + cluster_version = 'v1.31.1' + cluster_image = f"kindest/node:{cluster_version}" + self.cluster = KindCluster(name=self.cluster_name, image=cluster_image) if self.config is None: self.cluster.create() else: diff --git a/Tests/run_plugin.py b/Tests/run_plugin.py deleted file mode 100755 index d0e5f4d85..000000000 --- a/Tests/run_plugin.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python3 -# vim: set ts=4 sw=4 et: -# -# scs-test-runner.py -# -# (c) Toni Finger -# SPDX-License-Identifier: Apache-2.0 - -from kaas.plugin.plugin_kind import PluginKind -from kaas.plugin.plugin_static import PluginStatic -import click -import sys - - -def init_plugin(plugin_kind, config=None): - if plugin_kind == "static": - plugin = PluginStatic(config) - - elif plugin_kind == "kind": - plugin = PluginKind(config) - - else: - raise ValueError(f"unknown plugin '{plugin_kind}'") - - return plugin - - -@click.group() -@click.pass_obj -def cli(cfg): - pass - - -@cli.result_callback() -def process_pipeline(rc, *args, **kwargs): - sys.exit(rc) - - -@cli.command() -@click.option('--plugin', 'plugin', type=str, default="kind", help='type of the plugin to be used') -@click.option('--clusterid', 'cluster_id', type=str, default=None, help='Id or name of the cluster to be identified for the deletion process') -@click.option('--version', 'k8s_version', type=str, default=None, help='Kubernetes version of the cluster to create') -@click.option('--kubeconfig', 'kubeconfig_filepath', type=click.Path(exists=False), default="./kubeconfig.yaml", help='path/to/kubeconfig_file.yaml') -@click.option('--config', 'config', type=str, default=None, help='k8s cluster provisioner specific configuration') -@click.pass_obj -def create(cfg, plugin, cluster_id, k8s_version, kubeconfig_filepath, config): - plugin = init_plugin(plugin, config) - plugin.create(cluster_id, k8s_version, kubeconfig_filepath) - return 0 - - -@cli.command() -@click.option('--plugin', 'plugin', type=str, default="kind", help='type of the plugin to be used') -@click.option('--clusterid', 'cluster_id', type=str, default="", help='Id or name of the cluster to be identified for the deletion process') -@click.pass_obj -def delete(cfg, plugin, cluster_id): - plugin = init_plugin(plugin) - plugin.delete(cluster_id) - return 0 - - -if __name__ == '__main__': - cli() diff --git a/Tests/scs-test-runner.py b/Tests/scs-test-runner.py index d59155e4a..a0af649a7 100755 --- a/Tests/scs-test-runner.py +++ b/Tests/scs-test-runner.py @@ -33,8 +33,7 @@ def __init__(self): self.cwd = os.path.abspath(os.path.dirname(sys.argv[0]) or os.getcwd()) self.scs_compliance_check = os.path.join(self.cwd, 'scs-compliance-check.py') self.cleanup_py = os.path.join(self.cwd, 'cleanup.py') - # self.run_plugin_py = os.path.join(self.cwd, 'kaas', 'plugin', 'run_plugin.py') #TODO:!!! rebase artefact - self.run_plugin = os.path.join(self.cwd, 'run_plugin.py') + self.run_plugin_py = os.path.join(self.cwd, 'kaas', 'plugin', 'run_plugin.py') self.ssh_keygen = shutil.which('ssh-keygen') self.curl = shutil.which('curl') self.secrets = {} @@ -72,24 +71,6 @@ def get_kubernetes_setup(self, subject): kubernetes_setup.update(self.subjects.get(subject, {}).get('kubernetes_setup', {})) return kubernetes_setup - #TODO:!!! rebase artefact --> merge with functions `build_provision_command` and `build_unprovision_command` - #def build_create_cluster_command(self, plugin_type, cluster_id, k8s_version, kubeconfig_path): - # return [ - # self.run_plugin, - # "create", - # '--plugin', plugin_type, - # '--clusterid', cluster_id, - # '--version', k8s_version, - # '--kubeconfig', kubeconfig_path - # ] - #def build_delete_cluster_command(self, plugin_type, cluster_id): - # return [ - # self.run_plugin, - # "delete", - # '--plugin', plugin_type, - # '--clusterid', cluster_id - # ] - def abspath(self, path): return os.path.join(self.cwd, path) @@ -236,9 +217,8 @@ def run(cfg, scopes, subjects, preset, num_workers, monitor_url, report_yaml): with tempfile.TemporaryDirectory(dir=cfg.cwd) as tdirname: report_yaml_tmp = os.path.join(tdirname, 'report.yaml') jobs = [(scope, subject) for scope in scopes for subject in subjects] - logger.debug("Create clusters and provide kubeconfig") outputs = [os.path.join(tdirname, f'report-{idx}.yaml') for idx in range(len(jobs))] - commands = [cfg.build_check_command(job, output) for job, output in zip(jobs, outputs)] + commands = [cfg.build_check_command(job[0], job[1], output) for job, output in zip(jobs, outputs)] _run_commands(commands, num_workers=num_workers) _concat_files(outputs, report_yaml_tmp) subprocess.run(cfg.build_sign_command(report_yaml_tmp)) From 7fb5f32be78219cc6db7009413d29cf24061a98c Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Mon, 4 Nov 2024 10:18:01 +0100 Subject: [PATCH 12/36] Draft: split up clusterspec file In order to have one clusterspec file for each k8s version Signed-off-by: Toni Finger --- Tests/config.toml | 5 +++-- Tests/kaas/clusterspec_default.yaml | 5 +++++ Tests/kaas/clusterspec_v1.29.yaml | 5 +++++ Tests/kaas/clusterspec_v1.30.yaml | 5 +++++ 4 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 Tests/kaas/clusterspec_default.yaml create mode 100644 Tests/kaas/clusterspec_v1.29.yaml create mode 100644 Tests/kaas/clusterspec_v1.30.yaml diff --git a/Tests/config.toml b/Tests/config.toml index 1c913210a..87caa13b3 100644 --- a/Tests/config.toml +++ b/Tests/config.toml @@ -38,8 +38,9 @@ scopes = [ "scs-compatible-kaas", ] subjects = [ - "cloud-kaas-a", - #"cloud-kaas-b", + "cspA-default", + "cspA-1.29", + "cspA-1.30", ] workers = 4 diff --git a/Tests/kaas/clusterspec_default.yaml b/Tests/kaas/clusterspec_default.yaml new file mode 100644 index 000000000..1ee17a724 --- /dev/null +++ b/Tests/kaas/clusterspec_default.yaml @@ -0,0 +1,5 @@ +# this file specifies all clusters that have to be provisioned for the tests to run +clusters: + default: + branch: default + kubeconfig: kubeconfig.yaml diff --git a/Tests/kaas/clusterspec_v1.29.yaml b/Tests/kaas/clusterspec_v1.29.yaml new file mode 100644 index 000000000..a6fc25171 --- /dev/null +++ b/Tests/kaas/clusterspec_v1.29.yaml @@ -0,0 +1,5 @@ +# this file specifies all clusters that have to be provisioned for the tests to run +clusters: + v1.29: + branch: "1.29" + kubeconfig: kubecfg129.yaml diff --git a/Tests/kaas/clusterspec_v1.30.yaml b/Tests/kaas/clusterspec_v1.30.yaml new file mode 100644 index 000000000..a7dafceb9 --- /dev/null +++ b/Tests/kaas/clusterspec_v1.30.yaml @@ -0,0 +1,5 @@ +# this file specifies all clusters that have to be provisioned for the tests to run +clusters: + v1.30: + branch: "1.30" + kubeconfig: kubecfg130.yaml From 9a2809afbd17b51a5c09ee9a03f7c4caab78529e Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Mon, 4 Nov 2024 13:58:16 +0100 Subject: [PATCH 13/36] Merge to back to usage of one single clustersspec Signed-off-by: Toni Finger --- Tests/config.toml | 26 ++++++++++++++++++++++---- Tests/kaas/clusterspec_default.yaml | 5 ----- Tests/kaas/clusterspec_v1.29.yaml | 5 ----- Tests/kaas/clusterspec_v1.30.yaml | 5 ----- Tests/kaas/plugin/run_plugin.py | 20 ++++++++++---------- Tests/scs-compatible-kaas.yaml | 12 +----------- Tests/scs-test-runner.py | 2 ++ 7 files changed, 35 insertions(+), 40 deletions(-) delete mode 100644 Tests/kaas/clusterspec_default.yaml delete mode 100644 Tests/kaas/clusterspec_v1.29.yaml delete mode 100644 Tests/kaas/clusterspec_v1.30.yaml diff --git a/Tests/config.toml b/Tests/config.toml index 87caa13b3..c1d6cbd77 100644 --- a/Tests/config.toml +++ b/Tests/config.toml @@ -39,8 +39,8 @@ scopes = [ ] subjects = [ "cspA-default", - "cspA-1.29", - "cspA-1.30", + "cspA-v1.29", + "cspA-v1.30", ] workers = 4 @@ -48,6 +48,9 @@ workers = 4 [scopes.scs-compatible-iaas] spec = "./scs-compatible-iaas.yaml" +[scopes.scs-compatible-kaas] +spec = "./scs-compatible-kaas.yaml" + # default subject (not a real subject, but used to declare a default mapping) # (this is the only mapping declaration that supports using Python string interpolation) @@ -57,5 +60,20 @@ subject_root = "{subject}" [subjects._.kubernetes_setup] -kube_plugin = "kind" -clusterspec = "kaas/clusterspec.yaml" +kube_plugin = "kind" +clusterspec = "kaas/clusterspec.yaml" +clusterspec_cluster = "default" + + + +[subjects."cspA-v1.29".kubernetes_setup] +kube_plugin = "kind" +clusterspec_cluster = "v1.29" + + + +[subjects."cspA-v1.30".kubernetes_setup] +kube_plugin = "kind" +clusterspec_cluster = "v1.30" + + diff --git a/Tests/kaas/clusterspec_default.yaml b/Tests/kaas/clusterspec_default.yaml deleted file mode 100644 index 1ee17a724..000000000 --- a/Tests/kaas/clusterspec_default.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# this file specifies all clusters that have to be provisioned for the tests to run -clusters: - default: - branch: default - kubeconfig: kubeconfig.yaml diff --git a/Tests/kaas/clusterspec_v1.29.yaml b/Tests/kaas/clusterspec_v1.29.yaml deleted file mode 100644 index a6fc25171..000000000 --- a/Tests/kaas/clusterspec_v1.29.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# this file specifies all clusters that have to be provisioned for the tests to run -clusters: - v1.29: - branch: "1.29" - kubeconfig: kubecfg129.yaml diff --git a/Tests/kaas/clusterspec_v1.30.yaml b/Tests/kaas/clusterspec_v1.30.yaml deleted file mode 100644 index a7dafceb9..000000000 --- a/Tests/kaas/clusterspec_v1.30.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# this file specifies all clusters that have to be provisioned for the tests to run -clusters: - v1.30: - branch: "1.30" - kubeconfig: kubecfg130.yaml diff --git a/Tests/kaas/plugin/run_plugin.py b/Tests/kaas/plugin/run_plugin.py index 26b9a8194..dd55c3f66 100755 --- a/Tests/kaas/plugin/run_plugin.py +++ b/Tests/kaas/plugin/run_plugin.py @@ -22,16 +22,14 @@ def init_plugin(plugin_kind, config=None): return plugin_maker(config) -def run_plugin_create(plugin_kind, clusterspec): +def run_plugin_create(plugin_kind, clusterspec_cluster, clusterspec): plugin = init_plugin(plugin_kind) - for cluster_id, cluster_info in clusterspec.items(): - plugin.create(cluster_id, cluster_info['branch'], os.path.abspath(cluster_info['kubeconfig'])) + plugin.create(clusterspec_cluster, clusterspec[clusterspec_cluster]['branch'], os.path.abspath(clusterspec[clusterspec_cluster]['kubeconfig'])) -def run_plugin_delete(plugin_kind, clusterspec): +def run_plugin_delete(plugin_kind, clusterspec_cluster, clusterspec): plugin = init_plugin(plugin_kind) - for cluster_id in clusterspec: - plugin.delete(cluster_id) + plugin.delete(clusterspec_cluster) def load_spec(clusterspec_path): @@ -47,17 +45,19 @@ def cli(): @cli.command() @click.argument('plugin_kind', type=click.Choice(list(PLUGIN_LOOKUP), case_sensitive=False)) @click.argument('clusterspec_path', type=click.Path(exists=True, dir_okay=False)) -def create(plugin_kind, clusterspec_path): +@click.argument('clusterspec_cluster', type=str, default="default") +def create(plugin_kind, clusterspec_path, clusterspec_cluster): clusterspec = load_spec(clusterspec_path)['clusters'] - run_plugin_create(plugin_kind, clusterspec) + run_plugin_create(plugin_kind, clusterspec_cluster, clusterspec) @cli.command() @click.argument('plugin_kind', type=click.Choice(list(PLUGIN_LOOKUP), case_sensitive=False)) @click.argument('clusterspec_path', type=click.Path(exists=True, dir_okay=False)) -def delete(plugin_kind, clusterspec_path): +@click.argument('clusterspec_cluster', type=str, default="default") +def delete(plugin_kind, clusterspec_path, clusterspec_cluster): clusterspec = load_spec(clusterspec_path)['clusters'] - run_plugin_delete(plugin_kind, clusterspec) + run_plugin_delete(plugin_kind, clusterspec_cluster, clusterspec) if __name__ == '__main__': diff --git a/Tests/scs-compatible-kaas.yaml b/Tests/scs-compatible-kaas.yaml index 942121b80..e74ce07a2 100644 --- a/Tests/scs-compatible-kaas.yaml +++ b/Tests/scs-compatible-kaas.yaml @@ -47,18 +47,8 @@ versions: include: - ref: cncf-k8s-conformance parameters: - kubeconfig: kubeconfig.yaml + result_dir_name: "sonobuoy_cncf_results" - ref: scs-0210-v2 - parameters: - kubeconfig: kubeconfig.yaml - - ref: scs-0210-v2 - parameters: - kubeconfig: kubecfg129.yaml - - ref: scs-0210-v2 - parameters: - kubeconfig: kubecfg130.yaml - ref: scs-0214-v2 - parameters: - kubeconfig: kubeconfig.yaml targets: main: mandatory diff --git a/Tests/scs-test-runner.py b/Tests/scs-test-runner.py index a0af649a7..89f116064 100755 --- a/Tests/scs-test-runner.py +++ b/Tests/scs-test-runner.py @@ -94,6 +94,7 @@ def build_provision_command(self, subject): 'create', kubernetes_setup['kube_plugin'], self.abspath(kubernetes_setup['clusterspec']), + kubernetes_setup['clusterspec_cluster'], ], 'cwd': subject_root, } @@ -108,6 +109,7 @@ def build_unprovision_command(self, subject): 'delete', kubernetes_setup['kube_plugin'], self.abspath(kubernetes_setup['clusterspec']), + kubernetes_setup['clusterspec_cluster'], ], 'cwd': subject_root, } From bfa6ba4fd86754b8ddb8a4d03d9f20547e9c734c Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Tue, 5 Nov 2024 11:33:10 +0100 Subject: [PATCH 14/36] Rearange configuration files Signed-off-by: Toni Finger --- Tests/config.toml | 20 ++++++++------------ Tests/kaas/clusterspec.yaml | 10 +++++----- Tests/scs-compatible-kaas.yaml | 14 +++++--------- 3 files changed, 18 insertions(+), 26 deletions(-) diff --git a/Tests/config.toml b/Tests/config.toml index c1d6cbd77..f5d4b6ade 100644 --- a/Tests/config.toml +++ b/Tests/config.toml @@ -48,6 +48,7 @@ workers = 4 [scopes.scs-compatible-iaas] spec = "./scs-compatible-iaas.yaml" + [scopes.scs-compatible-kaas] spec = "./scs-compatible-kaas.yaml" @@ -55,25 +56,20 @@ spec = "./scs-compatible-kaas.yaml" # default subject (not a real subject, but used to declare a default mapping) # (this is the only mapping declaration that supports using Python string interpolation) [subjects._.mapping] -os_cloud = "{subject}" -subject_root = "{subject}" +os_cloud = "{subject}" +subject_root = "{subject}" [subjects._.kubernetes_setup] kube_plugin = "kind" clusterspec = "kaas/clusterspec.yaml" -clusterspec_cluster = "default" - - - -[subjects."cspA-v1.29".kubernetes_setup] -kube_plugin = "kind" -clusterspec_cluster = "v1.29" - +clusterspec_cluster = "current-k8s-release" [subjects."cspA-v1.30".kubernetes_setup] -kube_plugin = "kind" -clusterspec_cluster = "v1.30" +clusterspec_cluster = "current-k8s-release-1" +[subjects."cspA-v1.29".kubernetes_setup] +clusterspec_cluster = "current-k8s-release-2" + diff --git a/Tests/kaas/clusterspec.yaml b/Tests/kaas/clusterspec.yaml index b802613cc..ff801549f 100644 --- a/Tests/kaas/clusterspec.yaml +++ b/Tests/kaas/clusterspec.yaml @@ -1,11 +1,11 @@ # this file specifies all clusters that have to be provisioned for the tests to run clusters: - default: + current-k8s-release: branch: default kubeconfig: kubeconfig.yaml - v1.29: + current-k8s-release-1: branch: "1.29" - kubeconfig: kubecfg129.yaml - v1.30: + kubeconfig: kubeconfig.yaml + current-k8s-release-2: branch: "1.30" - kubeconfig: kubecfg130.yaml + kubeconfig: kubeconfig.yaml diff --git a/Tests/scs-compatible-kaas.yaml b/Tests/scs-compatible-kaas.yaml index e74ce07a2..e7480cb7e 100644 --- a/Tests/scs-compatible-kaas.yaml +++ b/Tests/scs-compatible-kaas.yaml @@ -6,34 +6,30 @@ variables: modules: - id: cncf-k8s-conformance name: CNCF Kubernetes conformance - url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0210-v2-k8s-version-policy.md # TODO:!!! this standard mentions cncf but is is the right reference? + url: https://github.com/cncf/k8s-conformance/tree/master parameters: - kubeconfig: relative path to kubeconfig file + result_dir_name: run: - executable: ./kaas/plugin/run_sonobuoy_executor.py - args: -k {subject_root}/{kubeconfig} -r {result_dir_name} -c 'cncf-conformance' + args: -k {subject_root}/kubeconfig.yaml -r {subject_root}/{result_dir_name} -c 'cncf-conformance' testcases: - id: cncf-k8s-conformance tags: [mandatory] - id: scs-0210-v2 name: Kubernetes version policy url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0210-v2-k8s-version-policy.md - parameters: - kubeconfig: relative path to kubeconfig file run: - executable: ./kaas/k8s-version-policy/k8s_version_policy.py - args: -k {subject_root}/{kubeconfig} + args: -k {subject_root}/kubeconfig.yaml testcases: - id: version-policy-check tags: [mandatory] - id: scs-0214-v2 name: Kubernetes node distribution and availability url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0214-v1-k8s-node-distribution.md - parameters: - kubeconfig: relative path to kubeconfig file run: - executable: ./kaas/k8s-node-distribution/k8s_node_distribution_check.py - args: -k {subject_root}/{kubeconfig} + args: -k {subject_root}/kubeconfig.yaml testcases: - id: node-distribution-check tags: [mandatory] From 1cfee94bfe1472e7facda8ab5dc5a6ab41fe8886 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Tue, 5 Nov 2024 12:10:45 +0100 Subject: [PATCH 15/36] Fixup configuration files Signed-off-by: Toni Finger --- Tests/config.toml | 6 +++--- Tests/kaas/clusterspec.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Tests/config.toml b/Tests/config.toml index f5d4b6ade..d761058af 100644 --- a/Tests/config.toml +++ b/Tests/config.toml @@ -38,9 +38,9 @@ scopes = [ "scs-compatible-kaas", ] subjects = [ - "cspA-default", - "cspA-v1.29", - "cspA-v1.30", + "cspA-current", + "cspA-current-1", + "cspA-current-2", ] workers = 4 diff --git a/Tests/kaas/clusterspec.yaml b/Tests/kaas/clusterspec.yaml index ff801549f..c8439a89f 100644 --- a/Tests/kaas/clusterspec.yaml +++ b/Tests/kaas/clusterspec.yaml @@ -1,11 +1,11 @@ # this file specifies all clusters that have to be provisioned for the tests to run clusters: current-k8s-release: - branch: default + branch: "1.31" kubeconfig: kubeconfig.yaml current-k8s-release-1: - branch: "1.29" + branch: "1.30" kubeconfig: kubeconfig.yaml current-k8s-release-2: - branch: "1.30" + branch: "1.29" kubeconfig: kubeconfig.yaml From 9f7fe4ccede0a3c48b9d4d0bcc404d840c35acd1 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Tue, 5 Nov 2024 12:28:30 +0100 Subject: [PATCH 16/36] Fixup configuration files Signed-off-by: Toni Finger --- Tests/config.toml | 4 ++-- Tests/scs-compatible-kaas.yaml | 10 +++------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/Tests/config.toml b/Tests/config.toml index d761058af..a7b82ab70 100644 --- a/Tests/config.toml +++ b/Tests/config.toml @@ -66,10 +66,10 @@ clusterspec = "kaas/clusterspec.yaml" clusterspec_cluster = "current-k8s-release" -[subjects."cspA-v1.30".kubernetes_setup] +[subjects.cspA-current-1.kubernetes_setup] clusterspec_cluster = "current-k8s-release-1" -[subjects."cspA-v1.29".kubernetes_setup] +[subjects.cspA-current-2.kubernetes_setup] clusterspec_cluster = "current-k8s-release-2" diff --git a/Tests/scs-compatible-kaas.yaml b/Tests/scs-compatible-kaas.yaml index e7480cb7e..130183cfa 100644 --- a/Tests/scs-compatible-kaas.yaml +++ b/Tests/scs-compatible-kaas.yaml @@ -7,11 +7,9 @@ modules: - id: cncf-k8s-conformance name: CNCF Kubernetes conformance url: https://github.com/cncf/k8s-conformance/tree/master - parameters: - result_dir_name: - run: - - executable: ./kaas/plugin/run_sonobuoy_executor.py - args: -k {subject_root}/kubeconfig.yaml -r {subject_root}/{result_dir_name} -c 'cncf-conformance' + #run: + # - executable: ./kaas/plugin/run_sonobuoy_executor.py + # args: -k {subject_root}/kubeconfig.yaml -r {result_dir_path} -c 'cncf-conformance' testcases: - id: cncf-k8s-conformance tags: [mandatory] @@ -42,8 +40,6 @@ versions: - version: v1 include: - ref: cncf-k8s-conformance - parameters: - result_dir_name: "sonobuoy_cncf_results" - ref: scs-0210-v2 - ref: scs-0214-v2 targets: From 3e5357af7b7c04007f2d2ad030693d6eefdb2928 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Tue, 5 Nov 2024 14:10:32 +0100 Subject: [PATCH 17/36] revert sonobuoy executor handling form this PR Signed-off-by: Toni Finger --- Tests/kaas/plugin/run_sonobuoy_executor.py | 48 ------ Tests/kaas/plugin/sonobuoy_executor.py | 176 --------------------- 2 files changed, 224 deletions(-) delete mode 100755 Tests/kaas/plugin/run_sonobuoy_executor.py delete mode 100644 Tests/kaas/plugin/sonobuoy_executor.py diff --git a/Tests/kaas/plugin/run_sonobuoy_executor.py b/Tests/kaas/plugin/run_sonobuoy_executor.py deleted file mode 100755 index 162ae7545..000000000 --- a/Tests/kaas/plugin/run_sonobuoy_executor.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 -# vim: set ts=4 sw=4 et: -# - -from sonobuoy_executor import SonobuoyExecutor -import sys -import click -import logging - -logging_config = { - "level": "INFO", - "version": 1, - "disable_existing_loggers": False, - "formatters": { - "sonobuoy_logs": { - "format": "%(levelname)s: %(message)s" - } - }, - "handlers": { - "console": { - "class": "logging.StreamHandler", - "formatter": "sonobuoy_logs", - "stream": "ext://sys.stdout" - } - }, - "root": { - "handlers": ["console"] - } -} - - -logger = logging.getLogger(__name__) - - -@click.command() -@click.option('-k', '--kubeconfig', 'kubeconfig', type=click.Path(exists=False), default=None, help='path/to/kubeconfig_file.yaml') -@click.option('-r', '--result_dir_name', 'result_dir_name', type=str, default="sonobuoy_results", help='directory name to store results at') -@click.option('-c', '--check', 'check_name', type=str, default="sonobuoy_executor", help='name of the check to p') # TODO:!!! this should be eighter 'scs-kaas-tests' or 'cncf-conformance' -@click.option('--debug/--no-debug', default=False) # TODO: Not Yet Implemented -def sonobuoy_run(kubeconfig, result_dir_name, check_name, debug): - logger.info("Run sonobuoy_executor") - sonobuoy_executor = SonobuoyExecutor(check_name, kubeconfig, result_dir_name) - return_code = sonobuoy_executor.run() - sys.exit(return_code) - - -if __name__ == "__main__": - sonobuoy_run() diff --git a/Tests/kaas/plugin/sonobuoy_executor.py b/Tests/kaas/plugin/sonobuoy_executor.py deleted file mode 100644 index 28e10ab54..000000000 --- a/Tests/kaas/plugin/sonobuoy_executor.py +++ /dev/null @@ -1,176 +0,0 @@ -from typing import final -from kubernetes import client, config -import os -import logging -from junitparser import JUnitXml - -logger = logging.getLogger("sonobuoy_executor") - - -def setup_k8s_client(kubeconfigfile=None): - - if kubeconfigfile: - logger.debug(f"loading kubeconfig file '{kubeconfigfile}'") - config.load_kube_config(kubeconfigfile) - logger.info("kubeconfigfile loaded successfully") - else: - logger.error("no kubeconfig file provided") - return None - - k8s_api_client = client.CoreV1Api() - - return k8s_api_client - - -class SonobuoyExecutor: - """ - An abstract base class for custom Kubernetes cluster provider plugins. - It represents an interface class from which the api provider-specific - plugins must be derived as child classes - """ - - kubeconfig_path = None - working_directory = None - - @final - def __init__( - self, - check_name="sonobuoy_executor", - kubeconfig=None, - result_dir_name="sonobuoy_results", - ): - self.check_name = check_name - logger.info(f"Inital SonobuoyExecutor for {self.check_name}") - logger.debug(f"kubeconfig: {kubeconfig} ") - if kubeconfig is None: - raise Exception("No kubeconfig provided") - else: - self.kubeconfig_path = kubeconfig - self.working_directory = os.getcwd() - self.result_dir_name = result_dir_name - logger.debug( - f"Working from {self.working_directory} placing results at {self.result_dir_name}" - ) - - @final - def _preflight_check(self): - """ - Prefligth test to ensure that everything is set up correctly for execution - :param: None - :return: None - """ - logger.info("check kubeconfig") - self.k8s_api_client = setup_k8s_client(self.kubeconfig_path) - - for api in client.ApisApi().get_api_versions().groups: - versions = [] - for v in api.versions: - name = "" - if v.version == api.preferred_version.version and len(api.versions) > 1: - name += "*" - name += v.version - versions.append(name) - logger.debug(f"[supported api]: {api.name:<40} {','.join(versions)}") - - logger.debug("checks if sonobuoy is availabe") - return_value = os.system( - f"sonobuoy version --kubeconfig='{self.kubeconfig_path}'" - ) - if return_value != 0: - raise Exception("sonobuoy is not installed") - - @final - def _test_k8s_cncf_conformance(self): - """ - This method invokes the conformance tests with sononbuoy - :param: None - :return: None - """ - logger.info(" invoke cncf conformance test") - # ~ os.system(f"sonobuoy run --wait --mode=certified-conformance --kubeconfig='{self.kubeconfig_path}'") - # TODO:!!! switch to the real test on the final merge !!! - # Only one test is currently being carried out for development purposes - os.system( - f"sonobuoy run --wait --plugin-env e2e.E2E_FOCUS=pods --plugin-env e2e.E2E_DRYRUN=true --kubeconfig='{self.kubeconfig_path}'" - ) - - @final - def _test_scs_kaas_conformance(self): - """ - This method invokes SCS's very own conformance tests by using sononbuoy - :param: None - :return: None - """ - raise NotImplementedError - - @final - def _cleanup_sonobuoy_resources(self): - """ - This method deletes all resources that sonobuoy has created in a k8s cluster for a test - :param: None - :return: None - """ - logger.info("removing sonobuoy tests from cluster") - os.system(f"sonobuoy delete --wait --kubeconfig='{self.kubeconfig_path}'") - - @final - def _retrieve_result(self): - """ - This method invokes sonobouy to store the results in a subdirectory of - the working directory. The Junit results file contained in it is then - analyzed in order to interpret the relevant information it containes - :param: result_file_name: - :return: None - """ - logger.debug(f"retrieving results to {self.result_dir_name}") - result_dir = self.working_directory + "/" + self.result_dir_name - if os.path.exists(result_dir): - os.system(f"rm -rf {result_dir}/*") - else: - os.mkdir(result_dir) - os.system( - f"sonobuoy retrieve {result_dir} -x --filename='{result_dir}' --kubeconfig='{self.kubeconfig_path}'" - ) - logger.debug( - f"parsing JUnit result from {result_dir + '/plugins/e2e/results/global/junit_01.xml'} " - ) - xml = JUnitXml.fromfile(result_dir + "/plugins/e2e/results/global/junit_01.xml") - failed_test_cases = 0 - passed_test_cases = 0 - skipped_test_cases = 0 - for suite in xml: - for case in suite: - if case.is_passed is True: - passed_test_cases += 1 - elif case.is_skipped is True: - skipped_test_cases += 1 - # ~ logger.warning(f"SKIPPED:{case.name}") # TODO:!!! decide if skipped is error or warning only ? - else: - failed_test_cases += 1 - logger.error(f"ERROR: {case.name}") - - result_message = f" {passed_test_cases} passed, {failed_test_cases} failed, {skipped_test_cases} skipped" - if failed_test_cases == 0 and skipped_test_cases == 0: - logger.info(result_message) - self.return_code = 0 - else: - logger.error("ERROR:" + result_message) - self.return_code = 3 - - @final - def run(self): - """ - This method is to be called to run the plugin - """ - self.return_code = 11 - try: - self._preflight_check() - self._test_k8s_cncf_conformance() - self._retrieve_result() - except Exception as e: - logging.error(e) - self.return_code = 1 - finally: - self._cleanup_sonobuoy_resources() - print(self.check_name + ": " + ("PASS", "FAIL")[min(1, self.return_code)]) - return self.return_code From 99c450fb3b29b295d069b32a9b55f665c5818b49 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Tue, 5 Nov 2024 15:08:57 +0100 Subject: [PATCH 18/36] Apply cluster configuration file handling Signed-off-by: Toni Finger --- Tests/config.toml | 1 + Tests/kaas/plugin/interface.py | 15 +++++---------- Tests/kaas/plugin/run_plugin.py | 9 +++++---- Tests/scs-test-runner.py | 1 + 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/Tests/config.toml b/Tests/config.toml index a7b82ab70..bd8493834 100644 --- a/Tests/config.toml +++ b/Tests/config.toml @@ -62,6 +62,7 @@ subject_root = "{subject}" [subjects._.kubernetes_setup] kube_plugin = "kind" +kube_plugin_config = "../playbooks/kubernetes_cluster_configs/kind_cluster_config.yaml" clusterspec = "kaas/clusterspec.yaml" clusterspec_cluster = "current-k8s-release" diff --git a/Tests/kaas/plugin/interface.py b/Tests/kaas/plugin/interface.py index dc53592a7..439578002 100644 --- a/Tests/kaas/plugin/interface.py +++ b/Tests/kaas/plugin/interface.py @@ -1,4 +1,4 @@ -from abc import ABC, abstractmethod +from abc import abstractmethod from typing import final import os import os.path @@ -8,7 +8,7 @@ logger = logging.getLogger("interface") -class KubernetesClusterPlugin(ABC): +class KubernetesClusterPlugin(): """ An abstract base class for custom Kubernetes cluster provider plugins. It represents an interface class from which the api provider-specific @@ -21,9 +21,9 @@ class KubernetesClusterPlugin(ABC): working_directory = None @final - def __init__(self, config=None): + def __init__(self, config_file=None): logger.info(f"Init provider plug-in of type {self.__class__.__name__}") - self.config = config + self.config = config_file logger.debug(self.config) self.working_directory = os.getcwd() logger.debug(f"Working from {self.working_directory}") @@ -56,12 +56,7 @@ def create(self, name="scs-cluster", version=None, kubeconfig_filepath=None): self.cluster_name = name self.cluster_version = version - self._create_cluster() # TODO: maybe we do not need to use try exept here? - # try: - # self._create_cluster() - # except Exception as e: - # logging.exception(e) - # self._delete_cluster() + self._create_cluster() if kubeconfig_filepath: shutil.move(self.kubeconfig, kubeconfig_filepath) diff --git a/Tests/kaas/plugin/run_plugin.py b/Tests/kaas/plugin/run_plugin.py index dd55c3f66..c757b3db0 100755 --- a/Tests/kaas/plugin/run_plugin.py +++ b/Tests/kaas/plugin/run_plugin.py @@ -22,8 +22,8 @@ def init_plugin(plugin_kind, config=None): return plugin_maker(config) -def run_plugin_create(plugin_kind, clusterspec_cluster, clusterspec): - plugin = init_plugin(plugin_kind) +def run_plugin_create(plugin_kind, plugin_config, clusterspec_cluster, clusterspec): + plugin = init_plugin(plugin_kind, plugin_config) plugin.create(clusterspec_cluster, clusterspec[clusterspec_cluster]['branch'], os.path.abspath(clusterspec[clusterspec_cluster]['kubeconfig'])) @@ -44,11 +44,12 @@ def cli(): @cli.command() @click.argument('plugin_kind', type=click.Choice(list(PLUGIN_LOOKUP), case_sensitive=False)) +@click.argument('plugin_config', type=click.Path(exists=True, dir_okay=False)) @click.argument('clusterspec_path', type=click.Path(exists=True, dir_okay=False)) @click.argument('clusterspec_cluster', type=str, default="default") -def create(plugin_kind, clusterspec_path, clusterspec_cluster): +def create(plugin_kind, plugin_config, clusterspec_path, clusterspec_cluster): clusterspec = load_spec(clusterspec_path)['clusters'] - run_plugin_create(plugin_kind, clusterspec_cluster, clusterspec) + run_plugin_create(plugin_kind, plugin_config, clusterspec_cluster, clusterspec) @cli.command() diff --git a/Tests/scs-test-runner.py b/Tests/scs-test-runner.py index 89f116064..9cddabfd7 100755 --- a/Tests/scs-test-runner.py +++ b/Tests/scs-test-runner.py @@ -93,6 +93,7 @@ def build_provision_command(self, subject): sys.executable, self.run_plugin_py, 'create', kubernetes_setup['kube_plugin'], + self.abspath(kubernetes_setup['kube_plugin_config']), self.abspath(kubernetes_setup['clusterspec']), kubernetes_setup['clusterspec_cluster'], ], From 5adb9f6d097ae2f0249063422e9b142489ae5c18 Mon Sep 17 00:00:00 2001 From: tonifinger <129007376+tonifinger@users.noreply.github.com> Date: Tue, 5 Nov 2024 16:04:56 +0100 Subject: [PATCH 19/36] Update Tests/kaas/plugin/run_plugin.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Büchse Signed-off-by: tonifinger <129007376+tonifinger@users.noreply.github.com> --- Tests/kaas/plugin/run_plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/kaas/plugin/run_plugin.py b/Tests/kaas/plugin/run_plugin.py index c757b3db0..49fdc6b0d 100755 --- a/Tests/kaas/plugin/run_plugin.py +++ b/Tests/kaas/plugin/run_plugin.py @@ -10,8 +10,8 @@ PLUGIN_LOOKUP = { - "kind": PluginKind, - "static": PluginStatic, + "kind": PluginKind, + "static": PluginStatic, } From 2127323d0b0345ed959daf3d2093a32aabf3adeb Mon Sep 17 00:00:00 2001 From: tonifinger <129007376+tonifinger@users.noreply.github.com> Date: Tue, 5 Nov 2024 16:11:19 +0100 Subject: [PATCH 20/36] Update Tests/config.toml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Büchse Signed-off-by: tonifinger <129007376+tonifinger@users.noreply.github.com> --- Tests/config.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Tests/config.toml b/Tests/config.toml index bd8493834..e7efbc16a 100644 --- a/Tests/config.toml +++ b/Tests/config.toml @@ -64,6 +64,9 @@ subject_root = "{subject}" kube_plugin = "kind" kube_plugin_config = "../playbooks/kubernetes_cluster_configs/kind_cluster_config.yaml" clusterspec = "kaas/clusterspec.yaml" + + +[subjects.cspA-current.kubernetes_setup] clusterspec_cluster = "current-k8s-release" From 5392c5e49611f0f1f9912c5e6f27d769c6a95ec2 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Tue, 5 Nov 2024 16:56:13 +0100 Subject: [PATCH 21/36] Fixup Add plugin_config to delete process as well Add missing file `kind_cluster_config.yaml` as configuration example Signed-off-by: Toni Finger --- Tests/kaas/plugin/run_plugin.py | 9 +++++---- Tests/scs-test-runner.py | 1 + .../kubernetes_cluster_configs/kind_cluster_config.yaml | 5 +++++ 3 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 playbooks/kubernetes_cluster_configs/kind_cluster_config.yaml diff --git a/Tests/kaas/plugin/run_plugin.py b/Tests/kaas/plugin/run_plugin.py index 49fdc6b0d..6c0423478 100755 --- a/Tests/kaas/plugin/run_plugin.py +++ b/Tests/kaas/plugin/run_plugin.py @@ -27,8 +27,8 @@ def run_plugin_create(plugin_kind, plugin_config, clusterspec_cluster, clustersp plugin.create(clusterspec_cluster, clusterspec[clusterspec_cluster]['branch'], os.path.abspath(clusterspec[clusterspec_cluster]['kubeconfig'])) -def run_plugin_delete(plugin_kind, clusterspec_cluster, clusterspec): - plugin = init_plugin(plugin_kind) +def run_plugin_delete(plugin_kind, plugin_config, clusterspec_cluster, clusterspec): + plugin = init_plugin(plugin_kind, plugin_config) plugin.delete(clusterspec_cluster) @@ -54,11 +54,12 @@ def create(plugin_kind, plugin_config, clusterspec_path, clusterspec_cluster): @cli.command() @click.argument('plugin_kind', type=click.Choice(list(PLUGIN_LOOKUP), case_sensitive=False)) +@click.argument('plugin_config', type=click.Path(exists=True, dir_okay=False)) @click.argument('clusterspec_path', type=click.Path(exists=True, dir_okay=False)) @click.argument('clusterspec_cluster', type=str, default="default") -def delete(plugin_kind, clusterspec_path, clusterspec_cluster): +def delete(plugin_kind, plugin_config, clusterspec_path, clusterspec_cluster): clusterspec = load_spec(clusterspec_path)['clusters'] - run_plugin_delete(plugin_kind, clusterspec_cluster, clusterspec) + run_plugin_delete(plugin_kind, plugin_config, clusterspec_cluster, clusterspec) if __name__ == '__main__': diff --git a/Tests/scs-test-runner.py b/Tests/scs-test-runner.py index 9cddabfd7..3f21f6c17 100755 --- a/Tests/scs-test-runner.py +++ b/Tests/scs-test-runner.py @@ -109,6 +109,7 @@ def build_unprovision_command(self, subject): sys.executable, self.run_plugin_py, 'delete', kubernetes_setup['kube_plugin'], + self.abspath(kubernetes_setup['kube_plugin_config']), self.abspath(kubernetes_setup['clusterspec']), kubernetes_setup['clusterspec_cluster'], ], diff --git a/playbooks/kubernetes_cluster_configs/kind_cluster_config.yaml b/playbooks/kubernetes_cluster_configs/kind_cluster_config.yaml new file mode 100644 index 000000000..ead21eb72 --- /dev/null +++ b/playbooks/kubernetes_cluster_configs/kind_cluster_config.yaml @@ -0,0 +1,5 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +- role: worker From 73edf993a97cee170b613ed0318d144de6507306 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Wed, 6 Nov 2024 15:17:41 +0100 Subject: [PATCH 22/36] Restructure abstract method handling in plugin Signed-off-by: Toni Finger --- Tests/kaas/plugin/interface.py | 64 +++----------------------------- Tests/kaas/plugin/plugin_kind.py | 31 +++++++++++++--- Tests/kaas/plugin/run_plugin.py | 6 +-- 3 files changed, 35 insertions(+), 66 deletions(-) diff --git a/Tests/kaas/plugin/interface.py b/Tests/kaas/plugin/interface.py index 439578002..9129d54b2 100644 --- a/Tests/kaas/plugin/interface.py +++ b/Tests/kaas/plugin/interface.py @@ -1,11 +1,4 @@ from abc import abstractmethod -from typing import final -import os -import os.path -import shutil -import logging - -logger = logging.getLogger("interface") class KubernetesClusterPlugin(): @@ -15,64 +8,19 @@ class KubernetesClusterPlugin(): plugins must be derived as child classes """ - kubeconfig = None - cluster_name = None - k8s_api_client = None - working_directory = None - - @final - def __init__(self, config_file=None): - logger.info(f"Init provider plug-in of type {self.__class__.__name__}") - self.config = config_file - logger.debug(self.config) - self.working_directory = os.getcwd() - logger.debug(f"Working from {self.working_directory}") - @abstractmethod - def _create_cluster(self, cluster_name) -> (str, int): + def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig_filepath=None) -> (str): """ - Create a Kubernetes cluster to test aggainst. + This method is to be called to create a k8s cluster :param: cluster_name: - :return: kubeconfig: kubeconfig of the cluster used for testing + :param: version: + :param: kubeconfig_filepath: + :return: kubeconfig_filepath """ - pass @abstractmethod - def _delete_cluster(self, cluster_name) -> (str, int): - """ - Delete the Kubernetes cluster. - :param: cluster_name: - :return: None - """ - pass - - @final - def create(self, name="scs-cluster", version=None, kubeconfig_filepath=None): - """ - This method is to be called to create a k8s cluster - :param: kubernetes_version: - :return: uuid - """ - self.cluster_name = name - self.cluster_version = version - - self._create_cluster() - - if kubeconfig_filepath: - shutil.move(self.kubeconfig, kubeconfig_filepath) - else: - kubeconfig_filepath = str(self.kubeconfig) - - return kubeconfig_filepath - - @final - def delete(self, cluster_name=None): + def delete_cluster(self, cluster_name=None): """ This method is to be called in order to unprovision a cluster :param: cluster_uuid: """ - self.cluster_name = cluster_name - try: - self._delete_cluster() - except Exception as e: - logging.exception(e) diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py index e9279ed0a..c6d8b0fe6 100644 --- a/Tests/kaas/plugin/plugin_kind.py +++ b/Tests/kaas/plugin/plugin_kind.py @@ -1,5 +1,10 @@ +import shutil +import os +import os.path from interface import KubernetesClusterPlugin from pytest_kind import KindCluster +import logging +logger = logging.getLogger("plugin_kind") class PluginKind(KubernetesClusterPlugin): @@ -7,9 +12,20 @@ class PluginKind(KubernetesClusterPlugin): Plugin to handle the provisioning of kubernetes cluster for conformance testing purpose with the use of Kind """ + def __init__(self, config_file=None): + logger.info(f"Init provider plug-in of type {self.__class__.__name__}") + self.config = config_file + logger.debug(self.config) + self.working_directory = os.getcwd() + logger.debug(f"Working from {self.working_directory}") - def _create_cluster(self): - cluster_version = self.cluster_version + def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig_filepath=None): + """ + This method is to be called to create a k8s cluster + :param: kubernetes_version: + :return: kubeconfig_filepath + """ + cluster_version = version if cluster_version == '1.29': cluster_version = 'v1.29.8' elif cluster_version == '1.30': @@ -17,13 +33,18 @@ def _create_cluster(self): elif cluster_version == '1.31' or cluster_version == 'default': cluster_version = 'v1.31.1' cluster_image = f"kindest/node:{cluster_version}" - self.cluster = KindCluster(name=self.cluster_name, image=cluster_image) + self.cluster = KindCluster(name=cluster_name, image=cluster_image) if self.config is None: self.cluster.create() else: self.cluster.create(self.config) self.kubeconfig = str(self.cluster.kubeconfig_path.resolve()) + if kubeconfig_filepath: + shutil.move(self.kubeconfig, kubeconfig_filepath) + else: + kubeconfig_filepath = str(self.kubeconfig) + return kubeconfig_filepath - def _delete_cluster(self): - self.cluster = KindCluster(self.cluster_name) + def delete_cluster(self, cluster_name=None): + self.cluster = KindCluster(cluster_name) self.cluster.delete() diff --git a/Tests/kaas/plugin/run_plugin.py b/Tests/kaas/plugin/run_plugin.py index 6c0423478..55dc6946a 100755 --- a/Tests/kaas/plugin/run_plugin.py +++ b/Tests/kaas/plugin/run_plugin.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 import logging import os.path - import click import yaml + from plugin_kind import PluginKind from plugin_static import PluginStatic @@ -24,12 +24,12 @@ def init_plugin(plugin_kind, config=None): def run_plugin_create(plugin_kind, plugin_config, clusterspec_cluster, clusterspec): plugin = init_plugin(plugin_kind, plugin_config) - plugin.create(clusterspec_cluster, clusterspec[clusterspec_cluster]['branch'], os.path.abspath(clusterspec[clusterspec_cluster]['kubeconfig'])) + plugin.create_cluster(clusterspec_cluster, clusterspec[clusterspec_cluster]['branch'], os.path.abspath(clusterspec[clusterspec_cluster]['kubeconfig'])) def run_plugin_delete(plugin_kind, plugin_config, clusterspec_cluster, clusterspec): plugin = init_plugin(plugin_kind, plugin_config) - plugin.delete(clusterspec_cluster) + plugin.delete_cluster(clusterspec_cluster) def load_spec(clusterspec_path): From 90d958e81cb04c6caf4d75584335b5e6efdc9a1a Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Wed, 6 Nov 2024 15:56:40 +0100 Subject: [PATCH 23/36] Add example plugin implementation to interface.py Signed-off-by: Toni Finger --- Tests/kaas/plugin/interface.py | 31 +++++++++++++++++++++++++++++- Tests/kaas/plugin/plugin_static.py | 4 ++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/Tests/kaas/plugin/interface.py b/Tests/kaas/plugin/interface.py index 9129d54b2..9628fb0d4 100644 --- a/Tests/kaas/plugin/interface.py +++ b/Tests/kaas/plugin/interface.py @@ -6,6 +6,35 @@ class KubernetesClusterPlugin(): An abstract base class for custom Kubernetes cluster provider plugins. It represents an interface class from which the api provider-specific plugins must be derived as child classes + + To implement fill the methods `create_cluster` and `delete_cluster` with + api provider-specific functionalities for creating and deleting clusters. + + - Implement `create_cluster` and `delete_cluster` methods + - Create `__init__(self, config_file=None)` method to handle api specific + configurations. + + Example: + .. code:: python + + from interface import KubernetesClusterPlugin + from apiX_library import cluster_api_class as ClusterAPI + + class PluginX(KubernetesClusterPlugin): + + def __init__(self, config_file=None): + self.config = config_file + + def create_cluster(self): + self.cluster = ClusterAPI(name=cluster_name, image=cluster_image) + self.cluster.create(self.config) + kubeconfig_filepath = str(self.cluster.kubeconfig_path.resolve()) + return self.kubeconfig_filepath + + def delete_cluster(self): + self.cluster = ClusterAPI(cluster_name) + self.cluster.delete() + .. """ @abstractmethod @@ -22,5 +51,5 @@ def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig_fi def delete_cluster(self, cluster_name=None): """ This method is to be called in order to unprovision a cluster - :param: cluster_uuid: + :param: cluster_name: """ diff --git a/Tests/kaas/plugin/plugin_static.py b/Tests/kaas/plugin/plugin_static.py index f28f148fd..469880daa 100644 --- a/Tests/kaas/plugin/plugin_static.py +++ b/Tests/kaas/plugin/plugin_static.py @@ -7,8 +7,8 @@ class PluginStatic(KubernetesClusterPlugin): using a kubeconfig file """ - def _create_cluster(self): + def create_cluster(self): self.kubeconfig = self.kubeconfig - def _delete_cluster(self): + def delete_cluster(self): pass From fc78a7cc29a036ac63589bdddbccd070cb625c88 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Wed, 6 Nov 2024 15:59:04 +0100 Subject: [PATCH 24/36] Fixup: change kube_plugin_config directory name Signed-off-by: Toni Finger --- .../kind_cluster_config.yaml => k8s_configs/kind_config.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename playbooks/{kubernetes_cluster_configs/kind_cluster_config.yaml => k8s_configs/kind_config.yaml} (100%) diff --git a/playbooks/kubernetes_cluster_configs/kind_cluster_config.yaml b/playbooks/k8s_configs/kind_config.yaml similarity index 100% rename from playbooks/kubernetes_cluster_configs/kind_cluster_config.yaml rename to playbooks/k8s_configs/kind_config.yaml From 76134d6a3c8801419efa4ce366047fccc5c10398 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Wed, 6 Nov 2024 16:00:16 +0100 Subject: [PATCH 25/36] Fixup: missing adjustment to config.toml Signed-off-by: Toni Finger --- Tests/config.toml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Tests/config.toml b/Tests/config.toml index e7efbc16a..6833fc5ad 100644 --- a/Tests/config.toml +++ b/Tests/config.toml @@ -61,19 +61,23 @@ subject_root = "{subject}" [subjects._.kubernetes_setup] -kube_plugin = "kind" -kube_plugin_config = "../playbooks/kubernetes_cluster_configs/kind_cluster_config.yaml" clusterspec = "kaas/clusterspec.yaml" [subjects.cspA-current.kubernetes_setup] +kube_plugin = "kind" +kube_plugin_config = "../playbooks/k8s_configs/kind_config.yaml" clusterspec_cluster = "current-k8s-release" [subjects.cspA-current-1.kubernetes_setup] +kube_plugin = "kind" +kube_plugin_config = "../playbooks/k8s_configs/kind_config.yaml" clusterspec_cluster = "current-k8s-release-1" [subjects.cspA-current-2.kubernetes_setup] +kube_plugin = "kind" +kube_plugin_config = "../playbooks/k8s_configs/kind_config.yaml" clusterspec_cluster = "current-k8s-release-2" From cfa113c717153e9429a50ab730d0e5b10a6b8ec1 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Thu, 7 Nov 2024 10:10:58 +0100 Subject: [PATCH 26/36] Remove abstract base clase handling and directly make use of Not ImplementedError Signed-off-by: Toni Finger --- Tests/kaas/plugin/interface.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Tests/kaas/plugin/interface.py b/Tests/kaas/plugin/interface.py index 9628fb0d4..7b3f0ff58 100644 --- a/Tests/kaas/plugin/interface.py +++ b/Tests/kaas/plugin/interface.py @@ -1,4 +1,3 @@ -from abc import abstractmethod class KubernetesClusterPlugin(): @@ -37,7 +36,6 @@ def delete_cluster(self): .. """ - @abstractmethod def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig_filepath=None) -> (str): """ This method is to be called to create a k8s cluster @@ -46,10 +44,11 @@ def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig_fi :param: kubeconfig_filepath: :return: kubeconfig_filepath """ + raise NotImplementedError - @abstractmethod def delete_cluster(self, cluster_name=None): """ This method is to be called in order to unprovision a cluster :param: cluster_name: """ + raise NotImplementedError From d43a38dc2126608cada574ada573754d84fbd952 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Thu, 7 Nov 2024 10:37:41 +0100 Subject: [PATCH 27/36] Update 'kind_plugin.py' kubeconfig filepath handling Signed-off-by: Toni Finger --- Tests/kaas/plugin/plugin_kind.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py index c6d8b0fe6..c3c888845 100644 --- a/Tests/kaas/plugin/plugin_kind.py +++ b/Tests/kaas/plugin/plugin_kind.py @@ -1,6 +1,6 @@ -import shutil import os import os.path +from pathlib import Path from interface import KubernetesClusterPlugin from pytest_kind import KindCluster import logging @@ -19,7 +19,7 @@ def __init__(self, config_file=None): self.working_directory = os.getcwd() logger.debug(f"Working from {self.working_directory}") - def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig_filepath=None): + def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig=None): """ This method is to be called to create a k8s cluster :param: kubernetes_version: @@ -33,17 +33,16 @@ def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig_fi elif cluster_version == '1.31' or cluster_version == 'default': cluster_version = 'v1.31.1' cluster_image = f"kindest/node:{cluster_version}" - self.cluster = KindCluster(name=cluster_name, image=cluster_image) + kubeconfig_filepath = Path(kubeconfig) + if kubeconfig_filepath is None: + raise ValueError("kubeconfig_filepath is missing") + else: + self.cluster = KindCluster(name=cluster_name, image=cluster_image, kubeconfig=kubeconfig_filepath) if self.config is None: self.cluster.create() else: self.cluster.create(self.config) - self.kubeconfig = str(self.cluster.kubeconfig_path.resolve()) - if kubeconfig_filepath: - shutil.move(self.kubeconfig, kubeconfig_filepath) - else: - kubeconfig_filepath = str(self.kubeconfig) - return kubeconfig_filepath + return str(self.cluster.kubeconfig_path.resolve()) def delete_cluster(self, cluster_name=None): self.cluster = KindCluster(cluster_name) From 8f4970b97cc6de54e16f5dae52c98b1beda047d9 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Thu, 7 Nov 2024 10:56:33 +0100 Subject: [PATCH 28/36] Sort python imports Signed-off-by: Toni Finger --- Tests/kaas/plugin/plugin_kind.py | 4 +++- Tests/kaas/plugin/run_plugin.py | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py index c3c888845..0d932eca4 100644 --- a/Tests/kaas/plugin/plugin_kind.py +++ b/Tests/kaas/plugin/plugin_kind.py @@ -1,9 +1,11 @@ +import logging import os import os.path from pathlib import Path + from interface import KubernetesClusterPlugin from pytest_kind import KindCluster -import logging + logger = logging.getLogger("plugin_kind") diff --git a/Tests/kaas/plugin/run_plugin.py b/Tests/kaas/plugin/run_plugin.py index 55dc6946a..5a220ecbe 100755 --- a/Tests/kaas/plugin/run_plugin.py +++ b/Tests/kaas/plugin/run_plugin.py @@ -1,14 +1,13 @@ #!/usr/bin/env python3 import logging import os.path + import click import yaml - from plugin_kind import PluginKind from plugin_static import PluginStatic - PLUGIN_LOOKUP = { "kind": PluginKind, "static": PluginStatic, From 5a8f8ca41ed6e551e4b08f33f065e5cfe43959d7 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Thu, 7 Nov 2024 11:06:45 +0100 Subject: [PATCH 29/36] Fixup: use '__name__' for logging handler Signed-off-by: Toni Finger --- Tests/kaas/plugin/plugin_kind.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py index 0d932eca4..d7df61dfa 100644 --- a/Tests/kaas/plugin/plugin_kind.py +++ b/Tests/kaas/plugin/plugin_kind.py @@ -6,7 +6,7 @@ from interface import KubernetesClusterPlugin from pytest_kind import KindCluster -logger = logging.getLogger("plugin_kind") +logger = logging.getLogger(__name__) class PluginKind(KubernetesClusterPlugin): From 7d6a0cb6a05cc48948c05ff844153d529b499968 Mon Sep 17 00:00:00 2001 From: tonifinger <129007376+tonifinger@users.noreply.github.com> Date: Thu, 7 Nov 2024 11:19:16 +0100 Subject: [PATCH 30/36] Update Tests/kaas/plugin/run_plugin.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Büchse Signed-off-by: tonifinger <129007376+tonifinger@users.noreply.github.com> --- Tests/kaas/plugin/run_plugin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tests/kaas/plugin/run_plugin.py b/Tests/kaas/plugin/run_plugin.py index 5a220ecbe..ebece4f08 100755 --- a/Tests/kaas/plugin/run_plugin.py +++ b/Tests/kaas/plugin/run_plugin.py @@ -23,7 +23,8 @@ def init_plugin(plugin_kind, config=None): def run_plugin_create(plugin_kind, plugin_config, clusterspec_cluster, clusterspec): plugin = init_plugin(plugin_kind, plugin_config) - plugin.create_cluster(clusterspec_cluster, clusterspec[clusterspec_cluster]['branch'], os.path.abspath(clusterspec[clusterspec_cluster]['kubeconfig'])) + clusterinfo = clusterspec[clusterspec_cluster] + plugin.create_cluster(clusterspec_cluster, clusterinfo['branch'], os.path.abspath(clusterinfo['kubeconfig'])) def run_plugin_delete(plugin_kind, plugin_config, clusterspec_cluster, clusterspec): From c4b261198c3711382ee27a86b9374b32d05fe893 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Thu, 7 Nov 2024 12:08:33 +0100 Subject: [PATCH 31/36] Update plugin kubeconfig handling Removed the return of the kubeconfig filepath from the `create_cluster` method as we do not use this handling. Signed-off-by: Toni Finger --- Tests/kaas/plugin/interface.py | 17 +++++++++-------- Tests/kaas/plugin/plugin_kind.py | 5 ++--- Tests/kaas/plugin/run_plugin.py | 21 ++++++++------------- 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/Tests/kaas/plugin/interface.py b/Tests/kaas/plugin/interface.py index 7b3f0ff58..c534c0dfc 100644 --- a/Tests/kaas/plugin/interface.py +++ b/Tests/kaas/plugin/interface.py @@ -8,6 +8,9 @@ class KubernetesClusterPlugin(): To implement fill the methods `create_cluster` and `delete_cluster` with api provider-specific functionalities for creating and deleting clusters. + The `create_cluster` method must ensure that the kubeconfigfile is provided + at the position in the file system defined by the parameter + `kubeconfig_filepath` - Implement `create_cluster` and `delete_cluster` methods - Create `__init__(self, config_file=None)` method to handle api specific @@ -24,31 +27,29 @@ class PluginX(KubernetesClusterPlugin): def __init__(self, config_file=None): self.config = config_file - def create_cluster(self): - self.cluster = ClusterAPI(name=cluster_name, image=cluster_image) + def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig_filepath=None): + self.cluster = ClusterAPI(name=cluster_name, image=cluster_image, kubeconfig_filepath) self.cluster.create(self.config) - kubeconfig_filepath = str(self.cluster.kubeconfig_path.resolve()) - return self.kubeconfig_filepath - def delete_cluster(self): + def delete_cluster(self, cluster_name=None, version=None): self.cluster = ClusterAPI(cluster_name) self.cluster.delete() .. """ - def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig_filepath=None) -> (str): + def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig_filepath=None): """ This method is to be called to create a k8s cluster :param: cluster_name: :param: version: :param: kubeconfig_filepath: - :return: kubeconfig_filepath """ raise NotImplementedError - def delete_cluster(self, cluster_name=None): + def delete_cluster(self, cluster_name=None, version=None): """ This method is to be called in order to unprovision a cluster :param: cluster_name: + :param: version: """ raise NotImplementedError diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py index d7df61dfa..6dcc46474 100644 --- a/Tests/kaas/plugin/plugin_kind.py +++ b/Tests/kaas/plugin/plugin_kind.py @@ -15,7 +15,7 @@ class PluginKind(KubernetesClusterPlugin): conformance testing purpose with the use of Kind """ def __init__(self, config_file=None): - logger.info(f"Init provider plug-in of type {self.__class__.__name__}") + logger.info(f"Init PluginKind") self.config = config_file logger.debug(self.config) self.working_directory = os.getcwd() @@ -44,8 +44,7 @@ def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig=No self.cluster.create() else: self.cluster.create(self.config) - return str(self.cluster.kubeconfig_path.resolve()) - def delete_cluster(self, cluster_name=None): + def delete_cluster(self, cluster_name=None, version=None): self.cluster = KindCluster(cluster_name) self.cluster.delete() diff --git a/Tests/kaas/plugin/run_plugin.py b/Tests/kaas/plugin/run_plugin.py index ebece4f08..518d1c62c 100755 --- a/Tests/kaas/plugin/run_plugin.py +++ b/Tests/kaas/plugin/run_plugin.py @@ -21,17 +21,6 @@ def init_plugin(plugin_kind, config=None): return plugin_maker(config) -def run_plugin_create(plugin_kind, plugin_config, clusterspec_cluster, clusterspec): - plugin = init_plugin(plugin_kind, plugin_config) - clusterinfo = clusterspec[clusterspec_cluster] - plugin.create_cluster(clusterspec_cluster, clusterinfo['branch'], os.path.abspath(clusterinfo['kubeconfig'])) - - -def run_plugin_delete(plugin_kind, plugin_config, clusterspec_cluster, clusterspec): - plugin = init_plugin(plugin_kind, plugin_config) - plugin.delete_cluster(clusterspec_cluster) - - def load_spec(clusterspec_path): with open(clusterspec_path, "rb") as fileobj: return yaml.load(fileobj, Loader=yaml.SafeLoader) @@ -49,7 +38,10 @@ def cli(): @click.argument('clusterspec_cluster', type=str, default="default") def create(plugin_kind, plugin_config, clusterspec_path, clusterspec_cluster): clusterspec = load_spec(clusterspec_path)['clusters'] - run_plugin_create(plugin_kind, plugin_config, clusterspec_cluster, clusterspec) + plugin = init_plugin(plugin_kind, plugin_config) + clusterinfo = clusterspec[clusterspec_cluster] + cluster_id = clusterspec_cluster + plugin.create_cluster(cluster_id, clusterinfo['branch'], os.path.abspath(clusterinfo['kubeconfig'])) @cli.command() @@ -59,7 +51,10 @@ def create(plugin_kind, plugin_config, clusterspec_path, clusterspec_cluster): @click.argument('clusterspec_cluster', type=str, default="default") def delete(plugin_kind, plugin_config, clusterspec_path, clusterspec_cluster): clusterspec = load_spec(clusterspec_path)['clusters'] - run_plugin_delete(plugin_kind, plugin_config, clusterspec_cluster, clusterspec) + clusterinfo = clusterspec[clusterspec_cluster] + cluster_id = clusterspec_cluster + plugin = init_plugin(plugin_kind, plugin_config) + plugin.delete_cluster(cluster_id, clusterinfo['branch']) if __name__ == '__main__': From 78fd6946290ca8840c48d05c52b2ce1dec552bbb Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Thu, 7 Nov 2024 17:56:03 +0100 Subject: [PATCH 32/36] Update plugin_static.py Signed-off-by: Toni Finger --- Tests/kaas/plugin/plugin_static.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Tests/kaas/plugin/plugin_static.py b/Tests/kaas/plugin/plugin_static.py index 469880daa..6a68bf183 100644 --- a/Tests/kaas/plugin/plugin_static.py +++ b/Tests/kaas/plugin/plugin_static.py @@ -1,3 +1,5 @@ +import shutil + from interface import KubernetesClusterPlugin @@ -7,8 +9,11 @@ class PluginStatic(KubernetesClusterPlugin): using a kubeconfig file """ - def create_cluster(self): - self.kubeconfig = self.kubeconfig + def __init__(self, config_file=None): + self.kubeconfig_path = config_file + + def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig=None): + shutil.copyfile(self.kubeconfig_path, kubeconfig) - def delete_cluster(self): + def delete_cluster(self, cluster_name=None, version=None): pass From 2301ba3350a8a5a0a0049841c2444d098ca37848 Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Thu, 7 Nov 2024 18:03:43 +0100 Subject: [PATCH 33/36] Fixup remove obsolete f-string Signed-off-by: Toni Finger --- Tests/kaas/plugin/plugin_kind.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py index 6dcc46474..b65b5af60 100644 --- a/Tests/kaas/plugin/plugin_kind.py +++ b/Tests/kaas/plugin/plugin_kind.py @@ -15,7 +15,7 @@ class PluginKind(KubernetesClusterPlugin): conformance testing purpose with the use of Kind """ def __init__(self, config_file=None): - logger.info(f"Init PluginKind") + logger.info("Init PluginKind") self.config = config_file logger.debug(self.config) self.working_directory = os.getcwd() From fa0247e38bc9a54e2457ee3b7e72f5a0097c784d Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Fri, 8 Nov 2024 12:55:31 +0100 Subject: [PATCH 34/36] Fixup Remove default values to prevent parameters from being optional Signed-off-by: Toni Finger --- Tests/kaas/plugin/interface.py | 10 +++++----- Tests/kaas/plugin/plugin_kind.py | 6 +++--- Tests/kaas/plugin/plugin_static.py | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Tests/kaas/plugin/interface.py b/Tests/kaas/plugin/interface.py index c534c0dfc..79e7c3596 100644 --- a/Tests/kaas/plugin/interface.py +++ b/Tests/kaas/plugin/interface.py @@ -24,20 +24,20 @@ class KubernetesClusterPlugin(): class PluginX(KubernetesClusterPlugin): - def __init__(self, config_file=None): + def __init__(self, config_file): self.config = config_file - def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig_filepath=None): + def create_cluster(self, cluster_name, version, kubeconfig_filepath): self.cluster = ClusterAPI(name=cluster_name, image=cluster_image, kubeconfig_filepath) self.cluster.create(self.config) - def delete_cluster(self, cluster_name=None, version=None): + def delete_cluster(self, cluster_name, version): self.cluster = ClusterAPI(cluster_name) self.cluster.delete() .. """ - def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig_filepath=None): + def create_cluster(self, cluster_name, version, kubeconfig_filepath): """ This method is to be called to create a k8s cluster :param: cluster_name: @@ -46,7 +46,7 @@ def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig_fi """ raise NotImplementedError - def delete_cluster(self, cluster_name=None, version=None): + def delete_cluster(self, cluster_name, version): """ This method is to be called in order to unprovision a cluster :param: cluster_name: diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py index b65b5af60..4bf091b43 100644 --- a/Tests/kaas/plugin/plugin_kind.py +++ b/Tests/kaas/plugin/plugin_kind.py @@ -14,14 +14,14 @@ class PluginKind(KubernetesClusterPlugin): Plugin to handle the provisioning of kubernetes cluster for conformance testing purpose with the use of Kind """ - def __init__(self, config_file=None): + def __init__(self, config_file): logger.info("Init PluginKind") self.config = config_file logger.debug(self.config) self.working_directory = os.getcwd() logger.debug(f"Working from {self.working_directory}") - def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig=None): + def create_cluster(self, cluster_name, version, kubeconfig): """ This method is to be called to create a k8s cluster :param: kubernetes_version: @@ -45,6 +45,6 @@ def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig=No else: self.cluster.create(self.config) - def delete_cluster(self, cluster_name=None, version=None): + def delete_cluster(self, cluster_name, version): self.cluster = KindCluster(cluster_name) self.cluster.delete() diff --git a/Tests/kaas/plugin/plugin_static.py b/Tests/kaas/plugin/plugin_static.py index 6a68bf183..1c3a5222c 100644 --- a/Tests/kaas/plugin/plugin_static.py +++ b/Tests/kaas/plugin/plugin_static.py @@ -9,11 +9,11 @@ class PluginStatic(KubernetesClusterPlugin): using a kubeconfig file """ - def __init__(self, config_file=None): + def __init__(self, config_file): self.kubeconfig_path = config_file - def create_cluster(self, cluster_name="scs-cluster", version=None, kubeconfig=None): + def create_cluster(self, cluster_name, version, kubeconfig): shutil.copyfile(self.kubeconfig_path, kubeconfig) - def delete_cluster(self, cluster_name=None, version=None): + def delete_cluster(self, cluster_name, version): pass From 932ed7aaac1ebd60b6f0dc82bbd1901253d7a10d Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Fri, 8 Nov 2024 12:58:50 +0100 Subject: [PATCH 35/36] Fixup: remove default value in description Signed-off-by: Toni Finger --- Tests/kaas/plugin/interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/kaas/plugin/interface.py b/Tests/kaas/plugin/interface.py index 79e7c3596..ba6f8134b 100644 --- a/Tests/kaas/plugin/interface.py +++ b/Tests/kaas/plugin/interface.py @@ -13,7 +13,7 @@ class KubernetesClusterPlugin(): `kubeconfig_filepath` - Implement `create_cluster` and `delete_cluster` methods - - Create `__init__(self, config_file=None)` method to handle api specific + - Create `__init__(self, config_file)` method to handle api specific configurations. Example: From a96047d9dafe75b340ff18b653d8cc7cf10c8d4b Mon Sep 17 00:00:00 2001 From: Toni Finger Date: Mon, 11 Nov 2024 12:10:56 +0100 Subject: [PATCH 36/36] Fixup: remove version parameter form delete Signed-off-by: Toni Finger --- Tests/kaas/plugin/interface.py | 5 ++--- Tests/kaas/plugin/plugin_kind.py | 2 +- Tests/kaas/plugin/run_plugin.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Tests/kaas/plugin/interface.py b/Tests/kaas/plugin/interface.py index ba6f8134b..f62e3b3e2 100644 --- a/Tests/kaas/plugin/interface.py +++ b/Tests/kaas/plugin/interface.py @@ -31,7 +31,7 @@ def create_cluster(self, cluster_name, version, kubeconfig_filepath): self.cluster = ClusterAPI(name=cluster_name, image=cluster_image, kubeconfig_filepath) self.cluster.create(self.config) - def delete_cluster(self, cluster_name, version): + def delete_cluster(self, cluster_name): self.cluster = ClusterAPI(cluster_name) self.cluster.delete() .. @@ -46,10 +46,9 @@ def create_cluster(self, cluster_name, version, kubeconfig_filepath): """ raise NotImplementedError - def delete_cluster(self, cluster_name, version): + def delete_cluster(self, cluster_name): """ This method is to be called in order to unprovision a cluster :param: cluster_name: - :param: version: """ raise NotImplementedError diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py index 4bf091b43..0b2d78bc9 100644 --- a/Tests/kaas/plugin/plugin_kind.py +++ b/Tests/kaas/plugin/plugin_kind.py @@ -45,6 +45,6 @@ def create_cluster(self, cluster_name, version, kubeconfig): else: self.cluster.create(self.config) - def delete_cluster(self, cluster_name, version): + def delete_cluster(self, cluster_name): self.cluster = KindCluster(cluster_name) self.cluster.delete() diff --git a/Tests/kaas/plugin/run_plugin.py b/Tests/kaas/plugin/run_plugin.py index 518d1c62c..b9807e28a 100755 --- a/Tests/kaas/plugin/run_plugin.py +++ b/Tests/kaas/plugin/run_plugin.py @@ -54,7 +54,7 @@ def delete(plugin_kind, plugin_config, clusterspec_path, clusterspec_cluster): clusterinfo = clusterspec[clusterspec_cluster] cluster_id = clusterspec_cluster plugin = init_plugin(plugin_kind, plugin_config) - plugin.delete_cluster(cluster_id, clusterinfo['branch']) + plugin.delete_cluster(cluster_id) if __name__ == '__main__':