diff --git a/Tests/config.toml b/Tests/config.toml index 0f270cd4b..6833fc5ad 100644 --- a/Tests/config.toml +++ b/Tests/config.toml @@ -33,11 +33,51 @@ subjects = [ workers = 4 +[presets.all-kaas] +scopes = [ + "scs-compatible-kaas", +] +subjects = [ + "cspA-current", + "cspA-current-1", + "cspA-current-2", +] +workers = 4 + + [scopes.scs-compatible-iaas] spec = "./scs-compatible-iaas.yaml" +[scopes.scs-compatible-kaas] +spec = "./scs-compatible-kaas.yaml" + + # default subject (not a real subject, but used to declare a default mapping) # (this is the only mapping declaration that supports using Python string interpolation) [subjects._.mapping] -os_cloud = "{subject}" +os_cloud = "{subject}" +subject_root = "{subject}" + + +[subjects._.kubernetes_setup] +clusterspec = "kaas/clusterspec.yaml" + + +[subjects.cspA-current.kubernetes_setup] +kube_plugin = "kind" +kube_plugin_config = "../playbooks/k8s_configs/kind_config.yaml" +clusterspec_cluster = "current-k8s-release" + + +[subjects.cspA-current-1.kubernetes_setup] +kube_plugin = "kind" +kube_plugin_config = "../playbooks/k8s_configs/kind_config.yaml" +clusterspec_cluster = "current-k8s-release-1" + + +[subjects.cspA-current-2.kubernetes_setup] +kube_plugin = "kind" +kube_plugin_config = "../playbooks/k8s_configs/kind_config.yaml" +clusterspec_cluster = "current-k8s-release-2" + diff --git a/Tests/kaas/clusterspec.yaml b/Tests/kaas/clusterspec.yaml new file mode 100644 index 000000000..c8439a89f --- /dev/null +++ b/Tests/kaas/clusterspec.yaml @@ -0,0 +1,11 @@ +# this file specifies all clusters that have to be provisioned for the tests to run +clusters: + current-k8s-release: + branch: "1.31" + kubeconfig: kubeconfig.yaml + current-k8s-release-1: + branch: "1.30" + kubeconfig: kubeconfig.yaml + current-k8s-release-2: + branch: "1.29" + kubeconfig: kubeconfig.yaml diff --git a/Tests/kaas/plugin/README.md b/Tests/kaas/plugin/README.md new file mode 100644 index 000000000..28cfa83ce --- /dev/null +++ b/Tests/kaas/plugin/README.md @@ -0,0 +1,39 @@ +# Plugin for provisioning k8s clusters and performing conformance tests on these clusters + +## Development environment + +### requirements + +* [docker](https://docs.docker.com/engine/install/) +* [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) +* [sonobuoy](https://sonobuoy.io/docs/v0.57.1/#installation) + +### setup for development + +1. Generate python 3.10 env + + ```bash + sudo apt-get install python3.10-dev + virtualenv -p /usr/bin/python3.10 venv + echo "*" >> venv/.gitignore + source venv/bin/activate + (venv) curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 + (venv) python3.10 -m pip install --upgrade pip + (venv) python3.10 -m pip --version + + ``` + +2. Install dependencies: + + ```bash + (venv) pip install pip-tools + (venv) pip-compile requirements.in + (venv) pip-sync requirements.txt + ``` + +3. Set environment variables and launch the process: + + ```bash + (venv) export CLUSTER_PROVIDER="kind" + (venv) python run.py + ``` diff --git a/Tests/kaas/plugin/interface.py b/Tests/kaas/plugin/interface.py new file mode 100644 index 000000000..f62e3b3e2 --- /dev/null +++ b/Tests/kaas/plugin/interface.py @@ -0,0 +1,54 @@ + + +class KubernetesClusterPlugin(): + """ + An abstract base class for custom Kubernetes cluster provider plugins. + It represents an interface class from which the api provider-specific + plugins must be derived as child classes + + To implement fill the methods `create_cluster` and `delete_cluster` with + api provider-specific functionalities for creating and deleting clusters. + The `create_cluster` method must ensure that the kubeconfigfile is provided + at the position in the file system defined by the parameter + `kubeconfig_filepath` + + - Implement `create_cluster` and `delete_cluster` methods + - Create `__init__(self, config_file)` method to handle api specific + configurations. + + Example: + .. code:: python + + from interface import KubernetesClusterPlugin + from apiX_library import cluster_api_class as ClusterAPI + + class PluginX(KubernetesClusterPlugin): + + def __init__(self, config_file): + self.config = config_file + + def create_cluster(self, cluster_name, version, kubeconfig_filepath): + self.cluster = ClusterAPI(name=cluster_name, image=cluster_image, kubeconfig_filepath) + self.cluster.create(self.config) + + def delete_cluster(self, cluster_name): + self.cluster = ClusterAPI(cluster_name) + self.cluster.delete() + .. + """ + + def create_cluster(self, cluster_name, version, kubeconfig_filepath): + """ + This method is to be called to create a k8s cluster + :param: cluster_name: + :param: version: + :param: kubeconfig_filepath: + """ + raise NotImplementedError + + def delete_cluster(self, cluster_name): + """ + This method is to be called in order to unprovision a cluster + :param: cluster_name: + """ + raise NotImplementedError diff --git a/Tests/kaas/plugin/plugin_kind.py b/Tests/kaas/plugin/plugin_kind.py new file mode 100644 index 000000000..0b2d78bc9 --- /dev/null +++ b/Tests/kaas/plugin/plugin_kind.py @@ -0,0 +1,50 @@ +import logging +import os +import os.path +from pathlib import Path + +from interface import KubernetesClusterPlugin +from pytest_kind import KindCluster + +logger = logging.getLogger(__name__) + + +class PluginKind(KubernetesClusterPlugin): + """ + Plugin to handle the provisioning of kubernetes cluster for + conformance testing purpose with the use of Kind + """ + def __init__(self, config_file): + logger.info("Init PluginKind") + self.config = config_file + logger.debug(self.config) + self.working_directory = os.getcwd() + logger.debug(f"Working from {self.working_directory}") + + def create_cluster(self, cluster_name, version, kubeconfig): + """ + This method is to be called to create a k8s cluster + :param: kubernetes_version: + :return: kubeconfig_filepath + """ + cluster_version = version + if cluster_version == '1.29': + cluster_version = 'v1.29.8' + elif cluster_version == '1.30': + cluster_version = 'v1.30.4' + elif cluster_version == '1.31' or cluster_version == 'default': + cluster_version = 'v1.31.1' + cluster_image = f"kindest/node:{cluster_version}" + kubeconfig_filepath = Path(kubeconfig) + if kubeconfig_filepath is None: + raise ValueError("kubeconfig_filepath is missing") + else: + self.cluster = KindCluster(name=cluster_name, image=cluster_image, kubeconfig=kubeconfig_filepath) + if self.config is None: + self.cluster.create() + else: + self.cluster.create(self.config) + + def delete_cluster(self, cluster_name): + self.cluster = KindCluster(cluster_name) + self.cluster.delete() diff --git a/Tests/kaas/plugin/plugin_static.py b/Tests/kaas/plugin/plugin_static.py new file mode 100644 index 000000000..1c3a5222c --- /dev/null +++ b/Tests/kaas/plugin/plugin_static.py @@ -0,0 +1,19 @@ +import shutil + +from interface import KubernetesClusterPlugin + + +class PluginStatic(KubernetesClusterPlugin): + """ + Plugin to handle the provisioning of kubernetes + using a kubeconfig file + """ + + def __init__(self, config_file): + self.kubeconfig_path = config_file + + def create_cluster(self, cluster_name, version, kubeconfig): + shutil.copyfile(self.kubeconfig_path, kubeconfig) + + def delete_cluster(self, cluster_name, version): + pass diff --git a/Tests/kaas/plugin/requirements.in b/Tests/kaas/plugin/requirements.in new file mode 100644 index 000000000..640831e54 --- /dev/null +++ b/Tests/kaas/plugin/requirements.in @@ -0,0 +1,3 @@ +pytest-kind +kubernetes +junitparser diff --git a/Tests/kaas/plugin/requirements.txt b/Tests/kaas/plugin/requirements.txt new file mode 100644 index 000000000..e99ac2285 --- /dev/null +++ b/Tests/kaas/plugin/requirements.txt @@ -0,0 +1,62 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile requirements.in +# +cachetools==5.5.0 + # via google-auth +certifi==2024.8.30 + # via + # kubernetes + # requests +charset-normalizer==3.3.2 + # via requests +google-auth==2.34.0 + # via kubernetes +idna==3.8 + # via requests +junitparser==3.2.0 + # via -r requirements.in +kubernetes==30.1.0 + # via -r requirements.in +oauthlib==3.2.2 + # via + # kubernetes + # requests-oauthlib +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.0 + # via google-auth +pykube-ng==23.6.0 + # via pytest-kind +pytest-kind==22.11.1 + # via -r requirements.in +python-dateutil==2.9.0.post0 + # via kubernetes +pyyaml==6.0.2 + # via + # kubernetes + # pykube-ng +requests==2.32.3 + # via + # kubernetes + # pykube-ng + # requests-oauthlib +requests-oauthlib==2.0.0 + # via kubernetes +rsa==4.9 + # via google-auth +six==1.16.0 + # via + # kubernetes + # python-dateutil +urllib3==2.2.2 + # via + # kubernetes + # pykube-ng + # requests +websocket-client==1.8.0 + # via kubernetes diff --git a/Tests/kaas/plugin/run_plugin.py b/Tests/kaas/plugin/run_plugin.py new file mode 100755 index 000000000..b9807e28a --- /dev/null +++ b/Tests/kaas/plugin/run_plugin.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +import logging +import os.path + +import click +import yaml + +from plugin_kind import PluginKind +from plugin_static import PluginStatic + +PLUGIN_LOOKUP = { + "kind": PluginKind, + "static": PluginStatic, +} + + +def init_plugin(plugin_kind, config=None): + plugin_maker = PLUGIN_LOOKUP.get(plugin_kind) + if plugin_maker is None: + raise ValueError(f"unknown plugin '{plugin_kind}'") + return plugin_maker(config) + + +def load_spec(clusterspec_path): + with open(clusterspec_path, "rb") as fileobj: + return yaml.load(fileobj, Loader=yaml.SafeLoader) + + +@click.group() +def cli(): + pass + + +@cli.command() +@click.argument('plugin_kind', type=click.Choice(list(PLUGIN_LOOKUP), case_sensitive=False)) +@click.argument('plugin_config', type=click.Path(exists=True, dir_okay=False)) +@click.argument('clusterspec_path', type=click.Path(exists=True, dir_okay=False)) +@click.argument('clusterspec_cluster', type=str, default="default") +def create(plugin_kind, plugin_config, clusterspec_path, clusterspec_cluster): + clusterspec = load_spec(clusterspec_path)['clusters'] + plugin = init_plugin(plugin_kind, plugin_config) + clusterinfo = clusterspec[clusterspec_cluster] + cluster_id = clusterspec_cluster + plugin.create_cluster(cluster_id, clusterinfo['branch'], os.path.abspath(clusterinfo['kubeconfig'])) + + +@cli.command() +@click.argument('plugin_kind', type=click.Choice(list(PLUGIN_LOOKUP), case_sensitive=False)) +@click.argument('plugin_config', type=click.Path(exists=True, dir_okay=False)) +@click.argument('clusterspec_path', type=click.Path(exists=True, dir_okay=False)) +@click.argument('clusterspec_cluster', type=str, default="default") +def delete(plugin_kind, plugin_config, clusterspec_path, clusterspec_cluster): + clusterspec = load_spec(clusterspec_path)['clusters'] + clusterinfo = clusterspec[clusterspec_cluster] + cluster_id = clusterspec_cluster + plugin = init_plugin(plugin_kind, plugin_config) + plugin.delete_cluster(cluster_id) + + +if __name__ == '__main__': + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) + cli() diff --git a/Tests/scs-compatible-kaas.yaml b/Tests/scs-compatible-kaas.yaml index 4aa540999..130183cfa 100644 --- a/Tests/scs-compatible-kaas.yaml +++ b/Tests/scs-compatible-kaas.yaml @@ -2,11 +2,14 @@ name: SCS-compatible KaaS uuid: 1fffebe6-fd4b-44d3-a36c-fc58b4bb0180 url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Tests/scs-compatible-kaas.yaml variables: - - kubeconfig + - subject_root # directory containing the kubeconfig files for the subject under test modules: - id: cncf-k8s-conformance name: CNCF Kubernetes conformance url: https://github.com/cncf/k8s-conformance/tree/master + #run: + # - executable: ./kaas/plugin/run_sonobuoy_executor.py + # args: -k {subject_root}/kubeconfig.yaml -r {result_dir_path} -c 'cncf-conformance' testcases: - id: cncf-k8s-conformance tags: [mandatory] @@ -15,7 +18,7 @@ modules: url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0210-v2-k8s-version-policy.md run: - executable: ./kaas/k8s-version-policy/k8s_version_policy.py - args: -k {kubeconfig} + args: -k {subject_root}/kubeconfig.yaml testcases: - id: version-policy-check tags: [mandatory] @@ -24,7 +27,7 @@ modules: url: https://raw.githubusercontent.com/SovereignCloudStack/standards/main/Standards/scs-0214-v1-k8s-node-distribution.md run: - executable: ./kaas/k8s-node-distribution/k8s_node_distribution_check.py - args: -k {kubeconfig} + args: -k {subject_root}/kubeconfig.yaml testcases: - id: node-distribution-check tags: [mandatory] @@ -33,18 +36,11 @@ timeline: - date: 2024-02-28 versions: v1: draft - v2: draft versions: - - version: v2 - include: - - cncf-k8s-conformance - - scs-0210-v2 - - scs-0214-v2 - targets: - main: mandatory - version: v1 include: - - scs-0210-v2 - - scs-0214-v2 + - ref: cncf-k8s-conformance + - ref: scs-0210-v2 + - ref: scs-0214-v2 targets: main: mandatory diff --git a/Tests/scs-test-runner.py b/Tests/scs-test-runner.py index de7152428..3f21f6c17 100755 --- a/Tests/scs-test-runner.py +++ b/Tests/scs-test-runner.py @@ -17,16 +17,23 @@ import click import tomli - logger = logging.getLogger(__name__) MONITOR_URL = "https://compliance.sovereignit.cloud/" +def ensure_dir(path): + try: + os.makedirs(path) + except FileExistsError: + pass + + class Config: def __init__(self): self.cwd = os.path.abspath(os.path.dirname(sys.argv[0]) or os.getcwd()) self.scs_compliance_check = os.path.join(self.cwd, 'scs-compliance-check.py') self.cleanup_py = os.path.join(self.cwd, 'cleanup.py') + self.run_plugin_py = os.path.join(self.cwd, 'kaas', 'plugin', 'run_plugin.py') self.ssh_keygen = shutil.which('ssh-keygen') self.curl = shutil.which('curl') self.secrets = {} @@ -58,42 +65,80 @@ def get_subject_mapping(self, subject): mapping.update(self.subjects.get(subject, {}).get('mapping', {})) return mapping + def get_kubernetes_setup(self, subject): + default_kubernetes_setup = self.subjects.get('_', {}).get('kubernetes_setup', {}) + kubernetes_setup = dict(default_kubernetes_setup) + kubernetes_setup.update(self.subjects.get(subject, {}).get('kubernetes_setup', {})) + return kubernetes_setup + def abspath(self, path): return os.path.join(self.cwd, path) def build_check_command(self, scope, subject, output): # TODO figure out when to supply --debug here (but keep separated from our --debug) - cmd = [ + args = [ sys.executable, self.scs_compliance_check, self.abspath(self.scopes[scope]['spec']), '--debug', '-C', '-o', output, '-s', subject, ] for key, value in self.get_subject_mapping(subject).items(): - cmd.extend(['-a', f'{key}={value}']) - return cmd + args.extend(['-a', f'{key}={value}']) + return {'args': args} + + def build_provision_command(self, subject): + kubernetes_setup = self.get_kubernetes_setup(subject) + subject_root = self.abspath(self.get_subject_mapping(subject).get('subject_root') or '.') + ensure_dir(subject_root) + return { + 'args': [ + sys.executable, self.run_plugin_py, + 'create', + kubernetes_setup['kube_plugin'], + self.abspath(kubernetes_setup['kube_plugin_config']), + self.abspath(kubernetes_setup['clusterspec']), + kubernetes_setup['clusterspec_cluster'], + ], + 'cwd': subject_root, + } + + def build_unprovision_command(self, subject): + kubernetes_setup = self.get_kubernetes_setup(subject) + subject_root = self.abspath(self.get_subject_mapping(subject).get('subject_root') or '.') + ensure_dir(subject_root) + return { + 'args': [ + sys.executable, self.run_plugin_py, + 'delete', + kubernetes_setup['kube_plugin'], + self.abspath(kubernetes_setup['kube_plugin_config']), + self.abspath(kubernetes_setup['clusterspec']), + kubernetes_setup['clusterspec_cluster'], + ], + 'cwd': subject_root, + } def build_cleanup_command(self, subject): # TODO figure out when to supply --debug here (but keep separated from our --debug) - return [ + return {'args': [ sys.executable, self.cleanup_py, '-c', self.get_subject_mapping(subject)['os_cloud'], '--prefix', '_scs-', '--ipaddr', '10.1.0.', '--debug', - ] + ]} def build_sign_command(self, target_path): - return [ + return {'args': [ self.ssh_keygen, '-Y', 'sign', '-f', self.abspath(self.secrets['keyfile']), '-n', 'report', target_path, - ] + ]} def build_upload_command(self, target_path, monitor_url): if not monitor_url.endswith('/'): monitor_url += '/' - return [ + return {'args': [ self.curl, '--fail-with-body', '--data-binary', f'@{target_path}.sig', @@ -101,7 +146,7 @@ def build_upload_command(self, target_path, monitor_url): '-H', 'Content-Type: application/x-signed-yaml', '-H', f'Authorization: Basic {self.auth_token}', f'{monitor_url}reports', - ] + ]} @click.group() @@ -123,7 +168,7 @@ def _run_commands(commands, num_workers=5): processes = [] while commands or processes: while commands and len(processes) < num_workers: - processes.append(subprocess.Popen(commands.pop())) + processes.append(subprocess.Popen(**commands.pop())) processes[:] = [p for p in processes if p.poll() is None] time.sleep(0.5) @@ -187,15 +232,7 @@ def run(cfg, scopes, subjects, preset, num_workers, monitor_url, report_yaml): return 0 -@cli.command() -@click.option('--subject', 'subjects', type=str) -@click.option('--preset', 'preset', type=str) -@click.option('--num-workers', 'num_workers', type=int, default=5) -@click.pass_obj -def cleanup(cfg, subjects, preset, num_workers): - """ - clean up any lingering resources - """ +def _run_command_for_subjects(cfg, subjects, preset, num_workers, command): if not subjects and not preset: preset = 'default' if preset: @@ -208,12 +245,49 @@ def cleanup(cfg, subjects, preset, num_workers): subjects = [subject.strip() for subject in subjects.split(',')] if subjects else [] if not subjects: raise click.UsageError('subject(s) must be non-empty') - logger.debug(f'cleaning up for subject(s) {", ".join(subjects)}, num_workers: {num_workers}') - commands = [cfg.build_cleanup_command(subject) for subject in subjects] + logger.debug(f'running {command} for subject(s) {", ".join(subjects)}, num_workers: {num_workers}') + m = getattr(cfg, f'build_{command}_command') + commands = [m(subject) for subject in subjects] _run_commands(commands, num_workers=num_workers) return 0 +@cli.command() +@click.option('--subject', 'subjects', type=str) +@click.option('--preset', 'preset', type=str) +@click.option('--num-workers', 'num_workers', type=int, default=5) +@click.pass_obj +def cleanup(cfg, subjects, preset, num_workers): + """ + clean up any lingering IaaS resources + """ + return _run_command_for_subjects(cfg, subjects, preset, num_workers, "cleanup") + + +@cli.command() +@click.option('--subject', 'subjects', type=str) +@click.option('--preset', 'preset', type=str) +@click.option('--num-workers', 'num_workers', type=int, default=5) +@click.pass_obj +def provision(cfg, subjects, preset, num_workers): + """ + create k8s clusters + """ + return _run_command_for_subjects(cfg, subjects, preset, num_workers, "provision") + + +@cli.command() +@click.option('--subject', 'subjects', type=str) +@click.option('--preset', 'preset', type=str) +@click.option('--num-workers', 'num_workers', type=int, default=5) +@click.pass_obj +def unprovision(cfg, subjects, preset, num_workers): + """ + clean up k8s clusters + """ + return _run_command_for_subjects(cfg, subjects, preset, num_workers, "unprovision") + + if __name__ == '__main__': logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) cli(obj=Config()) diff --git a/playbooks/k8s_configs/kind_config.yaml b/playbooks/k8s_configs/kind_config.yaml new file mode 100644 index 000000000..ead21eb72 --- /dev/null +++ b/playbooks/k8s_configs/kind_config.yaml @@ -0,0 +1,5 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +- role: worker