Test self-built rook containers also with SES

config/ses.toml

@@ -1,12 +1,24 @@
 [ses]
 
+# Whether or not to build rook from git. If not, the target version of SES images
+# are used.
+build_rook_from_git = false
+
+# Rook repo url
+repo = "https://github.com/SUSE/rook"
+
+# The rook version you want to test. This is the tag to be checked out so it must
+# exist for real within rook repo
+version = "master"
+
 # The target version of SES to install. This must match a key below that
 # contains the repositories and yaml substitutions.
 
 target = "devel7"
 
   [ses.ses7]
     rook_ceph_chart = "registry.suse.de/suse/sle-15-sp2/update/products/ses7/charts/rook-ceph:latest"
+    ceph_image = "registry.suse.de/devel/storage/7.0/containers/ses/7/rook/ceph:latest"
     [ses.ses7.yaml_substitutions]
       # "suse.com" = "suse.de/devel/storage/7.0/containers"
 
@@ -17,6 +29,7 @@ target = "devel7"
 
   [ses.devel7]
     rook_ceph_chart = "registry.suse.de/devel/storage/7.0/charts/ses/7/charts/rook-ceph:latest"
+    ceph_image = "registry.suse.de/devel/storage/7.0/containers/ses/7/rook/ceph:latest"
 
     [ses.devel7.yaml_substitutions]
       "registry.suse.com" = "registry.suse.de/devel/storage/7.0/containers"

tests/assets/ansible/playbook_rook_ses_upstream.yaml

@@ -0,0 +1,21 @@
+---
+- hosts: all
+  # TODO(toabctl): find a better structure for that
+  tasks:
+    - name: add podman packages
+      vars:
+        pkg:
+          - podman
+      zypper:
+          name: '{{ pkg }}'
+          state: present
+          update_cache: yes
+
+    - name: copy rook ceph image to cluster nodes
+      copy:
+        src: "{{ rookcheck_workspace_dir }}/build/rook/rook-ceph.tar.gz"
+        dest: "/root/.images/"
+        mode: '0644'
+
+    - name: load rook ceph image with podman  # noqa 301
+      command: podman load --input /root/.images/rook-ceph.tar.gz

tests/lib/rook/base.py

@@ -15,9 +15,11 @@
 import logging
 import os
 import re
+import requests
 
-from tests.config import settings
+from tests.config import settings, converter
 from tests.lib import common
+from tests.lib.common import execute
 from abc import ABC, abstractmethod
 
 logger = logging.getLogger(__name__)
@@ -29,6 +31,8 @@ def __init__(self, workspace, kubernetes):
         self.kubernetes = kubernetes
         self.toolbox_pod = None
         self.ceph_dir = None
+        self.rook_image = None
+        self.build_dir = os.path.join(self.workspace.build_dir, 'rook')
         logger.info(f"rook init on {self.kubernetes.hardware}")
 
     @property
@@ -37,9 +41,28 @@ def workspace(self):
 
     @abstractmethod
     def build(self):
-        # Having the method in child classes allows easier test writing
-        # when using threads to speed things up
-        pass
+        self.get_rook()
+
+        self.get_golang()
+        logger.info("Compiling rook...")
+        execute(
+            command=f"make --directory {self.build_dir} "
+                    f"-j BUILD_REGISTRY='rook-build' IMAGES='ceph'",
+            env={"PATH": f"{self.workspace.bin_dir}/go/bin:"
+                         f"{os.environ['PATH']}",
+                 "TMPDIR": self.workspace.tmp_dir,
+                 "GOCACHE": self.workspace.tmp_dir,
+                 "GOPATH": self.workspace.build_dir},
+            log_stderr=False)
+
+        logger.info(f"Tag image as {self.rook_image}")
+        execute(f'docker tag "rook-build/ceph-amd64" {self.rook_image}')
+
+        logger.info("Save image tar")
+        # TODO(jhesketh): build arch may differ
+        execute(f"docker save {self.rook_image} | gzip > %s"
+                % os.path.join(self.build_dir, 'rook-ceph.tar.gz'))
+        self._rook_built = True
 
     @abstractmethod
     def preinstall(self):
@@ -69,6 +92,10 @@ def execute_in_ceph_toolbox(self, command, log_stdout=False):
         return self.kubernetes.execute_in_pod(
             command, self.toolbox_pod, log_stdout=False)
 
+    @abstractmethod
+    def get_rook(self):
+        pass
+
     @abstractmethod
     def _get_charts(self):
         pass
@@ -81,10 +108,22 @@ def _get_helm(self):
     def _install_operator_helm(self):
         pass
 
+    @abstractmethod
+    def upload_rook_image(self):
+        if converter('@bool', settings.UPSTREAM_ROOK.BUILD_ROOK_FROM_GIT):
+            pass
+        else:
+            return
+
     def install(self):
         self.kubernetes.kubectl("create namespace rook-ceph")
         self._install_operator()
 
+        # CRDs are in an own file in rook >= 1.5
+        if os.path.isfile(os.path.join(self.ceph_dir, 'crds.yaml')):
+            self.kubernetes.kubectl_apply(
+                os.path.join(self.ceph_dir, 'crds.yaml'))
+
         # reduce wait time to discover devices
         self.kubernetes.kubectl(
             "-n rook-ceph set env "
@@ -124,7 +163,9 @@ def _install_operator(self):
         """
         Install operator using either kubectl of helm
         """
-        if settings.OPERATOR_INSTALLER == "helm":
+        if (settings.OPERATOR_INSTALLER == "helm" and not
+                converter('@bool',
+                          settings.UPSTREAM_ROOK.BUILD_ROOK_FROM_GIT)):
             logger.info('Deploying rook operator - using Helm')
             self._install_operator_helm()
         else:
@@ -190,6 +231,14 @@ def get_number_of_mons(self):
         logger.info("cluster has %s mon pods running", mons)
         return mons
 
+    def get_golang(self):
+        url = 'https://golang.org/VERSION?m=text'
+        version = requests.get(url).content.decode("utf-8")
+        self.workspace.get_unpack(
+            "https://dl.google.com/go/%s.linux-amd64.tar.gz" % version,
+            unpack_folder=self.workspace.bin_dir
+        )
+
     def __enter__(self):
         return self
 

tests/lib/rook/ses.py

@@ -14,11 +14,12 @@
 
 import logging
 import os
+import yaml
 
 from tests.lib.common import execute, recursive_replace
 from tests.lib.rook.base import RookBase
 
-from tests.config import settings
+from tests.config import settings, converter
 
 logger = logging.getLogger(__name__)
 
@@ -32,21 +33,54 @@ def __init__(self, workspace, kubernetes):
             f"SES.{settings.SES.TARGET}.rook_ceph_chart")
 
     def build(self):
+        if not converter('@bool', settings.SES.BUILD_ROOK_FROM_GIT):
+            return
+
+        image = 'localhost/rook/ceph'
+        tag = f"{settings.SES.VERSION}-rookcheck"
+        self.rook_image = f"{image}:{tag}"
         super().build()
-        logger.info('SES based rook does not require building')
+
+    def upload_rook_image(self):
+        super().upload_rook_image()
+        self.kubernetes.hardware.ansible_run_playbook(
+            "playbook_rook_ses_upstream.yaml")
 
     def preinstall(self):
         super().preinstall()
+        if converter('@bool', settings.SES.BUILD_ROOK_FROM_GIT):
+            self.upload_rook_image()
         repo_vars = {
             'ses_repositories':
                 settings(f'SES.{settings.SES.TARGET}.repositories')
         }
         self.kubernetes.hardware.ansible_run_playbook(
             'playbook_rook_ses.yaml', extra_vars=repo_vars)
-        self._get_rook()
+        self._get_rook_yaml()
         self._fix_yaml()
 
-    def _get_rook(self):
+    def get_rook(self):
+        if not converter('@bool', settings.SES.BUILD_ROOK_FROM_GIT):
+            return
+
+        super().get_rook()
+        logger.info("Clone rook version %s from repo %s" % (
+            settings.SES.VERSION,
+            settings.SES.REPO))
+        execute(
+            "git clone -b %s %s %s" % (
+                settings.SES.VERSION,
+                settings.SES.REPO,
+                self.build_dir),
+            log_stderr=False
+        )
+
+    def _get_rook_yaml(self):
+        # do not use rpm-package for self-built rook images
+        if converter('@bool', settings.SES.BUILD_ROOK_FROM_GIT):
+            self.ceph_dir = os.path.join(
+                self.build_dir, 'cluster/examples/kubernetes/ceph')
+            return
         # TODO (bleon)
         # This is not optima. Need to retrieve RPM directly and extract files
         # out of it. RPM URL should be configurable
@@ -58,9 +92,37 @@ def _get_rook(self):
     def _fix_yaml(self):
         # Replacements are to point container paths and/or versions to the
         # expected ones to test.
-        replacements = settings(
-            f'SES.{settings.SES.TARGET}.yaml_substitutions')
-        recursive_replace(self.ceph_dir, replacements)
+        if converter('@bool', settings.SES.BUILD_ROOK_FROM_GIT):
+            # Replace image reference if we built it in this run
+            with open(os.path.join(self.ceph_dir, 'operator.yaml')) as file:
+                docs = yaml.load_all(file, Loader=yaml.FullLoader)
+                for doc in docs:
+                    try:
+                        image = doc['spec']['template']['spec'][
+                                'containers'][0]['image']
+                        break
+                    except KeyError:
+                        pass
+                logger.info("replacing %s by %s", image, self.rook_image)
+                replacements = {image: self.rook_image}
+                recursive_replace(self.ceph_dir, replacements)
+            with open(os.path.join(self.ceph_dir, 'cluster.yaml')) as file:
+                docs = yaml.load_all(file, Loader=yaml.FullLoader)
+                for doc in docs:
+                    try:
+                        image = doc['spec']['cephVersion']['image']
+                        break
+                    except KeyError:
+                        pass
+                logger.info("replacing %s by %s", image, settings(
+                    f"SES.{settings.SES.TARGET}.ceph_image"))
+                replacements = {image: settings(
+                    f"SES.{settings.SES.TARGET}.ceph_image")}
+                recursive_replace(self.ceph_dir, replacements)
+        else:
+            replacements = settings(
+                f'SES.{settings.SES.TARGET}.yaml_substitutions')
+            recursive_replace(self.ceph_dir, replacements)
 
     def _get_charts(self):
         super()._get_charts()

tests/lib/rook/upstream.py

@@ -29,40 +29,18 @@ class RookCluster(RookBase):
     def __init__(self, workspace, kubernetes):
         super().__init__(workspace, kubernetes)
         self._rook_built = False
-        self.build_dir = os.path.join(self.workspace.build_dir, 'rook')
         self.ceph_dir = os.path.join(
             self.build_dir, 'cluster/examples/kubernetes/ceph')
         self.rook_chart = settings.UPSTREAM_ROOK.ROOK_CEPH_CHART
 
     def build(self):
-        super().build()
-        self.get_rook()
         if not converter('@bool', settings.UPSTREAM_ROOK.BUILD_ROOK_FROM_GIT):
             return
 
-        self.get_golang()
-        logger.info("Compiling rook...")
-        execute(
-            command=f"make --directory {self.build_dir} "
-                    f"-j BUILD_REGISTRY='rook-build' IMAGES='ceph'",
-            env={"PATH": f"{self.workspace.bin_dir}/go/bin:"
-                         f"{os.environ['PATH']}",
-                 "TMPDIR": self.workspace.tmp_dir,
-                 "GOCACHE": self.workspace.tmp_dir,
-                 "GOPATH": self.workspace.build_dir},
-            log_stderr=False)
-
         image = 'rook/ceph'
         tag = f"{settings.UPSTREAM_ROOK.VERSION}-rookcheck"
         self.rook_image = f"{image}:{tag}"
-        logger.info(f"Tag image as {image}:{tag}")
-        execute(f'docker tag "rook-build/ceph-amd64" {image}:{tag}')
-
-        logger.info("Save image tar")
-        # TODO(jhesketh): build arch may differ
-        execute(f"docker save {image}:{tag} | gzip > %s"
-                % os.path.join(self.build_dir, 'rook-ceph.tar.gz'))
-        self._rook_built = True
+        super().build()
 
     def preinstall(self):
         super().preinstall()
@@ -75,26 +53,6 @@ def _get_charts(self):
         logger.info(f"Adding rook chart helm repo {self.rook_chart}")
         self.kubernetes.helm(f"repo add rook-upstream {self.rook_chart}")
 
-    def get_rook(self):
-        logger.info("Clone rook version %s from repo %s" % (
-            settings.UPSTREAM_ROOK.VERSION,
-            settings.UPSTREAM_ROOK.REPO))
-        execute(
-            "git clone -b %s %s %s" % (
-                settings.UPSTREAM_ROOK.VERSION,
-                settings.UPSTREAM_ROOK.REPO,
-                self.build_dir),
-            log_stderr=False
-        )
-
-    def get_golang(self):
-        url = 'https://golang.org/VERSION?m=text'
-        version = requests.get(url).content.decode("utf-8")
-        self.workspace.get_unpack(
-            "https://dl.google.com/go/%s.linux-amd64.tar.gz" % version,
-            unpack_folder=self.workspace.bin_dir
-        )
-
     def _get_helm(self):
         super()._get_helm()
         url = "https://api.github.com/repos/helm/helm/releases/latest"
@@ -119,6 +77,7 @@ def _fix_yaml(self):
         recursive_replace(dir=self.ceph_dir, replacements=replacements)
 
     def upload_rook_image(self):
+        super().upload_rook_image()
         self.kubernetes.hardware.ansible_run_playbook(
             "playbook_rook_upstream.yaml")
 
@@ -133,3 +92,19 @@ def _install_operator_helm(self):
         self.kubernetes.helm(
             f"install -n rook-ceph rook-ceph rook-upstream/rook-ceph"
             f" {version}")
+
+    def get_rook(self):
+        if not converter('@bool', settings.UPSTREAM_ROOK.BUILD_ROOK_FROM_GIT):
+            return
+
+        super().get_rook()
+        logger.info("Clone rook version %s from repo %s" % (
+            settings.UPSTREAM_ROOK.VERSION,
+            settings.UPSTREAM_ROOK.REPO))
+        execute(
+            "git clone -b %s %s %s" % (
+                settings.UPSTREAM_ROOK.VERSION,
+                settings.UPSTREAM_ROOK.REPO,
+                self.build_dir),
+            log_stderr=False
+        )

tests/test_basic.py

@@ -323,6 +323,7 @@ def test_add_remove_node(rook_cluster):
     # NodeRole.WORKER adds the disk for us
     rook_cluster.kubernetes.hardware.node_add(node)
     rook_cluster.kubernetes.hardware.prepare_nodes(limit_to_nodes=[node])
+    rook_cluster.upload_rook_image()
     # add the node the k8s cluster
     rook_cluster.kubernetes.join([node])
 
@@ -368,7 +369,7 @@ def test_add_remove_node(rook_cluster):
 
     i = 0
     while osds_current != workers_current:
-        if i == 10:
+        if i == 90:
             pytest.fail("rook did not remove additional OSD "
                         "after node removal")
         time.sleep(10)