fix(e2e): use backup target API if settings are removed

ref: longhorn/longhorn 5411, 10043

Signed-off-by: James Lu <[email protected]>

e2e/libs/backupstore/base.py

@@ -1,12 +1,42 @@
 from abc import ABC, abstractmethod
 import hashlib
 import os
+import re
 
 from kubernetes import client
 
+from utility.utility import get_longhorn_client
+
+SECOND = 1
+MINUTE = 60 * SECOND
+HOUR = 60 * MINUTE
+
+duration_u = {
+    "s":  SECOND,
+    "m":  MINUTE,
+    "h":  HOUR,
+}
+
+
+def from_k8s_duration_to_seconds(duration_s):
+    # k8s duration string format such as "3h5m30s"
+    total = 0
+    pattern_str = r'([0-9]+)([smh]+)'
+    pattern = re.compile(pattern_str)
+    matches = pattern.findall(duration_s)
+    if not len(matches):
+        raise Exception("Invalid duration {}".format(duration_s))
+
+    for v, u in matches:
+        total += int(v) * duration_u[u]
+
+    return total
+
 
 class Base(ABC):
 
+    DEFAULT_BACKUPTARGET = "default"
+
     def __init__(self):
         self.core_api = client.CoreV1Api()
         backupstore = os.environ.get('LONGHORN_BACKUPSTORE')
@@ -34,6 +64,54 @@ def backup_volume_path(self, volume_name):
 
         return backupstore_bv_path
 
+    def get_backupstore_url(self):
+        return get_longhorn_client().by_id_backupTarget(
+                            self.DEFAULT_BACKUPTARGET).backupTargetURL
+
+    def get_backupstore_secret(self):
+        return get_longhorn_client().by_id_backupTarget(
+                            self.DEFAULT_BACKUPTARGET).credentialSecret
+
+    def get_backupstore_poll_interval(self):
+        k8s_duration = get_longhorn_client().by_id_backupTarget(
+                            self.DEFAULT_BACKUPTARGET).pollInterval
+        return from_k8s_duration_to_seconds(k8s_duration)
+
+    def set_backupstore_url(self, url):
+        bt = get_longhorn_client().by_id_backupTarget(
+                            self.DEFAULT_BACKUPTARGET)
+        p_interval = from_k8s_duration_to_seconds(bt.pollInterval)
+        self.update_backupstore(url=url,
+                                credential_secret=bt.credentialSecret,
+                                poll_interval=p_interval)
+
+    def set_backupstore_secret(self, secret):
+        bt = get_longhorn_client().by_id_backupTarget(
+                            self.DEFAULT_BACKUPTARGET)
+        p_interval = from_k8s_duration_to_seconds(bt.pollInterval)
+        self.update_backupstore(url=bt.backupTargetURL,
+                                credential_secret=secret,
+                                poll_interval=p_interval)
+
+    def set_backupstore_poll_interval(self, poll_interval):
+        bt = get_longhorn_client().by_id_backupTarget(
+                            self.DEFAULT_BACKUPTARGET)
+        self.update_backupstore(url=bt.backupTargetURL,
+                                credential_secret=bt.credentialSecret,
+                                poll_interval=poll_interval)
+
+    def reset_backupstore(self):
+        self.update_backupstore()
+
+    def update_backupstore(self,
+                           url="", credential_secret="", poll_interval=300):
+        backup_target = get_longhorn_client().by_id_backupTarget(
+                            self.DEFAULT_BACKUPTARGET)
+        backup_target.backupTargetUpdate(name=self.DEFAULT_BACKUPTARGET,
+                                         backupTargetURL=url,
+                                         credentialSecret=credential_secret,
+                                         pollInterval=str(poll_interval))
+
     @abstractmethod
     def get_backup_volume_prefix(self, volume_name):
         return NotImplemented
@@ -77,3 +155,42 @@ def delete_random_backup_block(self):
     @abstractmethod
     def count_backup_block_files(self):
         return NotImplemented
+
+
+class BackupStore(Base):
+
+    def __init__(self):
+        super().__init__()
+
+    def get_backup_volume_prefix(self, volume_name):
+        return NotImplemented
+
+    def get_backup_cfg_file_path(self, volume_name, backup_name):
+        return NotImplemented
+
+    def get_volume_cfg_file_path(self, volume_name):
+        return NotImplemented
+
+    def get_backup_blocks_dir(self, volume_name):
+        return NotImplemented
+
+    def create_file_in_backupstore(self):
+        return NotImplemented
+
+    def write_backup_cfg_file(self, volume_name, backup_name, data):
+        return NotImplemented
+
+    def delete_file_in_backupstore(self):
+        return NotImplemented
+
+    def delete_backup_cfg_file(self):
+        return NotImplemented
+
+    def delete_volume_cfg_file(self):
+        return NotImplemented
+
+    def delete_random_backup_block(self):
+        return NotImplemented
+
+    def count_backup_block_files(self):
+        return NotImplemented

e2e/libs/setting/setting.py

@@ -5,14 +5,25 @@
 from utility.utility import get_retry_count_and_interval
 from utility.utility import logging
 
+from backupstore.base import BackupStore
+
+
 class Setting:
 
     SETTING_BACKUP_TARGET = "backup-target"
     SETTING_BACKUP_TARGET_CREDENTIAL_SECRET = "backup-target-credential-secret"
     SETTING_BACKUPSTORE_POLL_INTERVAL = "backupstore-poll-interval"
 
+    SETTING_BACKUP_TARGET_NOT_SUPPORTED = \
+        f"setting {SETTING_BACKUP_TARGET} is not supported"
+    SETTING_BACKUP_TARGET_CREDENTIAL_SECRET_NOT_SUPPORTED = \
+        f"setting {SETTING_BACKUP_TARGET_CREDENTIAL_SECRET} is not supported"
+    SETTING_SETTING_BACKUPSTORE_POLL_INTERVAL_NOT_SUPPORTED = \
+        f"setting {SETTING_BACKUPSTORE_POLL_INTERVAL} is not supported"
+
     def __init__(self):
         self.retry_count, self.retry_interval = get_retry_count_and_interval()
+        self.backupstore = BackupStore()
 
     def update_setting(self, key, value, retry=True):
         if retry:
@@ -43,23 +54,65 @@ def set_backupstore(self):
         backupstore = self.get_backupstore_url()
         if backupstore:
             backupsettings = backupstore.split("$")
-            self.update_setting(self.SETTING_BACKUP_TARGET, backupsettings[0])
-            if len(backupsettings) > 1:
-                self.update_setting(self.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET, backupsettings[1])
-
             poll_interval = self.get_backupstore_poll_interval()
-            self.update_setting(self.SETTING_BACKUPSTORE_POLL_INTERVAL, poll_interval)
+            try:
+                self.update_setting(self.SETTING_BACKUP_TARGET,
+                                    backupsettings[0],
+                                    retry=False)
+                if len(backupsettings) > 1:
+                    self.update_setting(
+                        self.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET,
+                        backupsettings[1],
+                        retry=False)
+
+                self.update_setting(self.SETTING_BACKUPSTORE_POLL_INTERVAL,
+                                    poll_interval,
+                                    retry=False)
+            except Exception as e:
+                if self.SETTING_BACKUP_TARGET_NOT_SUPPORTED in e.error.message:
+                    self.backupstore.set_backupstore_url(backupsettings[0])
+                    if len(backupsettings) > 1:
+                        self.backupstore.set_backupstore_secret(
+                            backupsettings[1])
+                    self.backupstore.set_backupstore_poll_interval(
+                        poll_interval)
+                else:
+                    logging(e)
 
     def reset_backupstore(self):
-        self.update_setting(self.SETTING_BACKUP_TARGET, "")
-        self.update_setting(self.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET, "")
-        self.update_setting(self.SETTING_BACKUPSTORE_POLL_INTERVAL, "300")
+        try:
+            self.update_setting(self.SETTING_BACKUP_TARGET, "", retry=False)
+            self.update_setting(self.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET,
+                                "",
+                                retry=False)
+            self.update_setting(self.SETTING_BACKUPSTORE_POLL_INTERVAL,
+                                "300",
+                                retry=False)
+        except Exception as e:
+            if self.SETTING_BACKUP_TARGET_NOT_SUPPORTED in e.error.message:
+                self.backupstore.reset_backupstore()
+            else:
+                logging(e)
 
     def get_backup_target(self):
-        return self.get_setting(self.SETTING_BACKUP_TARGET)
+        try:
+            return self.get_setting(self.SETTING_BACKUP_TARGET)
+        except Exception as e:
+            if self.SETTING_BACKUP_TARGET_NOT_SUPPORTED in e.error.message:
+                return self.backupstore.get_backupstore_url()
+            else:
+                logging(e)
 
     def get_secret(self):
-        return self.get_setting(self.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
+        try:
+            return self.get_setting(
+                self.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
+        except Exception as e:
+            if (self.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET_NOT_SUPPORTED
+                    in e.error.message):
+                return self.backupstore.get_backupstore_secret()
+            else:
+                logging(e)
 
     def reset_settings(self):
         client = get_longhorn_client()

manager/integration/tests/backupstore.py

@@ -15,8 +15,8 @@
 from common import SETTING_BACKUP_TARGET_CREDENTIAL_SECRET
 from common import SETTING_BACKUPSTORE_POLL_INTERVAL
 from common import LONGHORN_NAMESPACE
-from common import NODE_UPDATE_RETRY_COUNT
-from common import NODE_UPDATE_RETRY_INTERVAL
+from common import RETRY_COUNTS_SHORT
+from common import RETRY_INTERVAL
 from common import cleanup_all_volumes
 from common import is_backupTarget_s3
 from common import is_backupTarget_nfs
@@ -37,7 +37,7 @@
 DEFAULT_BACKUPTARGET = "default"
 object_has_been_modified = "the object has been modified; " + \
     "please apply your changes to the latest version and try again"
-    
+
 SETTING_BACKUP_TARGET_NOT_SUPPORTED = \
     f"setting {SETTING_BACKUP_TARGET} is not supported"
 SETTING_BACKUP_TARGET_CREDENTIAL_SECRET_NOT_SUPPORTED = \
@@ -60,14 +60,14 @@
 }
 
 
-def from_k8s_duration_str_to_seconds(duration_str):
+def from_k8s_duration_to_seconds(duration_s):
     # k8s duration string format such as "3h5m30s"
     total = 0
     pattern_str = r'([0-9]+)([smh]+)'
     pattern = re.compile(pattern_str)
-    matches = pattern.findall(duration_str)
+    matches = pattern.findall(duration_s)
     if not len(matches):
-        raise Exception("Invalid duration {}".format(duration_str))
+        raise Exception("Invalid duration {}".format(duration_s))
 
     for v, u in matches:
         total += int(v) * duration_u[u]
@@ -146,19 +146,20 @@ def set_backupstore_setting_by_api(client,
                                    credential_secret="",
                                    poll_interval=300):
     bt = client.by_id_backupTarget(DEFAULT_BACKUPTARGET)
-    bt = bt.backupTargetUpdate(name=DEFAULT_BACKUPTARGET,
-                               backupTargetURL=url,
-                               credentialSecret=credential_secret,
-                               pollInterval=str(poll_interval))
+    bt.backupTargetUpdate(name=DEFAULT_BACKUPTARGET,
+                          backupTargetURL=url,
+                          credentialSecret=credential_secret,
+                          pollInterval=str(poll_interval))
 
-    for _ in range(NODE_UPDATE_RETRY_COUNT):
+    for _ in range(RETRY_COUNTS_SHORT):
         bt = client.by_id_backupTarget(DEFAULT_BACKUPTARGET)
-        p_interval = from_k8s_duration_str_to_seconds(str(bt.pollInterval))
-        if bt.backupTargetURL == url and \
-                bt.credentialSecret == credential_secret and \
-                p_interval == poll_interval:
+        p_interval = from_k8s_duration_to_seconds(str(bt.pollInterval))
+        bt_url = str(bt.backupTargetURL)
+        bt_secret = str(bt.credentialSecret)
+        if bt_url == url and bt_secret == credential_secret and \
+                p_interval == int(poll_interval):
             break
-        time.sleep(NODE_UPDATE_RETRY_INTERVAL)
+        time.sleep(RETRY_INTERVAL)
 
 
 def set_backupstore_invalid(client):
@@ -221,7 +222,7 @@ def set_backupstore_url(client, url):
     except Exception as e:
         if SETTING_BACKUP_TARGET_NOT_SUPPORTED in e.error.message:
             bt = client.by_id_backupTarget(DEFAULT_BACKUPTARGET)
-            poll_interval = from_k8s_duration_str_to_seconds(bt.pollInterval)
+            poll_interval = from_k8s_duration_to_seconds(bt.pollInterval)
             set_backupstore_setting_by_api(
                 client,
                 url=url,
@@ -233,13 +234,13 @@ def set_backupstore_url(client, url):
 
 def set_backupstore_url_by_setting(client, url):
     backup_target_setting = client.by_id_setting(SETTING_BACKUP_TARGET)
-    for _ in range(NODE_UPDATE_RETRY_COUNT):
+    for _ in range(RETRY_COUNTS_SHORT):
         try:
             backup_target_setting = client.update(backup_target_setting,
                                                   value=url)
         except Exception as e:
             if object_has_been_modified in str(e.error.message):
-                time.sleep(NODE_UPDATE_RETRY_INTERVAL)
+                time.sleep(RETRY_INTERVAL)
                 continue
             print(e)
             raise e
@@ -255,7 +256,7 @@ def set_backupstore_credential_secret(client, credential_secret):
         if SETTING_BACKUP_TARGET_CREDENTIAL_SECRET_NOT_SUPPORTED \
                 in e.error.message:
             bt = client.by_id_backupTarget(DEFAULT_BACKUPTARGET)
-            poll_interval = from_k8s_duration_str_to_seconds(bt.pollInterval)
+            poll_interval = from_k8s_duration_to_seconds(bt.pollInterval)
             set_backupstore_setting_by_api(
                 client,
                 url=bt.backupTargetURL,
@@ -268,13 +269,13 @@ def set_backupstore_credential_secret(client, credential_secret):
 def set_backupstore_credential_secret_by_setting(client, credential_secret):
     backup_target_credential_setting = client.by_id_setting(
         SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
-    for _ in range(NODE_UPDATE_RETRY_COUNT):
+    for _ in range(RETRY_COUNTS_SHORT):
         try:
             backup_target_credential_setting = client.update(
                 backup_target_credential_setting, value=credential_secret)
         except Exception as e:
             if object_has_been_modified in str(e.error.message):
-                time.sleep(NODE_UPDATE_RETRY_INTERVAL)
+                time.sleep(RETRY_INTERVAL)
                 continue
             print(e)
             raise
@@ -302,13 +303,13 @@ def set_backupstore_poll_interval(client, poll_interval):
 def set_backupstore_poll_interval_by_setting(client, poll_interval):
     backup_store_poll_interval_setting = client.by_id_setting(
         SETTING_BACKUPSTORE_POLL_INTERVAL)
-    for _ in range(NODE_UPDATE_RETRY_COUNT):
+    for _ in range(RETRY_COUNTS_SHORT):
         try:
             backup_target_poll_interal_setting = client.update(
                 backup_store_poll_interval_setting, value=poll_interval)
         except Exception as e:
             if object_has_been_modified in str(e.error.message):
-                time.sleep(NODE_UPDATE_RETRY_INTERVAL)
+                time.sleep(RETRY_INTERVAL)
                 continue
             print(e)
             raise
@@ -498,7 +499,7 @@ def backupstore_get_secret(client):
                 in e.error.message:
             bt = client.by_id_backupTarget(DEFAULT_BACKUPTARGET)
             return bt.credentialSecret
-        else:  
+        else:
             raise e
 
 

manager/integration/tests/test_ha.py

@@ -1374,7 +1374,7 @@ def test_restore_volume_with_invalid_backupstore(client, volume_name, backupstor
         if SETTING_BACKUP_TARGET_NOT_SUPPORTED in e.error.message:
             set_backupstore_url(client, invalid_backup_target_url)
         else:
-            raise e 
+            raise e
 
     # make fromBackup URL consistent to the the invalid target URL
     url = invalid_backup_target_url + b.url.split("?")[1]