diff --git a/e2e/libs/backupstore/base.py b/e2e/libs/backupstore/base.py index 05775406e8..8671f86998 100644 --- a/e2e/libs/backupstore/base.py +++ b/e2e/libs/backupstore/base.py @@ -1,12 +1,42 @@ from abc import ABC, abstractmethod import hashlib import os +import re from kubernetes import client +from utility.utility import get_longhorn_client + +SECOND = 1 +MINUTE = 60 * SECOND +HOUR = 60 * MINUTE + +duration_u = { + "s": SECOND, + "m": MINUTE, + "h": HOUR, +} + + +def from_k8s_duration_to_seconds(duration_s): + # k8s duration string format such as "3h5m30s" + total = 0 + pattern_str = r'([0-9]+)([smh]+)' + pattern = re.compile(pattern_str) + matches = pattern.findall(duration_s) + if not len(matches): + raise Exception("Invalid duration {}".format(duration_s)) + + for v, u in matches: + total += int(v) * duration_u[u] + + return total + class Base(ABC): + DEFAULT_BACKUPTARGET = "default" + def __init__(self): self.core_api = client.CoreV1Api() backupstore = os.environ.get('LONGHORN_BACKUPSTORE') @@ -34,6 +64,54 @@ def backup_volume_path(self, volume_name): return backupstore_bv_path + def get_backupstore_url(self): + return get_longhorn_client().by_id_backupTarget( + self.DEFAULT_BACKUPTARGET).backupTargetURL + + def get_backupstore_secret(self): + return get_longhorn_client().by_id_backupTarget( + self.DEFAULT_BACKUPTARGET).credentialSecret + + def get_backupstore_poll_interval(self): + k8s_duration = get_longhorn_client().by_id_backupTarget( + self.DEFAULT_BACKUPTARGET).pollInterval + return from_k8s_duration_to_seconds(k8s_duration) + + def set_backupstore_url(self, url): + bt = get_longhorn_client().by_id_backupTarget( + self.DEFAULT_BACKUPTARGET) + p_interval = from_k8s_duration_to_seconds(bt.pollInterval) + self.update_backupstore(url=url, + credential_secret=bt.credentialSecret, + poll_interval=p_interval) + + def set_backupstore_secret(self, secret): + bt = get_longhorn_client().by_id_backupTarget( + self.DEFAULT_BACKUPTARGET) + p_interval = from_k8s_duration_to_seconds(bt.pollInterval) + self.update_backupstore(url=bt.backupTargetURL, + credential_secret=secret, + poll_interval=p_interval) + + def set_backupstore_poll_interval(self, poll_interval): + bt = get_longhorn_client().by_id_backupTarget( + self.DEFAULT_BACKUPTARGET) + self.update_backupstore(url=bt.backupTargetURL, + credential_secret=bt.credentialSecret, + poll_interval=poll_interval) + + def reset_backupstore(self): + self.update_backupstore() + + def update_backupstore(self, + url="", credential_secret="", poll_interval=300): + backup_target = get_longhorn_client().by_id_backupTarget( + self.DEFAULT_BACKUPTARGET) + backup_target.backupTargetUpdate(name=self.DEFAULT_BACKUPTARGET, + backupTargetURL=url, + credentialSecret=credential_secret, + pollInterval=str(poll_interval)) + @abstractmethod def get_backup_volume_prefix(self, volume_name): return NotImplemented @@ -77,3 +155,42 @@ def delete_random_backup_block(self): @abstractmethod def count_backup_block_files(self): return NotImplemented + + +class BackupStore(Base): + + def __init__(self): + super().__init__() + + def get_backup_volume_prefix(self, volume_name): + return NotImplemented + + def get_backup_cfg_file_path(self, volume_name, backup_name): + return NotImplemented + + def get_volume_cfg_file_path(self, volume_name): + return NotImplemented + + def get_backup_blocks_dir(self, volume_name): + return NotImplemented + + def create_file_in_backupstore(self): + return NotImplemented + + def write_backup_cfg_file(self, volume_name, backup_name, data): + return NotImplemented + + def delete_file_in_backupstore(self): + return NotImplemented + + def delete_backup_cfg_file(self): + return NotImplemented + + def delete_volume_cfg_file(self): + return NotImplemented + + def delete_random_backup_block(self): + return NotImplemented + + def count_backup_block_files(self): + return NotImplemented diff --git a/e2e/libs/setting/setting.py b/e2e/libs/setting/setting.py index d479b11f56..a33295466e 100644 --- a/e2e/libs/setting/setting.py +++ b/e2e/libs/setting/setting.py @@ -5,14 +5,25 @@ from utility.utility import get_retry_count_and_interval from utility.utility import logging +from backupstore.base import BackupStore + + class Setting: SETTING_BACKUP_TARGET = "backup-target" SETTING_BACKUP_TARGET_CREDENTIAL_SECRET = "backup-target-credential-secret" SETTING_BACKUPSTORE_POLL_INTERVAL = "backupstore-poll-interval" + SETTING_BACKUP_TARGET_NOT_SUPPORTED = \ + f"setting {SETTING_BACKUP_TARGET} is not supported" + SETTING_BACKUP_TARGET_CREDENTIAL_SECRET_NOT_SUPPORTED = \ + f"setting {SETTING_BACKUP_TARGET_CREDENTIAL_SECRET} is not supported" + SETTING_SETTING_BACKUPSTORE_POLL_INTERVAL_NOT_SUPPORTED = \ + f"setting {SETTING_BACKUPSTORE_POLL_INTERVAL} is not supported" + def __init__(self): self.retry_count, self.retry_interval = get_retry_count_and_interval() + self.backupstore = BackupStore() def update_setting(self, key, value, retry=True): if retry: @@ -43,23 +54,65 @@ def set_backupstore(self): backupstore = self.get_backupstore_url() if backupstore: backupsettings = backupstore.split("$") - self.update_setting(self.SETTING_BACKUP_TARGET, backupsettings[0]) - if len(backupsettings) > 1: - self.update_setting(self.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET, backupsettings[1]) - poll_interval = self.get_backupstore_poll_interval() - self.update_setting(self.SETTING_BACKUPSTORE_POLL_INTERVAL, poll_interval) + try: + self.update_setting(self.SETTING_BACKUP_TARGET, + backupsettings[0], + retry=False) + if len(backupsettings) > 1: + self.update_setting( + self.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET, + backupsettings[1], + retry=False) + + self.update_setting(self.SETTING_BACKUPSTORE_POLL_INTERVAL, + poll_interval, + retry=False) + except Exception as e: + if self.SETTING_BACKUP_TARGET_NOT_SUPPORTED in e.error.message: + self.backupstore.set_backupstore_url(backupsettings[0]) + if len(backupsettings) > 1: + self.backupstore.set_backupstore_secret( + backupsettings[1]) + self.backupstore.set_backupstore_poll_interval( + poll_interval) + else: + logging(e) def reset_backupstore(self): - self.update_setting(self.SETTING_BACKUP_TARGET, "") - self.update_setting(self.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET, "") - self.update_setting(self.SETTING_BACKUPSTORE_POLL_INTERVAL, "300") + try: + self.update_setting(self.SETTING_BACKUP_TARGET, "", retry=False) + self.update_setting(self.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET, + "", + retry=False) + self.update_setting(self.SETTING_BACKUPSTORE_POLL_INTERVAL, + "300", + retry=False) + except Exception as e: + if self.SETTING_BACKUP_TARGET_NOT_SUPPORTED in e.error.message: + self.backupstore.reset_backupstore() + else: + logging(e) def get_backup_target(self): - return self.get_setting(self.SETTING_BACKUP_TARGET) + try: + return self.get_setting(self.SETTING_BACKUP_TARGET) + except Exception as e: + if self.SETTING_BACKUP_TARGET_NOT_SUPPORTED in e.error.message: + return self.backupstore.get_backupstore_url() + else: + logging(e) def get_secret(self): - return self.get_setting(self.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) + try: + return self.get_setting( + self.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) + except Exception as e: + if (self.SETTING_BACKUP_TARGET_CREDENTIAL_SECRET_NOT_SUPPORTED + in e.error.message): + return self.backupstore.get_backupstore_secret() + else: + logging(e) def reset_settings(self): client = get_longhorn_client() diff --git a/manager/integration/tests/backupstore.py b/manager/integration/tests/backupstore.py index 00e34cd094..03525df759 100644 --- a/manager/integration/tests/backupstore.py +++ b/manager/integration/tests/backupstore.py @@ -15,8 +15,8 @@ from common import SETTING_BACKUP_TARGET_CREDENTIAL_SECRET from common import SETTING_BACKUPSTORE_POLL_INTERVAL from common import LONGHORN_NAMESPACE -from common import NODE_UPDATE_RETRY_COUNT -from common import NODE_UPDATE_RETRY_INTERVAL +from common import RETRY_COUNTS_SHORT +from common import RETRY_INTERVAL from common import cleanup_all_volumes from common import is_backupTarget_s3 from common import is_backupTarget_nfs @@ -37,7 +37,7 @@ DEFAULT_BACKUPTARGET = "default" object_has_been_modified = "the object has been modified; " + \ "please apply your changes to the latest version and try again" - + SETTING_BACKUP_TARGET_NOT_SUPPORTED = \ f"setting {SETTING_BACKUP_TARGET} is not supported" SETTING_BACKUP_TARGET_CREDENTIAL_SECRET_NOT_SUPPORTED = \ @@ -60,14 +60,14 @@ } -def from_k8s_duration_str_to_seconds(duration_str): +def from_k8s_duration_to_seconds(duration_s): # k8s duration string format such as "3h5m30s" total = 0 pattern_str = r'([0-9]+)([smh]+)' pattern = re.compile(pattern_str) - matches = pattern.findall(duration_str) + matches = pattern.findall(duration_s) if not len(matches): - raise Exception("Invalid duration {}".format(duration_str)) + raise Exception("Invalid duration {}".format(duration_s)) for v, u in matches: total += int(v) * duration_u[u] @@ -146,19 +146,20 @@ def set_backupstore_setting_by_api(client, credential_secret="", poll_interval=300): bt = client.by_id_backupTarget(DEFAULT_BACKUPTARGET) - bt = bt.backupTargetUpdate(name=DEFAULT_BACKUPTARGET, - backupTargetURL=url, - credentialSecret=credential_secret, - pollInterval=str(poll_interval)) + bt.backupTargetUpdate(name=DEFAULT_BACKUPTARGET, + backupTargetURL=url, + credentialSecret=credential_secret, + pollInterval=str(poll_interval)) - for _ in range(NODE_UPDATE_RETRY_COUNT): + for _ in range(RETRY_COUNTS_SHORT): bt = client.by_id_backupTarget(DEFAULT_BACKUPTARGET) - p_interval = from_k8s_duration_str_to_seconds(str(bt.pollInterval)) - if bt.backupTargetURL == url and \ - bt.credentialSecret == credential_secret and \ - p_interval == poll_interval: + p_interval = from_k8s_duration_to_seconds(str(bt.pollInterval)) + bt_url = str(bt.backupTargetURL) + bt_secret = str(bt.credentialSecret) + if bt_url == url and bt_secret == credential_secret and \ + p_interval == int(poll_interval): break - time.sleep(NODE_UPDATE_RETRY_INTERVAL) + time.sleep(RETRY_INTERVAL) def set_backupstore_invalid(client): @@ -221,7 +222,7 @@ def set_backupstore_url(client, url): except Exception as e: if SETTING_BACKUP_TARGET_NOT_SUPPORTED in e.error.message: bt = client.by_id_backupTarget(DEFAULT_BACKUPTARGET) - poll_interval = from_k8s_duration_str_to_seconds(bt.pollInterval) + poll_interval = from_k8s_duration_to_seconds(bt.pollInterval) set_backupstore_setting_by_api( client, url=url, @@ -233,13 +234,13 @@ def set_backupstore_url(client, url): def set_backupstore_url_by_setting(client, url): backup_target_setting = client.by_id_setting(SETTING_BACKUP_TARGET) - for _ in range(NODE_UPDATE_RETRY_COUNT): + for _ in range(RETRY_COUNTS_SHORT): try: backup_target_setting = client.update(backup_target_setting, value=url) except Exception as e: if object_has_been_modified in str(e.error.message): - time.sleep(NODE_UPDATE_RETRY_INTERVAL) + time.sleep(RETRY_INTERVAL) continue print(e) raise e @@ -255,7 +256,7 @@ def set_backupstore_credential_secret(client, credential_secret): if SETTING_BACKUP_TARGET_CREDENTIAL_SECRET_NOT_SUPPORTED \ in e.error.message: bt = client.by_id_backupTarget(DEFAULT_BACKUPTARGET) - poll_interval = from_k8s_duration_str_to_seconds(bt.pollInterval) + poll_interval = from_k8s_duration_to_seconds(bt.pollInterval) set_backupstore_setting_by_api( client, url=bt.backupTargetURL, @@ -268,13 +269,13 @@ def set_backupstore_credential_secret(client, credential_secret): def set_backupstore_credential_secret_by_setting(client, credential_secret): backup_target_credential_setting = client.by_id_setting( SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) - for _ in range(NODE_UPDATE_RETRY_COUNT): + for _ in range(RETRY_COUNTS_SHORT): try: backup_target_credential_setting = client.update( backup_target_credential_setting, value=credential_secret) except Exception as e: if object_has_been_modified in str(e.error.message): - time.sleep(NODE_UPDATE_RETRY_INTERVAL) + time.sleep(RETRY_INTERVAL) continue print(e) raise @@ -302,13 +303,13 @@ def set_backupstore_poll_interval(client, poll_interval): def set_backupstore_poll_interval_by_setting(client, poll_interval): backup_store_poll_interval_setting = client.by_id_setting( SETTING_BACKUPSTORE_POLL_INTERVAL) - for _ in range(NODE_UPDATE_RETRY_COUNT): + for _ in range(RETRY_COUNTS_SHORT): try: backup_target_poll_interal_setting = client.update( backup_store_poll_interval_setting, value=poll_interval) except Exception as e: if object_has_been_modified in str(e.error.message): - time.sleep(NODE_UPDATE_RETRY_INTERVAL) + time.sleep(RETRY_INTERVAL) continue print(e) raise @@ -498,7 +499,7 @@ def backupstore_get_secret(client): in e.error.message: bt = client.by_id_backupTarget(DEFAULT_BACKUPTARGET) return bt.credentialSecret - else: + else: raise e diff --git a/manager/integration/tests/test_ha.py b/manager/integration/tests/test_ha.py index f214a2a4a8..661580795d 100644 --- a/manager/integration/tests/test_ha.py +++ b/manager/integration/tests/test_ha.py @@ -1374,7 +1374,7 @@ def test_restore_volume_with_invalid_backupstore(client, volume_name, backupstor if SETTING_BACKUP_TARGET_NOT_SUPPORTED in e.error.message: set_backupstore_url(client, invalid_backup_target_url) else: - raise e + raise e # make fromBackup URL consistent to the the invalid target URL url = invalid_backup_target_url + b.url.split("?")[1]