feat(backup): fix test cases for multiple backup stores

ref: longhorn/longhorn 5411

Signed-off-by: James Lu <[email protected]>

manager/integration/tests/backupstore.py

@@ -14,6 +14,8 @@
 from common import SETTING_BACKUP_TARGET_CREDENTIAL_SECRET
 from common import SETTING_BACKUPSTORE_POLL_INTERVAL
 from common import LONGHORN_NAMESPACE
+from common import NODE_UPDATE_RETRY_COUNT
+from common import NODE_UPDATE_RETRY_INTERVAL
 from common import cleanup_all_volumes
 from common import is_backupTarget_s3
 from common import is_backupTarget_nfs
@@ -31,6 +33,9 @@
 
 BACKUPSTORE_BV_PREFIX = "/backupstore/volumes/"
 BACKUPSTORE_LOCK_DURATION = 150
+DEFAULT_BACKUPTARGET = "default"
+object_has_been_modified = "the object has been modified; " + \
+    "please apply your changes to the latest version and try again"
 
 TEMP_FILE_PATH = "/tmp/temp_file"
 
@@ -91,6 +96,13 @@ def reset_backupstore_setting(client):
         SETTING_BACKUPSTORE_POLL_INTERVAL)
     client.update(backup_store_poll_interval, value="300")
 
+    for _ in range(NODE_UPDATE_RETRY_COUNT):
+        bt = client.by_id_backupTarget(DEFAULT_BACKUPTARGET)
+        if bt.backupTargetURL == "" and \
+                bt.credentialSecret == "" and bt.pollInterval == "5m0s":
+            break
+        time.sleep(NODE_UPDATE_RETRY_INTERVAL)
+
 
 def set_backupstore_invalid(client):
     poll_interval = get_backupstore_poll_interval()
@@ -148,24 +160,54 @@ def set_backupstore_azurite(client):
 
 def set_backupstore_url(client, url):
     backup_target_setting = client.by_id_setting(SETTING_BACKUP_TARGET)
-    backup_target_setting = client.update(backup_target_setting,
-                                          value=url)
+    for _ in range(NODE_UPDATE_RETRY_COUNT):
+        try:
+            backup_target_setting = client.update(backup_target_setting,
+                                                  value=url)
+        except Exception as e:
+            if object_has_been_modified in str(e.error.message):
+                time.sleep(NODE_UPDATE_RETRY_INTERVAL)
+                continue
+            print(e)
+            raise
+        else:
+            break
     assert backup_target_setting.value == url
 
 
 def set_backupstore_credential_secret(client, credential_secret):
     backup_target_credential_setting = client.by_id_setting(
         SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
-    backup_target_credential_setting = client.update(
-        backup_target_credential_setting, value=credential_secret)
+    for _ in range(NODE_UPDATE_RETRY_COUNT):
+        try:
+            backup_target_credential_setting = client.update(
+                backup_target_credential_setting, value=credential_secret)
+        except Exception as e:
+            if object_has_been_modified in str(e.error.message):
+                time.sleep(NODE_UPDATE_RETRY_INTERVAL)
+                continue
+            print(e)
+            raise
+        else:
+            break
     assert backup_target_credential_setting.value == credential_secret
 
 
 def set_backupstore_poll_interval(client, poll_interval):
     backup_store_poll_interval_setting = client.by_id_setting(
         SETTING_BACKUPSTORE_POLL_INTERVAL)
-    backup_target_poll_interal_setting = client.update(
-        backup_store_poll_interval_setting, value=poll_interval)
+    for _ in range(NODE_UPDATE_RETRY_COUNT):
+        try:
+            backup_target_poll_interal_setting = client.update(
+                backup_store_poll_interval_setting, value=poll_interval)
+        except Exception as e:
+            if object_has_been_modified in str(e.error.message):
+                time.sleep(NODE_UPDATE_RETRY_INTERVAL)
+                continue
+            print(e)
+            raise
+        else:
+            break
     assert backup_target_poll_interal_setting.value == poll_interval
 
 

manager/integration/tests/common.py

@@ -516,16 +516,15 @@ def wait_for_backup_count(backup_volume, number, retry_counts=120):
     assert ok
 
 
-def delete_backup(client, volume_name, backup_name):
-    backup_volume = client.by_id_backupVolume(volume_name)
-    backup_volume.backupDelete(name=backup_name)
-    wait_for_backup_delete(client, volume_name, backup_name)
+def delete_backup(client, bv, backup_name):
+    bv.backupDelete(name=backup_name)
+    wait_for_backup_delete(client, bv.volumeName, backup_name)
 
 
-def delete_backup_volume(client, volume_name):
-    bv = client.by_id_backupVolume(volume_name)
+def delete_backup_volume(client, backup_volume_name):
+    bv = client.by_id_backupVolume(backup_volume_name)
     client.delete(bv)
-    wait_for_backup_volume_delete(client, volume_name)
+    wait_for_backup_volume_delete(client, backup_volume_name)
 
 
 def delete_backup_backing_image(client, backing_image_name):
@@ -3219,11 +3218,13 @@ def check_volume_endpoint(v):
     return endpoint
 
 
-def find_backup_volume(client, volume_name):
-    bvs = client.list_backupVolume()
-    for bv in bvs:
-        if bv.name == volume_name and bv.created != "":
-            return bv
+def find_backup_volume(client, volume_name, retry=1):
+    for _ in range(retry):
+        bvs = client.list_backupVolume()
+        for bv in bvs:
+            if bv.volumeName == volume_name and bv.created != "":
+                return bv
+        time.sleep(RETRY_BACKUP_INTERVAL)
     return None
 
 
@@ -3952,17 +3953,17 @@ def is_backupTarget_azurite(s):
     return s.startswith("azblob://")
 
 
-def wait_for_backup_volume(client, vol_name, backing_image=""):
+def wait_for_backup_volume(client, bv_name, backing_image=""):
     for _ in range(RETRY_BACKUP_COUNTS):
-        bv = client.by_id_backupVolume(vol_name)
+        bv = client.by_id_backupVolume(bv_name)
         if bv is not None:
             if backing_image == "":
                 break
             if bv.backingImageName == backing_image \
                     and bv.backingImageChecksum != "":
                 break
         time.sleep(RETRY_BACKUP_INTERVAL)
-    assert bv is not None, "failed to find backup volume " + vol_name
+    assert bv is not None, "failed to find backup volume " + bv_name
 
 
 def wait_for_backup_target_available(client, available):
@@ -4877,7 +4878,7 @@ def wait_for_volume_restoration_start(client, volume_name, backup_name,
             started = True
         if started:
             break
-        time.sleep(RETRY_INTERVAL)
+        time.sleep(RETRY_INTERVAL_SHORT)
     assert started
     return status.replica