From d4e1c41f5852e3b810b8df1054964c6230196005 Mon Sep 17 00:00:00 2001 From: James Lu Date: Thu, 19 Dec 2024 11:22:04 +0800 Subject: [PATCH] feat(backup): fix test cases for multiple backup stores ref: longhorn/longhorn 5411 Signed-off-by: James Lu --- manager/integration/tests/backupstore.py | 54 ++++++++- manager/integration/tests/common.py | 33 ++--- manager/integration/tests/test_basic.py | 113 +++++++++--------- manager/integration/tests/test_csi.py | 2 +- .../integration/tests/test_csi_snapshotter.py | 2 + manager/integration/tests/test_kubernetes.py | 6 +- manager/integration/tests/test_metric.py | 10 +- .../integration/tests/test_recurring_job.py | 46 ++++--- manager/integration/tests/test_statefulset.py | 2 +- .../tests/test_system_backup_restore.py | 7 +- 10 files changed, 168 insertions(+), 107 deletions(-) diff --git a/manager/integration/tests/backupstore.py b/manager/integration/tests/backupstore.py index b938636d85..f927b0a168 100644 --- a/manager/integration/tests/backupstore.py +++ b/manager/integration/tests/backupstore.py @@ -14,6 +14,8 @@ from common import SETTING_BACKUP_TARGET_CREDENTIAL_SECRET from common import SETTING_BACKUPSTORE_POLL_INTERVAL from common import LONGHORN_NAMESPACE +from common import NODE_UPDATE_RETRY_COUNT +from common import NODE_UPDATE_RETRY_INTERVAL from common import cleanup_all_volumes from common import is_backupTarget_s3 from common import is_backupTarget_nfs @@ -31,6 +33,9 @@ BACKUPSTORE_BV_PREFIX = "/backupstore/volumes/" BACKUPSTORE_LOCK_DURATION = 150 +DEFAULT_BACKUPTARGET = "default" +object_has_been_modified = "the object has been modified; " + \ + "please apply your changes to the latest version and try again" TEMP_FILE_PATH = "/tmp/temp_file" @@ -91,6 +96,13 @@ def reset_backupstore_setting(client): SETTING_BACKUPSTORE_POLL_INTERVAL) client.update(backup_store_poll_interval, value="300") + for _ in range(NODE_UPDATE_RETRY_COUNT): + bt = client.by_id_backupTarget(DEFAULT_BACKUPTARGET) + if bt.backupTargetURL == "" and \ + bt.credentialSecret == "" and bt.pollInterval == "5m0s": + break + time.sleep(NODE_UPDATE_RETRY_INTERVAL) + def set_backupstore_invalid(client): poll_interval = get_backupstore_poll_interval() @@ -148,24 +160,54 @@ def set_backupstore_azurite(client): def set_backupstore_url(client, url): backup_target_setting = client.by_id_setting(SETTING_BACKUP_TARGET) - backup_target_setting = client.update(backup_target_setting, - value=url) + for _ in range(NODE_UPDATE_RETRY_COUNT): + try: + backup_target_setting = client.update(backup_target_setting, + value=url) + except Exception as e: + if object_has_been_modified in str(e.error.message): + time.sleep(NODE_UPDATE_RETRY_INTERVAL) + continue + print(e) + raise + else: + break assert backup_target_setting.value == url def set_backupstore_credential_secret(client, credential_secret): backup_target_credential_setting = client.by_id_setting( SETTING_BACKUP_TARGET_CREDENTIAL_SECRET) - backup_target_credential_setting = client.update( - backup_target_credential_setting, value=credential_secret) + for _ in range(NODE_UPDATE_RETRY_COUNT): + try: + backup_target_credential_setting = client.update( + backup_target_credential_setting, value=credential_secret) + except Exception as e: + if object_has_been_modified in str(e.error.message): + time.sleep(NODE_UPDATE_RETRY_INTERVAL) + continue + print(e) + raise + else: + break assert backup_target_credential_setting.value == credential_secret def set_backupstore_poll_interval(client, poll_interval): backup_store_poll_interval_setting = client.by_id_setting( SETTING_BACKUPSTORE_POLL_INTERVAL) - backup_target_poll_interal_setting = client.update( - backup_store_poll_interval_setting, value=poll_interval) + for _ in range(NODE_UPDATE_RETRY_COUNT): + try: + backup_target_poll_interal_setting = client.update( + backup_store_poll_interval_setting, value=poll_interval) + except Exception as e: + if object_has_been_modified in str(e.error.message): + time.sleep(NODE_UPDATE_RETRY_INTERVAL) + continue + print(e) + raise + else: + break assert backup_target_poll_interal_setting.value == poll_interval diff --git a/manager/integration/tests/common.py b/manager/integration/tests/common.py index 9101b49369..5293cd4ead 100644 --- a/manager/integration/tests/common.py +++ b/manager/integration/tests/common.py @@ -516,16 +516,15 @@ def wait_for_backup_count(backup_volume, number, retry_counts=120): assert ok -def delete_backup(client, volume_name, backup_name): - backup_volume = client.by_id_backupVolume(volume_name) - backup_volume.backupDelete(name=backup_name) - wait_for_backup_delete(client, volume_name, backup_name) +def delete_backup(client, bv, backup_name): + bv.backupDelete(name=backup_name) + wait_for_backup_delete(client, bv.volumeName, backup_name) -def delete_backup_volume(client, volume_name): - bv = client.by_id_backupVolume(volume_name) +def delete_backup_volume(client, backup_volume_name): + bv = client.by_id_backupVolume(backup_volume_name) client.delete(bv) - wait_for_backup_volume_delete(client, volume_name) + wait_for_backup_volume_delete(client, backup_volume_name) def delete_backup_backing_image(client, backing_image_name): @@ -3219,11 +3218,13 @@ def check_volume_endpoint(v): return endpoint -def find_backup_volume(client, volume_name): - bvs = client.list_backupVolume() - for bv in bvs: - if bv.name == volume_name and bv.created != "": - return bv +def find_backup_volume(client, volume_name, retry=1): + for _ in range(retry): + bvs = client.list_backupVolume() + for bv in bvs: + if bv.volumeName == volume_name and bv.created != "": + return bv + time.sleep(RETRY_BACKUP_INTERVAL) return None @@ -3952,9 +3953,9 @@ def is_backupTarget_azurite(s): return s.startswith("azblob://") -def wait_for_backup_volume(client, vol_name, backing_image=""): +def wait_for_backup_volume(client, bv_name, backing_image=""): for _ in range(RETRY_BACKUP_COUNTS): - bv = client.by_id_backupVolume(vol_name) + bv = client.by_id_backupVolume(bv_name) if bv is not None: if backing_image == "": break @@ -3962,7 +3963,7 @@ def wait_for_backup_volume(client, vol_name, backing_image=""): and bv.backingImageChecksum != "": break time.sleep(RETRY_BACKUP_INTERVAL) - assert bv is not None, "failed to find backup volume " + vol_name + assert bv is not None, "failed to find backup volume " + bv_name def wait_for_backup_target_available(client, available): @@ -4877,7 +4878,7 @@ def wait_for_volume_restoration_start(client, volume_name, backup_name, started = True if started: break - time.sleep(RETRY_INTERVAL) + time.sleep(RETRY_INTERVAL_SHORT) assert started return status.replica diff --git a/manager/integration/tests/test_basic.py b/manager/integration/tests/test_basic.py index 1c316713b3..9b00ed78af 100644 --- a/manager/integration/tests/test_basic.py +++ b/manager/integration/tests/test_basic.py @@ -625,7 +625,7 @@ def backup_failure_predicate(b): "allowScheduling", True) # delete the old backup - delete_backup(client, bv.name, b.name) + delete_backup(client, bv, b.name) volume = wait_for_volume_status(client, volume_name, "lastBackup", "") assert volume.lastBackupAt == "" @@ -634,7 +634,7 @@ def backup_failure_predicate(b): bv, b, _, _ = create_backup(client, volume_name) # delete the new backup - delete_backup(client, bv.name, b.name) + delete_backup(client, bv, b.name) volume = wait_for_volume_status(client, volume_name, "lastBackup", "") assert volume.lastBackupAt == "" @@ -694,19 +694,19 @@ def test_backup_block_deletion(set_random_backupstore, client, core_api, volume_ 'len': 2 * BACKUP_BLOCK_SIZE, 'content': common.generate_random_data(2 * BACKUP_BLOCK_SIZE)} - bv0, backup0, _, _ = create_backup(client, volume_name, data0) + _, backup0, _, _ = create_backup(client, volume_name, data0) data1 = {'pos': 0, 'len': BACKUP_BLOCK_SIZE, 'content': common.generate_random_data(BACKUP_BLOCK_SIZE)} - bv1, backup1, _, _ = create_backup(client, volume_name, data1) + _, backup1, _, _ = create_backup(client, volume_name, data1) data2 = {'pos': 0, 'len': BACKUP_BLOCK_SIZE, 'content': common.generate_random_data(BACKUP_BLOCK_SIZE)} - bv2, backup2, _, _ = create_backup(client, volume_name, data2) + backup_volume, backup2, _, _ = create_backup(client, volume_name, data2) backup_blocks_count = backupstore_count_backup_block_files(client, core_api, @@ -716,29 +716,29 @@ def test_backup_block_deletion(set_random_backupstore, client, core_api, volume_ bvs = client.list_backupVolume() for bv in bvs: - if bv['name'] == volume_name: + if bv['name'] == backup_volume.name: assert bv['dataStored'] == \ str(backup_blocks_count * BACKUP_BLOCK_SIZE) backupstore_create_dummy_in_progress_backup(client, core_api, volume_name) - delete_backup(client, volume_name, backup1.name) + delete_backup(client, backup_volume, backup1.name) assert backupstore_count_backup_block_files(client, core_api, volume_name) == 4 backupstore_delete_dummy_in_progress_backup(client, core_api, volume_name) - delete_backup(client, volume_name, backup0.name) + delete_backup(client, backup_volume, backup0.name) assert backupstore_count_backup_block_files(client, core_api, volume_name) == 2 - delete_backup(client, volume_name, backup2.name) + delete_backup(client, backup_volume, backup2.name) assert backupstore_count_backup_block_files(client, core_api, volume_name) == 0 - delete_backup_volume(client, volume_name) + delete_backup_volume(client, backup_volume.name) def test_dr_volume_activated_with_failed_replica(set_random_backupstore, client, core_api, volume_name): # NOQA @@ -858,7 +858,7 @@ def test_dr_volume_with_backup_block_deletion(set_random_backupstore, client, co data1 = {'pos': 0, 'len': BACKUP_BLOCK_SIZE, 'content': common.generate_random_data(BACKUP_BLOCK_SIZE)} - _, backup1, _, data1 = create_backup( + backup_volume, backup1, _, data1 = create_backup( client, volume_name, data1) backup_blocks_count = backupstore_count_backup_block_files(client, @@ -874,7 +874,7 @@ def test_dr_volume_with_backup_block_deletion(set_random_backupstore, client, co check_volume_last_backup(client, dr_vol_name, backup1.name) wait_for_backup_restore_completed(client, dr_vol_name, backup1.name) - delete_backup(client, volume_name, backup1.name) + delete_backup(client, backup_volume, backup1.name) assert backupstore_count_backup_block_files(client, core_api, volume_name) == 2 @@ -958,7 +958,7 @@ def test_dr_volume_with_backup_block_deletion_abort_during_backup_in_progress(se data1 = {'pos': 0, 'len': BACKUP_BLOCK_SIZE, 'content': common.generate_random_data(BACKUP_BLOCK_SIZE)} - _, backup1, _, data1 = create_backup( + bv, backup1, _, data1 = create_backup( client, volume_name, data1) backup_blocks_count = backupstore_count_backup_block_files(client, @@ -975,7 +975,7 @@ def test_dr_volume_with_backup_block_deletion_abort_during_backup_in_progress(se wait_for_backup_restore_completed(client, dr_vol_name, backup1.name) backupstore_create_dummy_in_progress_backup(client, core_api, volume_name) - delete_backup(client, volume_name, backup1.name) + delete_backup(client, bv, backup1.name) assert backupstore_count_backup_block_files(client, core_api, volume_name) == 3 @@ -1047,7 +1047,7 @@ def test_dr_volume_with_backup_and_backup_volume_deleted(set_random_backupstore, client, volume_name, data0) data1 = {'pos': 0, 'len': 2 * BACKUP_BLOCK_SIZE, 'content': common.generate_random_data(2 * BACKUP_BLOCK_SIZE)} - _, backup1, _, data1 = create_backup( + bv, backup1, _, data1 = create_backup( client, volume_name, data1) backup_blocks_count = backupstore_count_backup_block_files(client, @@ -1070,7 +1070,7 @@ def test_dr_volume_with_backup_and_backup_volume_deleted(set_random_backupstore, check_volume_last_backup(client, dr_vol_name2, backup1.name) wait_for_backup_restore_completed(client, dr_vol_name2, backup1.name) - delete_backup(client, volume_name, backup1.name) + delete_backup(client, bv, backup1.name) assert backupstore_count_backup_block_files(client, core_api, volume_name) == 2 @@ -1085,13 +1085,13 @@ def test_dr_volume_with_backup_and_backup_volume_deleted(set_random_backupstore, dr_vol1 = common.wait_for_volume_healthy(client, dr_vol_name1) check_volume_data(dr_vol1, data0, False) - delete_backup(client, volume_name, backup0.name) + delete_backup(client, bv, backup0.name) assert backupstore_count_backup_block_files(client, core_api, volume_name) == 0 check_volume_last_backup(client, dr_vol_name2, "") - delete_backup_volume(client, volume_name) + delete_backup_volume(client, bv.name) activate_standby_volume(client, dr_vol_name2) dr_vol2 = client.by_id_volume(dr_vol_name2) @@ -1156,12 +1156,12 @@ def verify_no_err(): verified_bvs = set() backup_volume_list = client.list_backupVolume() for bv in backup_volume_list: - if bv.name in (volume1_name, volume2_name): + if bv.name in (bv1.name, bv2.name): assert not bv['messages'] for b in bv.backupList().data: - if bv.name == volume1_name \ + if bv.name == bv1.name \ and b.name == backup1.name \ - or bv.name == volume2_name \ + or bv.name == bv2.name \ and b.name == backup2.name: verified_bvs.add(bv.name) if len(verified_bvs) == 2: @@ -1278,8 +1278,8 @@ def test_backup_metadata_deletion(set_random_backupstore, client, core_api, volu v2b2_new = v2bv.backupGet(name=v2b2.name) assert_backup_state(v2b2, v2b2_new) - delete_backup(client, volume1_name, v1b1.name) - delete_backup(client, volume2_name, v2b1.name) + delete_backup(client, v1bv, v1b1.name) + delete_backup(client, v2bv, v2b1.name) for i in range(RETRY_COUNTS): found1 = found2 = found3 = found4 = False @@ -1308,7 +1308,7 @@ def test_backup_metadata_deletion(set_random_backupstore, client, core_api, volu backupstore_delete_volume_cfg_file(client, core_api, volume2_name) - delete_backup(client, volume2_name, v2b2.name) + delete_backup(client, v2bv, v2b2.name) assert len(v2bv.backupList()) == 0 delete_backup_volume(client, v2bv.name) @@ -1346,7 +1346,7 @@ def test_backup_metadata_deletion(set_random_backupstore, client, core_api, volu assert_backup_state(v1b2, v1b2_new) assert v1b2_new.messages == v1b2.messages is None - delete_backup(client, volume1_name, v1b2.name) + delete_backup(client, v1bv, v1b2.name) for i in range(RETRY_COUNTS): if backupstore_count_backup_block_files(client, core_api, @@ -1440,7 +1440,7 @@ def backupstore_test(client, host_id, volname, size, compression_method): # NOQ volume = volume.detach() volume = common.wait_for_volume_detached(client, restore_name) - delete_backup(client, bv.name, b.name) + delete_backup(client, bv, b.name) volume = wait_for_volume_status(client, volume.name, "lastBackup", "") assert volume.lastBackupAt == "" @@ -1483,7 +1483,7 @@ def backup_labels_test(client, random_labels, volume_name, size=SIZE, backing_im assert len(backup.labels) == len(random_labels) + 1 assert random_labels["key"] == backup.labels["key"] assert "longhorn.io/volume-access-mode" in backup.labels.keys() - wait_for_backup_volume(client, volume_name, backing_image) + wait_for_backup_volume(client, bv.name, backing_image) @pytest.mark.coretest # NOQA @@ -1736,9 +1736,9 @@ def test_deleting_backup_volume(set_random_backupstore, client, volume_name): # volume = common.wait_for_volume_healthy(client, volume_name) create_backup(client, volume_name) - create_backup(client, volume_name) + bv, _, _, _ = create_backup(client, volume_name) - delete_backup_volume(client, volume_name) + delete_backup_volume(client, bv.name) cleanup_volume(client, volume) @@ -1844,14 +1844,14 @@ def test_listing_backup_volume(client, backing_image=""): # NOQA subprocess.check_output(cmd) subprocess.check_output(["sync"]) - bv1, b1 = common.find_backup(client, volume1_name, snap1.name) - common.delete_backup(client, volume1_name, b1.name) + bv1, b1 = find_backup(client, volume1_name, snap1.name) + delete_backup(client, bv1, b1.name) - bv2, b2 = common.find_backup(client, volume2_name, snap2.name) - common.delete_backup(client, volume2_name, b2.name) + bv2, b2 = find_backup(client, volume2_name, snap2.name) + delete_backup(client, bv2, b2.name) # corrupt backup for snap4 - bv4, b4 = common.find_backup(client, volume3_name, snap4.name) + bv4, b4 = find_backup(client, volume3_name, snap4.name) b4_cfg_name = "backup_" + b4["name"] + ".cfg" cmd = ["find", "/mnt/nfs", "-type", "d", "-name", volume3_name] v3_backup_path = subprocess.check_output(cmd).strip().decode('utf-8') @@ -1880,15 +1880,15 @@ def test_listing_backup_volume(client, backing_image=""): # NOQA os.rename(b4_tmp_cfg_path, b4_cfg_path) subprocess.check_output(["sync"]) - bv3, b3 = common.find_backup(client, volume3_name, snap3.name) - common.delete_backup(client, volume3_name, b3.name) - bv4, b4 = common.find_backup(client, volume3_name, snap4.name) - common.delete_backup(client, volume3_name, b4.name) - bv5, b5 = common.find_backup(client, volume3_name, snap5.name) - common.delete_backup(client, volume3_name, b5.name) + bv3, b3 = find_backup(client, volume3_name, snap3.name) + delete_backup(client, bv3, b3.name) + bv4, b4 = find_backup(client, volume3_name, snap4.name) + delete_backup(client, bv3, b4.name) + bv5, b5 = find_backup(client, volume3_name, snap5.name) + delete_backup(client, bv3, b5.name) - common.delete_backup_volume(client, volume3_name) - common.wait_for_backup_volume_delete(client, volume3_name) + delete_backup_volume(client, bv3.name) + common.wait_for_backup_volume_delete(client, bv3.name) volume1.detach() volume1 = common.wait_for_volume_detached(client, volume1_name) @@ -3238,13 +3238,13 @@ def test_backup_lock_deletion_during_restoration(set_random_backupstore, client, std_volume.snapshotBackup(name=snap1.name) wait_for_backup_completion(client, std_volume_name, snap1.name) - _, b = common.find_backup(client, std_volume_name, snap1.name) + bv, b = find_backup(client, std_volume_name, snap1.name) client.create_volume(name=restore_volume_name, fromBackup=b.url, numberOfReplicas=3, dataEngine=DATA_ENGINE) wait_for_volume_restoration_start(client, restore_volume_name, b.name) - backup_volume = client.by_id_backupVolume(std_volume_name) + backup_volume = client.by_id_backupVolume(bv.name) backup_volume.backupDelete(name=b.name) wait_for_volume_restoration_completed(client, restore_volume_name) @@ -3311,7 +3311,7 @@ def test_backup_lock_deletion_during_backup(set_random_backupstore, client, core snap1 = create_snapshot(client, std_volume_name) std_volume.snapshotBackup(name=snap1.name) wait_for_backup_completion(client, std_volume_name, snap1.name) - _, b1 = common.find_backup(client, std_volume_name, snap1.name) + bv, b1 = find_backup(client, std_volume_name, snap1.name) write_pod_volume_random_data(core_api, std_pod_name, "/data/test", common.DATA_SIZE_IN_MB_4) @@ -3321,18 +3321,18 @@ def test_backup_lock_deletion_during_backup(set_random_backupstore, client, core std_volume.snapshotBackup(name=snap2.name) wait_for_backup_to_start(client, std_volume_name, snapshot_name=snap2.name) - backup_volume = client.by_id_backupVolume(std_volume_name) + backup_volume = client.by_id_backupVolume(bv.name) backup_volume.backupDelete(name=b1.name) wait_for_backup_completion(client, std_volume_name, snap2.name, retry_count=600) wait_for_backup_delete(client, std_volume_name, b1.name) - _, b2 = common.find_backup(client, std_volume_name, snap2.name) + _, b2 = find_backup(client, std_volume_name, snap2.name) assert b2 is not None try: - _, b1 = common.find_backup(client, std_volume_name, snap1.name) + _, b1 = find_backup(client, std_volume_name, snap1.name) except AssertionError: b1 = None assert b1 is None @@ -3395,13 +3395,13 @@ def test_backup_lock_creation_during_deletion(set_random_backupstore, client, co snap1 = create_snapshot(client, std_volume_name) std_volume.snapshotBackup(name=snap1.name) wait_for_backup_completion(client, std_volume_name, snap1.name) - _, b1 = common.find_backup(client, std_volume_name, snap1.name) + bv, b1 = common.find_backup(client, std_volume_name, snap1.name) # create second snapshot snap2 = create_snapshot(client, std_volume_name) # delete first backup - backup_volume = client.by_id_backupVolume(std_volume_name) + backup_volume = client.by_id_backupVolume(bv.name) backup_volume.backupDelete(name=b1.name) # create second backup immediately @@ -3455,8 +3455,7 @@ def test_backup_lock_restoration_during_deletion(set_random_backupstore, client, std_volume.snapshotBackup(name=snap1.name) wait_for_backup_completion(client, std_volume_name, snap1.name) std_volume.snapshotBackup(name=snap1.name) - backup_volume = client.by_id_backupVolume(std_volume_name) - _, b1 = common.find_backup(client, std_volume_name, snap1.name) + backup_volume, b1 = find_backup(client, std_volume_name, snap1.name) write_pod_volume_random_data(core_api, std_pod_name, "/data/test2", 1500) @@ -3464,7 +3463,7 @@ def test_backup_lock_restoration_during_deletion(set_random_backupstore, client, std_volume.snapshotBackup(name=snap2.name) wait_for_backup_completion(client, std_volume_name, snap2.name, retry_count=1200) - _, b2 = common.find_backup(client, std_volume_name, snap2.name) + _, b2 = find_backup(client, std_volume_name, snap2.name) backup_volume.backupDelete(name=b2.name) @@ -3476,11 +3475,11 @@ def test_backup_lock_restoration_during_deletion(set_random_backupstore, client, wait_for_backup_delete(client, volume_name, b2.name) - _, b1 = common.find_backup(client, std_volume_name, snap1.name) + _, b1 = find_backup(client, std_volume_name, snap1.name) assert b1 is not None try: - _, b2 = common.find_backup(client, std_volume_name, snap2.name) + _, b2 = find_backup(client, std_volume_name, snap2.name) except AssertionError: b2 = None assert b2 is None @@ -4660,7 +4659,7 @@ def test_restore_basic(set_random_backupstore, client, core_api, volume_name, po delete_and_wait_pod(core_api, restore_pod_name) # Delete the 2nd backup - delete_backup(client, backup_volume.name, backup2.name) + delete_backup(client, backup_volume, backup2.name) # restore 3rd backup again restored_data_checksum3, output, restore_pod_name = \ @@ -5420,7 +5419,7 @@ def test_delete_backup_during_restoring_volume(set_random_backupstore, client): fromBackup=b.url, dataEngine=DATA_ENGINE) - delete_backup(client, bv.name, b.name) + delete_backup(client, bv, b.name) volume = wait_for_volume_status(client, vol_v1_name, "lastBackup", "") assert volume.lastBackupAt == "" diff --git a/manager/integration/tests/test_csi.py b/manager/integration/tests/test_csi.py index 58e4743577..57c46ff5f2 100644 --- a/manager/integration/tests/test_csi.py +++ b/manager/integration/tests/test_csi.py @@ -246,7 +246,7 @@ def backupstore_test(client, core_api, csi_pv, pvc, pod_make, pod_name, vol_name resp = read_volume_data(core_api, pod2_name) assert resp == test_data - delete_backup(client, bv.name, b.name) + delete_backup(client, bv, b.name) delete_and_wait_pod(core_api, pod2_name) client.delete(volume2) diff --git a/manager/integration/tests/test_csi_snapshotter.py b/manager/integration/tests/test_csi_snapshotter.py index df0a3f54e7..b57b485860 100644 --- a/manager/integration/tests/test_csi_snapshotter.py +++ b/manager/integration/tests/test_csi_snapshotter.py @@ -1365,6 +1365,7 @@ def test_csi_volumesnapshot_backing_image_basic(client, # NOQA pvc, # NOQA request, # NOQA volume_name, # NOQA + storage_class, # NOQA volumesnapshotclass, # NOQA volumesnapshot): # NOQA """ @@ -1488,6 +1489,7 @@ def test_csi_volumesnapshot_backing_image_basic(client, # NOQA csivolsnap, restore_pvc_name, restore_pvc_size, + storage_class, wait_for_restore=False) restore_pod = pod_make() diff --git a/manager/integration/tests/test_kubernetes.py b/manager/integration/tests/test_kubernetes.py index bf2f0164e7..3e86272475 100644 --- a/manager/integration/tests/test_kubernetes.py +++ b/manager/integration/tests/test_kubernetes.py @@ -585,13 +585,13 @@ def test_backup_kubernetes_status(set_random_backupstore, client, core_api, pod, snap = create_snapshot(client, volume_name) volume.snapshotBackup(name=snap.name) wait_for_backup_completion(client, volume_name, snap.name) - _, b = find_backup(client, volume_name, snap.name) + volbv, b = find_backup(client, volume_name, snap.name) # Check backup label status = loads(b.labels.get(KUBERNETES_STATUS_LABEL)) assert status == ks # Check backup volume label for _ in range(RETRY_COUNTS): - bv = client.by_id_backupVolume(volume_name) + bv = client.by_id_backupVolume(volbv.name) if bv is not None and bv.labels is not None: break time.sleep(RETRY_INTERVAL) @@ -629,7 +629,7 @@ def test_backup_kubernetes_status(set_random_backupstore, client, core_api, pod, assert restore.kubernetesStatus.lastPodRefAt == ks["lastPodRefAt"] assert restore.kubernetesStatus.lastPVCRefAt == ks["lastPVCRefAt"] - delete_backup(client, bv.name, b.name) + delete_backup(client, bv, b.name) client.delete(restore) wait_for_volume_delete(client, restore_name) delete_and_wait_pod(core_api, pod_name) diff --git a/manager/integration/tests/test_metric.py b/manager/integration/tests/test_metric.py index 763dd736f4..56beca5337 100644 --- a/manager/integration/tests/test_metric.py +++ b/manager/integration/tests/test_metric.py @@ -40,6 +40,7 @@ from common import wait_for_cron_job_count from common import create_backup from common import wait_for_backup_count +from common import find_backup_volume from common import delete_backup_volume from common import get_longhorn_api_client from common import remount_volume_read_only @@ -717,8 +718,7 @@ def test_metric_longhorn_backup(set_random_backupstore, client, core_api, batch_ 'len': data_size, 'content': generate_random_data(data_size)} write_volume_data(volume, backup_data) - create_backup(client, volume_name) - bv = client.by_id_backupVolume(volume_name) + bv, _, _, _ = create_backup(client, volume_name) wait_for_backup_count(bv, 1) # get the backup size. @@ -747,7 +747,7 @@ def test_metric_longhorn_backup(set_random_backupstore, client, core_api, batch_ 3) # delete the existing backup before creating a recurring backup job. - delete_backup_volume(client, volume_name) + delete_backup_volume(client, bv.name) # create a recurring backup job. recurring_jobs = { @@ -765,8 +765,8 @@ def test_metric_longhorn_backup(set_random_backupstore, client, core_api, batch_ wait_for_cron_job_count(batch_v1_api, 1) # wait for the recurring backup job to run. - time.sleep(60) - bv = client.by_id_backupVolume(volume_name) + time.sleep(90) + bv = find_backup_volume(client, volume_name) wait_for_backup_count(bv, 1) # get the recurring backup size. diff --git a/manager/integration/tests/test_recurring_job.py b/manager/integration/tests/test_recurring_job.py index 676b5cc528..711f7c7d81 100644 --- a/manager/integration/tests/test_recurring_job.py +++ b/manager/integration/tests/test_recurring_job.py @@ -53,6 +53,7 @@ from common import crash_engine_process_with_sigkill from common import find_backup +from common import find_backup_volume from common import wait_for_backup_volume from common import wait_for_backup_completion from common import wait_for_backup_count @@ -75,6 +76,7 @@ from common import JOB_LABEL from common import KUBERNETES_STATUS_LABEL from common import LONGHORN_NAMESPACE +from common import RETRY_COUNTS_SHORT from common import RETRY_BACKUP_COUNTS from common import RETRY_BACKUP_INTERVAL from common import SETTING_RECURRING_JOB_WHILE_VOLUME_DETACHED @@ -511,7 +513,7 @@ def recurring_job_labels_test(client, labels, volume_name, size=SIZE, backing_im wait_for_snapshot_count(volume, 2) # Verify the Labels on the actual Backup. - bv = client.by_id_backupVolume(volume_name) + bv = find_backup_volume(client, volume_name) wait_for_backup_count(bv, 1) backups = bv.backupList().data @@ -523,7 +525,7 @@ def recurring_job_labels_test(client, labels, volume_name, size=SIZE, backing_im # Longhorn will automatically add a label `longhorn.io/volume-access-mode` # to a newly created backup assert len(b.labels) == len(labels) + 2 - wait_for_backup_volume(client, volume_name, backing_image) + wait_for_backup_volume(client, bv.name, backing_image) @pytest.mark.v2_volume_test # NOQA @@ -584,7 +586,7 @@ def test_recurring_job_kubernetes_status(set_random_backupstore, client, core_ap wait_for_snapshot_count(volume, 2) # Verify the Labels on the actual Backup. - bv = client.by_id_backupVolume(volume_name) + bv = find_backup_volume(client, volume_name) backups = bv.backupList().data wait_for_backup_count(bv, 1) @@ -781,7 +783,7 @@ def test_recurring_jobs_allow_detached_volume(set_random_backupstore, client, co wait_for_backup_completion(client, volume.name) for _ in range(4): - bv = client.by_id_backupVolume(volume.name) + bv = find_backup_volume(client, volume.name) wait_for_backup_count(bv, 1) time.sleep(30) @@ -1070,12 +1072,16 @@ def test_recurring_job_groups(set_random_backupstore, client, batch_v1_api): # wait_for_snapshot_count(volume1, 3) # volume-head,snapshot,backup-snapshot wait_for_snapshot_count(volume2, 2) # volume-head,snapshot - wait_for_backup_count(client.by_id_backupVolume(volume1_name), 1) + wait_for_backup_count(find_backup_volume(client, + volume1_name, + retry=RETRY_COUNTS_SHORT), 1) backup_created = True try: - wait_for_backup_count(client.by_id_backupVolume(volume2_name), 1, + wait_for_backup_count(find_backup_volume(client, volume2_name), + 1, retry_counts=60) - except AssertionError: + # AttributeError: BackupVolume is 'NoneType' object has no attribute 'backupList' + except (AssertionError, AttributeError): backup_created = False assert not backup_created @@ -1575,7 +1581,10 @@ def test_recurring_job_multiple_volumes(set_random_backupstore, client, batch_v1 write_volume_random_data(volume1) wait_for_snapshot_count(volume1, 2) - wait_for_backup_count(client.by_id_backupVolume(volume1_name), 1) + wait_for_backup_count(find_backup_volume(client, + volume1_name, + retry=RETRY_COUNTS_SHORT), + 1) volume2_name = "test-job-2" client.create_volume(name=volume2_name, size=SIZE, @@ -1587,7 +1596,10 @@ def test_recurring_job_multiple_volumes(set_random_backupstore, client, batch_v1 write_volume_random_data(volume2) wait_for_snapshot_count(volume2, 2) - wait_for_backup_count(client.by_id_backupVolume(volume2_name), 1) + wait_for_backup_count(find_backup_volume(client, + volume2_name, + retry=RETRY_COUNTS_SHORT), + 1) volume2.recurringJobAdd(name=back2, isGroup=False) wait_for_volume_recurring_job_update(volume1, @@ -1598,8 +1610,8 @@ def test_recurring_job_multiple_volumes(set_random_backupstore, client, batch_v1 write_volume_random_data(volume1) write_volume_random_data(volume2) time.sleep(70 - WRITE_DATA_INTERVAL) - wait_for_backup_count(client.by_id_backupVolume(volume2_name), 2) - wait_for_backup_count(client.by_id_backupVolume(volume1_name), 1) + wait_for_backup_count(find_backup_volume(client, volume2_name), 2) + wait_for_backup_count(find_backup_volume(client, volume1_name), 1) @pytest.mark.v2_volume_test # NOQA @@ -1997,15 +2009,19 @@ def test_recurring_job_backup(set_random_backupstore, client, batch_v1_api): # write_volume_random_data(volume1) write_volume_random_data(volume2) time.sleep(60 - WRITE_DATA_INTERVAL) - wait_for_backup_count(client.by_id_backupVolume(volume1_name), 1) - wait_for_backup_count(client.by_id_backupVolume(volume2_name), 1) + wait_for_backup_count( + find_backup_volume(client, volume1_name, retry=RETRY_COUNTS_SHORT), 1) + wait_for_backup_count( + find_backup_volume(client, volume2_name, retry=RETRY_COUNTS_SHORT), 1) # 2nd job write_volume_random_data(volume1) write_volume_random_data(volume2) time.sleep(60 - WRITE_DATA_INTERVAL) - wait_for_backup_count(client.by_id_backupVolume(volume1_name), 2) - wait_for_backup_count(client.by_id_backupVolume(volume2_name), 2) + wait_for_backup_count( + find_backup_volume(client, volume1_name, retry=RETRY_COUNTS_SHORT), 2) + wait_for_backup_count( + find_backup_volume(client, volume2_name, retry=RETRY_COUNTS_SHORT), 2) @pytest.mark.v2_volume_test # NOQA diff --git a/manager/integration/tests/test_statefulset.py b/manager/integration/tests/test_statefulset.py index 428119cfae..ce516c9c9d 100644 --- a/manager/integration/tests/test_statefulset.py +++ b/manager/integration/tests/test_statefulset.py @@ -57,7 +57,7 @@ def create_and_test_backups(api, cli, pod_info): for i in range(DEFAULT_BACKUP_TIMEOUT): backup_volumes = cli.list_backupVolume() for bv in backup_volumes: - if bv.name == pod['pv_name']: + if bv.volumeName == pod['pv_name']: found = True break if found: diff --git a/manager/integration/tests/test_system_backup_restore.py b/manager/integration/tests/test_system_backup_restore.py index 1299363ad2..cab62f2d44 100644 --- a/manager/integration/tests/test_system_backup_restore.py +++ b/manager/integration/tests/test_system_backup_restore.py @@ -9,6 +9,7 @@ from common import cleanup_volume from common import create_and_check_volume from common import create_backup +from common import find_backup_volume from common import get_self_host_id from common import system_backups_cleanup from common import system_backup_random_name @@ -255,7 +256,7 @@ def create_system_backup_and_assert_volume_backup_count(count): system_backup_wait_for_state("Ready", system_backup_name, client) - backup_volume = client.by_id_backupVolume(volume_name) + backup_volume = find_backup_volume(client, volume_name) wait_for_backup_count(backup_volume, count) create_system_backup_and_assert_volume_backup_count(1) @@ -311,7 +312,7 @@ def test_system_backup_with_volume_backup_policy_always(client, volume_name, set system_backup_wait_for_state("Ready", system_backup_name, client) - backup_volume = client.by_id_backupVolume(volume_name) + backup_volume = find_backup_volume(client, volume_name) wait_for_backup_count(backup_volume, 2) system_backups_cleanup(client) @@ -321,7 +322,7 @@ def test_system_backup_with_volume_backup_policy_always(client, volume_name, set system_backup_wait_for_state("Ready", system_backup_name, client) - backup_volume = client.by_id_backupVolume(volume_name) + backup_volume = find_backup_volume(client, volume_name) wait_for_backup_count(backup_volume, 3)