Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(backup): multiple backupstores support #1600

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

feat(backup): multiple backupstores support #1600

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ee2e760
feat(backup): multiple backup store
mantissahz Dec 15, 2023
8074f80
feat(backup): test case of multiple backup store
mantissahz Dec 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
203 changes: 140 additions & 63 deletions manager/integration/tests/backupstore.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,9 @@
from minio.error import ResponseError
from urllib.parse import urlparse

from common import SETTING_BACKUP_TARGET
from common import SETTING_BACKUP_TARGET_CREDENTIAL_SECRET
from common import SETTING_BACKUPSTORE_POLL_INTERVAL
from common import LONGHORN_NAMESPACE
from common import DEFAULT_BACKUPSTORE_NAME
from common import DEFAULT_BACKUPSTORE_POLL_INTERVAL
from common import cleanup_all_volumes
from common import is_backupTarget_s3
from common import is_backupTarget_nfs
Expand All @@ -24,6 +23,9 @@
from common import get_backupstores
from common import system_backups_cleanup
from common import get_custom_object_api_client
from common import golang_duration_str_to_seconds
from common import wait_for_backup_target_status
from common import wait_for_backup_target_creation

BACKUPSTORE_BV_PREFIX = "/backupstore/volumes/"
BACKUPSTORE_LOCK_DURATION = 150
Expand Down Expand Up @@ -57,6 +59,10 @@ def backupstore_nfs(client):

@pytest.fixture(params=BACKUPSTORE)
def set_random_backupstore(request, client):
bts = client.list_backup_target()
if len(bts) == 0:
create_default_backup_target(client)

if request.param == "s3":
set_backupstore_s3(client)
elif request.param == "nfs":
Expand All @@ -73,15 +79,33 @@ def set_random_backupstore(request, client):
umount_nfs_backupstore(client)


def create_default_backup_target(client):
client.create_backupTarget(
Name=DEFAULT_BACKUPSTORE_NAME,
BackupTargetURL="",
CredentialSecret="",
PollInterval=str(DEFAULT_BACKUPSTORE_POLL_INTERVAL),
Default=True,
ReadOnly=False
)


def create_backup_target(client, backup_target_name):
client.create_backupTarget(
Name=backup_target_name,
BackupTargetURL="",
CredentialSecret="",
PollInterval=str(DEFAULT_BACKUPSTORE_POLL_INTERVAL),
Default=True,
ReadOnly=False
)
wait_for_backup_target_creation(client, backup_target_name)


def reset_backupstore_setting(client):
backup_target_setting = client.by_id_setting(SETTING_BACKUP_TARGET)
client.update(backup_target_setting, value="")
backup_target_credential_setting = client.by_id_setting(
SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
client.update(backup_target_credential_setting, value="")
backup_store_poll_interval = client.by_id_setting(
SETTING_BACKUPSTORE_POLL_INTERVAL)
client.update(backup_store_poll_interval, value="300")
set_backupstore_poll_interval(client, DEFAULT_BACKUPSTORE_POLL_INTERVAL)
set_backupstore_credential_secret(client, "")
set_backupstore_url(client, "")


def set_backupstore_invalid(client):
Expand All @@ -91,56 +115,106 @@ def set_backupstore_invalid(client):
set_backupstore_poll_interval(client, poll_interval)


def set_backupstore_s3(client):
def set_backupstore_s3(client, backup_target_name=""):
backupstores = get_backupstore_url()
poll_interval = get_backupstore_poll_interval()
for backupstore in backupstores:
if is_backupTarget_s3(backupstore):
backupsettings = backupstore.split("$")
set_backupstore_url(client, backupsettings[0])
set_backupstore_credential_secret(client, backupsettings[1])
set_backupstore_poll_interval(client, poll_interval)
set_backupstore_url(client, backupsettings[0], backup_target_name)
set_backupstore_credential_secret(client,
backupsettings[1],
backup_target_name)
set_backupstore_poll_interval(client,
poll_interval,
backup_target_name)
break


def set_backupstore_nfs(client):
def set_backupstore_nfs(client, backup_target_name=""):
backupstores = get_backupstore_url()
poll_interval = get_backupstore_poll_interval()
for backupstore in backupstores:
if is_backupTarget_nfs(backupstore):
set_backupstore_url(client, backupstore)
set_backupstore_credential_secret(client, "")
set_backupstore_poll_interval(client, poll_interval)
set_backupstore_url(client, backupstore, backup_target_name)
set_backupstore_credential_secret(client, "", backup_target_name)
set_backupstore_poll_interval(client,
poll_interval,
backup_target_name)
break


def set_backupstore_url(client, url):
backup_target_setting = client.by_id_setting(SETTING_BACKUP_TARGET)
backup_target_setting = client.update(backup_target_setting,
value=url)
assert backup_target_setting.value == url


def set_backupstore_credential_secret(client, credential_secret):
backup_target_credential_setting = client.by_id_setting(
SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)
backup_target_credential_setting = client.update(
backup_target_credential_setting, value=credential_secret)
assert backup_target_credential_setting.value == credential_secret


def set_backupstore_poll_interval(client, poll_interval):
backup_store_poll_interval_setting = client.by_id_setting(
SETTING_BACKUPSTORE_POLL_INTERVAL)
backup_target_poll_interal_setting = client.update(
backup_store_poll_interval_setting, value=poll_interval)
assert backup_target_poll_interal_setting.value == poll_interval
def set_backupstore_url(client, url, backup_target_name=""):
bt_name = DEFAULT_BACKUPSTORE_NAME
if backup_target_name != "":
bt_name = backup_target_name
bt = client.by_id_backup_target(bt_name)
bt_poll_interval = golang_duration_str_to_seconds(bt.pollInterval)
bt = client.update(bt,
name=bt_name,
backupTargetURL=url,
credentialSecret=bt.credentialSecret,
default=bt.default,
pollInterval=str(bt_poll_interval),
readOnly=bt.readOnly)
assert bt.backupTargetURL == url


def set_backupstore_credential_secret(client, credential_secret, backup_target_name=""): # NOQA
bt_name = DEFAULT_BACKUPSTORE_NAME
if backup_target_name != "":
bt_name = backup_target_name
bt = client.by_id_backup_target(bt_name)
bt_poll_interval = golang_duration_str_to_seconds(bt.pollInterval)
bt = client.update(bt,
name=bt_name,
credentialSecret=credential_secret,
backupTargetURL=bt.backupTargetURL,
default=bt.default,
pollInterval=str(bt_poll_interval),
readOnly=bt.readOnly)
assert bt.credentialSecret == credential_secret


def set_backupstore_poll_interval(client, poll_interval, backup_target_name=""): # NOQA
bt_name = DEFAULT_BACKUPSTORE_NAME
if backup_target_name != "":
bt_name = backup_target_name
bt = client.by_id_backup_target(bt_name)
bt = client.update(bt,
name=bt_name,
pollInterval=str(poll_interval),
backupTargetURL=bt.backupTargetURL,
credentialSecret=bt.credentialSecret,
default=bt.default,
readOnly=bt.readOnly)
new_poll_interval = golang_duration_str_to_seconds(bt.pollInterval)
assert str(new_poll_interval) == str(poll_interval)


def set_backupstore_default(client, default=True, backup_target_name=""):
bt_name = DEFAULT_BACKUPSTORE_NAME
if backup_target_name != "":
bt_name = backup_target_name
bt = client.by_id_backup_target(bt_name)
bt_poll_interval = golang_duration_str_to_seconds(bt.pollInterval)
client.update(bt,
name=bt_name,
backupTargetURL=bt.backupTargetURL,
credentialSecret=bt.credentialSecret,
default=default,
pollInterval=str(bt_poll_interval),
readOnly=bt.readOnly)
wait_for_backup_target_status(client,
bt_name,
"default",
True)


def mount_nfs_backupstore(client, mount_path="/mnt/nfs"):
cmd = ["mkdir", "-p", mount_path]
subprocess.check_output(cmd)
nfs_backuptarget = client.by_id_setting(SETTING_BACKUP_TARGET).value
nfs_backuptarget = backupstore_get_backup_target(client)
nfs_url = urlparse(nfs_backuptarget).netloc + \
urlparse(nfs_backuptarget).path
cmd = ["mount", "-t", "nfs", "-o", "nfsvers=4.2", nfs_url, mount_path]
Expand Down Expand Up @@ -170,15 +244,25 @@ def backup_cleanup():
backup['metadata']['name'])


def backupstore_cleanup(client):
def backupstore_cleanup(client, backup_target_name=""):
backup_volumes = client.list_backup_volume()

# we delete the whole backup volume, which skips block gc
for backup_volume in backup_volumes:
delete_backup_volume(client, backup_volume.name)
if backup_target_name != "":
if backup_volume.backupTargetName != backup_target_name:
continue
delete_backup_volume(client,
backup_volume.volumeName,
backup_target_name)

backup_volumes = client.list_backup_volume()
assert backup_volumes.data == []
if backup_target_name == "":
assert backup_volumes.data == []
else:
for bv in backup_volumes:
if bv.backupTargetName == backup_target_name:
assert False, "Backup Volume {} is not deleted".format(bv)


def minio_get_api_client(client, core_api, minio_secret_name):
Expand Down Expand Up @@ -229,7 +313,7 @@ def minio_get_backupstore_path(client):


def get_nfs_mount_point(client):
nfs_backuptarget = client.by_id_setting(SETTING_BACKUP_TARGET).value
nfs_backuptarget = backupstore_get_backup_target(client)
nfs_url = urlparse(nfs_backuptarget).netloc + \
urlparse(nfs_backuptarget).path

Expand Down Expand Up @@ -282,15 +366,13 @@ def nfs_get_backup_volume_prefix(client, volume_name):


def backupstore_get_backup_target(client):
backup_target_setting = client.by_id_setting(SETTING_BACKUP_TARGET)
return backup_target_setting.value
backup_target = client.by_id_backup_target(DEFAULT_BACKUPSTORE_NAME)
return backup_target.backupTargetURL


def backupstore_get_secret(client):
backup_target_credential_setting = client.by_id_setting(
SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)

return backup_target_credential_setting.value
backup_target = client.by_id_backup_target(DEFAULT_BACKUPSTORE_NAME)
return backup_target.credentialSecret


def backupstore_get_backup_cfg_file_path(client, volume_name, backup_name):
Expand Down Expand Up @@ -363,8 +445,7 @@ def nfs_get_backup_blocks_dir(client, volume_name):


def backupstore_create_file(client, core_api, file_path, data={}):
backup_target_setting = client.by_id_setting(SETTING_BACKUP_TARGET)
backupstore = backup_target_setting.value
backupstore = backupstore_get_backup_target(client)

if is_backupTarget_s3(backupstore):
return mino_create_file_in_backupstore(client,
Expand All @@ -379,10 +460,8 @@ def backupstore_create_file(client, core_api, file_path, data={}):


def mino_create_file_in_backupstore(client, core_api, file_path, data={}): # NOQA
backup_target_credential_setting = client.by_id_setting(
SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)

secret_name = backup_target_credential_setting.value
secret_name = backupstore_get_secret(client)
assert secret_name != ''

minio_api = minio_get_api_client(client, core_api, secret_name)
bucket_name = minio_get_backupstore_bucket_name(client)
Expand All @@ -408,6 +487,7 @@ def nfs_create_file_in_backupstore(file_path, data={}):
with open(file_path, 'w') as cfg_file:
cfg_file.write(str(data))


def backupstore_write_backup_cfg_file(client, core_api, volume_name, backup_name, data): # NOQA
backupstore = backupstore_get_backup_target(client)

Expand Down Expand Up @@ -461,8 +541,7 @@ def minio_write_backup_cfg_file(client, core_api, volume_name, backup_name, back


def backupstore_delete_file(client, core_api, file_path):
backup_target_setting = client.by_id_setting(SETTING_BACKUP_TARGET)
backupstore = backup_target_setting.value
backupstore = backupstore_get_backup_target(client)

if is_backupTarget_s3(backupstore):
return mino_delete_file_in_backupstore(client,
Expand All @@ -477,10 +556,8 @@ def backupstore_delete_file(client, core_api, file_path):


def mino_delete_file_in_backupstore(client, core_api, file_path):
backup_target_credential_setting = client.by_id_setting(
SETTING_BACKUP_TARGET_CREDENTIAL_SECRET)

secret_name = backup_target_credential_setting.value
secret_name = backupstore_get_secret(client)
assert secret_name != ''

minio_api = minio_get_api_client(client, core_api, secret_name)
bucket_name = minio_get_backupstore_bucket_name(client)
Expand Down
Loading