From 87409b6f58253f8528f8b5ba432e443ba4288685 Mon Sep 17 00:00:00 2001 From: Jake Callahan Date: Thu, 24 Oct 2024 13:07:38 -0400 Subject: [PATCH] [POC] New-Style Upgrade Tests (#14111) * [POC] New-Style Upgrade Tests SharedResource: - Added the ability to validate the result of a given action function via an action_validator function. - Made an improvement to exiting under error conditions that improved tracking file cleanup. New directory for new-style upgrades located at tests/new_upgrades. This will help to keep changes isolated from the existing upgrade tests. new_upgrades/conftest: - Removed the requirement for all upgrade tests to be marked as pre/post - Introduced fixtures that coordinate checkout/checkin actions between multiple xdist workers. - Introduced a fixture that performs an upgrade on a target satellite - Introduced a fixture that is used for two test conversions in different modules. test conversions: - test_cv_upgrade_scenario and test_scenario_custom_repo_check converted - pre-upgrade tests are now pre-upgrade fixtures that perform setup and yield their data in Box objects instead of saving to disk - post-upgrade tests can now directly access the setup objects by inheriting the pre-upgrade fixture results * Get test_cv_upgrade_scenario into passing state * Get test_scenario_custom_repo_check into passing state * Revert hard-coded setting in conftest --------- Co-authored-by: synkd Co-authored-by: synkd <48261305+synkd@users.noreply.github.com> (cherry picked from commit e34528585ec1e991a8f5a3d3505d352c3aa0427e) --- conf/upgrade.yaml.template | 2 + robottelo/host_helpers/satellite_mixins.py | 4 +- robottelo/hosts.py | 2 +- robottelo/utils/shared_resource.py | 30 ++++- tests/new_upgrades/__init__.py | 0 tests/new_upgrades/conftest.py | 100 ++++++++++++++++ tests/new_upgrades/test_contentview.py | 133 +++++++++++++++++++++ tests/new_upgrades/test_repository.py | 129 ++++++++++++++++++++ tests/robottelo/test_shared_resource.py | 10 +- 9 files changed, 402 insertions(+), 8 deletions(-) create mode 100644 tests/new_upgrades/__init__.py create mode 100644 tests/new_upgrades/conftest.py create mode 100644 tests/new_upgrades/test_contentview.py create mode 100644 tests/new_upgrades/test_repository.py diff --git a/conf/upgrade.yaml.template b/conf/upgrade.yaml.template index 1b971ba1b30..d0962acc6b7 100644 --- a/conf/upgrade.yaml.template +++ b/conf/upgrade.yaml.template @@ -5,6 +5,8 @@ UPGRADE: TO_VERSION: "6.9" # Satellite, Capsule hosts RHEL operating system version. OS: "rhel7" + # The job template Broker should use to upgrade a Satellite + SATELLITE_UPGRADE_JOB_TEMPLATE: satellite-upgrade # Capsule's activation key will only be available when we spawn the VM using upgrade template. CAPSULE_AK: RHEL6: diff --git a/robottelo/host_helpers/satellite_mixins.py b/robottelo/host_helpers/satellite_mixins.py index aaa33566d0c..6d4ebdda0dc 100644 --- a/robottelo/host_helpers/satellite_mixins.py +++ b/robottelo/host_helpers/satellite_mixins.py @@ -181,7 +181,7 @@ def is_sca_mode_enabled(self, org_id): """ return self.api.Organization(id=org_id).read().simple_content_access - def publish_content_view(self, org, repo_list): + def publish_content_view(self, org, repo_list, name): """This method publishes the content view for a given organization and repository list. :param str org: The name of the organization to which the content view belongs @@ -190,7 +190,7 @@ def publish_content_view(self, org, repo_list): :return: A dictionary containing the details of the published content view. """ repo = repo_list if isinstance(repo_list, list) else [repo_list] - content_view = self.api.ContentView(organization=org, repository=repo).create() + content_view = self.api.ContentView(organization=org, repository=repo, name=name).create() content_view.publish() return content_view.read() diff --git a/robottelo/hosts.py b/robottelo/hosts.py index ffc60dc4f9d..0037fd52328 100644 --- a/robottelo/hosts.py +++ b/robottelo/hosts.py @@ -880,7 +880,7 @@ def put(self, local_path, remote_path=None): content_file.flush() self.session.sftp_write(source=content_file.name, destination=remote_path) else: - self.session.sftp_write(source=local_path, destination=remote_path) + self.session.sftp_write(source=str(local_path), destination=str(remote_path)) def put_ssh_key(self, source_key_path, destination_key_name): """Copy ssh key to virtual machine ssh path and ensure proper permission is set diff --git a/robottelo/utils/shared_resource.py b/robottelo/utils/shared_resource.py index fa17f3f0c14..1eaddd9b103 100644 --- a/robottelo/utils/shared_resource.py +++ b/robottelo/utils/shared_resource.py @@ -30,6 +30,10 @@ from broker.helpers import FileLock +class SharedResourceError(Exception): + """An exception class for SharedResource errors.""" + + class SharedResource: """A class representing a shared resource. @@ -44,19 +48,21 @@ class SharedResource: is_recovering (bool): Whether the current instance is recovering from an error or not. """ - def __init__(self, resource_name, action, *action_args, **action_kwargs): + def __init__(self, resource_name, action, *action_args, action_validator=None, **action_kwargs): """Initializes a new instance of the SharedResource class. Args: resource_name (str): The name of the shared resource. action (function): The function to be executed when the resource is ready. action_args (tuple): The arguments to be passed to the action function. + action_validator (function): The function to validate the action results. action_kwargs (dict): The keyword arguments to be passed to the action function. """ self.resource_file = Path(f"/tmp/{resource_name}.shared") self.lock_file = FileLock(self.resource_file) self.id = str(uuid4().fields[-1]) self.action = action + self.action_validator = action_validator self.action_is_recoverable = action_kwargs.pop("action_is_recoverable", False) self.action_args = action_args self.action_kwargs = action_kwargs @@ -152,6 +158,14 @@ def register(self): curr_data["statuses"][self.id] = "pending" self.resource_file.write_text(json.dumps(curr_data, indent=4)) + def unregister(self): + """Unregisters the current process as a watcher.""" + with self.lock_file: + curr_data = json.loads(self.resource_file.read_text()) + curr_data["watchers"].remove(self.id) + del curr_data["statuses"][self.id] + self.resource_file.write_text(json.dumps(curr_data, indent=4)) + def ready(self): """Marks the current process as ready to perform the action.""" self._update_status("ready") @@ -164,10 +178,13 @@ def done(self): def act(self): """Attempt to perform the action.""" try: - self.action(*self.action_args, **self.action_kwargs) + result = self.action(*self.action_args, **self.action_kwargs) except Exception as err: self._update_main_status("error") - raise err + raise SharedResourceError("Main worker failed during action") from err + # If the action_validator is a callable, use it to validate the result + if callable(self.action_validator) and not self.action_validator(result): + raise SharedResourceError(f"Action validation failed for {self.action} with {result=}") def wait(self): """Top-level wait function, separating behavior between main and non-main watchers.""" @@ -190,11 +207,16 @@ def __exit__(self, exc_type, exc_value, traceback): raise exc_value if exc_type is None: self.done() + self.unregister() if self.is_main: self._wait_for_status("done") self.resource_file.unlink() else: self._update_status("error") if self.is_main: - self._update_main_status("error") + if self._check_all_status("error"): + # All have failed, delete the file + self.resource_file.unlink() + else: + self._update_main_status("error") raise exc_value diff --git a/tests/new_upgrades/__init__.py b/tests/new_upgrades/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/new_upgrades/conftest.py b/tests/new_upgrades/conftest.py new file mode 100644 index 00000000000..4dfe4f565c2 --- /dev/null +++ b/tests/new_upgrades/conftest.py @@ -0,0 +1,100 @@ +""" +This module is intended to be used for upgrade tests that have a single run stage. +""" + +import datetime + +from broker import Broker +import pytest + +from robottelo.config import settings +from robottelo.hosts import Satellite +from robottelo.utils.shared_resource import SharedResource + +pre_upgrade_failed_tests = [] + + +PRE_UPGRADE_TESTS_FILE_OPTION = 'pre_upgrade_tests_file' +PRE_UPGRADE_TESTS_FILE_PATH = '/var/tmp/robottelo_pre_upgrade_failed_tests.json' +PRE_UPGRADE = False +POST_UPGRADE = False +PRE_UPGRADE_MARK = 'pre_upgrade' +POST_UPGRADE_MARK = 'post_upgrade' +TEST_NODE_ID_NAME = '__pytest_node_id' + + +def log(message, level="DEBUG"): + """Pytest has a limitation to use logging.logger from conftest.py + so we need to emulate the logger by std-out the output + """ + now = datetime.datetime.now() + full_message = "{date} - conftest - {level} - {message}\n".format( + date=now.strftime("%Y-%m-%d %H:%M:%S"), level=level, message=message + ) + print(full_message) # noqa + with open('robottelo.log', 'a') as log_file: + log_file.write(full_message) + + +def pytest_configure(config): + """Register custom markers to avoid warnings.""" + markers = [ + "content_upgrades: Upgrade tests that run under .", + ] + for marker in markers: + config.addinivalue_line("markers", marker) + + +def shared_checkout(shared_name): + Satellite(hostname="blank")._swap_nailgun(f"{settings.UPGRADE.FROM_VERSION}.z") + bx_inst = Broker( + workflow=settings.SERVER.deploy_workflows.product, + deploy_sat_version=settings.UPGRADE.FROM_VERSION, + host_class=Satellite, + upgrade_group=f"{shared_name}_shared_checkout", + ) + with SharedResource( + resource_name=f"{shared_name}_sat_checkout", + action=bx_inst.checkout, + action_validator=lambda result: isinstance(result, Satellite), + ) as sat_checkout: + sat_checkout.ready() + sat_instance = bx_inst.from_inventory( + filter=f'@inv._broker_args.upgrade_group == "{shared_name}_shared_checkout"' + )[0] + sat_instance.setup() + return sat_instance + + +def shared_checkin(sat_instance): + sat_instance.teardown() + with SharedResource( + resource_name=sat_instance.hostname + "_checkin", + action=Broker(hosts=[sat_instance]).checkin, + ) as sat_checkin: + sat_checkin.ready() + + +@pytest.fixture(scope='session') +def upgrade_action(): + def _upgrade_action(target_sat): + Broker( + job_template=settings.UPGRADE.SATELLITE_UPGRADE_JOB_TEMPLATE, + target_vm=target_sat.name, + sat_version=settings.UPGRADE.TO_VERSION, + upgrade_path="ystream", + tower_inventory=target_sat.tower_inventory, + ).execute() + + return _upgrade_action + + +@pytest.fixture +def content_upgrade_shared_satellite(): + """Mark tests using this fixture with pytest.mark.content_upgrades.""" + sat_instance = shared_checkout("content_upgrade") + with SharedResource( + "content_upgrade_tests", shared_checkin, sat_instance=sat_instance + ) as test_duration: + yield sat_instance + test_duration.ready() diff --git a/tests/new_upgrades/test_contentview.py b/tests/new_upgrades/test_contentview.py new file mode 100644 index 00000000000..ca5396e95f2 --- /dev/null +++ b/tests/new_upgrades/test_contentview.py @@ -0,0 +1,133 @@ +"""Test for Content View related Upgrade Scenario's + +:Requirement: UpgradedSatellite + +:CaseAutomation: Automated + +:CaseComponent: ContentViews + +:Team: Phoenix-content + +:CaseImportance: High + +""" + +from box import Box +from fauxfactory import gen_alpha +import pytest + +from robottelo.config import settings +from robottelo.constants import RPM_TO_UPLOAD, DataFile +from robottelo.utils.shared_resource import SharedResource + + +@pytest.fixture +def cv_upgrade_setup(content_upgrade_shared_satellite, upgrade_action): + """Pre-upgrade scenario that creates content-view with various repositories. + + :id: preupgrade-a4ebbfa1-106a-4962-9c7c-082833879ae8 + + :steps: + 1. Create custom repositories of yum and file type. + 2. Create content-view. + 3. Add yum and file repositories in the content view. + 4. Publish the content-view. + + :expectedresults: Content-view created with various repositories. + """ + target_sat = content_upgrade_shared_satellite + with SharedResource(target_sat.hostname, upgrade_action, target_sat=target_sat) as sat_upgrade: + test_data = Box( + { + 'target_sat': target_sat, + 'cv': None, + 'org': None, + 'product': None, + 'yum_repo': None, + 'file_repo': None, + } + ) + test_name = f'cv_upgrade_{gen_alpha()}' # unique name for the test + org = target_sat.api.Organization(name=f'{test_name}_org').create() + test_data.org = org + product = target_sat.api.Product(organization=org, name=f'{test_name}_prod').create() + test_data.product = product + yum_repository = target_sat.api.Repository( + product=product, + name=f'{test_name}_yum_repo', + url=settings.repos.yum_1.url, + content_type='yum', + ).create() + test_data.yum_repo = yum_repository + target_sat.api.Repository.sync(yum_repository) + file_repository = target_sat.api.Repository( + product=product, name=f'{test_name}_file_repo', content_type='file' + ).create() + test_data.file_repo = file_repository + remote_file_path = f'/tmp/{RPM_TO_UPLOAD}' + target_sat.put(DataFile.RPM_TO_UPLOAD, remote_file_path) + file_repository.upload_content(files={'content': DataFile.RPM_TO_UPLOAD.read_bytes()}) + assert 'content' in file_repository.files()['results'][0]['name'] + cv = target_sat.publish_content_view(org, [yum_repository, file_repository], test_name) + assert len(cv.read_json()['versions']) == 1 + test_data.cv = cv + sat_upgrade.ready() + target_sat._session = None + yield test_data + + +@pytest.mark.content_upgrades +def test_cv_upgrade_scenario(cv_upgrade_setup): + """After upgrade, the existing content-view(created before upgrade) should be updated. + + :id: postupgrade-a4ebbfa1-106a-4962-9c7c-082833879ae8 + + :steps: + 1. Check yum and file repository which was added in CV before upgrade. + 2. Check the content view which was was created before upgrade. + 3. Remove yum repository from existing CV. + 4. Create new yum repository in existing CV. + 5. Publish content-view + + :expectedresults: After upgrade, + 1. All the repositories should be intact. + 2. Content view created before upgrade should be intact. + 3. The new repository should be added/updated to the CV. + + """ + target_sat = cv_upgrade_setup.target_sat + org = target_sat.api.Organization().search( + query={'search': f'name="{cv_upgrade_setup.org.name}"'} + )[0] + product = target_sat.api.Product(organization=org.id).search( + query={'search': f'name="{cv_upgrade_setup.product.name}"'} + )[0] + cv = target_sat.api.ContentView(organization=org.id).search( + query={'search': f'name="{cv_upgrade_setup.cv.name}"'} + )[0] + target_sat.api.Repository(organization=org.id).search( + query={'search': f'name="{cv_upgrade_setup.yum_repo.name}"'} + )[0] + target_sat.api.Repository(organization=org.id).search( + query={'search': f'name="{cv_upgrade_setup.file_repo.name}"'} + )[0] + cv.repository = [] + cv.update(['repository']) + assert len(cv.read_json()['repositories']) == 0 + + yum_repository2 = target_sat.api.Repository( + product=product, + name='cv_upgrade_yum_repos2', + url=settings.repos.yum_2.url, + content_type='yum', + ).create() + yum_repository2.sync() + cv.repository = [yum_repository2] + cv.update(['repository']) + assert cv.read_json()['repositories'][0]['name'] == yum_repository2.name + + cv.publish() + assert len(cv.read_json()['versions']) == 2 + content_view_json = cv.read_json()['environments'][0] + cv.delete_from_environment(content_view_json['id']) + assert len(cv.read_json()['environments']) == 0 diff --git a/tests/new_upgrades/test_repository.py b/tests/new_upgrades/test_repository.py new file mode 100644 index 00000000000..24c2c368e04 --- /dev/null +++ b/tests/new_upgrades/test_repository.py @@ -0,0 +1,129 @@ +"""Test for Repository related Upgrade Scenarios + +:Requirement: UpgradedSatellite + +:CaseAutomation: Automated + +:CaseComponent: Repositories + +:Team: Phoenix-content + +:CaseImportance: High + +""" + +from box import Box +from fauxfactory import gen_alpha +import pytest + +from robottelo.config import settings +from robottelo.constants import ( + FAKE_0_CUSTOM_PACKAGE_NAME, + FAKE_4_CUSTOM_PACKAGE_NAME, +) +from robottelo.hosts import ContentHost +from robottelo.utils.shared_resource import SharedResource + + +@pytest.fixture +def custom_repo_check_setup(sat_upgrade_chost, content_upgrade_shared_satellite, upgrade_action): + """This is pre-upgrade scenario test to verify if we can create a + custom repository and consume it via content host. + + :id: preupgrade-eb6831b1-c5b6-4941-a325-994a09467478 + + :steps: + 1. Before Satellite upgrade. + 2. Create new Organization, Location. + 3. Create Product, custom repo, cv. + 4. Create activation key and add subscription. + 5. Create a content host, register and install package on it. + + :expectedresults: + + 1. Custom repo is created. + 2. Package is installed on Content host. + + """ + target_sat = content_upgrade_shared_satellite + with SharedResource(target_sat.hostname, upgrade_action, target_sat=target_sat) as sat_upgrade: + test_data = Box( + { + 'target_sat': target_sat, + 'rhel_client': sat_upgrade_chost, + 'lce': None, + 'repo': None, + 'content_view': None, + } + ) + test_name = f'repo_upgrade_{gen_alpha()}' # unique name for the test + org = target_sat.api.Organization(name=f'{test_name}_org').create() + lce = target_sat.api.LifecycleEnvironment( + organization=org, name=f'{test_name}_lce', prior=2 + ).create() + test_data.lce = lce + product = target_sat.api.Product(organization=org, name=f'{test_name}_prod').create() + repo = target_sat.api.Repository( + product=product.id, + name=f'{test_name}_repo', + url=settings.repos.yum_1.url, + content_type='yum', + ).create() + test_data.repo = repo + repo.sync() + content_view = target_sat.publish_content_view(org, repo, test_name) + test_data.content_view = content_view + content_view.version[0].promote(data={'environment_ids': lce.id}) + ak = target_sat.api.ActivationKey( + content_view=content_view, organization=org.id, environment=lce, name=test_name + ).create() + if not target_sat.is_sca_mode_enabled(org.id): + subscription = target_sat.api.Subscription(organization=org).search( + query={'search': f'name={product.name}'} + )[0] + ak.add_subscriptions(data={'subscription_id': subscription.id}) + sat_upgrade_chost.api_register( + target_sat, organization=org, activation_keys=[ak.name], location=None + ) + sat_upgrade_chost.execute('subscription-manager repos --enable=* && yum clean all') + result = sat_upgrade_chost.execute(f'yum install -y {FAKE_0_CUSTOM_PACKAGE_NAME}') + assert result.status == 0 + sat_upgrade.ready() + target_sat._session = None + yield test_data + + +@pytest.mark.content_upgrades +def test_scenario_custom_repo_check(custom_repo_check_setup): + """This is post-upgrade scenario test to verify if we can alter the + created custom repository and satellite will be able to sync back + the repo. + + :id: postupgrade-5c793577-e573-46a7-abbf-b6fd1f20b06e + + :steps: + 1. Remove old and add new package into custom repo. + 2. Sync repo , publish the new version of cv. + 3. Try to install new package on client. + + + :expectedresults: Content host should be able to pull the new rpm. + + """ + test_data = custom_repo_check_setup + target_sat = test_data.target_sat + repo = target_sat.api.Repository(name=test_data.repo.name).search()[0] + repo.sync() + + content_view = target_sat.api.ContentView(name=test_data.content_view.name).search()[0] + content_view.publish() + + content_view = target_sat.api.ContentView(name=test_data.content_view.name).search()[0] + latest_cvv_id = sorted(cvv.id for cvv in content_view.version)[-1] + target_sat.api.ContentViewVersion(id=latest_cvv_id).promote( + data={'environment_ids': test_data.lce.id} + ) + + rhel_client = ContentHost.get_host_by_hostname(test_data.rhel_client.hostname) + result = rhel_client.execute(f'yum install -y {FAKE_4_CUSTOM_PACKAGE_NAME}') + assert result.status == 0 diff --git a/tests/robottelo/test_shared_resource.py b/tests/robottelo/test_shared_resource.py index ff2146cd80a..d0745bc1fce 100644 --- a/tests/robottelo/test_shared_resource.py +++ b/tests/robottelo/test_shared_resource.py @@ -11,6 +11,7 @@ def upgrade_action(*args, **kwargs): print(f"Upgrading satellite with {args=} and {kwargs=}") time.sleep(1) print("Satellite upgraded!") + return True def run_resource(resource_name): @@ -24,7 +25,14 @@ def run_resource(resource_name): def test_shared_resource(): """Test the SharedResource class.""" - with SharedResource("test_resource", upgrade_action, 1, 2, 3, foo="bar") as resource: + + def action_validator(result): + return result is True + + shared_args = (1, 2, 3) + with SharedResource( + "test_resource", upgrade_action, *shared_args, action_validator=action_validator, foo="bar" + ) as resource: assert Path("/tmp/test_resource.shared").exists() assert resource.is_main assert not resource.is_recovering