diff --git a/repos/system_upgrade/common/actors/xfsinfoscanner/actor.py b/repos/system_upgrade/common/actors/xfsinfoscanner/actor.py index ebc7e17eee..a97e51fa3e 100644 --- a/repos/system_upgrade/common/actors/xfsinfoscanner/actor.py +++ b/repos/system_upgrade/common/actors/xfsinfoscanner/actor.py @@ -1,22 +1,32 @@ from leapp.actors import Actor from leapp.libraries.actor.xfsinfoscanner import scan_xfs -from leapp.models import StorageInfo, XFSPresence +from leapp.models import StorageInfo, XFSInfoFacts, XFSPresence from leapp.tags import FactsPhaseTag, IPUWorkflowTag class XFSInfoScanner(Actor): """ - This actor scans all mounted mountpoints for XFS information + This actor scans all mounted mountpoints for XFS information. + + The actor checks the `StorageInfo` message, which contains details about + the system's storage. For each mountpoint reported, it determines whether + the filesystem is XFS and collects information about its configuration. + Specifically, it identifies whether the XFS filesystem is using `ftype=0`, + which requires special handling for overlay filesystems. + + The actor produces two types of messages: + + - `XFSPresence`: Indicates whether any XFS use `ftype=0`, and lists the + mountpoints where `ftype=0` is used. + + - `XFSInfoFacts`: Contains detailed metadata about all XFS mountpoints. + This includes sections parsed from the `xfs_info` command. - The actor will check each mountpoint reported in the StorageInfo message, if the mountpoint is a partition with XFS - using ftype = 0. The actor will produce a message with the findings. - It will contain a list of all XFS mountpoints with ftype = 0 so that those mountpoints can be handled appropriately - for the overlayfs that is going to be created. """ name = 'xfs_info_scanner' consumes = (StorageInfo,) - produces = (XFSPresence,) + produces = (XFSPresence, XFSInfoFacts,) tags = (FactsPhaseTag, IPUWorkflowTag,) def process(self): diff --git a/repos/system_upgrade/common/actors/xfsinfoscanner/libraries/xfsinfoscanner.py b/repos/system_upgrade/common/actors/xfsinfoscanner/libraries/xfsinfoscanner.py index fafe456eaa..e930edf7d9 100644 --- a/repos/system_upgrade/common/actors/xfsinfoscanner/libraries/xfsinfoscanner.py +++ b/repos/system_upgrade/common/actors/xfsinfoscanner/libraries/xfsinfoscanner.py @@ -1,13 +1,74 @@ import os +import re from leapp.libraries.stdlib import api, CalledProcessError, run -from leapp.models import StorageInfo, XFSPresence +from leapp.models import ( + StorageInfo, + XFSInfo, + XFSInfoData, + XFSInfoFacts, + XFSInfoLog, + XFSInfoMetaData, + XFSInfoNaming, + XFSInfoRealtime, + XFSPresence +) + + +def scan_xfs(): + storage_info_msgs = api.consume(StorageInfo) + storage_info = next(storage_info_msgs, None) + + if list(storage_info_msgs): + api.current_logger().warning( + 'Unexpectedly received more than one StorageInfo message.' + ) + + fstab_data = set() + mount_data = set() + if storage_info: + fstab_data = scan_xfs_fstab(storage_info.fstab) + mount_data = scan_xfs_mount(storage_info.mount) + + mountpoints = fstab_data | mount_data + + xfs_infos = {} + for mountpoint in mountpoints: + content = read_xfs_info(mountpoint) + if content is None: + continue + + xfs_info = parse_xfs_info(content) + xfs_infos[mountpoint] = xfs_info + + mountpoints_ftype0 = [ + mountpoint + for mountpoint in xfs_infos + if is_without_ftype(xfs_infos[mountpoint]) + ] + + # By now, we only have XFS mountpoints and check whether or not it has + # ftype = 0 + api.produce(XFSPresence( + present=len(mountpoints) > 0, + without_ftype=len(mountpoints_ftype0) > 0, + mountpoints_without_ftype=mountpoints_ftype0, + )) + + api.produce( + XFSInfoFacts( + mountpoints=[ + generate_xfsinfo_for_mountpoint(xfs_infos[mountpoint], mountpoint) + for mountpoint in xfs_infos + ] + ) + ) def scan_xfs_fstab(data): mountpoints = set() for entry in data: - if entry.fs_vfstype == "xfs": + if entry.fs_vfstype == 'xfs': mountpoints.add(entry.fs_file) return mountpoints @@ -16,49 +77,110 @@ def scan_xfs_fstab(data): def scan_xfs_mount(data): mountpoints = set() for entry in data: - if entry.tp == "xfs": + if entry.tp == 'xfs': mountpoints.add(entry.mount) return mountpoints -def is_xfs_without_ftype(mp): - if not os.path.ismount(mp): - # Check if mp is actually a mountpoint - api.current_logger().warning('{} is not mounted'.format(mp)) - return False +def read_xfs_info(mp): + if not is_mountpoint(mp): + return None + try: - xfs_info = run(['/usr/sbin/xfs_info', '{}'.format(mp)], split=True) + result = run(['/usr/sbin/xfs_info', '{}'.format(mp)], split=True) except CalledProcessError as err: - api.current_logger().warning('Error during command execution: {}'.format(err)) - return False - - for l in xfs_info['stdout']: - if 'ftype=0' in l: - return True - - return False - + api.current_logger().warning( + 'Error during command execution: {}'.format(err) + ) + return None -def scan_xfs(): - storage_info_msgs = api.consume(StorageInfo) - storage_info = next(storage_info_msgs, None) + return result['stdout'] - if list(storage_info_msgs): - api.current_logger().warning('Unexpectedly received more than one StorageInfo message.') - fstab_data = set() - mount_data = set() - if storage_info: - fstab_data = scan_xfs_fstab(storage_info.fstab) - mount_data = scan_xfs_mount(storage_info.mount) - - mountpoints = fstab_data | mount_data - mountpoints_ftype0 = list(filter(is_xfs_without_ftype, mountpoints)) +def is_mountpoint(mp): + if not os.path.ismount(mp): + # Check if mp is actually a mountpoint + api.current_logger().warning('{} is not mounted'.format(mp)) + return False - # By now, we only have XFS mountpoints and check whether or not it has ftype = 0 - api.produce(XFSPresence( - present=len(mountpoints) > 0, - without_ftype=len(mountpoints_ftype0) > 0, - mountpoints_without_ftype=mountpoints_ftype0, - )) + return True + + +def parse_xfs_info(content): + """ + This parser reads the output of the ``xfs_info`` command. + + In general the pattern is:: + + section =sectionkey key1=value1 key2=value2, key3=value3 + = key4=value4 + nextsec =sectionkey sectionvalue key=value otherkey=othervalue + + Sections are continued over lines as per RFC822. The first equals + sign is column-aligned, and the first key=value is too, but the + rest seems to be comma separated. Specifiers come after the first + equals sign, and sometimes have a value property, but sometimes not. + + NOTE: This function is adapted from [1] + + [1]: https://github.com/RedHatInsights/insights-core/blob/master/insights/parsers/xfs_info.py + """ + + xfs_info = {} + + info_re = re.compile(r'^(?P
[\w-]+)?\s*' + + r'=(?:(?P\S+)(?:\s(?P\w+))?)?' + + r'\s+(?P\w.*\w)$' + ) + keyval_re = re.compile(r'(?P[\w-]+)=(?P\d+(?: blks)?)') + + sect_info = None + + for line in content: + match = info_re.search(line) + if match: + if match.group('section'): + # Change of section - make new sect_info dict and link + sect_info = {} + xfs_info[match.group('section')] = sect_info + if match.group('specifier'): + sect_info['specifier'] = match.group('specifier') + if match.group('specval'): + sect_info['specifier_value'] = match.group('specval') + for key, value in keyval_re.findall(match.group('keyvaldata')): + sect_info[key] = value + + # Normalize strings + xfs_info = { + str(section): { + str(attr): str(value) + for attr, value in sect_info.items() + } + for section, sect_info in xfs_info.items() + } + + return xfs_info + + +def is_without_ftype(xfs_info): + return xfs_info['naming'].get('ftype', '') == '0' + + +def generate_xfsinfo_for_mountpoint(xfs_info, mountpoint): + result = XFSInfo( + mountpoint=mountpoint, + meta_data=XFSInfoMetaData( + device=xfs_info['meta-data']['specifier'], + bigtime=xfs_info['meta-data'].get('bigtime'), + crc=xfs_info['meta-data'].get('crc'), + ), + data=XFSInfoData(), + naming=XFSInfoNaming( + ftype=xfs_info['naming']['ftype'] + ), + log=XFSInfoLog(), + realtime=XFSInfoRealtime(), + ) + + return result diff --git a/repos/system_upgrade/common/actors/xfsinfoscanner/tests/unit_test_xfsinfoscanner.py b/repos/system_upgrade/common/actors/xfsinfoscanner/tests/unit_test_xfsinfoscanner.py index 4ac6a0d16f..d39617c179 100644 --- a/repos/system_upgrade/common/actors/xfsinfoscanner/tests/unit_test_xfsinfoscanner.py +++ b/repos/system_upgrade/common/actors/xfsinfoscanner/tests/unit_test_xfsinfoscanner.py @@ -3,7 +3,148 @@ from leapp.libraries.actor import xfsinfoscanner from leapp.libraries.common.testutils import produce_mocked from leapp.libraries.stdlib import api, CalledProcessError -from leapp.models import FstabEntry, MountEntry, StorageInfo, SystemdMountEntry, XFSPresence +from leapp.models import ( + FstabEntry, + MountEntry, + StorageInfo, + SystemdMountEntry, + XFSInfo, + XFSInfoData, + XFSInfoFacts, + XFSInfoLog, + XFSInfoMetaData, + XFSInfoNaming, + XFSInfoRealtime, + XFSPresence +) + +TEST_XFS_INFO_FTYPE1 = """ +meta-data=/dev/loop0 isize=512 agcount=4, agsize=131072 blks + = sectsz=512 attr=2, projid32bit=1 + = crc=1 finobt=0 spinodes=0 +data = bsize=4096 blocks=524288, imaxpct=25 + = sunit=0 swidth=0 blks +naming =version 2 bsize=4096 ascii-ci=0 ftype=1 +log =internal bsize=4096 blocks=2560, version=2 + = sectsz=512 sunit=0 blks, lazy-count=1 +realtime =none extsz=4096 blocks=0, rtextents=0 +""" +TEST_XFS_INFO_FTYPE1_PARSED = { + 'meta-data': { + 'agcount': '4', + 'agsize': '131072 blks', + 'attr': '2', + 'crc': '1', + 'finobt': '0', + 'isize': '512', + 'projid32bit': '1', + 'sectsz': '512', + 'specifier': '/dev/loop0', + 'spinodes': '0' + }, + 'data': { + 'blocks': '524288', + 'bsize': '4096', + 'imaxpct': '25', + 'sunit': '0', + 'swidth': '0 blks' + }, + 'naming': { + 'ascii-ci': '0', + 'bsize': '4096', + 'ftype': '1', + 'specifier': 'version', + 'specifier_value': '2' + }, + 'log': { + 'blocks': '2560', + 'bsize': '4096', + 'lazy-count': '1', + 'sectsz': '512', + 'specifier': 'internal', + 'sunit': '0 blks', + 'version': '2' + }, + 'realtime': { + 'blocks': '0', + 'extsz': '4096', + 'rtextents': '0', + 'specifier': 'none' + }, +} + +TEST_XFS_INFO_FTYPE0 = """ +meta-data=/dev/loop0 isize=512 agcount=4, agsize=131072 blks + = sectsz=512 attr=2, projid32bit=1 + = crc=1 finobt=0 spinodes=0 +data = bsize=4096 blocks=524288, imaxpct=25 + = sunit=0 swidth=0 blks +naming =version 2 bsize=4096 ascii-ci=0 ftype=0 +log =internal bsize=4096 blocks=2560, version=2 + = sectsz=512 sunit=0 blks, lazy-count=1 +realtime =none extsz=4096 blocks=0, rtextents=0 +""" +TEST_XFS_INFO_FTYPE0_PARSED = { + 'meta-data': { + 'agcount': '4', + 'agsize': '131072 blks', + 'attr': '2', + 'crc': '1', + 'finobt': '0', + 'isize': '512', + 'projid32bit': '1', + 'sectsz': '512', + 'specifier': '/dev/loop0', + 'spinodes': '0' + }, + 'data': { + 'blocks': '524288', + 'bsize': '4096', + 'imaxpct': '25', + 'sunit': '0', + 'swidth': '0 blks' + }, + 'naming': { + 'ascii-ci': '0', + 'bsize': '4096', + 'ftype': '0', + 'specifier': 'version', + 'specifier_value': '2' + }, + 'log': { + 'blocks': '2560', + 'bsize': '4096', + 'lazy-count': '1', + 'sectsz': '512', + 'specifier': 'internal', + 'sunit': '0 blks', + 'version': '2' + }, + 'realtime': { + 'blocks': '0', + 'extsz': '4096', + 'rtextents': '0', + 'specifier': 'none' + } +} + +TEST_XFS_INFO_FTYPE1_MODEL = XFSInfo( + mountpoint='/', + meta_data=XFSInfoMetaData(device='/dev/loop0', bigtime=None, crc='1'), + data=XFSInfoData(), + naming=XFSInfoNaming(ftype='1'), + log=XFSInfoLog(), + realtime=XFSInfoRealtime() +) + +TEST_XFS_INFO_FTYPE0_MODEL = XFSInfo( + mountpoint='/var', + meta_data=XFSInfoMetaData(device='/dev/loop0', bigtime=None, crc='1'), + data=XFSInfoData(), + naming=XFSInfoNaming(ftype='0'), + log=XFSInfoLog(), + realtime=XFSInfoRealtime() +) class run_mocked(object): @@ -15,29 +156,10 @@ def __call__(self, args, split=True): self.called += 1 self.args = args - with_ftype = {'stdout': [ - "meta-data=/dev/loop0 isize=512 agcount=4, agsize=131072 blks", - " = sectsz=512 attr=2, projid32bit=1", - " = crc=1 finobt=0 spinodes=0", - "data = bsize=4096 blocks=524288, imaxpct=25", - " = sunit=0 swidth=0 blks", - "naming =version 2 bsize=4096 ascii-ci=0 ftype=1", - "log =internal bsize=4096 blocks=2560, version=2", - " = sectsz=512 sunit=0 blks, lazy-count=1", - "realtime =none extsz=4096 blocks=0, rtextents=0"]} - - without_ftype = {'stdout': [ - "meta-data=/dev/loop0 isize=512 agcount=4, agsize=131072 blks", - " = sectsz=512 attr=2, projid32bit=1", - " = crc=1 finobt=0 spinodes=0", - "data = bsize=4096 blocks=524288, imaxpct=25", - " = sunit=0 swidth=0 blks", - "naming =version 2 bsize=4096 ascii-ci=0 ftype=0", - "log =internal bsize=4096 blocks=2560, version=2", - " = sectsz=512 sunit=0 blks, lazy-count=1", - "realtime =none extsz=4096 blocks=0, rtextents=0"]} - - if "/var" in self.args: + with_ftype = {'stdout': TEST_XFS_INFO_FTYPE1.splitlines()} + without_ftype = {'stdout': TEST_XFS_INFO_FTYPE0.splitlines()} + + if '/var' in self.args: return without_ftype return with_ftype @@ -45,163 +167,228 @@ def __call__(self, args, split=True): def test_scan_xfs_fstab(monkeypatch): fstab_data_no_xfs = { - "fs_spec": "/dev/mapper/fedora-home", - "fs_file": "/home", - "fs_vfstype": "ext4", - "fs_mntops": "defaults,x-systemd.device-timeout=0", - "fs_freq": "1", - "fs_passno": "2"} + 'fs_spec': '/dev/mapper/fedora-home', + 'fs_file': '/home', + 'fs_vfstype': 'ext4', + 'fs_mntops': 'defaults,x-systemd.device-timeout=0', + 'fs_freq': '1', + 'fs_passno': '2'} mountpoints = xfsinfoscanner.scan_xfs_fstab([FstabEntry(**fstab_data_no_xfs)]) assert not mountpoints fstab_data_xfs = { - "fs_spec": "/dev/mapper/rhel-root", - "fs_file": "/", - "fs_vfstype": "xfs", - "fs_mntops": "defaults", - "fs_freq": "0", - "fs_passno": "0"} + 'fs_spec': '/dev/mapper/rhel-root', + 'fs_file': '/', + 'fs_vfstype': 'xfs', + 'fs_mntops': 'defaults', + 'fs_freq': '0', + 'fs_passno': '0'} mountpoints = xfsinfoscanner.scan_xfs_fstab([FstabEntry(**fstab_data_xfs)]) - assert mountpoints == {"/"} + assert mountpoints == {'/'} def test_scan_xfs_mount(monkeypatch): mount_data_no_xfs = { - "name": "tmpfs", - "mount": "/run/snapd/ns", - "tp": "tmpfs", - "options": "rw,nosuid,nodev,seclabel,mode=755"} + 'name': 'tmpfs', + 'mount': '/run/snapd/ns', + 'tp': 'tmpfs', + 'options': 'rw,nosuid,nodev,seclabel,mode=755'} mountpoints = xfsinfoscanner.scan_xfs_mount([MountEntry(**mount_data_no_xfs)]) assert not mountpoints mount_data_xfs = { - "name": "/dev/vda1", - "mount": "/boot", - "tp": "xfs", - "options": "rw,relatime,seclabel,attr2,inode64,noquota"} + 'name': '/dev/vda1', + 'mount': '/boot', + 'tp': 'xfs', + 'options': 'rw,relatime,seclabel,attr2,inode64,noquota'} mountpoints = xfsinfoscanner.scan_xfs_mount([MountEntry(**mount_data_xfs)]) - assert mountpoints == {"/boot"} - + assert mountpoints == {'/boot'} -def test_is_xfs_without_ftype(monkeypatch): - monkeypatch.setattr(xfsinfoscanner, "run", run_mocked()) - monkeypatch.setattr(os.path, "ismount", lambda _: True) - assert xfsinfoscanner.is_xfs_without_ftype("/var") - assert ' '.join(xfsinfoscanner.run.args) == "/usr/sbin/xfs_info /var" +def test_is_without_ftype(monkeypatch): + assert xfsinfoscanner.is_without_ftype(TEST_XFS_INFO_FTYPE0_PARSED) + assert not xfsinfoscanner.is_without_ftype(TEST_XFS_INFO_FTYPE1_PARSED) + assert not xfsinfoscanner.is_without_ftype({'naming': {}}) - assert not xfsinfoscanner.is_xfs_without_ftype("/boot") - assert ' '.join(xfsinfoscanner.run.args) == "/usr/sbin/xfs_info /boot" - -def test_is_xfs_command_failed(monkeypatch): +def test_read_xfs_info_failed(monkeypatch): def _run_mocked_exception(*args, **kwargs): - raise CalledProcessError(message="No such file or directory", command=["xfs_info", "/nosuchmountpoint"], + raise CalledProcessError(message='No such file or directory', command=['xfs_info', '/nosuchmountpoint'], result=1) # not a mountpoint - monkeypatch.setattr(os.path, "ismount", lambda _: False) - monkeypatch.setattr(xfsinfoscanner, "run", _run_mocked_exception) - assert not xfsinfoscanner.is_xfs_without_ftype("/nosuchmountpoint") + monkeypatch.setattr(os.path, 'ismount', lambda _: False) + monkeypatch.setattr(xfsinfoscanner, 'run', _run_mocked_exception) + assert xfsinfoscanner.read_xfs_info('/nosuchmountpoint') is None # a real mountpoint but something else caused command to fail - monkeypatch.setattr(os.path, "ismount", lambda _: True) - assert not xfsinfoscanner.is_xfs_without_ftype("/nosuchmountpoint") + monkeypatch.setattr(os.path, 'ismount', lambda _: True) + assert xfsinfoscanner.read_xfs_info('/nosuchmountpoint') is None -def test_scan_xfs(monkeypatch): - monkeypatch.setattr(xfsinfoscanner, "run", run_mocked()) - monkeypatch.setattr(os.path, "ismount", lambda _: True) +def test_scan_xfs_no_xfs(monkeypatch): + monkeypatch.setattr(xfsinfoscanner, 'run', run_mocked()) + monkeypatch.setattr(os.path, 'ismount', lambda _: True) def consume_no_xfs_message_mocked(*models): yield StorageInfo() - monkeypatch.setattr(api, "consume", consume_no_xfs_message_mocked) - monkeypatch.setattr(api, "produce", produce_mocked()) + monkeypatch.setattr(api, 'consume', consume_no_xfs_message_mocked) + monkeypatch.setattr(api, 'produce', produce_mocked()) xfsinfoscanner.scan_xfs() - assert api.produce.called == 1 - assert len(api.produce.model_instances) == 1 - assert isinstance(api.produce.model_instances[0], XFSPresence) - assert not api.produce.model_instances[0].present - assert not api.produce.model_instances[0].without_ftype - assert not api.produce.model_instances[0].mountpoints_without_ftype + + assert api.produce.called == 2 + assert len(api.produce.model_instances) == 2 + + xfs_presence = next(model for model in api.produce.model_instances if isinstance(model, XFSPresence)) + assert not xfs_presence.present + assert not xfs_presence.without_ftype + assert not xfs_presence.mountpoints_without_ftype + + xfs_info_facts = next(model for model in api.produce.model_instances if isinstance(model, XFSInfoFacts)) + assert xfs_info_facts.mountpoints == [] + + +def test_scan_xfs_ignored_xfs(monkeypatch): + monkeypatch.setattr(xfsinfoscanner, 'run', run_mocked()) + monkeypatch.setattr(os.path, 'ismount', lambda _: True) def consume_ignored_xfs_message_mocked(*models): mount_data = { - "name": "/dev/vda1", - "mount": "/boot", - "tp": "xfs", - "options": "rw,relatime,seclabel,attr2,inode64,noquota"} + 'name': '/dev/vda1', + 'mount': '/boot', + 'tp': 'xfs', + 'options': 'rw,relatime,seclabel,attr2,inode64,noquota' + } yield StorageInfo(mount=[MountEntry(**mount_data)]) - monkeypatch.setattr(api, "consume", consume_ignored_xfs_message_mocked) - monkeypatch.setattr(api, "produce", produce_mocked()) + monkeypatch.setattr(api, 'consume', consume_ignored_xfs_message_mocked) + monkeypatch.setattr(api, 'produce', produce_mocked()) xfsinfoscanner.scan_xfs() - assert api.produce.called == 1 - assert len(api.produce.model_instances) == 1 - assert isinstance(api.produce.model_instances[0], XFSPresence) - assert api.produce.model_instances[0].present - assert not api.produce.model_instances[0].without_ftype - assert not api.produce.model_instances[0].mountpoints_without_ftype + + assert api.produce.called == 2 + assert len(api.produce.model_instances) == 2 + + xfs_presence = next(model for model in api.produce.model_instances if isinstance(model, XFSPresence)) + assert xfs_presence.present + assert not xfs_presence.without_ftype + assert not xfs_presence.mountpoints_without_ftype + + xfs_info_facts = next(model for model in api.produce.model_instances if isinstance(model, XFSInfoFacts)) + assert len(xfs_info_facts.mountpoints) == 1 + assert xfs_info_facts.mountpoints[0].mountpoint == '/boot' + assert xfs_info_facts.mountpoints[0].meta_data == TEST_XFS_INFO_FTYPE1_MODEL.meta_data + assert xfs_info_facts.mountpoints[0].data == TEST_XFS_INFO_FTYPE1_MODEL.data + assert xfs_info_facts.mountpoints[0].naming == TEST_XFS_INFO_FTYPE1_MODEL.naming + assert xfs_info_facts.mountpoints[0].log == TEST_XFS_INFO_FTYPE1_MODEL.log + assert xfs_info_facts.mountpoints[0].realtime == TEST_XFS_INFO_FTYPE1_MODEL.realtime + + +def test_scan_xfs_with_ftype(monkeypatch): + monkeypatch.setattr(xfsinfoscanner, 'run', run_mocked()) + monkeypatch.setattr(os.path, 'ismount', lambda _: True) def consume_xfs_with_ftype_message_mocked(*models): fstab_data = { - "fs_spec": "/dev/mapper/rhel-root", - "fs_file": "/", - "fs_vfstype": "xfs", - "fs_mntops": "defaults", - "fs_freq": "0", - "fs_passno": "0"} + 'fs_spec': '/dev/mapper/rhel-root', + 'fs_file': '/', + 'fs_vfstype': 'xfs', + 'fs_mntops': 'defaults', + 'fs_freq': '0', + 'fs_passno': '0'} yield StorageInfo(fstab=[FstabEntry(**fstab_data)]) - monkeypatch.setattr(api, "consume", consume_xfs_with_ftype_message_mocked) - monkeypatch.setattr(api, "produce", produce_mocked()) + monkeypatch.setattr(api, 'consume', consume_xfs_with_ftype_message_mocked) + monkeypatch.setattr(api, 'produce', produce_mocked()) xfsinfoscanner.scan_xfs() - assert api.produce.called == 1 - assert len(api.produce.model_instances) == 1 - assert isinstance(api.produce.model_instances[0], XFSPresence) - assert api.produce.model_instances[0].present - assert not api.produce.model_instances[0].without_ftype - assert not api.produce.model_instances[0].mountpoints_without_ftype + + assert api.produce.called == 2 + assert len(api.produce.model_instances) == 2 + + xfs_presence = next(model for model in api.produce.model_instances if isinstance(model, XFSPresence)) + assert xfs_presence.present + assert not xfs_presence.without_ftype + assert not xfs_presence.mountpoints_without_ftype + + xfs_info_facts = next(model for model in api.produce.model_instances if isinstance(model, XFSInfoFacts)) + assert len(xfs_info_facts.mountpoints) == 1 + assert xfs_info_facts.mountpoints[0].mountpoint == '/' + assert xfs_info_facts.mountpoints[0].meta_data == TEST_XFS_INFO_FTYPE1_MODEL.meta_data + assert xfs_info_facts.mountpoints[0].data == TEST_XFS_INFO_FTYPE1_MODEL.data + assert xfs_info_facts.mountpoints[0].naming == TEST_XFS_INFO_FTYPE1_MODEL.naming + assert xfs_info_facts.mountpoints[0].log == TEST_XFS_INFO_FTYPE1_MODEL.log + assert xfs_info_facts.mountpoints[0].realtime == TEST_XFS_INFO_FTYPE1_MODEL.realtime + + +def test_scan_xfs_without_ftype(monkeypatch): + monkeypatch.setattr(xfsinfoscanner, 'run', run_mocked()) + monkeypatch.setattr(os.path, 'ismount', lambda _: True) def consume_xfs_without_ftype_message_mocked(*models): fstab_data = { - "fs_spec": "/dev/mapper/rhel-root", - "fs_file": "/var", - "fs_vfstype": "xfs", - "fs_mntops": "defaults", - "fs_freq": "0", - "fs_passno": "0"} + 'fs_spec': '/dev/mapper/rhel-root', + 'fs_file': '/var', + 'fs_vfstype': 'xfs', + 'fs_mntops': 'defaults', + 'fs_freq': '0', + 'fs_passno': '0'} yield StorageInfo(fstab=[FstabEntry(**fstab_data)]) - monkeypatch.setattr(api, "consume", consume_xfs_without_ftype_message_mocked) - monkeypatch.setattr(api, "produce", produce_mocked()) + monkeypatch.setattr(api, 'consume', consume_xfs_without_ftype_message_mocked) + monkeypatch.setattr(api, 'produce', produce_mocked()) xfsinfoscanner.scan_xfs() - assert api.produce.called == 1 - assert len(api.produce.model_instances) == 1 - assert isinstance(api.produce.model_instances[0], XFSPresence) - assert api.produce.model_instances[0].present - assert api.produce.model_instances[0].without_ftype - assert api.produce.model_instances[0].mountpoints_without_ftype - assert len(api.produce.model_instances[0].mountpoints_without_ftype) == 1 - assert api.produce.model_instances[0].mountpoints_without_ftype[0] == '/var' + + assert api.produce.called == 2 + assert len(api.produce.model_instances) == 2 + + xfs_presence = next(model for model in api.produce.model_instances if isinstance(model, XFSPresence)) + assert xfs_presence.present + assert xfs_presence.without_ftype + assert xfs_presence.mountpoints_without_ftype + + xfs_info_facts = next(model for model in api.produce.model_instances if isinstance(model, XFSInfoFacts)) + assert len(xfs_info_facts.mountpoints) == 1 + assert xfs_info_facts.mountpoints[0].mountpoint == '/var' + assert xfs_info_facts.mountpoints[0].meta_data == TEST_XFS_INFO_FTYPE0_MODEL.meta_data + assert xfs_info_facts.mountpoints[0].data == TEST_XFS_INFO_FTYPE0_MODEL.data + assert xfs_info_facts.mountpoints[0].naming == TEST_XFS_INFO_FTYPE0_MODEL.naming + assert xfs_info_facts.mountpoints[0].log == TEST_XFS_INFO_FTYPE0_MODEL.log + assert xfs_info_facts.mountpoints[0].realtime == TEST_XFS_INFO_FTYPE0_MODEL.realtime + + +def test_scan_xfs_no_message(monkeypatch): + monkeypatch.setattr(xfsinfoscanner, 'run', run_mocked()) + monkeypatch.setattr(os.path, 'ismount', lambda _: True) def consume_no_message_mocked(*models): yield None - monkeypatch.setattr(api, "consume", consume_no_message_mocked) - monkeypatch.setattr(api, "produce", produce_mocked()) + monkeypatch.setattr(api, 'consume', consume_no_message_mocked) + monkeypatch.setattr(api, 'produce', produce_mocked()) xfsinfoscanner.scan_xfs() - assert api.produce.called == 1 - assert len(api.produce.model_instances) == 1 - assert isinstance(api.produce.model_instances[0], XFSPresence) - assert not api.produce.model_instances[0].present - assert not api.produce.model_instances[0].without_ftype - assert not api.produce.model_instances[0].mountpoints_without_ftype + + assert api.produce.called == 2 + assert len(api.produce.model_instances) == 2 + + xfs_presence = next(model for model in api.produce.model_instances if isinstance(model, XFSPresence)) + assert not xfs_presence.present + assert not xfs_presence.without_ftype + assert not xfs_presence.mountpoints_without_ftype + + xfs_info_facts = next(model for model in api.produce.model_instances if isinstance(model, XFSInfoFacts)) + assert not xfs_info_facts.mountpoints + + +def test_parse_xfs_info(monkeypatch): + xfs_info = xfsinfoscanner.parse_xfs_info(TEST_XFS_INFO_FTYPE0.splitlines()) + assert xfs_info == TEST_XFS_INFO_FTYPE0_PARSED + + xfs_info = xfsinfoscanner.parse_xfs_info(TEST_XFS_INFO_FTYPE1.splitlines()) + assert xfs_info == TEST_XFS_INFO_FTYPE1_PARSED diff --git a/repos/system_upgrade/common/models/xfsinfo.py b/repos/system_upgrade/common/models/xfsinfo.py new file mode 100644 index 0000000000..62aa72d6cd --- /dev/null +++ b/repos/system_upgrade/common/models/xfsinfo.py @@ -0,0 +1,143 @@ +from leapp.models import fields, Model +from leapp.topics import SystemInfoTopic + + +class XFSInfoSection(Model): + """ + Represents a section of `xfs_info`. + """ + topic = SystemInfoTopic + + +class XFSInfoMetaData(XFSInfoSection): + """ + Represents the `meta-data` section of `xfs_info`. + """ + + device = fields.String() + bigtime = fields.Nullable(fields.String()) + crc = fields.Nullable(fields.String()) + + # NOTE(dkubek): meta-data section might also contain the following fields + # which are not being used right now + + # isize = fields.String() + # agcount = fields.String() + # agsize = fields.String() + # sectsz = fields.String() + # attr = fields.String() + # projid32bit = fields.String() + # finobt = fields.String() + # sparse = fields.String() + # rmapbt = fields.String() + # reflink = fields.String() + # inobtcount = fields.String() + # nrext64 = fields.String() + + +class XFSInfoData(XFSInfoSection): + """ + Represents the `data` section of `xfs_info`. + """ + + # NOTE(dkubek): data section might also contain the following fields + # which are not being used right now + + # blocks = fields.String() + # bsize = fields.String() + # imaxpct = fields.String() + # sunit = fields.String() + # swidth = fields.String() + + +class XFSInfoNaming(XFSInfoSection): + """ + Represents the `naming` section of `xfs_info`. + """ + ftype = fields.Nullable(fields.String()) + + # NOTE(dkubek): naming section might also contain the following fields + # which are not being used right now + + # version = fields.String() + # bsize = fields.String() + # ascii_ci = fields.String() + + +class XFSInfoLog(XFSInfoSection): + """ + Represents the `log` section of `xfs_info`. + """ + + # NOTE(dkubek): log section might also contain the following fields + # which are not being used right now + + # internal = fields.String() + # bsize = fields.String() + # blocks = fields.String() + # version = fields.String() + # sectsz = fields.String() + # sunit = fields.String() + # lazy_count = fields.String() + + +class XFSInfoRealtime(XFSInfoSection): + """ + Represents the `realtime` section of `xfs_info`. + """ + + # NOTE(dkubek): realtime section might also contain the following fields + # which are not being used right now + + # extsz = fields.String() + # blocks = fields.String() + # rtextents = fields.String() + + +class XFSInfo(Model): + """ + A message containing the parsed results from `xfs_info` command for given mountpoint. + + Attributes are stored as key-value pairs. Optional section attribute is + stored under the identifier 'specifier'. + """ + topic = SystemInfoTopic + + mountpoint = fields.String() + """ + Mountpoint containing the XFS filesystem. + """ + + meta_data = fields.Model(XFSInfoMetaData) + """ + Attributes of 'meta-data' section. + """ + + data = fields.Model(XFSInfoData) + """ + Attributes of 'data' section. + """ + + naming = fields.Model(XFSInfoNaming) + """ + Attributes of 'naming' section. + """ + + log = fields.Model(XFSInfoLog) + """ + Attributes of 'log' section. + """ + + realtime = fields.Model(XFSInfoRealtime) + """ + Attributes of 'realtime' section. + """ + + +class XFSInfoFacts(Model): + """ + Message containing the xfs info for all mounted XFS filesystems. + """ + topic = SystemInfoTopic + + mountpoints = fields.List(fields.Model(XFSInfo)) diff --git a/repos/system_upgrade/el9toel10/actors/checkoldxfs/actor.py b/repos/system_upgrade/el9toel10/actors/checkoldxfs/actor.py new file mode 100644 index 0000000000..630a0d712e --- /dev/null +++ b/repos/system_upgrade/el9toel10/actors/checkoldxfs/actor.py @@ -0,0 +1,24 @@ +import leapp.libraries.actor.checkoldxfs as checkoldxfs +from leapp.actors import Actor +from leapp.models import XFSInfoFacts +from leapp.reporting import Report +from leapp.tags import ChecksPhaseTag, IPUWorkflowTag + + +class CheckOldXFS(Actor): + """ + Inhibit upgrade if XFS requirements for RHEL 10 are not satisfied. + + RHEL 10 introduces stricter requirements for XFS filesystems. If any XFS + filesystem on the system lack these required features, the upgrade will be + inhibited. + + """ + + name = 'check_old_xfs' + consumes = (XFSInfoFacts,) + produces = (Report,) + tags = (ChecksPhaseTag, IPUWorkflowTag,) + + def process(self): + checkoldxfs.process() diff --git a/repos/system_upgrade/el9toel10/actors/checkoldxfs/libraries/checkoldxfs.py b/repos/system_upgrade/el9toel10/actors/checkoldxfs/libraries/checkoldxfs.py new file mode 100644 index 0000000000..454bcc3700 --- /dev/null +++ b/repos/system_upgrade/el9toel10/actors/checkoldxfs/libraries/checkoldxfs.py @@ -0,0 +1,124 @@ +from leapp import reporting +from leapp.exceptions import StopActorExecutionError +from leapp.libraries.stdlib import api +from leapp.models import XFSInfoFacts + +RHEL_9_TO_10_BACKUP_RESTORE_LINK = 'https://red.ht/rhel_9_to_10_backup_restore_xfs' + +FMT_LIST_SEPARATOR = '\n - ' + + +def _formatted_list_output(input_list, sep=FMT_LIST_SEPARATOR): + return ['{}{}'.format(sep, item) for item in input_list] + + +def process(): + xfs_info_facts = _get_xfs_info_facts() + + invalid_bigtime = [] + invalid_crc = [] + for xfs_info in xfs_info_facts.mountpoints: + if not _has_valid_bigtime(xfs_info): + api.current_logger().debug( + 'Mountpoint {} has invalid bigtime'.format(xfs_info.mountpoint) + ) + invalid_bigtime.append(xfs_info.mountpoint) + + if not _has_valid_crc(xfs_info): + api.current_logger().debug( + 'Mountpoint {} has invalid crc'.format(xfs_info.mountpoint) + ) + invalid_crc.append(xfs_info.mountpoint) + + if invalid_bigtime or invalid_crc: + _inhibit_upgrade(invalid_bigtime, invalid_crc) + + return + + api.current_logger().debug('All XFS system detected are valid.') + + +def _get_xfs_info_facts(): + msgs = api.consume(XFSInfoFacts) + + xfs_info_facts = next(msgs, None) + if xfs_info_facts is None: + raise StopActorExecutionError('Could not retrieve XFSInfoFacts!') + + if next(msgs, None): + api.current_logger().warning( + 'Unexpectedly received more than one XFSInfoFacts message.') + + return xfs_info_facts + + +def _has_valid_bigtime(xfs_info): + return xfs_info.meta_data.bigtime == '1' + + +def _has_valid_crc(xfs_info): + return xfs_info.meta_data.crc == '1' + + +def _inhibit_upgrade(invalid_bigtime, invalid_crc): + title = 'Detected XFS filesystems incompatible with RHEL 10.' + summary = [] + remediation_hint = [] + if invalid_crc: + summary.append(( + 'XFS v4 format has been deprecated and it has been removed from' + ' the target kernel. Such filesystems cannot be mounted by target' + ' system kernel and so the upgrade cannot proceed successfully.' + ' Following XFS filesystems have v4 format:{}' + .format(''.join(_formatted_list_output(invalid_crc))) + )) + remediation_hint.append(( + 'Migrate XFS v4 filesystems to new XFS v5 format.' + ' For filesystems hosting data, perform a back up, reformat, and restore procedure.' + ' Refer to official documentation for details.' + ' For filesystems hosting the system a clean installation is recommended instead.' + )) + if invalid_bigtime: + summary.append(( + 'XFS v5 filesystem format brings the "bigtime" feature since RHEL 9' + ' to support timestamp beyond year 2038. XFS filesystems without' + ' enabled "bigtime" feature are not considered to be compatible with' + ' the target system to prevent any future problems.' + ' Following XFS file systems have not enabled the "bigtime" feature:{}' + .format(''.join(_formatted_list_output(invalid_bigtime))) + )) + # NOTE(pstodulk): This will affect any system which upgraded from RHEL 8 + # so it is clear that such FS will have to be modified offline e.g. from + # initramfs - and that we speak about significant number of systems. + # So this should be improved yet. E.g. to update the initramfs having + # xfs_admin inside and working: + # # dracut -I "/usr/sbin/xfs_admin /usr/bin/expr" -f + # Note that it seems that it could be done without xfs_admin, using xfs_db + # only - which is present already. + remediation_hint.append(( + 'Enable the "bigtime" feature on XFS v5 filesystems using the command:' + '\n\txfs_admin -O bigtime=1 \n\n' + 'Note that for older XFS v5 filesystems this step can only be done offline' + ' right now (i.e. without the filesystem mounted).' + )) + + + reporting.create_report([ + reporting.Title(title), + reporting.Summary('\n\n'.join(summary)), + reporting.Remediation(hint='\n\n'.join(remediation_hint)), + reporting.ExternalLink( + title='Backing up an XFS file system', + url='https://red.ht/rhel-9-xfs-backup', + ), + reporting.ExternalLink( + title='Restoring an XFS file system from backup', + url='https://red.ht/rhel-9-xfs-restore-from-backup', + ), + reporting.ExternalLink( + title='XFS supports bigtime feature', + url='https://red.ht/rhel-9-xfs-bigtime', + ), + reporting.Severity(reporting.Severity.HIGH), + reporting.Groups([reporting.Groups.INHIBITOR]), + ]) diff --git a/repos/system_upgrade/el9toel10/actors/checkoldxfs/tests/test_checkoldxfs.py b/repos/system_upgrade/el9toel10/actors/checkoldxfs/tests/test_checkoldxfs.py new file mode 100644 index 0000000000..1632a644b2 --- /dev/null +++ b/repos/system_upgrade/el9toel10/actors/checkoldxfs/tests/test_checkoldxfs.py @@ -0,0 +1,198 @@ +import pytest + +from leapp import reporting +from leapp.exceptions import StopActorExecutionError +from leapp.libraries.actor import checkoldxfs +from leapp.libraries.common.testutils import create_report_mocked, CurrentActorMocked, logger_mocked +from leapp.libraries.stdlib import api +from leapp.models import ( + XFSInfo, + XFSInfoData, + XFSInfoFacts, + XFSInfoLog, + XFSInfoMetaData, + XFSInfoNaming, + XFSInfoRealtime +) +from leapp.utils.report import is_inhibitor + + +def test_has_valid_bigtime_passes(): + """ + Test _has_valid_bigtime passes for correct attributes. + """ + + xfs_info = XFSInfo( + mountpoint='/MOUNTPOINT', + meta_data=XFSInfoMetaData(bigtime='1', crc=None, device='/dev/vda'), + data=XFSInfoData(), + naming=XFSInfoNaming(), + log=XFSInfoLog(), + realtime=XFSInfoRealtime(), + ) + + assert checkoldxfs._has_valid_bigtime(xfs_info) + + +@pytest.mark.parametrize("bigtime", ['0', '', '', None]) +def test_has_valid_bigtime_fail(bigtime): + """ + Test _has_valid_bigtime fails for incorrect attributes. + """ + + xfs_info = XFSInfo( + mountpoint='/MOUNTPOINT', + meta_data=( + XFSInfoMetaData(bigtime=bigtime, crc=None, device='/dev/vda') + if bigtime + else XFSInfoMetaData(device='/dev/vda') + ), + data=XFSInfoData(), + naming=XFSInfoNaming(), + log=XFSInfoLog(), + realtime=XFSInfoRealtime(), + ) + + assert not checkoldxfs._has_valid_bigtime(xfs_info) + + +def test_has_valid_crc_passes(): + """ + Test _has_valid_crc passes for correct attributes. + """ + + xfs_info = XFSInfo( + mountpoint='/MOUNTPOINT', + meta_data=XFSInfoMetaData(crc='1', bigtime=None, device='/dev/vda'), + data=XFSInfoData(), + naming=XFSInfoNaming(), + log=XFSInfoLog(), + realtime=XFSInfoRealtime(), + ) + + assert checkoldxfs._has_valid_crc(xfs_info) + + +@pytest.mark.parametrize("crc", ['0', '', '', None]) +def test_has_valid_crc_fail(crc): + """ + Test _has_valid_crc fails for incorrect attributes. + """ + + xfs_info = XFSInfo( + mountpoint='/MOUNTPOINT', + meta_data=( + XFSInfoMetaData(crc=crc, bigtime=None, device='/dev/vda') + if crc + else XFSInfoMetaData(device='/dev/vda') + ), + data=XFSInfoData(), + naming=XFSInfoNaming(), + log=XFSInfoLog(), + realtime=XFSInfoRealtime(), + ) + + assert not checkoldxfs._has_valid_crc(xfs_info) + + +def test_get_xfs_info_facts_info_single_entry(monkeypatch): + xfs_info_facts = XFSInfoFacts(mountpoints=[]) + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(msgs=[xfs_info_facts])) + + result = checkoldxfs._get_xfs_info_facts() + assert result == xfs_info_facts + + +def test_get_workaround_efi_info_multiple_entries(monkeypatch): + logger = logger_mocked() + xfs_info_facts = XFSInfoFacts(mountpoints=[]) + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked( + msgs=[xfs_info_facts, xfs_info_facts])) + monkeypatch.setattr(api, 'current_logger', logger) + + result = checkoldxfs._get_xfs_info_facts() + assert result == xfs_info_facts + assert 'Unexpectedly received more than one XFSInfoFacts message.' in logger.warnmsg + + +def test_get_workaround_efi_info_no_entry(monkeypatch): + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(msgs=[])) + + with pytest.raises(StopActorExecutionError, match='Could not retrieve XFSInfoFacts!'): + checkoldxfs._get_xfs_info_facts() + + +def test_valid_xfs_passes(monkeypatch): + """ + Test no report is generated for valid XFS mountpoint + """ + + logger = logger_mocked() + monkeypatch.setattr(reporting, "create_report", create_report_mocked()) + monkeypatch.setattr(api, 'current_logger', logger) + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(msgs=[ + XFSInfoFacts( + mountpoints=[ + XFSInfo( + mountpoint='/MOUNTPOINT', + meta_data=XFSInfoMetaData(crc='1', bigtime='1', device='/dev/vda'), + data=XFSInfoData(), + naming=XFSInfoNaming(), + log=XFSInfoLog(), + realtime=XFSInfoRealtime(), + ), + ] + ) + ])) + + checkoldxfs.process() + + assert 'All XFS system detected are valid.' in logger.dbgmsg[0] + assert not reporting.create_report.called + + +@pytest.mark.parametrize( + 'valid_crc,valid_bigtime', + [ + (False, True), + (True, False), + (False, False), + ] +) +def test_unsupported_xfs(monkeypatch, valid_crc, valid_bigtime): + """ + Test report is generated for unsupported XFS mountpoint + """ + + logger = logger_mocked() + monkeypatch.setattr(reporting, "create_report", create_report_mocked()) + monkeypatch.setattr(api, 'current_logger', logger) + monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(msgs=[ + XFSInfoFacts( + mountpoints=[ + XFSInfo( + mountpoint='/MOUNTPOINT', + meta_data=XFSInfoMetaData( + crc='1' if valid_crc else '0', + bigtime='1' if valid_bigtime else '0', + device='/dev/vda', + ), + data=XFSInfoData(), + naming=XFSInfoNaming(), + log=XFSInfoLog(), + realtime=XFSInfoRealtime(), + ), + ] + ) + ])) + + checkoldxfs.process() + + produced_title = reporting.create_report.report_fields.get('title') + produced_summary = reporting.create_report.report_fields.get('summary') + + assert reporting.create_report.called == 1 + assert 'inhibited due to incompatible XFS filesystems' in produced_title + assert 'Some XFS filesystems' in produced_summary + assert reporting.create_report.report_fields['severity'] == reporting.Severity.HIGH + assert is_inhibitor(reporting.create_report.report_fields)