From 1ff714655560e3bf19424fe854b74031a336c09e Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Tue, 8 Dec 2020 12:24:37 +0100 Subject: [PATCH 01/26] safe_dbus: Add function to get all properties for an interface --- blivet/safe_dbus.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/blivet/safe_dbus.py b/blivet/safe_dbus.py index d99143e52..92d2ea80e 100644 --- a/blivet/safe_dbus.py +++ b/blivet/safe_dbus.py @@ -186,6 +186,33 @@ def get_property_sync(service, obj_path, iface, prop_name, return ret +def get_properties_sync(service, obj_path, iface, connection=None): + """ + Get all properties of a given object provided by a given service. + + :param service: DBus service to use + :type service: str + :param obj_path: object path + :type obj_path: str + :param iface: interface to use + :type iface: str + :param connection: connection to use (if None, a new connection is + established) + :type connection: Gio.DBusConnection + :return: unpacked value of the property + :rtype: tuple with elements that depend on the type of the property + :raise DBusCallError: when the internal dbus_call_safe_sync invocation + raises an exception + + """ + + args = GLib.Variant('(s)', (iface,)) + ret = call_sync(service, obj_path, DBUS_PROPS_IFACE, "GetAll", args, + connection) + + return ret + + def check_object_available(service, obj_path, iface=None): intro_data = call_sync(service, obj_path, DBUS_INTRO_IFACE, "Introspect", None) node_info = Gio.DBusNodeInfo.new_for_xml(intro_data[0]) From a94e180d5a7965e9ef601fba7f579f9ebbeeb54e Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Tue, 8 Dec 2020 12:25:35 +0100 Subject: [PATCH 02/26] Ignore all "private" devices during populate Currently only Stratis private devices are being ignored. --- blivet/populator/populator.py | 4 ++++ blivet/udev.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/blivet/populator/populator.py b/blivet/populator/populator.py index 75bb1741d..80d0bf0d1 100644 --- a/blivet/populator/populator.py +++ b/blivet/populator/populator.py @@ -253,6 +253,10 @@ def handle_device(self, info, update_orig_fmt=False): log.info("device %s is marked as hidden in sysfs, ignoring", name) return + if udev.device_is_private(info): + log.info("device %s is private, ignoring", name) + return + # make sure we note the name of every device we see self._add_name(name) device = self.get_device_by_name(name) diff --git a/blivet/udev.py b/blivet/udev.py index a8297f3f6..693d6e6fa 100644 --- a/blivet/udev.py +++ b/blivet/udev.py @@ -710,6 +710,11 @@ def device_is_dm_livecd(info): device_get_name(info).startswith("live")) +def device_is_dm_stratis(info): + """ Return True if the device is a Stratis pool or filesystem. """ + return device_dm_subsystem_match(info, "stratis") + + def device_is_biosraid_member(info): # Note that this function does *not* identify raid sets. # Tests to see if device is part of a dmraid set. @@ -1024,3 +1029,18 @@ def device_is_hidden(info): return False return bool(int(hidden)) + + +def device_is_stratis_private(info): + if not device_is_dm_stratis(info): + return False + try: + return info.get("DM_UUID", "").split("-")[2] == "private" + except IndexError: + return False + + +def device_is_private(info): + if device_is_dm_stratis(info): + return device_is_stratis_private(info) + return False From 3a9c4372e8af3590e92c771f9069aab04526f899 Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Tue, 8 Dec 2020 12:52:54 +0100 Subject: [PATCH 03/26] Add basic support for Stratis devices New StratisBlockdev format and two new devices StratisPoolDevice and StratisFilesystemDevice has been added. Stratis devices are populated using data from Stratis DBus API (cached in static_data.stratis_info). --- blivet/devicelibs/stratis.py | 26 ++++ blivet/devices/__init__.py | 1 + blivet/devices/stratis.py | 72 ++++++++++ blivet/formats/__init__.py | 2 +- blivet/formats/stratis.py | 81 +++++++++++ blivet/populator/helpers/__init__.py | 1 + blivet/populator/helpers/dm.py | 3 +- blivet/populator/helpers/stratis.py | 121 +++++++++++++++++ blivet/populator/populator.py | 22 ++- blivet/static_data/__init__.py | 1 + blivet/static_data/stratis_info.py | 192 +++++++++++++++++++++++++++ blivet/udev.py | 10 ++ tests/populator_test.py | 2 + 13 files changed, 526 insertions(+), 8 deletions(-) create mode 100644 blivet/devicelibs/stratis.py create mode 100644 blivet/devices/stratis.py create mode 100644 blivet/formats/stratis.py create mode 100644 blivet/populator/helpers/stratis.py create mode 100644 blivet/static_data/stratis_info.py diff --git a/blivet/devicelibs/stratis.py b/blivet/devicelibs/stratis.py new file mode 100644 index 000000000..983073235 --- /dev/null +++ b/blivet/devicelibs/stratis.py @@ -0,0 +1,26 @@ +# +# stratis.py +# stratis functions +# +# Copyright (C) 2020 Red Hat, Inc. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# Author(s): Vojtech Trefny +# + +from ..size import Size + + +STRATIS_FS_SIZE = Size("1 TiB") diff --git a/blivet/devices/__init__.py b/blivet/devices/__init__.py index 984e5af66..8bb0a979e 100644 --- a/blivet/devices/__init__.py +++ b/blivet/devices/__init__.py @@ -35,3 +35,4 @@ from .nodev import NoDevice, TmpFSDevice from .network import NetworkStorageDevice from .nfs import NFSDevice +from .stratis import StratisPoolDevice, StratisFilesystemDevice diff --git a/blivet/devices/stratis.py b/blivet/devices/stratis.py new file mode 100644 index 000000000..40009638f --- /dev/null +++ b/blivet/devices/stratis.py @@ -0,0 +1,72 @@ +# devices/stratis.py +# +# Copyright (C) 2020 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# the GNU General Public License v.2, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the +# GNU General Public License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the +# source code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission of +# Red Hat, Inc. +# +# Red Hat Author(s): Vojtech Trefny +# + +import logging +log = logging.getLogger("blivet") + +from .storage import StorageDevice +from ..static_data import stratis_info +from ..size import Size + + +class StratisPoolDevice(StorageDevice): + """ A stratis pool device """ + + _type = "stratis pool" + _resizable = False + _packages = ["stratisd", "stratis-cli"] + _dev_dir = "/dev/stratis" + + def read_current_size(self): + size = Size(0) + if self.exists and self.uuid in stratis_info.pools.keys(): + size = stratis_info.pools[self.uuid].physical_size + + return size + + +class StratisFilesystemDevice(StorageDevice): + """ A stratis pool device """ + + _type = "stratis filesystem" + _resizable = False + _packages = ["stratisd", "stratis-cli"] + _dev_dir = "/dev/stratis" + + def _get_name(self): + """ This device's name. """ + if self.pool is not None: + return "%s/%s" % (self.pool.name, self._name) + else: + return super(StratisFilesystemDevice, self)._get_name() + + @property + def fsname(self): + """ The Stratis filesystem name (not including pool name). """ + return self._name + + @property + def pool(self): + if not self.parents: + # this should never happen but just to be sure + return None + + return self.parents[0] diff --git a/blivet/formats/__init__.py b/blivet/formats/__init__.py index 0dbbae936..eb62e5887 100644 --- a/blivet/formats/__init__.py +++ b/blivet/formats/__init__.py @@ -742,4 +742,4 @@ def populate_ksdata(self, data): register_device_format(DeviceFormat) # import the format modules (which register their device formats) -from . import biosboot, disklabel, dmraid, fslib, fs, luks, lvmpv, mdraid, multipath, prepboot, swap +from . import biosboot, disklabel, dmraid, fslib, fs, luks, lvmpv, mdraid, multipath, prepboot, swap, stratis diff --git a/blivet/formats/stratis.py b/blivet/formats/stratis.py new file mode 100644 index 000000000..b8eecb31a --- /dev/null +++ b/blivet/formats/stratis.py @@ -0,0 +1,81 @@ +# stratis.py +# Device format classes for anaconda's storage configuration module. +# +# Copyright (C) 2020 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# the GNU General Public License v.2, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the +# GNU General Public License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the +# source code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission of +# Red Hat, Inc. +# +# Red Hat Author(s): Vojtech Trefny +# + +from ..storage_log import log_method_call +from ..i18n import N_ +from ..size import Size +from . import DeviceFormat, register_device_format + +import logging +log = logging.getLogger("blivet") + + +class StratisBlockdev(DeviceFormat): + """ A Stratis block device """ + + _type = "stratis" + _name = N_("Stratis block device") + _udev_types = ["stratis"] + _formattable = False # can be formatted + _supported = True # is supported + _linux_native = True # for clearpart + _min_size = Size("1 GiB") + _packages = ["stratisd"] # required packages + _resizable = False + + def __init__(self, **kwargs): + """ + :keyword device: path to the block device node + :keyword uuid: this Stratis block device UUID (not the pool UUID) + :keyword exists: indicates whether this is an existing format + :type exists: bool + :keyword pool_name: the name of the pool this block device belongs to + :keyword pool_uuid: the UUID of the pool this block device belongs to + + .. note:: + + The 'device' kwarg is required for existing formats. For non- + existent formats, it is only necessary that the :attr:`device` + attribute be set before the :meth:`create` method runs. Note + that you can specify the device at the last moment by specifying + it via the 'device' kwarg to the :meth:`create` method. + """ + log_method_call(self, **kwargs) + DeviceFormat.__init__(self, **kwargs) + + self.pool_name = kwargs.get("pool_name") + self.pool_uuid = kwargs.get("pool_uuid") + + def __repr__(self): + s = DeviceFormat.__repr__(self) + s += (" pool_name = %(pool_name)s pool_uuid = %(pool_uuid)s" % + {"pool_name": self.pool_name, "pool_uuid": self.pool_uuid}) + return s + + @property + def dict(self): + d = super(StratisBlockdev, self).dict + d.update({"pool_name": self.pool_name, "pool_uuid": self.pool_uuid}) + return d + + +register_device_format(StratisBlockdev) diff --git a/blivet/populator/helpers/__init__.py b/blivet/populator/helpers/__init__.py index aaf28c5cd..c01a6da8b 100644 --- a/blivet/populator/helpers/__init__.py +++ b/blivet/populator/helpers/__init__.py @@ -17,6 +17,7 @@ from .multipath import MultipathDevicePopulator, MultipathFormatPopulator from .optical import OpticalDevicePopulator from .partition import PartitionDevicePopulator +from .stratis import StratisFormatPopulator __all__ = ["get_device_helper", "get_format_helper"] diff --git a/blivet/populator/helpers/dm.py b/blivet/populator/helpers/dm.py index 30e99aa19..4721390ec 100644 --- a/blivet/populator/helpers/dm.py +++ b/blivet/populator/helpers/dm.py @@ -40,7 +40,8 @@ def match(cls, data): not udev.device_is_dm_integrity(data) and not udev.device_is_dm_lvm(data) and not udev.device_is_dm_mpath(data) and - not udev.device_is_dm_raid(data)) + not udev.device_is_dm_raid(data) and + not udev.device_is_dm_stratis(data)) def run(self): name = udev.device_get_name(self.data) diff --git a/blivet/populator/helpers/stratis.py b/blivet/populator/helpers/stratis.py new file mode 100644 index 000000000..eae7d85c2 --- /dev/null +++ b/blivet/populator/helpers/stratis.py @@ -0,0 +1,121 @@ +# populator/helpers/stratis.py +# Stratis backend code for populating a DeviceTree. +# +# Copyright (C) 2020 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# the GNU Lesser General Public License v.2, or (at your option) any later +# version. This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY expressed or implied, including the implied +# warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See +# the GNU Lesser General Public License for more details. You should have +# received a copy of the GNU Lesser General Public License along with this +# program; if not, write to the Free Software Foundation, Inc., 51 Franklin +# Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat trademarks +# that are incorporated in the source code or documentation are not subject +# to the GNU Lesser General Public License and may only be used or +# replicated with the express permission of Red Hat, Inc. +# +# Red Hat Author(s): Vojtech Trefny +# + +import copy + +from ...callbacks import callbacks +from ... import udev +from ...devices.stratis import StratisPoolDevice, StratisFilesystemDevice +from ...devicelibs.stratis import STRATIS_FS_SIZE +from ...storage_log import log_method_call +from .formatpopulator import FormatPopulator + +from ...static_data import stratis_info + +import logging +log = logging.getLogger("blivet") + + +class StratisFormatPopulator(FormatPopulator): + priority = 100 + _type_specifier = "stratis" + + def _get_kwargs(self): + kwargs = super(StratisFormatPopulator, self)._get_kwargs() + + bd_info = stratis_info.blockdevs.get(self.device.path) + + name = udev.device_get_name(self.data) + if bd_info: + if bd_info.pool_name: + kwargs["pool_name"] = bd_info.pool_name + else: + log.warning("Stratis block device %s has no pool_name", name) + if bd_info.pool_uuid: + kwargs["pool_uuid"] = bd_info.pool_uuid + else: + log.warning("Stratis block device %s has no pool_uuid", name) + + return kwargs + + def _add_pool_device(self): + bd_info = stratis_info.blockdevs.get(self.device.format.uuid) + if not bd_info: + # no info about the stratis block device -> we're done + return + + if not bd_info.pool_name: + log.info("stratis block device %s has no pool", self.device.name) + return + + pool_info = stratis_info.pools.get(bd_info.pool_uuid) + if pool_info is None: + log.warning("Failed to get information about Stratis pool %s (%s)", + bd_info.pool_name, bd_info.pool_uuid) + return + + pool_device = self._devicetree.get_device_by_uuid(bd_info.pool_uuid) + if pool_device and self.device not in pool_device.parents: + pool_device.parents.append(self.device) + callbacks.parent_added(device=pool_device, parent=self.device) + elif pool_device is None: + # TODO: stratis duplicate pool name + + pool_device = StratisPoolDevice(pool_info.name, + parents=[self.device], + uuid=pool_info.uuid, + size=pool_info.physical_size, + exists=True) + self._devicetree._add_device(pool_device) + + # now add filesystems on this pool + for fs_info in stratis_info.filesystems.values(): + if fs_info.pool_uuid != pool_info.uuid: + continue + + fs_device = self._devicetree.get_device_by_uuid(fs_info.uuid) + if fs_device is not None: + log.debug("stratis filesystem already added %s", fs_info.name) + continue + + pool_device = self._devicetree.get_device_by_uuid(fs_info.pool_uuid) + if not pool_device: + log.info("stratis pool %s has not been added yet", fs_info.pool_name) + return + + fs_device = StratisFilesystemDevice(fs_info.name, parents=[pool_device], + uuid=fs_info.uuid, size=STRATIS_FS_SIZE, + exists=True) + self._devicetree._add_device(fs_device) + + # do format handling now + udev_info = udev.get_device(fs_device.sysfs_path) + if not udev_info: + return + + self._devicetree.handle_format(udev_info, fs_device) + fs_device.original_format = copy.deepcopy(fs_device.format) + + def run(self): + log_method_call(self, pv=self.device.name) + super(StratisFormatPopulator, self).run() + self._add_pool_device() diff --git a/blivet/populator/populator.py b/blivet/populator/populator.py index 80d0bf0d1..84e14ee57 100644 --- a/blivet/populator/populator.py +++ b/blivet/populator/populator.py @@ -47,7 +47,7 @@ from ..tasks import availability from ..threads import SynchronizedMeta from .helpers import get_device_helper, get_format_helper -from ..static_data import lvs_info, pvs_info, vgs_info, luks_data, mpath_members +from ..static_data import lvs_info, pvs_info, vgs_info, luks_data, mpath_members, stratis_info from ..callbacks import callbacks import logging @@ -88,7 +88,7 @@ def reset(self, disk_images=None): self.set_disk_images(disk_images) # initialize attributes that may later hold cached lvm info - self.drop_lvm_cache() + self.drop_device_info_cache() self._cleanup = False @@ -146,6 +146,10 @@ def _add_name(self, name): if name not in self.names: self.names.append(name) + def _remove_name(self, name): + if name in self.names: + self.names.remove(name) + def _reason_to_skip_device(self, info): sysfs_path = udev.device_get_sysfs_path(info) uuid = udev.device_get_uuid(info) @@ -257,6 +261,11 @@ def handle_device(self, info, update_orig_fmt=False): log.info("device %s is private, ignoring", name) return + if udev.device_is_stratis_filesystem(info): + log.debug("skipping %s stratis filesystem, will be handled with corresponding " + "stratis block device", name) + return + # make sure we note the name of every device we see self._add_name(name) device = self.get_device_by_name(name) @@ -430,8 +439,7 @@ def _populate(self): self.ignored_disks, self.exclusive_disks) disklib.update_volume_info() - self.drop_lvm_cache() - mpath_members.drop_cache() + self.drop_device_info_cache() if flags.auto_dev_updates and availability.BLOCKDEV_MPATH_PLUGIN.available: blockdev.mpath.set_friendly_names(flags.multipath_friendly_names) @@ -471,11 +479,13 @@ def _populate(self): # inconsistencies are ignored or resolved. self._handle_inconsistencies() - def drop_lvm_cache(self): - """ Drop cached lvm information. """ + def drop_device_info_cache(self): + """ Drop cached device information. """ lvs_info.drop_cache() pvs_info.drop_cache() vgs_info.drop_cache() + mpath_members.drop_cache() + stratis_info.drop_cache() def handle_nodev_filesystems(self): for line in open("/proc/mounts").readlines(): diff --git a/blivet/static_data/__init__.py b/blivet/static_data/__init__.py index 321dae0fd..b5dd925e4 100644 --- a/blivet/static_data/__init__.py +++ b/blivet/static_data/__init__.py @@ -2,3 +2,4 @@ from .luks_data import luks_data from .mpath_info import mpath_members from .nvdimm import nvdimm +from .stratis_info import stratis_info diff --git a/blivet/static_data/stratis_info.py b/blivet/static_data/stratis_info.py new file mode 100644 index 000000000..5cfbea33e --- /dev/null +++ b/blivet/static_data/stratis_info.py @@ -0,0 +1,192 @@ +# stratis_info.py +# Backend code for populating a DeviceTree. +# +# Copyright (C) 2020 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# the GNU Lesser General Public License v.2, or (at your option) any later +# version. This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY expressed or implied, including the implied +# warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See +# the GNU Lesser General Public License for more details. You should have +# received a copy of the GNU Lesser General Public License along with this +# program; if not, write to the Free Software Foundation, Inc., 51 Franklin +# Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat trademarks +# that are incorporated in the source code or documentation are not subject +# to the GNU Lesser General Public License and may only be used or +# replicated with the express permission of Red Hat, Inc. +# +# Red Hat Author(s): Vojtech Trefny +# + +from collections import namedtuple + +from .. import safe_dbus +from ..size import Size + +import logging +log = logging.getLogger("blivet") + + +STRATIS_SERVICE = "org.storage.stratis2" +STRATIS_PATH = "/org/storage/stratis2" +STRATIS_POOL_INTF = STRATIS_SERVICE + ".pool" +STRATIS_FILESYSTEM_INTF = STRATIS_SERVICE + ".filesystem" +STRATIS_BLOCKDEV_INTF = STRATIS_SERVICE + ".blockdev" +STRATIS_PROPS_INTF = STRATIS_SERVICE + ".FetchProperties" + + +StratisPoolInfo = namedtuple("StratisPoolInfo", ["name", "uuid", "physical_size"]) +StratisFilesystemInfo = namedtuple("StratisFilesystemInfo", ["name", "uuid", "pool_name", "pool_uuid"]) +StratisBlockdevInfo = namedtuple("StratisBlockdevInfo", ["path", "uuid", "pool_name", "pool_uuid"]) + + +class StratisInfo(object): + """ Class to be used as a singleton. + Maintains the Stratis devices info cache. + """ + + def __init__(self): + self._info_cache = None + + def _get_pool_info(self, pool_path): + try: + properties = safe_dbus.get_properties_sync(STRATIS_SERVICE, + pool_path, + STRATIS_POOL_INTF)[0] + except safe_dbus.DBusPropertyError as e: + log.error("Error when getting DBus properties of '%s': %s", + pool_path, str(e)) + + if not properties: + log.error("Failed to get DBus properties of '%s'", pool_path) + return None + + all_props = safe_dbus.call_sync(STRATIS_SERVICE, + pool_path, + STRATIS_PROPS_INTF, + "GetAllProperties", + None)[0] + if all_props: + valid, pool_size = all_props.get("TotalPhysicalSize", + (False, "TotalPhysicalSize not available")) + if not valid: + log.warning("Failed to get Stratis pool physical size for %s: %s", + properties["Name"], pool_size) + pool_size = 0 + else: + log.error("Failed to get Stratis pool properties for %s.", properties["Name"]) + pool_size = 0 + + return StratisPoolInfo(name=properties["Name"], uuid=properties["Uuid"], + physical_size=Size(pool_size)) + + def _get_filesystem_info(self, filesystem_path): + try: + properties = safe_dbus.get_properties_sync(STRATIS_SERVICE, + filesystem_path, + STRATIS_FILESYSTEM_INTF)[0] + except safe_dbus.DBusPropertyError as e: + log.error("Error when getting DBus properties of '%s': %s", + filesystem_path, str(e)) + + if not properties: + log.error("Failed to get DBus properties of '%s'", filesystem_path) + return None + + pool_info = self._get_pool_info(properties["Pool"]) + if not pool_info: + return None + + return StratisFilesystemInfo(name=properties["Name"], uuid=properties["Uuid"], + pool_name=pool_info.name, pool_uuid=pool_info.uuid) + + def _get_blockdev_info(self, blockdev_path): + try: + properties = safe_dbus.get_properties_sync(STRATIS_SERVICE, + blockdev_path, + STRATIS_BLOCKDEV_INTF)[0] + except safe_dbus.DBusPropertyError as e: + log.error("Error when getting DBus properties of '%s': %s", + blockdev_path, str(e)) + + if not properties: + log.error("Failed to get DBus properties of '%s'", blockdev_path) + return None + + pool_path = properties["Pool"] + if pool_path == "/": + pool_name = "" + else: + pool_info = self._get_pool_info(properties["Pool"]) + if not pool_info: + return None + pool_name = pool_info.name + + return StratisBlockdevInfo(path=properties["Devnode"], uuid=properties["Uuid"], + pool_name=pool_name, pool_uuid=pool_info.uuid) + + def _get_stratis_info(self): + self._info_cache = dict() + self._info_cache["pools"] = dict() + self._info_cache["blockdevs"] = dict() + self._info_cache["filesystems"] = dict() + + try: + ret = safe_dbus.check_object_available(STRATIS_SERVICE, STRATIS_PATH) + except safe_dbus.DBusCallError: + log.warning("Stratis DBus service is not running") + return + else: + if not ret: + log.warning("Stratis DBus service is not available") + + objects = safe_dbus.call_sync(STRATIS_SERVICE, + STRATIS_PATH, + "org.freedesktop.DBus.ObjectManager", + "GetManagedObjects", + None)[0] + + for path, interfaces in objects.items(): + if STRATIS_POOL_INTF in interfaces.keys(): + pool_info = self._get_pool_info(path) + if pool_info: + self._info_cache["pools"][pool_info.uuid] = pool_info + + if STRATIS_FILESYSTEM_INTF in interfaces.keys(): + fs_info = self._get_filesystem_info(path) + if fs_info: + self._info_cache["filesystems"][fs_info.uuid] = fs_info + + if STRATIS_BLOCKDEV_INTF in interfaces.keys(): + bd_info = self._get_blockdev_info(path) + if bd_info: + self._info_cache["blockdevs"][bd_info.uuid] = bd_info + + @property + def pools(self): + if self._info_cache is None: + self._get_stratis_info() + + return self._info_cache["pools"] + + @property + def filesystems(self): + if self._info_cache is None: + self._get_stratis_info() + + return self._info_cache["filesystems"] + + @property + def blockdevs(self): + if self._info_cache is None: + self._get_stratis_info() + + return self._info_cache["blockdevs"] + + def drop_cache(self): + self._info_cache = None + + +stratis_info = StratisInfo() diff --git a/blivet/udev.py b/blivet/udev.py index 693d6e6fa..97283dc73 100644 --- a/blivet/udev.py +++ b/blivet/udev.py @@ -373,6 +373,7 @@ def device_is_disk(info): device_is_dm_partition(info) or device_is_dm_lvm(info) or device_is_dm_crypt(info) or + device_is_dm_stratis(info) or (device_is_md(info) and (not device_get_md_container(info) and not all(device_is_disk(d) for d in device_get_parents(info)))))) @@ -1031,6 +1032,15 @@ def device_is_hidden(info): return bool(int(hidden)) +def device_is_stratis_filesystem(info): + if not device_is_dm_stratis(info): + return False + try: + return info.get("DM_UUID", "").split("-")[4] == "fs" + except IndexError: + return False + + def device_is_stratis_private(info): if not device_is_dm_stratis(info): return False diff --git a/tests/populator_test.py b/tests/populator_test.py index 2a8532f0b..b63780f95 100644 --- a/tests/populator_test.py +++ b/tests/populator_test.py @@ -40,6 +40,7 @@ class DMDevicePopulatorTestCase(PopulatorHelperTestCase): @patch("blivet.udev.device_is_dm_mpath", return_value=False) @patch("blivet.udev.device_is_dm_partition", return_value=False) @patch("blivet.udev.device_is_dm_raid", return_value=False) + @patch("blivet.udev.device_is_dm_stratis", return_value=False) @patch("blivet.udev.device_is_dm", return_value=True) def test_match(self, *args): """Test matching of dm device populator.""" @@ -60,6 +61,7 @@ def test_match(self, *args): @patch("blivet.udev.device_is_dm_mpath", return_value=False) @patch("blivet.udev.device_is_dm_partition", return_value=False) @patch("blivet.udev.device_is_dm_raid", return_value=False) + @patch("blivet.udev.device_is_dm_stratis", return_value=False) @patch("blivet.udev.device_is_md", return_value=False) @patch("blivet.udev.device_is_loop", return_value=False) @patch("blivet.udev.device_is_dm", return_value=True) From 30f14fd1f719f5ae6c1ec9ce7b8df8f638ff5571 Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Tue, 8 Dec 2020 14:30:31 +0100 Subject: [PATCH 04/26] Add support for removing Stratis devices using DBus API We are ignoring a lot of stratis private device mapper devices now so we can no longer use libblockdev DM plugin to remove Stratis storage stack and we need to use Stratis DBus API instead. --- blivet/devicelibs/stratis.py | 70 ++++++++++++++++++++++++++++++ blivet/devices/stratis.py | 13 +++++- blivet/errors.py | 5 +++ blivet/static_data/stratis_info.py | 23 ++++------ 4 files changed, 96 insertions(+), 15 deletions(-) diff --git a/blivet/devicelibs/stratis.py b/blivet/devicelibs/stratis.py index 983073235..507fd31ed 100644 --- a/blivet/devicelibs/stratis.py +++ b/blivet/devicelibs/stratis.py @@ -20,7 +20,77 @@ # Author(s): Vojtech Trefny # +import gi +gi.require_version("GLib", "2.0") + +from gi.repository import GLib + +from ..errors import StratisError from ..size import Size +from ..static_data import stratis_info +from .. import safe_dbus + + +STRATIS_SERVICE = "org.storage.stratis2" +STRATIS_PATH = "/org/storage/stratis2" +STRATIS_POOL_INTF = STRATIS_SERVICE + ".pool" +STRATIS_FILESYSTEM_INTF = STRATIS_SERVICE + ".filesystem" +STRATIS_BLOCKDEV_INTF = STRATIS_SERVICE + ".blockdev" +STRATIS_PROPS_INTF = STRATIS_SERVICE + ".FetchProperties" +STRATIS_MANAGER_INTF = STRATIS_SERVICE + ".Manager" STRATIS_FS_SIZE = Size("1 TiB") + + +def remove_pool(pool_uuid): + if not safe_dbus.check_object_available(STRATIS_SERVICE, STRATIS_PATH): + raise StratisError("Stratis DBus service not available") + + # repopulate the stratis info cache just to be sure all values are still valid + stratis_info.drop_cache() + + if pool_uuid not in stratis_info.pools.keys(): + raise StratisError("Stratis pool with UUID %s not found" % pool_uuid) + + pool_info = stratis_info.pools[pool_uuid] + + try: + (succ, _uuid), rc, err = safe_dbus.call_sync(STRATIS_SERVICE, + STRATIS_PATH, + STRATIS_MANAGER_INTF, + "DestroyPool", + GLib.Variant("(o)", (pool_info.object_path,))) + except safe_dbus.DBusCallError as e: + raise StratisError("Failed to remove stratis pool: %s" % str(e)) + else: + if not succ: + raise StratisError("Failed to remove stratis pool: %s (%d)" % (err, rc)) + + +def remove_filesystem(pool_uuid, fs_uuid): + if not safe_dbus.check_object_available(STRATIS_SERVICE, STRATIS_PATH): + raise StratisError("Stratis DBus service not available") + + # repopulate the stratis info cache just to be sure all values are still valid + stratis_info.drop_cache() + + if pool_uuid not in stratis_info.pools.keys(): + raise StratisError("Stratis pool with UUID %s not found" % pool_uuid) + if fs_uuid not in stratis_info.filesystems.keys(): + raise StratisError("Stratis filesystem with UUID %s not found" % fs_uuid) + + pool_info = stratis_info.pools[pool_uuid] + fs_info = stratis_info.filesystems[fs_uuid] + + try: + (succ, _uuid), rc, err = safe_dbus.call_sync(STRATIS_SERVICE, + pool_info.object_path, + STRATIS_POOL_INTF, + "DestroyFilesystems", + GLib.Variant("(ao)", ([fs_info.object_path],))) + except safe_dbus.DBusCallError as e: + raise StratisError("Failed to remove stratis filesystem: %s" % str(e)) + else: + if not succ: + raise StratisError("Failed to remove stratis filesystem: %s (%d)" % (err, rc)) diff --git a/blivet/devices/stratis.py b/blivet/devices/stratis.py index 40009638f..19f26d847 100644 --- a/blivet/devices/stratis.py +++ b/blivet/devices/stratis.py @@ -25,6 +25,8 @@ from .storage import StorageDevice from ..static_data import stratis_info from ..size import Size +from ..storage_log import log_method_call +from .. import devicelibs class StratisPoolDevice(StorageDevice): @@ -39,9 +41,13 @@ def read_current_size(self): size = Size(0) if self.exists and self.uuid in stratis_info.pools.keys(): size = stratis_info.pools[self.uuid].physical_size - return size + def _destroy(self): + """ Destroy the device. """ + log_method_call(self, self.name, status=self.status) + devicelibs.stratis.remove_pool(self.uuid) + class StratisFilesystemDevice(StorageDevice): """ A stratis pool device """ @@ -70,3 +76,8 @@ def pool(self): return None return self.parents[0] + + def _destroy(self): + """ Destroy the device. """ + log_method_call(self, self.name, status=self.status) + devicelibs.stratis.remove_filesystem(self.pool.uuid, self.uuid) diff --git a/blivet/errors.py b/blivet/errors.py index fd51283f7..8a26eadda 100644 --- a/blivet/errors.py +++ b/blivet/errors.py @@ -185,6 +185,11 @@ class BTRFSError(StorageError): class BTRFSValueError(BTRFSError, ValueError): pass + +class StratisError(StorageError): + pass + + # DeviceTree diff --git a/blivet/static_data/stratis_info.py b/blivet/static_data/stratis_info.py index 5cfbea33e..c5409c46c 100644 --- a/blivet/static_data/stratis_info.py +++ b/blivet/static_data/stratis_info.py @@ -24,22 +24,15 @@ from .. import safe_dbus from ..size import Size +from ..devicelibs.stratis import STRATIS_SERVICE, STRATIS_PATH, STRATIS_POOL_INTF, STRATIS_FILESYSTEM_INTF, STRATIS_BLOCKDEV_INTF, STRATIS_PROPS_INTF import logging log = logging.getLogger("blivet") -STRATIS_SERVICE = "org.storage.stratis2" -STRATIS_PATH = "/org/storage/stratis2" -STRATIS_POOL_INTF = STRATIS_SERVICE + ".pool" -STRATIS_FILESYSTEM_INTF = STRATIS_SERVICE + ".filesystem" -STRATIS_BLOCKDEV_INTF = STRATIS_SERVICE + ".blockdev" -STRATIS_PROPS_INTF = STRATIS_SERVICE + ".FetchProperties" - - -StratisPoolInfo = namedtuple("StratisPoolInfo", ["name", "uuid", "physical_size"]) -StratisFilesystemInfo = namedtuple("StratisFilesystemInfo", ["name", "uuid", "pool_name", "pool_uuid"]) -StratisBlockdevInfo = namedtuple("StratisBlockdevInfo", ["path", "uuid", "pool_name", "pool_uuid"]) +StratisPoolInfo = namedtuple("StratisPoolInfo", ["name", "uuid", "physical_size", "object_path"]) +StratisFilesystemInfo = namedtuple("StratisFilesystemInfo", ["name", "uuid", "pool_name", "pool_uuid", "object_path"]) +StratisBlockdevInfo = namedtuple("StratisBlockdevInfo", ["path", "uuid", "pool_name", "pool_uuid", "object_path"]) class StratisInfo(object): @@ -80,7 +73,7 @@ def _get_pool_info(self, pool_path): pool_size = 0 return StratisPoolInfo(name=properties["Name"], uuid=properties["Uuid"], - physical_size=Size(pool_size)) + physical_size=Size(pool_size), object_path=pool_path) def _get_filesystem_info(self, filesystem_path): try: @@ -100,7 +93,8 @@ def _get_filesystem_info(self, filesystem_path): return None return StratisFilesystemInfo(name=properties["Name"], uuid=properties["Uuid"], - pool_name=pool_info.name, pool_uuid=pool_info.uuid) + pool_name=pool_info.name, pool_uuid=pool_info.uuid, + object_path=filesystem_path) def _get_blockdev_info(self, blockdev_path): try: @@ -125,7 +119,8 @@ def _get_blockdev_info(self, blockdev_path): pool_name = pool_info.name return StratisBlockdevInfo(path=properties["Devnode"], uuid=properties["Uuid"], - pool_name=pool_name, pool_uuid=pool_info.uuid) + pool_name=pool_name, pool_uuid=pool_info.uuid, + object_path=blockdev_path) def _get_stratis_info(self): self._info_cache = dict() From 44c47986f2d5445632abbb32c7d4c72ed5c8f00c Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Thu, 14 Jan 2021 14:19:57 +0100 Subject: [PATCH 05/26] Add dracut setup args for Stratis devices --- blivet/devices/stratis.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/blivet/devices/stratis.py b/blivet/devices/stratis.py index 19f26d847..ac7499afb 100644 --- a/blivet/devices/stratis.py +++ b/blivet/devices/stratis.py @@ -48,6 +48,9 @@ def _destroy(self): log_method_call(self, self.name, status=self.status) devicelibs.stratis.remove_pool(self.uuid) + def dracut_setup_args(self): + return set(["stratis.rootfs.pool_uuid=%s" % self.uuid]) + class StratisFilesystemDevice(StorageDevice): """ A stratis pool device """ @@ -81,3 +84,6 @@ def _destroy(self): """ Destroy the device. """ log_method_call(self, self.name, status=self.status) devicelibs.stratis.remove_filesystem(self.pool.uuid, self.uuid) + + def dracut_setup_args(self): + return set(["root=%s" % self.path]) From 4cb3903a017e98adde6bfe8884759a746259736c Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Thu, 28 Jan 2021 13:31:45 +0100 Subject: [PATCH 06/26] Avoid circular depency when in static_data/stratis_info.py devicelibs.stratis uses stratis_info so we can't import constants from it here without breaking Python. --- blivet/static_data/stratis_info.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/blivet/static_data/stratis_info.py b/blivet/static_data/stratis_info.py index c5409c46c..8a7ad68fa 100644 --- a/blivet/static_data/stratis_info.py +++ b/blivet/static_data/stratis_info.py @@ -24,12 +24,21 @@ from .. import safe_dbus from ..size import Size -from ..devicelibs.stratis import STRATIS_SERVICE, STRATIS_PATH, STRATIS_POOL_INTF, STRATIS_FILESYSTEM_INTF, STRATIS_BLOCKDEV_INTF, STRATIS_PROPS_INTF import logging log = logging.getLogger("blivet") +# XXX we can't import these from devicelibs.stratis, circular imports make python mad +STRATIS_SERVICE = "org.storage.stratis2" +STRATIS_PATH = "/org/storage/stratis2" +STRATIS_POOL_INTF = STRATIS_SERVICE + ".pool" +STRATIS_FILESYSTEM_INTF = STRATIS_SERVICE + ".filesystem" +STRATIS_BLOCKDEV_INTF = STRATIS_SERVICE + ".blockdev" +STRATIS_PROPS_INTF = STRATIS_SERVICE + ".FetchProperties" +STRATIS_MANAGER_INTF = STRATIS_SERVICE + ".Manager.r2" + + StratisPoolInfo = namedtuple("StratisPoolInfo", ["name", "uuid", "physical_size", "object_path"]) StratisFilesystemInfo = namedtuple("StratisFilesystemInfo", ["name", "uuid", "pool_name", "pool_uuid", "object_path"]) StratisBlockdevInfo = namedtuple("StratisBlockdevInfo", ["path", "uuid", "pool_name", "pool_uuid", "object_path"]) From 8bbe58100c92915eb919eb182d083e38252ed192 Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Mon, 8 Feb 2021 10:14:22 +0100 Subject: [PATCH 07/26] Add a special "XFS Stratis" filesystem for Stratis filesystem devices This will allow us handling the XFS filesystem on Stratis in a better way to avoid creating and removing it separately (it should be created and removed automatically together with the Stratis filesystem block device). --- blivet/formats/fs.py | 24 ++++++++++++++++++ blivet/populator/helpers/__init__.py | 2 +- blivet/populator/helpers/formatpopulator.py | 5 +++- blivet/populator/helpers/stratis.py | 27 +++++++++++++++++++++ 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/blivet/formats/fs.py b/blivet/formats/fs.py index 23c6f968a..749af0864 100644 --- a/blivet/formats/fs.py +++ b/blivet/formats/fs.py @@ -1132,6 +1132,30 @@ def write_uuid(self): register_device_format(XFS) +class StratisXFS(XFS): + """ XFS on top of Stratis filesystem device """ + + _type = "stratis xfs" + + _resize_class = fsresize.UnimplementedFSResize + _size_info_class = fssize.UnimplementedFSSize + _info_class = fsinfo.UnimplementedFSInfo + _minsize_class = fsminsize.UnimplementedFSMinSize + _writelabel_class = fswritelabel.UnimplementedFSWriteLabel + _writeuuid_class = fswriteuuid.UnimplementedFSWriteUUID + + def _create(self, **kwargs): # pylint: disable=unused-argument + # format is created together with the stratis filesystem device + pass + + def _destroy(self, **kwargs): # pylint: disable=unused-argument + # format is destroyed together with the stratis filesystem device + pass + + +register_device_format(StratisXFS) + + class HFS(FS): _type = "hfs" _modules = ["hfs"] diff --git a/blivet/populator/helpers/__init__.py b/blivet/populator/helpers/__init__.py index c01a6da8b..c5ac412fa 100644 --- a/blivet/populator/helpers/__init__.py +++ b/blivet/populator/helpers/__init__.py @@ -17,7 +17,7 @@ from .multipath import MultipathDevicePopulator, MultipathFormatPopulator from .optical import OpticalDevicePopulator from .partition import PartitionDevicePopulator -from .stratis import StratisFormatPopulator +from .stratis import StratisFormatPopulator, StratisXFSFormatPopulator __all__ = ["get_device_helper", "get_format_helper"] diff --git a/blivet/populator/helpers/formatpopulator.py b/blivet/populator/helpers/formatpopulator.py index e9f71650f..8d78bf0ca 100644 --- a/blivet/populator/helpers/formatpopulator.py +++ b/blivet/populator/helpers/formatpopulator.py @@ -45,7 +45,10 @@ def match(cls, data, device): # pylint: disable=arguments-differ,unused-argumen :returns: whether this class is appropriate for the specified device :rtype: bool """ - ret = False + if device.type == "stratis filesystem": + # XFS on stratis filesystem device handled in StratisXFSFormatPopulator + return False + if cls is FormatPopulator: ret = True else: diff --git a/blivet/populator/helpers/stratis.py b/blivet/populator/helpers/stratis.py index eae7d85c2..01d86c447 100644 --- a/blivet/populator/helpers/stratis.py +++ b/blivet/populator/helpers/stratis.py @@ -24,6 +24,7 @@ from ...callbacks import callbacks from ... import udev +from ...formats import get_format from ...devices.stratis import StratisPoolDevice, StratisFilesystemDevice from ...devicelibs.stratis import STRATIS_FS_SIZE from ...storage_log import log_method_call @@ -119,3 +120,29 @@ def run(self): log_method_call(self, pv=self.device.name) super(StratisFormatPopulator, self).run() self._add_pool_device() + + +class StratisXFSFormatPopulator(FormatPopulator): + priority = 100 + _type_specifier = "stratis xfs" + + @classmethod + def match(cls, data, device): # pylint: disable=arguments-differ,unused-argument + """ Return True if this helper is appropriate for the given device. + + :param :class:`pyudev.Device` data: udev data describing a device + :param device: device instance corresponding to the udev data + :type device: :class:`~.devices.StorageDevice` + :returns: whether this class is appropriate for the specified device + :rtype: bool + """ + if device.type == "stratis filesystem" and udev.device_get_format(data) == "xfs": + return True + + return False + + def run(self): + """ Create a format instance and associate it with the device instance. """ + kwargs = self._get_kwargs() + log.info("type detected on '%s' is '%s'", self.device.name, self.type_spec) + self.device.format = get_format(self.type_spec, **kwargs) From bfefdd1da99c16f7dd88801e78e143a87bf21c35 Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Thu, 28 Jan 2021 15:02:44 +0100 Subject: [PATCH 08/26] Add basic support for creating Stratis devices --- blivet/blivet.py | 103 ++++++++++++++++++++++++++--- blivet/devicefactory.py | 1 + blivet/devicelibs/stratis.py | 57 +++++++++++++++- blivet/devices/stratis.py | 50 ++++++++++++-- blivet/formats/stratis.py | 2 +- blivet/static_data/stratis_info.py | 17 +++++ 6 files changed, 214 insertions(+), 16 deletions(-) diff --git a/blivet/blivet.py b/blivet/blivet.py index f4ce6a48a..b1d32d5e2 100644 --- a/blivet/blivet.py +++ b/blivet/blivet.py @@ -31,6 +31,7 @@ from .devices import BTRFSSubVolumeDevice, BTRFSVolumeDevice from .devices import LVMLogicalVolumeDevice, LVMVolumeGroupDevice from .devices import MDRaidArrayDevice, PartitionDevice, TmpFSDevice, device_path_to_name +from .devices import StratisPoolDevice, StratisFilesystemDevice from .deviceaction import ActionCreateDevice, ActionCreateFormat, ActionDestroyDevice from .deviceaction import ActionDestroyFormat, ActionResizeDevice, ActionResizeFormat from .devicelibs.edd import get_edd_dict @@ -770,6 +771,77 @@ def new_btrfs_sub_volume(self, *args, **kwargs): kwargs["subvol"] = True return self.new_btrfs(*args, **kwargs) + def new_stratis_pool(self, *args, **kwargs): + """ Return a new StratisPoolDevice instance. + + :returns: the new Stratis pool device + :rtype: :class:`~.devices.StratisPoolDevice` + + All arguments are passed on to the + :class:`~.devices.StratisPoolDevice` constructor. + + If a name is not specified, one will be generated based on the + hostname, and/or product name. + """ + blockdevs = kwargs.pop("parents", []) + + name = kwargs.pop("name", None) + if name: + safe_name = self.safe_device_name(name, devicefactory.DEVICE_TYPE_STRATIS) + if safe_name != name: + log.warning("using '%s' instead of specified name '%s'", + safe_name, name) + name = safe_name + else: + name = self.suggest_container_name(container_type=devicefactory.DEVICE_TYPE_STRATIS) + + if name in self.names: + raise ValueError("name '%s' is already in use" % name) + + return StratisPoolDevice(name, parents=blockdevs, *args, **kwargs) + + def new_stratis_filesystem(self, *args, **kwargs): + """ Return a new StratisFilesystemDevice instance. + + :keyword mountpoint: mountpoint for filesystem + :type mountpoint: str + :returns: the new device + :rtype: :class:`~.devices.StratisFilesystemDevice` + + All other arguments are passed on to the appropriate + :class:`~.devices.StratisFilesystemDevice` constructor. + + If a name is not specified, one will be generated based on the + format type and/or mountpoint. + """ + pool = kwargs.get("parents", [None])[0] + + mountpoint = kwargs.pop("mountpoint", None) + name = kwargs.pop("name", None) + if name: + # make sure the specified name is sensible + full_name = "%s/%s" % (pool.name, name) + safe_name = self.safe_device_name(full_name, devicefactory.DEVICE_TYPE_STRATIS) + if safe_name != full_name: + new_name = safe_name[len(pool.name) + 1:] + log.warning("using '%s' instead of specified name '%s'", + new_name, name) + name = new_name + else: + name = self.suggest_device_name(parent=pool, + mountpoint=mountpoint, + device_type=devicefactory.DEVICE_TYPE_STRATIS) + + if "%s/%s" % (pool.name, name) in self.names: + raise ValueError("name '%s' is already in use" % name) + + device = StratisFilesystemDevice(name, *args, **kwargs) + + # XFS will be created automatically on the device so lets just add it here + device.format = get_format("stratis xfs", mountpoint=mountpoint) + + return device + def new_tmp_fs(self, *args, **kwargs): """ Return a new TmpFSDevice. """ return TmpFSDevice(*args, **kwargs) @@ -917,6 +989,8 @@ def safe_device_name(self, name, device_type=None): allowed = devicelibs.mdraid.safe_name_characters elif device_type == devicefactory.DEVICE_TYPE_BTRFS: allowed = devicelibs.btrfs.safe_name_characters + elif device_type == devicefactory.DEVICE_TYPE_STRATIS: + allowed = devicelibs.stratis.safe_name_characters else: allowed = "0-9a-zA-Z._-" @@ -940,17 +1014,23 @@ def safe_device_name(self, name, device_type=None): return tmp - def unique_device_name(self, name, parent=None, name_set=True): + def unique_device_name(self, name, parent=None, name_set=True, device_type=None): """ Turn given name into a unique one by adding numeric suffix to it """ + + if device_type == devicefactory.DEVICE_TYPE_STRATIS: + parent_separator = "/" + else: + parent_separator = "-" + if name_set: - if parent and "%s-%s" % (parent.name, name) not in self.names: + if parent and "%s%s%s" % (parent.name, parent_separator, name) not in self.names: return name elif not parent and name not in self.names: return name for suffix in range(100): if parent: - if "%s-%s%02d" % (parent.name, name, suffix) not in self.names: + if "%s%s%s%02d" % (parent.name, parent_separator, name, suffix) not in self.names: return "%s%02d" % (name, suffix) else: if "%s%02d" % (name, suffix) not in self.names: @@ -974,7 +1054,7 @@ def suggest_container_name(self, prefix="", container_type=None): name = self._get_container_name_template(prefix=prefix) if name in self.names: try: - name = self.unique_device_name(name) + name = self.unique_device_name(name, device_type=container_type) except RuntimeError: log.error("failed to create device name based on template '%s'", name) raise @@ -982,7 +1062,8 @@ def suggest_container_name(self, prefix="", container_type=None): return name def suggest_device_name(self, parent=None, swap=None, - mountpoint=None, prefix=""): + mountpoint=None, prefix="", + device_type=None): """ Return a suitable, unused name for a new device. :keyword parent: the parent device @@ -1008,12 +1089,18 @@ def suggest_device_name(self, parent=None, swap=None, if prefix and body: body = "_" + body - name = self.safe_device_name(prefix + body) - full_name = "%s-%s" % (parent.name, name) if parent else name + name = self.safe_device_name(prefix + body, device_type) + + if device_type == devicefactory.DEVICE_TYPE_STRATIS: + parent_separator = "/" + else: + parent_separator = "-" + + full_name = "%s%s%s" % (parent.name, parent_separator, name) if parent else name if full_name in self.names or not body: try: - name = self.unique_device_name(name, parent, bool(body)) + name = self.unique_device_name(name, parent, bool(body), device_type) except RuntimeError: log.error("failed to create device name based on parent '%s', " "prefix '%s', mountpoint '%s', swap '%s'", diff --git a/blivet/devicefactory.py b/blivet/devicefactory.py index efadd2559..08d2c2a9d 100644 --- a/blivet/devicefactory.py +++ b/blivet/devicefactory.py @@ -59,6 +59,7 @@ DEVICE_TYPE_DISK = 4 DEVICE_TYPE_LVM_THINP = 5 DEVICE_TYPE_LVM_VDO = 6 +DEVICE_TYPE_STRATIS = 7 def is_supported_device_type(device_type): diff --git a/blivet/devicelibs/stratis.py b/blivet/devicelibs/stratis.py index 507fd31ed..4fbb90609 100644 --- a/blivet/devicelibs/stratis.py +++ b/blivet/devicelibs/stratis.py @@ -37,12 +37,15 @@ STRATIS_FILESYSTEM_INTF = STRATIS_SERVICE + ".filesystem" STRATIS_BLOCKDEV_INTF = STRATIS_SERVICE + ".blockdev" STRATIS_PROPS_INTF = STRATIS_SERVICE + ".FetchProperties" -STRATIS_MANAGER_INTF = STRATIS_SERVICE + ".Manager" +STRATIS_MANAGER_INTF = STRATIS_SERVICE + ".Manager.r2" STRATIS_FS_SIZE = Size("1 TiB") +safe_name_characters = "0-9a-zA-Z._-" + + def remove_pool(pool_uuid): if not safe_dbus.check_object_available(STRATIS_SERVICE, STRATIS_PATH): raise StratisError("Stratis DBus service not available") @@ -94,3 +97,55 @@ def remove_filesystem(pool_uuid, fs_uuid): else: if not succ: raise StratisError("Failed to remove stratis filesystem: %s (%d)" % (err, rc)) + + +def create_pool(name, devices): + if not safe_dbus.check_object_available(STRATIS_SERVICE, STRATIS_PATH): + raise StratisError("Stratis DBus service not available") + + raid_opt = GLib.Variant("(bq)", (False, 0)) + key_opt = GLib.Variant("(bs)", (False, "")) + + try: + ((succ, _paths), rc, err) = safe_dbus.call_sync(STRATIS_SERVICE, + STRATIS_PATH, + STRATIS_MANAGER_INTF, + "CreatePool", + GLib.Variant("(s(bq)as(bs))", (name, raid_opt, + devices, key_opt))) + except safe_dbus.DBusCallError as e: + raise StratisError("Failed to create stratis pool: %s" % str(e)) + else: + if not succ: + raise StratisError("Failed to create stratis pool: %s (%d)" % (err, rc)) + + # repopulate the stratis info cache so the new pool will be added + stratis_info.drop_cache() + + +def create_filesystem(name, pool_uuid): + if not safe_dbus.check_object_available(STRATIS_SERVICE, STRATIS_PATH): + raise StratisError("Stratis DBus service not available") + + # repopulate the stratis info cache just to be sure all values are still valid + stratis_info.drop_cache() + + if pool_uuid not in stratis_info.pools.keys(): + raise StratisError("Stratis pool with UUID %s not found" % pool_uuid) + + pool_info = stratis_info.pools[pool_uuid] + + try: + ((succ, _paths), rc, err) = safe_dbus.call_sync(STRATIS_SERVICE, + pool_info.object_path, + STRATIS_POOL_INTF, + "CreateFilesystems", + GLib.Variant("(as)", ([name],))) + except safe_dbus.DBusCallError as e: + raise StratisError("Failed to create stratis filesystem on '%s': %s" % (pool_info.name, str(e))) + else: + if not succ: + raise StratisError("Failed to create stratis filesystem on '%s': %s (%d)" % (pool_info.name, err, rc)) + + # repopulate the stratis info cache so the new filesystem will be added + stratis_info.drop_cache() diff --git a/blivet/devices/stratis.py b/blivet/devices/stratis.py index ac7499afb..c2be27e54 100644 --- a/blivet/devices/stratis.py +++ b/blivet/devices/stratis.py @@ -24,8 +24,8 @@ from .storage import StorageDevice from ..static_data import stratis_info -from ..size import Size from ..storage_log import log_method_call +from ..errors import DeviceError from .. import devicelibs @@ -37,11 +37,30 @@ class StratisPoolDevice(StorageDevice): _packages = ["stratisd", "stratis-cli"] _dev_dir = "/dev/stratis" - def read_current_size(self): - size = Size(0) - if self.exists and self.uuid in stratis_info.pools.keys(): - size = stratis_info.pools[self.uuid].physical_size - return size + @property + def size(self): + """ The size of this pool """ + # sum up the sizes of the block devices + return sum(parent.size for parent in self.parents) + + def _create(self): + """ Create the device. """ + log_method_call(self, self.name, status=self.status) + bd_list = [bd.path for bd in self.parents] + devicelibs.stratis.create_pool(self.name, bd_list) + + def _post_create(self): + super(StratisPoolDevice, self)._post_create() + self.format.exists = True + + pool_info = stratis_info.get_pool_info(self.name) + if not pool_info: + raise DeviceError("Failed to get information about newly created pool %s" % self.name) + self.uuid = pool_info.uuid + + for parent in self.parents: + parent.format.pool_name = self.name + parent.format.pool_uuid = self.uuid def _destroy(self): """ Destroy the device. """ @@ -60,6 +79,12 @@ class StratisFilesystemDevice(StorageDevice): _packages = ["stratisd", "stratis-cli"] _dev_dir = "/dev/stratis" + def __init__(self, *args, **kwargs): + if kwargs.get("size") is None and not kwargs.get("exists"): + kwargs["size"] = devicelibs.stratis.STRATIS_FS_SIZE + + super(StratisFilesystemDevice, self).__init__(*args, **kwargs) + def _get_name(self): """ This device's name. """ if self.pool is not None: @@ -80,6 +105,19 @@ def pool(self): return self.parents[0] + def _create(self): + """ Create the device. """ + log_method_call(self, self.name, status=self.status) + devicelibs.stratis.create_filesystem(self.fsname, self.pool.uuid) + + def _post_create(self): + super(StratisFilesystemDevice, self)._post_create() + + fs_info = stratis_info.get_filesystem_info(self.pool.name, self.fsname) + if not fs_info: + raise DeviceError("Failed to get information about newly created filesystem %s" % self.name) + self.uuid = fs_info.uuid + def _destroy(self): """ Destroy the device. """ log_method_call(self, self.name, status=self.status) diff --git a/blivet/formats/stratis.py b/blivet/formats/stratis.py index b8eecb31a..4101a7759 100644 --- a/blivet/formats/stratis.py +++ b/blivet/formats/stratis.py @@ -35,7 +35,7 @@ class StratisBlockdev(DeviceFormat): _type = "stratis" _name = N_("Stratis block device") _udev_types = ["stratis"] - _formattable = False # can be formatted + _formattable = True # can be formatted _supported = True # is supported _linux_native = True # for clearpart _min_size = Size("1 GiB") diff --git a/blivet/static_data/stratis_info.py b/blivet/static_data/stratis_info.py index 8a7ad68fa..1f1f05937 100644 --- a/blivet/static_data/stratis_info.py +++ b/blivet/static_data/stratis_info.py @@ -20,6 +20,8 @@ # Red Hat Author(s): Vojtech Trefny # +import os + from collections import namedtuple from .. import safe_dbus @@ -192,5 +194,20 @@ def blockdevs(self): def drop_cache(self): self._info_cache = None + def get_pool_info(self, pool_name): + for pool in self.pools.values(): + if pool.name == pool_name: + return pool + + def get_filesystem_info(self, pool_name, fs_name): + for fs in self.filesystems.values(): + if fs.pool_name == pool_name and fs.name == fs_name: + return fs + + def get_blockdev_info(self, bd_path): + for bd in self.blockdevs.values(): + if bd.path == bd_path or bd.path == os.path.realpath(bd_path): + return bd + stratis_info = StratisInfo() From d18d85f85f91440a886969bdbd20bf4578e6b9e7 Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Thu, 28 Jan 2021 15:03:07 +0100 Subject: [PATCH 09/26] Add simple test case for Stratis --- tests/devices_test/stratis_test.py | 37 ++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/devices_test/stratis_test.py diff --git a/tests/devices_test/stratis_test.py b/tests/devices_test/stratis_test.py new file mode 100644 index 000000000..1e1cc0ac9 --- /dev/null +++ b/tests/devices_test/stratis_test.py @@ -0,0 +1,37 @@ +import test_compat # pylint: disable=unused-import + +import unittest + +import blivet + +from blivet.devices import StorageDevice +from blivet.devices import StratisPoolDevice +from blivet.devices import StratisFilesystemDevice +from blivet.size import Size + + +DEVICE_CLASSES = [ + StratisPoolDevice, + StratisFilesystemDevice, + StorageDevice +] + + +@unittest.skipUnless(not any(x.unavailable_type_dependencies() for x in DEVICE_CLASSES), "some unsupported device classes required for this test") +class BlivetNewStratisDeviceTest(unittest.TestCase): + def test_new_stratis(self): + b = blivet.Blivet() + bd = StorageDevice("bd1", fmt=blivet.formats.get_format("stratis"), + size=Size("1 GiB"), exists=True) + + pool = b.new_stratis_pool(name="testpool", parents=[bd]) + self.assertEqual(pool.name, "testpool") + self.assertEqual(pool.size, bd.size) + + fs = b.new_stratis_filesystem(name="testfs", parents=[pool]) + + self.assertEqual(fs.name, "testpool/testfs") + self.assertEqual(fs.path, "/dev/stratis/%s" % fs.name) + self.assertEqual(fs.size, Size("1 TiB")) + self.assertEqual(fs.pool, pool) + self.assertEqual(fs.format.type, "stratis xfs") From 82ae46d36240d044bc2316bc843b1b6382f4b69c Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Thu, 28 Jan 2021 15:03:33 +0100 Subject: [PATCH 10/26] Add Stratis example --- examples/stratis.py | 50 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 examples/stratis.py diff --git a/examples/stratis.py b/examples/stratis.py new file mode 100644 index 000000000..2e56db962 --- /dev/null +++ b/examples/stratis.py @@ -0,0 +1,50 @@ +import os + +import blivet +from blivet.size import Size +from blivet.util import set_up_logging, create_sparse_tempfile + +set_up_logging() +b = blivet.Blivet() # create an instance of Blivet (don't add system devices) + +# create a disk image file on which to create new devices +disk1_file = create_sparse_tempfile("disk1", Size("100GiB")) +b.disk_images["disk1"] = disk1_file +disk2_file = create_sparse_tempfile("disk2", Size("100GiB")) +b.disk_images["disk2"] = disk2_file + +b.reset() + +try: + disk1 = b.devicetree.get_device_by_name("disk1") + disk2 = b.devicetree.get_device_by_name("disk2") + + b.initialize_disk(disk1) + b.initialize_disk(disk2) + + bd = b.new_partition(size=Size("50GiB"), fmt_type="stratis", parents=[disk1]) + b.create_device(bd) + bd2 = b.new_partition(size=Size("50GiB"), fmt_type="stratis", parents=[disk2]) + b.create_device(bd2) + + # allocate the partitions (decide where and on which disks they'll reside) + blivet.partitioning.do_partitioning(b) + + pool = b.new_stratis_pool(name="stratis_pool", parents=[bd, bd2]) + b.create_device(pool) + + fs = b.new_stratis_filesystem(name="stratis_filesystem", parents=[pool]) + b.create_device(fs) + + print(b.devicetree) + + # write the new partitions to disk and format them as specified + b.do_it() + print(b.devicetree) + input("Check the state and hit ENTER to trigger cleanup") +finally: + b.devicetree.recursive_remove(pool) + b.do_it() + b.devicetree.teardown_disk_images() + os.unlink(disk1_file) + os.unlink(disk2_file) From 418e29fead3a9cebd1bdef74573c950bfcea8896 Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Fri, 12 Feb 2021 13:17:13 +0100 Subject: [PATCH 11/26] Mark format on Stratis pool devices as immutable --- blivet/devices/stratis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/blivet/devices/stratis.py b/blivet/devices/stratis.py index c2be27e54..c4f005af9 100644 --- a/blivet/devices/stratis.py +++ b/blivet/devices/stratis.py @@ -36,6 +36,7 @@ class StratisPoolDevice(StorageDevice): _resizable = False _packages = ["stratisd", "stratis-cli"] _dev_dir = "/dev/stratis" + _format_immutable = True @property def size(self): From 5e3ea0c2a713becbf884f14ae6e148a084c06749 Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Fri, 12 Feb 2021 13:18:03 +0100 Subject: [PATCH 12/26] Add Stratis device factory --- blivet/blivet.py | 11 +++ blivet/devicefactory.py | 150 +++++++++++++++++++++++++++++++++++- tests/devicefactory_test.py | 106 ++++++++++++++++++++++++- 3 files changed, 264 insertions(+), 3 deletions(-) diff --git a/blivet/blivet.py b/blivet/blivet.py index b1d32d5e2..48b7f5ecc 100644 --- a/blivet/blivet.py +++ b/blivet/blivet.py @@ -314,6 +314,17 @@ def btrfs_volumes(self): return sorted((d for d in self.devices if d.type == "btrfs volume"), key=lambda d: d.name) + @property + def stratis_pools(self): + """ A list of the Stratis pools in the device tree. + + This is based on the current state of the device tree and + does not necessarily reflect the actual on-disk state of the + system's disks. + """ + return sorted((d for d in self.devices if d.type == "stratis pool"), + key=lambda d: d.name) + @property def swaps(self): """ A list of the swap devices in the device tree. diff --git a/blivet/devicefactory.py b/blivet/devicefactory.py index 08d2c2a9d..ea127f3d5 100644 --- a/blivet/devicefactory.py +++ b/blivet/devicefactory.py @@ -28,6 +28,7 @@ from .devices import LUKSDevice, LVMLogicalVolumeDevice from .devices import PartitionDevice, MDRaidArrayDevice from .devices.lvm import LVMVDOPoolMixin, LVMVDOLogicalVolumeMixin, DEFAULT_THPOOL_RESERVE +from .devices import StratisFilesystemDevice, StratisPoolDevice from .formats import get_format from .devicelibs import btrfs from .devicelibs import mdraid @@ -84,6 +85,8 @@ def is_supported_device_type(device_type): devices = [MDRaidArrayDevice] elif device_type == DEVICE_TYPE_LVM_VDO: devices = [LVMLogicalVolumeDevice, LVMVDOPoolMixin, LVMVDOLogicalVolumeMixin] + elif device_type == DEVICE_TYPE_STRATIS: + devices = [StratisFilesystemDevice, StratisPoolDevice] return not any(c.unavailable_type_dependencies() for c in devices) @@ -124,7 +127,8 @@ def get_device_type(device): "lvmvdopool": DEVICE_TYPE_LVM, "btrfs subvolume": DEVICE_TYPE_BTRFS, "btrfs volume": DEVICE_TYPE_BTRFS, - "mdarray": DEVICE_TYPE_MD} + "mdarray": DEVICE_TYPE_MD, + "stratis filesystem": DEVICE_TYPE_STRATIS} use_dev = device.raw_device if use_dev.is_disk: @@ -143,7 +147,8 @@ def get_device_factory(blivet, device_type=DEVICE_TYPE_LVM, **kwargs): DEVICE_TYPE_MD: MDFactory, DEVICE_TYPE_LVM_THINP: LVMThinPFactory, DEVICE_TYPE_LVM_VDO: LVMVDOFactory, - DEVICE_TYPE_DISK: DeviceFactory} + DEVICE_TYPE_DISK: DeviceFactory, + DEVICE_TYPE_STRATIS: StratisFactory} factory_class = class_table[device_type] log.debug("instantiating %s: %s, %s, %s", factory_class, @@ -598,6 +603,8 @@ def get_container(self, device=None, name=None, allow_existing=False): container = device.volume elif hasattr(device, "subvolumes"): container = device + elif device.type == "stratis filesystem": + container = device.pool elif name: for c in self.storage.devices: if c.name == name and c in self.container_list: @@ -2014,3 +2021,142 @@ def _reconfigure_device(self): return super(BTRFSFactory, self)._reconfigure_device() + + +class StratisFactory(DeviceFactory): + + """ Factory for creating Stratis filesystems with partition block devices. """ + child_factory_class = PartitionSetFactory + child_factory_fstype = "stratis" + size_set_class = TotalSizeSet + + def __init__(self, storage, **kwargs): + # override default size policy, AUTO doesn't make sense for Stratis + self._default_settings["container_size"] = SIZE_POLICY_MAX + super(StratisFactory, self).__init__(storage, **kwargs) + + @property + def pool(self): + return self.container + + @property + def container_list(self): + return self.storage.stratis_pools[:] + + def _reconfigure_container(self): + """ Reconfigure a defined container required by this factory device. """ + if getattr(self.container, "exists", False): + return + + self._set_container_members() + + # + # methods related to device size and disk space requirements + # + def _get_total_space(self): + """ Return the total space need for this factory's device/container. + + This is used for the size argument to the child factory constructor + and also to construct the size set in PartitionSetFactory.configure. + """ + + if self.container_size == SIZE_POLICY_AUTO: + raise DeviceFactoryError("Automatic size is not support for Stratis pools") + elif self.container_size == SIZE_POLICY_MAX: + space = Size(0) + # grow the container as large as possible + if self.pool: + space += sum(p.size for p in self.pool.parents) + log.debug("size bumped to %s to include Stratis pool parents", space) + + space += self._get_free_disk_space() + log.debug("size bumped to %s to include free disk space", space) + + return space + else: + return self.container_size + + def _get_device_space(self): + """ The total disk space required for the factory device. """ + return Size(0) # FIXME + + def _normalize_size(self): + pass + + def _handle_no_size(self): + """ Set device size so that it grows to the largest size possible. """ + + def _get_new_container(self, *args, **kwargs): + return self.storage.new_stratis_pool(*args, **kwargs) + + # + # methods to configure the factory's device + # + def _create_device(self): + """ Create the factory device. """ + if self.device_name: + kwa = {"name": self.device_name} + else: + kwa = {} + + parents = self._get_parent_devices() + try: + # pylint: disable=assignment-from-no-return + device = self._get_new_device(parents=parents, + mountpoint=self.mountpoint, + **kwa) + except (StorageError, ValueError) as e: + log.error("device instance creation failed: %s", e) + raise + + self.storage.create_device(device) + try: + self._post_create() + except (StorageError, blockdev.BlockDevError) as e: + log.error("device post-create method failed: %s", e) + self.storage.destroy_device(device) + raise_from(StorageError(e), e) + else: + if not device.size: + self.storage.destroy_device(device) + raise StorageError("failed to create device") + + self.device = device + + def _get_new_device(self, *args, **kwargs): + """ Create and return the factory device as a StorageDevice. """ + + name = kwargs.pop("name", "") + kwargs["name"] = self.storage.unique_device_name(name, parent=self.pool, name_set=True) + + return self.storage.new_stratis_filesystem(*args, **kwargs) + + def _set_name(self): + if not self.device_name: + # pylint: disable=attribute-defined-outside-init + self.device_name = self.storage.suggest_device_name( + parent=self.pool, + swap=False, + mountpoint=self.mountpoint) + + fsname = "%s/%s" % (self.pool.name, self.device_name) + safe_new_name = self.storage.safe_device_name(fsname, DEVICE_TYPE_STRATIS) + if self.device.name != safe_new_name: + if safe_new_name in self.storage.names: + log.error("not renaming '%s' to in-use name '%s'", + self.device.name, safe_new_name) + return + + if not safe_new_name.startswith(self.pool.name): + log.error("device rename failure (%s)", safe_new_name) + return + + # strip off the vg name before setting + safe_new_name = safe_new_name[len(self.pool.name) + 1:] + log.debug("renaming device '%s' to '%s'", + self.device.name, safe_new_name) + self.raw_device.name = safe_new_name + + def _configure(self): + self._set_container() # just sets self.container based on the specs + super(StratisFactory, self)._configure() diff --git a/tests/devicefactory_test.py b/tests/devicefactory_test.py index 93c8bdb71..14a2fbc11 100644 --- a/tests/devicefactory_test.py +++ b/tests/devicefactory_test.py @@ -12,13 +12,14 @@ import blivet from blivet import devicefactory -from blivet.devicelibs import raid, crypto +from blivet.devicelibs import raid, crypto, stratis from blivet.devices import DiskDevice from blivet.devices import DiskFile from blivet.devices import LUKSDevice from blivet.devices import LVMLogicalVolumeDevice from blivet.devices import MDRaidArrayDevice from blivet.devices import PartitionDevice +from blivet.devices import StratisFilesystemDevice from blivet.devices.lvm import DEFAULT_THPOOL_RESERVE from blivet.errors import RaidError from blivet.formats import get_format @@ -893,3 +894,106 @@ def test_mdfactory(self, *args): # pylint: disable=unused-argument self.assertEqual(factory2.container_list, []) self.assertIsNone(factory2.get_container()) + + +class StratisFactoryTestCase(DeviceFactoryTestCase): + device_class = StratisFilesystemDevice + device_type = devicefactory.DEVICE_TYPE_STRATIS + encryption_supported = False + factory_class = devicefactory.StratisFactory + + # pylint: disable=unused-argument + def _get_size_delta(self, devices=None): + """ Return size delta for a specific factory type. + + :keyword devices: list of factory-managed devices or None + :type devices: list(:class:`blivet.devices.StorageDevice`) or NoneType + """ + return Size("2 MiB") + + def _validate_factory_device(self, *args, **kwargs): + device = args[0] + + self.assertEqual(device.type, "stratis filesystem") + self.assertEqual(device.size, stratis.STRATIS_FS_SIZE) + self.assertTrue(hasattr(device, "pool")) + self.assertIsNotNone(device.pool) + self.assertEqual(device.pool.type, "stratis pool") + self.assertIsNotNone(device.format) + self.assertEqual(device.format.type, "stratis xfs") + self.assertEqual(device.format.mountpoint, kwargs.get("mountpoint")) + + if kwargs.get("name"): + self.assertEqual(device.fsname, kwargs.get("name")) + + self.assertTrue(set(device.disks).issubset(kwargs["disks"])) + + if kwargs.get("container_size"): + self.assertAlmostEqual(device.pool.size, + kwargs.get("container_size"), + delta=self._get_size_delta()) + else: + # if container size is not specified, we'll use all available space (not counting size taken by partitions) + self.assertAlmostEqual(device.pool.size, + sum(d.size - Size("2 MiB") for d in self.b.disks), + delta=self._get_size_delta()) + + return device + + @patch("blivet.devices.stratis.StratisFilesystemDevice.type_external_dependencies", return_value=set()) + @patch("blivet.devices.stratis.StratisPoolDevice.type_external_dependencies", return_value=set()) + def test_device_factory(self, *args): # pylint: disable=unused-argument,arguments-differ + device_type = self.device_type + kwargs = {"disks": self.b.disks, + "mountpoint": "/factorytest"} + device = self._factory_device(device_type, **kwargs) + self._validate_factory_device(device, device_type, **kwargs) + + # rename the device + kwargs["name"] = "stratisfs" + kwargs["device"] = device + device = self._factory_device(device_type, **kwargs) + self._validate_factory_device(device, device_type, **kwargs) + + # new mountpoint + kwargs["mountpoint"] = "/a/different/dir" + device = self._factory_device(device_type, **kwargs) + self._validate_factory_device(device, device_type, **kwargs) + + # change container size + kwargs = {"disks": self.b.disks, + "mountpoint": "/factorytest", + "container_size": Size("2.5 GiB")} + device = self._factory_device(device_type, **kwargs) + self._validate_factory_device(device, device_type, **kwargs) + + @patch("blivet.devices.stratis.StratisFilesystemDevice.type_external_dependencies", return_value=set()) + @patch("blivet.devices.stratis.StratisPoolDevice.type_external_dependencies", return_value=set()) + def test_normalize_size(self, *args): # pylint: disable=unused-argument + # size normalization doesn't make sense for stratis -- filesystems are always 1 TiB + pass + + @patch("blivet.devices.stratis.StratisFilesystemDevice.type_external_dependencies", return_value=set()) + @patch("blivet.devices.stratis.StratisPoolDevice.type_external_dependencies", return_value=set()) + def test_get_free_disk_space(self, *args): # pylint: disable=unused-argument + # get_free_disk_space should return the total free space on disks + kwargs = self._get_test_factory_args() + factory = devicefactory.get_device_factory(self.b, + self.device_type, + disks=self.b.disks, + **kwargs) + # disks contain empty disklabels, so free space is sum of disk sizes + self.assertAlmostEqual(factory._get_free_disk_space(), + sum(d.size for d in self.b.disks), + delta=self._get_size_delta()) + + factory.configure() + factory = devicefactory.get_device_factory(self.b, + self.device_type, + disks=self.b.disks, + **kwargs) + # default container size policy for Stratis factory is SIZE_POLICY_MAX so there should + # be (almost) no free space on the disks + self.assertAlmostEqual(factory._get_free_disk_space(), + Size("2 MiB"), + delta=self._get_size_delta()) From d6dae9a1049486ce408554b826998462b97b4180 Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Wed, 17 Feb 2021 17:32:55 +0100 Subject: [PATCH 13/26] Add support for creating encrypted Stratis pools --- blivet/devicelibs/stratis.py | 47 +++++++++++++++++++++++++++-- blivet/devices/stratis.py | 36 ++++++++++++++++++++-- blivet/populator/helpers/stratis.py | 3 +- blivet/safe_dbus.py | 12 +++++--- blivet/static_data/stratis_info.py | 7 +++-- examples/stratis.py | 5 +++ 6 files changed, 96 insertions(+), 14 deletions(-) diff --git a/blivet/devicelibs/stratis.py b/blivet/devicelibs/stratis.py index 4fbb90609..31f759d58 100644 --- a/blivet/devicelibs/stratis.py +++ b/blivet/devicelibs/stratis.py @@ -22,8 +22,11 @@ import gi gi.require_version("GLib", "2.0") +gi.require_version("Gio", "2.0") -from gi.repository import GLib +from gi.repository import GLib, Gio + +import os from ..errors import StratisError from ..size import Size @@ -99,12 +102,50 @@ def remove_filesystem(pool_uuid, fs_uuid): raise StratisError("Failed to remove stratis filesystem: %s (%d)" % (err, rc)) -def create_pool(name, devices): +def set_key(key_desc, passphrase, key_file): + if passphrase: + (read, write) = os.pipe() + os.write(write, passphrase.encode("utf-8")) + fd = read + elif key_file: + fd = os.open(key_file, os.O_RDONLY) + + fd_list = Gio.UnixFDList() + fd_list.append(fd) + + try: + ((_changed, _set), rc, err) = safe_dbus.call_sync(STRATIS_SERVICE, + STRATIS_PATH, + STRATIS_MANAGER_INTF, + "SetKey", + GLib.Variant("(shb)", (key_desc, 0, False)), fds=fd_list) + except safe_dbus.DBusCallError as e: + raise StratisError("Failed to set key for new pool: %s" % str(e)) + else: + if rc != 0: + raise StratisError("Failed to set key for new pool: %s (%d)" % (err, rc)) + finally: + if key_file: + os.close(fd) + if passphrase: + os.close(write) + + +def create_pool(name, devices, encrypted, passphrase, key_file): if not safe_dbus.check_object_available(STRATIS_SERVICE, STRATIS_PATH): raise StratisError("Stratis DBus service not available") + if encrypted and not (passphrase or key_file): + raise StratisError("Passphrase or key file must be specified for encrypted pool") + raid_opt = GLib.Variant("(bq)", (False, 0)) - key_opt = GLib.Variant("(bs)", (False, "")) + + if encrypted: + key_desc = "blivet-%s" % name # XXX what would be a good key description? + set_key(key_desc, passphrase, key_file) + key_opt = GLib.Variant("(bs)", (True, key_desc)) + else: + key_opt = GLib.Variant("(bs)", (False, "")) try: ((succ, _paths), rc, err) = safe_dbus.call_sync(STRATIS_SERVICE, diff --git a/blivet/devices/stratis.py b/blivet/devices/stratis.py index c4f005af9..2e4cfaa60 100644 --- a/blivet/devices/stratis.py +++ b/blivet/devices/stratis.py @@ -19,13 +19,15 @@ # Red Hat Author(s): Vojtech Trefny # +import os + import logging log = logging.getLogger("blivet") from .storage import StorageDevice from ..static_data import stratis_info from ..storage_log import log_method_call -from ..errors import DeviceError +from ..errors import DeviceError, StratisError from .. import devicelibs @@ -38,17 +40,47 @@ class StratisPoolDevice(StorageDevice): _dev_dir = "/dev/stratis" _format_immutable = True + def __init__(self, *args, **kwargs): + """ + :encrypted: whether this pool is encrypted or not + :type encrypted: bool + :keyword passphrase: device passphrase + :type passphrase: str + :keyword key_file: path to a file containing a key + :type key_file: str + """ + self._encrypted = kwargs.pop("encrypted", False) + self.__passphrase = kwargs.pop("passphrase", None) + self._key_file = kwargs.pop("key_file", None) + + super(StratisPoolDevice, self).__init__(*args, **kwargs) + @property def size(self): """ The size of this pool """ # sum up the sizes of the block devices return sum(parent.size for parent in self.parents) + @property + def has_key(self): + return ((self.__passphrase not in ["", None]) or + (self._key_file and os.access(self._key_file, os.R_OK))) + + def _pre_create(self, **kwargs): + super(StratisPoolDevice, self)._pre_create(**kwargs) + + if self._encrypted and not self.has_key: + raise StratisError("cannot create encrypted stratis pool without key") + def _create(self): """ Create the device. """ log_method_call(self, self.name, status=self.status) bd_list = [bd.path for bd in self.parents] - devicelibs.stratis.create_pool(self.name, bd_list) + devicelibs.stratis.create_pool(name=self.name, + devices=bd_list, + encrypted=self._encrypted, + passphrase=self.__passphrase, + key_file=self._key_file) def _post_create(self): super(StratisPoolDevice, self)._post_create() diff --git a/blivet/populator/helpers/stratis.py b/blivet/populator/helpers/stratis.py index 01d86c447..6d2c4fbb7 100644 --- a/blivet/populator/helpers/stratis.py +++ b/blivet/populator/helpers/stratis.py @@ -85,7 +85,8 @@ def _add_pool_device(self): parents=[self.device], uuid=pool_info.uuid, size=pool_info.physical_size, - exists=True) + exists=True, + encrypted=pool_info.encrypted) self._devicetree._add_device(pool_device) # now add filesystems on this pool diff --git a/blivet/safe_dbus.py b/blivet/safe_dbus.py index 92d2ea80e..76416f901 100644 --- a/blivet/safe_dbus.py +++ b/blivet/safe_dbus.py @@ -100,7 +100,7 @@ def get_new_session_connection(): def call_sync(service, obj_path, iface, method, args, - connection=None): + connection=None, fds=None): """ Safely call a given method on a given object of a given service over DBus passing given arguments. If a connection is given, it is used, otherwise a @@ -120,6 +120,8 @@ def call_sync(service, obj_path, iface, method, args, :param connection: connection to use (if None, a new connection is established) :type connection: Gio.DBusConnection + :param fds: list of file descriptors for the call + :type: Gio.UnixFDList :return: unpacked value returned by the method :rtype: tuple with elements that depend on the method :raise DBusCallError: if some DBus related error appears @@ -136,9 +138,9 @@ def call_sync(service, obj_path, iface, method, args, raise DBusCallError("Connection is closed") try: - ret = connection.call_sync(service, obj_path, iface, method, args, - None, Gio.DBusCallFlags.NONE, - DEFAULT_DBUS_TIMEOUT, None) + ret = connection.call_with_unix_fd_list_sync(service, obj_path, iface, method, args, + None, Gio.DBusCallFlags.NONE, + DEFAULT_DBUS_TIMEOUT, fds, None) except GLib.GError as gerr: msg = "Failed to call %s method on %s with %s arguments: %s" % \ (method, obj_path, args, gerr.message) # pylint: disable=no-member @@ -148,7 +150,7 @@ def call_sync(service, obj_path, iface, method, args, msg = "No return from %s method on %s with %s arguments" % (method, obj_path, args) raise DBusCallError(msg) - return ret.unpack() + return ret[0].unpack() def get_property_sync(service, obj_path, iface, prop_name, diff --git a/blivet/static_data/stratis_info.py b/blivet/static_data/stratis_info.py index 1f1f05937..827eb0c6d 100644 --- a/blivet/static_data/stratis_info.py +++ b/blivet/static_data/stratis_info.py @@ -34,14 +34,14 @@ # XXX we can't import these from devicelibs.stratis, circular imports make python mad STRATIS_SERVICE = "org.storage.stratis2" STRATIS_PATH = "/org/storage/stratis2" -STRATIS_POOL_INTF = STRATIS_SERVICE + ".pool" +STRATIS_POOL_INTF = STRATIS_SERVICE + ".pool.r1" STRATIS_FILESYSTEM_INTF = STRATIS_SERVICE + ".filesystem" STRATIS_BLOCKDEV_INTF = STRATIS_SERVICE + ".blockdev" STRATIS_PROPS_INTF = STRATIS_SERVICE + ".FetchProperties" STRATIS_MANAGER_INTF = STRATIS_SERVICE + ".Manager.r2" -StratisPoolInfo = namedtuple("StratisPoolInfo", ["name", "uuid", "physical_size", "object_path"]) +StratisPoolInfo = namedtuple("StratisPoolInfo", ["name", "uuid", "physical_size", "object_path", "encrypted"]) StratisFilesystemInfo = namedtuple("StratisFilesystemInfo", ["name", "uuid", "pool_name", "pool_uuid", "object_path"]) StratisBlockdevInfo = namedtuple("StratisBlockdevInfo", ["path", "uuid", "pool_name", "pool_uuid", "object_path"]) @@ -84,7 +84,8 @@ def _get_pool_info(self, pool_path): pool_size = 0 return StratisPoolInfo(name=properties["Name"], uuid=properties["Uuid"], - physical_size=Size(pool_size), object_path=pool_path) + physical_size=Size(pool_size), object_path=pool_path, + encrypted=properties["Encrypted"]) def _get_filesystem_info(self, filesystem_path): try: diff --git a/examples/stratis.py b/examples/stratis.py index 2e56db962..0b7219230 100644 --- a/examples/stratis.py +++ b/examples/stratis.py @@ -33,6 +33,11 @@ pool = b.new_stratis_pool(name="stratis_pool", parents=[bd, bd2]) b.create_device(pool) + # # encrypted stratis pool can be created by adding "encrypted" and "passphrase" + # # keywords, only the entire pool can be encrypted: + # pool = b.new_stratis_pool(name="stratis_pool", parents=[bd, bd2], encrypted=True, passphrase="secret") + # b.create_device(pool) + fs = b.new_stratis_filesystem(name="stratis_filesystem", parents=[pool]) b.create_device(fs) From b001fb8a7ae4c50250eac52aaf29f074dedffba8 Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Thu, 1 Apr 2021 13:41:41 +0200 Subject: [PATCH 14/26] Add support for working with locked Stratis pools Locked pools are tricky because Stratis uses LUKS but the "Stratis LUKS devices" can't be unlocked with cryptsetup, only with Stratis API. This implementation "hides" the LUKS device for locked pools and use the StratisBlockdev format for them instead. To unlock a locked pool, unlock_pool() function of the StratisBlockdev format must be used. --- blivet/devicelibs/stratis.py | 17 ++++++++++ blivet/formats/stratis.py | 48 +++++++++++++++++++++++++++-- blivet/populator/helpers/luks.py | 14 ++++++++- blivet/populator/helpers/stratis.py | 39 +++++++++++++++++++++-- blivet/static_data/stratis_info.py | 42 ++++++++++++++++++++++++- blivet/udev.py | 4 +-- 6 files changed, 155 insertions(+), 9 deletions(-) diff --git a/blivet/devicelibs/stratis.py b/blivet/devicelibs/stratis.py index 31f759d58..60587eb8a 100644 --- a/blivet/devicelibs/stratis.py +++ b/blivet/devicelibs/stratis.py @@ -131,6 +131,23 @@ def set_key(key_desc, passphrase, key_file): os.close(write) +def unlock_pool(pool_uuid): + if not safe_dbus.check_object_available(STRATIS_SERVICE, STRATIS_PATH): + raise StratisError("Stratis DBus service not available") + + try: + (succ, err, _blockdevs) = safe_dbus.call_sync(STRATIS_SERVICE, + STRATIS_PATH, + STRATIS_MANAGER_INTF, + "UnlockPool", + GLib.Variant("(ss)", (pool_uuid, "keyring"))) + except safe_dbus.DBusCallError as e: + raise StratisError("Failed to unlock pool: %s" % str(e)) + else: + if not succ: + raise StratisError("Failed to unlock pool: %s" % err) + + def create_pool(name, devices, encrypted, passphrase, key_file): if not safe_dbus.check_object_available(STRATIS_SERVICE, STRATIS_PATH): raise StratisError("Stratis DBus service not available") diff --git a/blivet/formats/stratis.py b/blivet/formats/stratis.py index 4101a7759..58202a0f6 100644 --- a/blivet/formats/stratis.py +++ b/blivet/formats/stratis.py @@ -20,9 +20,13 @@ # Red Hat Author(s): Vojtech Trefny # +import os + from ..storage_log import log_method_call from ..i18n import N_ from ..size import Size +from ..errors import StratisError +from ..devicelibs import stratis from . import DeviceFormat, register_device_format import logging @@ -50,6 +54,14 @@ def __init__(self, **kwargs): :type exists: bool :keyword pool_name: the name of the pool this block device belongs to :keyword pool_uuid: the UUID of the pool this block device belongs to + :keyword locked_pool: whether this block device belongs to a locked pool or not + :type locked_pool: bool + :keyword locked_pool_key_desc: kernel keyring description for locked pool + :type locked_pool_key_desc: str + :keyword passphrase: passphrase for the locked pool + :type passphrase: str + :keyword key_file: path to a file containing a key + :type key_file: str .. note:: @@ -64,11 +76,17 @@ def __init__(self, **kwargs): self.pool_name = kwargs.get("pool_name") self.pool_uuid = kwargs.get("pool_uuid") + self.locked_pool = kwargs.get("locked_pool") + self.locked_pool_key_desc = kwargs.get("locked_pool_key_desc") + + self.__passphrase = kwargs.get("passphrase") + self._key_file = kwargs.get("key_file") def __repr__(self): s = DeviceFormat.__repr__(self) - s += (" pool_name = %(pool_name)s pool_uuid = %(pool_uuid)s" % - {"pool_name": self.pool_name, "pool_uuid": self.pool_uuid}) + s += (" pool_name = %(pool_name)s pool_uuid = %(pool_uuid)s locked_pool = %(locked_pool)s" % + {"pool_name": self.pool_name, "pool_uuid": self.pool_uuid, + "locked_pool": self.locked_pool}) return s @property @@ -77,5 +95,31 @@ def dict(self): d.update({"pool_name": self.pool_name, "pool_uuid": self.pool_uuid}) return d + @property + def key_file(self): + """ Path to key file to be used in /etc/crypttab """ + return self._key_file + + def _set_passphrase(self, passphrase): + """ Set the passphrase used to access this device. """ + self.__passphrase = passphrase + + passphrase = property(fset=_set_passphrase) + + @property + def has_key(self): + return ((self.__passphrase not in ["", None]) or + (self._key_file and os.access(self._key_file, os.R_OK))) + + def unlock_pool(self): + if not self.locked_pool: + raise StratisError("This device doesn't contain a locked Stratis pool") + + if not self.has_key: + raise StratisError("No passphrase/key file for the locked Stratis pool") + + stratis.set_key(self.locked_pool_key_desc, self.__passphrase, self.key_file) + stratis.unlock_pool(self.pool_uuid) + register_device_format(StratisBlockdev) diff --git a/blivet/populator/helpers/luks.py b/blivet/populator/helpers/luks.py index 52795a98e..a2173a34c 100644 --- a/blivet/populator/helpers/luks.py +++ b/blivet/populator/helpers/luks.py @@ -31,7 +31,7 @@ from ...flags import flags from .devicepopulator import DevicePopulator from .formatpopulator import FormatPopulator -from ...static_data import luks_data +from ...static_data import luks_data, stratis_info import logging log = logging.getLogger("blivet") @@ -71,6 +71,18 @@ class LUKSFormatPopulator(FormatPopulator): priority = 100 _type_specifier = "luks" + @classmethod + def match(cls, data, device): # pylint: disable=arguments-differ,unused-argument + if not super(LUKSFormatPopulator, cls).match(data, device): + return False + + # locked stratis pools are managed in the StratisFormatPopulator + for pool in stratis_info.locked_pools: + if device.path in pool.devices: + return False + + return True + def _get_kwargs(self): kwargs = super(LUKSFormatPopulator, self)._get_kwargs() kwargs["name"] = "luks-%s" % udev.device_get_uuid(self.data) diff --git a/blivet/populator/helpers/stratis.py b/blivet/populator/helpers/stratis.py index 6d2c4fbb7..c323fb0ad 100644 --- a/blivet/populator/helpers/stratis.py +++ b/blivet/populator/helpers/stratis.py @@ -40,12 +40,43 @@ class StratisFormatPopulator(FormatPopulator): priority = 100 _type_specifier = "stratis" + @classmethod + def match(cls, data, device): # pylint: disable=arguments-differ,unused-argument + if super(StratisFormatPopulator, cls).match(data, device): + return True + + # locked stratis pools are managed here + for pool in stratis_info.locked_pools: + if device.path in pool.devices: + return True + + return False + + def _get_blockdev_uuid(self): + if udev.device_get_format(self.data) == "crypto_LUKS": + holders = udev.device_get_holders(self.data) + if holders: + return udev.device_get_uuid(holders[0]) + + return udev.device_get_uuid(self.data) + def _get_kwargs(self): kwargs = super(StratisFormatPopulator, self)._get_kwargs() - bd_info = stratis_info.blockdevs.get(self.device.path) - name = udev.device_get_name(self.data) + uuid = self._get_blockdev_uuid() + kwargs["uuid"] = uuid + + # stratis block device hosting an encrypted pool + kwargs["locked_pool"] = False + for pool in stratis_info.locked_pools: + if self.device.path in pool.devices: + kwargs["locked_pool"] = True + kwargs["pool_uuid"] = pool.uuid + kwargs["locked_pool_key_desc"] = pool.key_desc + return kwargs + + bd_info = stratis_info.blockdevs.get(uuid) if bd_info: if bd_info.pool_name: kwargs["pool_name"] = bd_info.pool_name @@ -120,7 +151,9 @@ def _add_pool_device(self): def run(self): log_method_call(self, pv=self.device.name) super(StratisFormatPopulator, self).run() - self._add_pool_device() + + if not self.device.format.locked_pool: + self._add_pool_device() class StratisXFSFormatPopulator(FormatPopulator): diff --git a/blivet/static_data/stratis_info.py b/blivet/static_data/stratis_info.py index 827eb0c6d..b58d90824 100644 --- a/blivet/static_data/stratis_info.py +++ b/blivet/static_data/stratis_info.py @@ -20,6 +20,11 @@ # Red Hat Author(s): Vojtech Trefny # +import gi +gi.require_version("GLib", "2.0") + +from gi.repository import GLib + import os from collections import namedtuple @@ -37,13 +42,14 @@ STRATIS_POOL_INTF = STRATIS_SERVICE + ".pool.r1" STRATIS_FILESYSTEM_INTF = STRATIS_SERVICE + ".filesystem" STRATIS_BLOCKDEV_INTF = STRATIS_SERVICE + ".blockdev" -STRATIS_PROPS_INTF = STRATIS_SERVICE + ".FetchProperties" +STRATIS_PROPS_INTF = STRATIS_SERVICE + ".FetchProperties.r4" STRATIS_MANAGER_INTF = STRATIS_SERVICE + ".Manager.r2" StratisPoolInfo = namedtuple("StratisPoolInfo", ["name", "uuid", "physical_size", "object_path", "encrypted"]) StratisFilesystemInfo = namedtuple("StratisFilesystemInfo", ["name", "uuid", "pool_name", "pool_uuid", "object_path"]) StratisBlockdevInfo = namedtuple("StratisBlockdevInfo", ["path", "uuid", "pool_name", "pool_uuid", "object_path"]) +StratisLockedPoolInfo = namedtuple("StratisLockedPoolInfo", ["uuid", "key_desc", "devices"]) class StratisInfo(object): @@ -134,11 +140,36 @@ def _get_blockdev_info(self, blockdev_path): pool_name=pool_name, pool_uuid=pool_info.uuid, object_path=blockdev_path) + def _get_locked_pools_info(self): + locked_pools = [] + + try: + props = safe_dbus.call_sync(STRATIS_SERVICE, + STRATIS_PATH, + STRATIS_PROPS_INTF, + "GetProperties", + GLib.Variant("(as)", (["LockedPoolsWithDevs"],)))[0] + except safe_dbus.DBusCallError as e: + log.error("Failed to get list of locked Stratis pools: %s", str(e)) + return locked_pools + + if props and "LockedPoolsWithDevs" in props.keys(): + valid, pools_info = props["LockedPoolsWithDevs"] + if valid: + for pool_uuid in pools_info.keys(): + info = StratisLockedPoolInfo(uuid=pool_uuid, + key_desc=pools_info[pool_uuid]["key_description"], + devices=[d["devnode"] for d in pools_info[pool_uuid]["devs"]]) + locked_pools.append(info) + + return locked_pools + def _get_stratis_info(self): self._info_cache = dict() self._info_cache["pools"] = dict() self._info_cache["blockdevs"] = dict() self._info_cache["filesystems"] = dict() + self._info_cache["locked_pools"] = [] try: ret = safe_dbus.check_object_available(STRATIS_SERVICE, STRATIS_PATH) @@ -171,6 +202,8 @@ def _get_stratis_info(self): if bd_info: self._info_cache["blockdevs"][bd_info.uuid] = bd_info + self._info_cache["locked_pools"] = self._get_locked_pools_info() + @property def pools(self): if self._info_cache is None: @@ -192,6 +225,13 @@ def blockdevs(self): return self._info_cache["blockdevs"] + @property + def locked_pools(self): + if self._info_cache is None: + self._get_stratis_info() + + return self._info_cache["locked_pools"] + def drop_cache(self): self._info_cache = None diff --git a/blivet/udev.py b/blivet/udev.py index 97283dc73..efbc53d6f 100644 --- a/blivet/udev.py +++ b/blivet/udev.py @@ -1042,10 +1042,10 @@ def device_is_stratis_filesystem(info): def device_is_stratis_private(info): - if not device_is_dm_stratis(info): + if not (device_is_dm_stratis(info) or device_is_dm_luks(info)): return False try: - return info.get("DM_UUID", "").split("-")[2] == "private" + return info.get("DM_NAME", "").split("-")[2] == "private" except IndexError: return False From 882dfef112d14c227ccff5e684cc0c41b7d080e0 Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Wed, 7 Apr 2021 15:17:52 +0200 Subject: [PATCH 15/26] Add support for creating encrypted Stratis devices with DeviceFactory --- blivet/devicefactory.py | 14 ++++++++++++-- blivet/devices/stratis.py | 24 ++++++++++++++++++++++-- tests/devicefactory_test.py | 12 ++++++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/blivet/devicefactory.py b/blivet/devicefactory.py index ea127f3d5..dfaf6d083 100644 --- a/blivet/devicefactory.py +++ b/blivet/devicefactory.py @@ -384,6 +384,9 @@ def __init__(self, storage, **kwargs): self.__actions = [] self.__roots = [] + def _is_container_encrypted(self): + return all(isinstance(p, LUKSDevice) for p in self.device.container.parents) + def _update_defaults_from_device(self): """ Update default settings based on passed in device, if provided. """ if self.device is None: @@ -414,8 +417,7 @@ def _update_defaults_from_device(self): len(self.device.container.pvs) == 1 and hasattr(self.device.container.pvs[0].raw_device, "level")): self.container_raid_level = self.device.container.pvs[0].raw_device.level - self.container_encrypted = all(isinstance(p, LUKSDevice) - for p in self.device.container.parents) + self.container_encrypted = self._is_container_encrypted() @property def encrypted(self): @@ -2048,6 +2050,7 @@ def _reconfigure_container(self): if getattr(self.container, "exists", False): return + self.container.encrypted = self.container_encrypted or False self._set_container_members() # @@ -2087,6 +2090,10 @@ def _handle_no_size(self): """ Set device size so that it grows to the largest size possible. """ def _get_new_container(self, *args, **kwargs): + if self.container_encrypted: + kwargs["encrypted"] = True + else: + kwargs["encrypted"] = False return self.storage.new_stratis_pool(*args, **kwargs) # @@ -2160,3 +2167,6 @@ def _set_name(self): def _configure(self): self._set_container() # just sets self.container based on the specs super(StratisFactory, self)._configure() + + def _is_container_encrypted(self): + return self.container.encrypted diff --git a/blivet/devices/stratis.py b/blivet/devices/stratis.py index 2e4cfaa60..7d19350bc 100644 --- a/blivet/devices/stratis.py +++ b/blivet/devices/stratis.py @@ -61,6 +61,26 @@ def size(self): # sum up the sizes of the block devices return sum(parent.size for parent in self.parents) + @property + def encrypted(self): + """ True if this device is encrypted. """ + return self._encrypted + + @encrypted.setter + def encrypted(self, encrypted): + self._encrypted = encrypted + + @property + def key_file(self): + """ Path to key file to be used in /etc/crypttab """ + return self._key_file + + def _set_passphrase(self, passphrase): + """ Set the passphrase used to access this device. """ + self.__passphrase = passphrase + + passphrase = property(fset=_set_passphrase) + @property def has_key(self): return ((self.__passphrase not in ["", None]) or @@ -69,7 +89,7 @@ def has_key(self): def _pre_create(self, **kwargs): super(StratisPoolDevice, self)._pre_create(**kwargs) - if self._encrypted and not self.has_key: + if self.encrypted and not self.has_key: raise StratisError("cannot create encrypted stratis pool without key") def _create(self): @@ -78,7 +98,7 @@ def _create(self): bd_list = [bd.path for bd in self.parents] devicelibs.stratis.create_pool(name=self.name, devices=bd_list, - encrypted=self._encrypted, + encrypted=self.encrypted, passphrase=self.__passphrase, key_file=self._key_file) diff --git a/tests/devicefactory_test.py b/tests/devicefactory_test.py index 14a2fbc11..2c2e6168b 100644 --- a/tests/devicefactory_test.py +++ b/tests/devicefactory_test.py @@ -938,6 +938,8 @@ def _validate_factory_device(self, *args, **kwargs): sum(d.size - Size("2 MiB") for d in self.b.disks), delta=self._get_size_delta()) + self.assertEqual(device.pool.encrypted, kwargs.get("container_encrypted", False)) + return device @patch("blivet.devices.stratis.StratisFilesystemDevice.type_external_dependencies", return_value=set()) @@ -967,6 +969,16 @@ def test_device_factory(self, *args): # pylint: disable=unused-argument,argumen device = self._factory_device(device_type, **kwargs) self._validate_factory_device(device, device_type, **kwargs) + # enable encryption on the container + kwargs["container_encrypted"] = True + device = self._factory_device(device_type, **kwargs) + self._validate_factory_device(device, device_type, **kwargs) + + # disable encryption on the container + kwargs["container_encrypted"] = False + device = self._factory_device(device_type, **kwargs) + self._validate_factory_device(device, device_type, **kwargs) + @patch("blivet.devices.stratis.StratisFilesystemDevice.type_external_dependencies", return_value=set()) @patch("blivet.devices.stratis.StratisPoolDevice.type_external_dependencies", return_value=set()) def test_normalize_size(self, *args): # pylint: disable=unused-argument From d4ba56334f4b77d203b8cff116cb4fbd5d83b4a0 Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Wed, 7 Apr 2021 15:42:18 +0200 Subject: [PATCH 16/26] Add more tests for creating Stratis devices --- tests/devices_test/stratis_test.py | 51 ++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/devices_test/stratis_test.py b/tests/devices_test/stratis_test.py index 1e1cc0ac9..9594a27be 100644 --- a/tests/devices_test/stratis_test.py +++ b/tests/devices_test/stratis_test.py @@ -1,6 +1,7 @@ import test_compat # pylint: disable=unused-import import unittest +from six.moves.mock import patch # pylint: disable=no-name-in-module,import-error import blivet @@ -24,6 +25,8 @@ def test_new_stratis(self): bd = StorageDevice("bd1", fmt=blivet.formats.get_format("stratis"), size=Size("1 GiB"), exists=True) + b.devicetree._add_device(bd) + pool = b.new_stratis_pool(name="testpool", parents=[bd]) self.assertEqual(pool.name, "testpool") self.assertEqual(pool.size, bd.size) @@ -35,3 +38,51 @@ def test_new_stratis(self): self.assertEqual(fs.size, Size("1 TiB")) self.assertEqual(fs.pool, pool) self.assertEqual(fs.format.type, "stratis xfs") + + b.create_device(pool) + b.create_device(fs) + + with patch("blivet.devicelibs.stratis") as stratis_dbus: + with patch.object(pool, "_pre_create"): + with patch.object(pool, "_post_create"): + pool.create() + stratis_dbus.create_pool.assert_called_with(name='testpool', + devices=['/dev/bd1'], + encrypted=False, + passphrase=None, + key_file=None) + + # we would get this from pool._post_create + pool.uuid = "c4fc9ebe-e173-4cab-8d81-cc6abddbe02d" + + with patch("blivet.devicelibs.stratis") as stratis_dbus: + with patch.object(fs, "_pre_create"): + with patch.object(fs, "_post_create"): + fs.create() + stratis_dbus.create_filesystem.assert_called_with("testfs", + "c4fc9ebe-e173-4cab-8d81-cc6abddbe02d") + + def test_new_encryted_stratis(self): + b = blivet.Blivet() + bd = StorageDevice("bd1", fmt=blivet.formats.get_format("stratis"), + size=Size("1 GiB"), exists=True) + + b.devicetree._add_device(bd) + + pool = b.new_stratis_pool(name="testpool", parents=[bd], encrypted=True, passphrase="secret") + self.assertEqual(pool.name, "testpool") + self.assertEqual(pool.size, bd.size) + self.assertTrue(pool.encrypted) + self.assertTrue(pool.has_key) + + b.create_device(pool) + + with patch("blivet.devicelibs.stratis") as stratis_dbus: + with patch.object(pool, "_pre_create"): + with patch.object(pool, "_post_create"): + pool.create() + stratis_dbus.create_pool.assert_called_with(name='testpool', + devices=['/dev/bd1'], + encrypted=True, + passphrase="secret", + key_file=None) From f174cc6fb6f0b3252e3325fd94d61a77321fc680 Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Thu, 8 Apr 2021 17:31:12 +0200 Subject: [PATCH 17/26] Set the StratisBlockdev format status based on whether it has a pool or not --- blivet/formats/stratis.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/blivet/formats/stratis.py b/blivet/formats/stratis.py index 58202a0f6..648c4cac4 100644 --- a/blivet/formats/stratis.py +++ b/blivet/formats/stratis.py @@ -89,6 +89,10 @@ def __repr__(self): "locked_pool": self.locked_pool}) return s + @property + def status(self): + return self.exists and (self.pool_name or self.pool_uuid) + @property def dict(self): d = super(StratisBlockdev, self).dict From fda17500f59f29b754aeddad9a7c4b5b538737a2 Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Thu, 8 Apr 2021 17:32:37 +0200 Subject: [PATCH 18/26] Set pool info on the block devices when adding/removing Stratis pool --- blivet/devices/stratis.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/blivet/devices/stratis.py b/blivet/devices/stratis.py index 7d19350bc..7da9274be 100644 --- a/blivet/devices/stratis.py +++ b/blivet/devices/stratis.py @@ -120,6 +120,23 @@ def _destroy(self): log_method_call(self, self.name, status=self.status) devicelibs.stratis.remove_pool(self.uuid) + def add_hook(self, new=True): + super(StratisPoolDevice, self).add_hook(new=new) + if new: + return + + for parent in self.parents: + parent.format.pool_name = self.name + parent.format.pool_uuid = self.uuid + + def remove_hook(self, modparent=True): + if modparent: + for parent in self.parents: + parent.format.pool_name = None + parent.format.pool_uuid = None + + super(StratisPoolDevice, self).remove_hook(modparent=modparent) + def dracut_setup_args(self): return set(["stratis.rootfs.pool_uuid=%s" % self.uuid]) From cc17684b207277028019f6f90f0e101dad766bdf Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Thu, 8 Apr 2021 17:33:16 +0200 Subject: [PATCH 19/26] Add property with list of Stratis block devices to StratisPoolDevice --- blivet/devices/stratis.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/blivet/devices/stratis.py b/blivet/devices/stratis.py index 7da9274be..446d4ac86 100644 --- a/blivet/devices/stratis.py +++ b/blivet/devices/stratis.py @@ -55,6 +55,11 @@ def __init__(self, *args, **kwargs): super(StratisPoolDevice, self).__init__(*args, **kwargs) + @property + def blockdevs(self): + """ A list of this pool block devices """ + return self.parents[:] + @property def size(self): """ The size of this pool """ From 07c8b013757f4431badb993b12b6a59db276463b Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Fri, 23 Apr 2021 15:27:32 +0200 Subject: [PATCH 20/26] Hide the private LUKS device for unlockded Stratis pools The internal private dm-crypt device is created and destroyed together with the pool so having it in the tree would make things even more complicated. --- blivet/populator/helpers/luks.py | 9 +++++++++ blivet/populator/helpers/stratis.py | 8 ++++++++ blivet/populator/populator.py | 18 ++++++++++++++++-- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/blivet/populator/helpers/luks.py b/blivet/populator/helpers/luks.py index a2173a34c..e6d7467b6 100644 --- a/blivet/populator/helpers/luks.py +++ b/blivet/populator/helpers/luks.py @@ -81,6 +81,15 @@ def match(cls, data, device): # pylint: disable=arguments-differ,unused-argumen if device.path in pool.devices: return False + # unlocked stratis pools are also managed in the StratisFormatPopulator + holders = udev.device_get_holders(data) + if holders: + fs = udev.device_get_format(holders[0]) + if fs == "stratis": + log.debug("ignoring LUKS format on %s, it appears to be an encrypted Stratis pool", + device.name) + return False + return True def _get_kwargs(self): diff --git a/blivet/populator/helpers/stratis.py b/blivet/populator/helpers/stratis.py index c323fb0ad..410f1b26f 100644 --- a/blivet/populator/helpers/stratis.py +++ b/blivet/populator/helpers/stratis.py @@ -50,6 +50,14 @@ def match(cls, data, device): # pylint: disable=arguments-differ,unused-argumen if device.path in pool.devices: return True + # unlocked encrypted pools are also managed here + if udev.device_get_format(data) == "crypto_LUKS": + holders = udev.device_get_holders(data) + if holders: + fs = udev.device_get_format(holders[0]) + if fs == cls._type_specifier: + return True + return False def _get_blockdev_uuid(self): diff --git a/blivet/populator/populator.py b/blivet/populator/populator.py index 84e14ee57..1173e45af 100644 --- a/blivet/populator/populator.py +++ b/blivet/populator/populator.py @@ -266,6 +266,20 @@ def handle_device(self, info, update_orig_fmt=False): "stratis block device", name) return + if udev.device_is_dm_luks(info) and udev.device_is_stratis_private(info): + log.debug("skipping %s stratis private crypto device, will be handled with " + "corresponding stratis block device", name) + + # XXX for newly unlocked stratis pools we need to force run handle_format + # on one of the parent block devices to add the pool to the tree + parents = udev.device_get_parents(info) + if parents: + bname = udev.device_get_name(parents[0]) + bdev = self.get_device_by_name(bname) + if bdev: + self.handle_format(info, bdev, force=True) + return + # make sure we note the name of every device we see self._add_name(name) device = self.get_device_by_name(name) @@ -298,7 +312,7 @@ def handle_device(self, info, update_orig_fmt=False): device.device_links = udev.device_get_symlinks(info) callbacks.device_scanned(device_name=name) - def handle_format(self, info, device): + def handle_format(self, info, device, force=False): log_method_call(self, name=getattr(device, "name", None)) if not info: @@ -313,7 +327,7 @@ def handle_format(self, info, device): name = udev.device_get_name(info) if (not device or (not udev.device_get_format(info) and not udev.device_get_disklabel_type(info)) or - device.format.type): + (device.format.type and not force)): # this device has no formatting or it has already been set up log.debug("no type or existing type for %s, bailing", name) return From 017d83be7caa5cc305e028a7f40e9e45ca375b45 Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Mon, 3 May 2021 12:39:01 +0200 Subject: [PATCH 21/26] Add Stratis devices and formats to the public API documentation --- doc/api/blivet.rst | 5 +++++ doc/api/devices.rst | 5 +++++ doc/api/formats.rst | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/doc/api/blivet.rst b/doc/api/blivet.rst index 2b9dbfc1a..c403369a8 100644 --- a/doc/api/blivet.rst +++ b/doc/api/blivet.rst @@ -48,6 +48,8 @@ blivet * :meth:`~blivet.blivet.Blivet.new_lv_from_lvs` * :meth:`~blivet.blivet.Blivet.new_mdarray` * :meth:`~blivet.blivet.Blivet.new_partition` + * :meth:`~blivet.blivet.Blivet.new_stratis_filesystem` + * :meth:`~blivet.blivet.Blivet.new_stratis_pool` * :meth:`~blivet.blivet.Blivet.new_tmp_fs` * :meth:`~blivet.blivet.Blivet.new_vg` * :attr:`~blivet.blivet.Blivet.partitioned` @@ -61,6 +63,7 @@ blivet * :meth:`~blivet.blivet.Blivet.save_passphrase` * :meth:`~blivet.blivet.Blivet.set_default_fstype` * :meth:`~blivet.blivet.Blivet.shutdown` + * :attr:`~blivet.blivet.Blivet.stratis_pools` * :meth:`~blivet.blivet.Blivet.suggest_container_name` * :meth:`~blivet.blivet.Blivet.suggest_device_name` * :attr:`~blivet.blivet.Blivet.swaps` @@ -86,6 +89,7 @@ blivet * :const:`~blivet.devicefactory.DEVICE_TYPE_DISK` * :const:`~blivet.devicefactory.DEVICE_TYPE_LVM_THINP` * :const:`~blivet.devicefactory.DEVICE_TYPE_LVM_VDO` + * :const:`~blivet.devicefactory.DEVICE_TYPE_STRATIS` * :func:`~blivet.devicefactory.is_supported_device_type` * :func:`~blivet.devicefactory.get_device_factory` * :func:`~blivet.devicefactory.get_device_type` @@ -179,6 +183,7 @@ blivet * :class:`~blivet.errors.SinglePhysicalVolumeError` * :class:`~blivet.errors.SizePlacesError` * :class:`~blivet.errors.StorageError` + * :class:`~blivet.errors.StratisError` * :class:`~blivet.errors.SwapSpaceError` * :class:`~blivet.errors.ThreadError` * :class:`~blivet.errors.UdevError` diff --git a/doc/api/devices.rst b/doc/api/devices.rst index 19fbdefc4..bd250cae9 100644 --- a/doc/api/devices.rst +++ b/doc/api/devices.rst @@ -137,3 +137,8 @@ devices * :attr:`~blivet.devices.storage.StorageDevice.target_size` * :meth:`~blivet.devices.storage.StorageDevice.teardown` * :attr:`~blivet.devices.storage.StorageDevice.uuid` + +* :mod:`~blivet.devices.stratis` + * :class:`~blivet.devices.stratis.StratisPoolDevice` (see :ref:`inherited public API `) + * :attr:`~blivet.devices.stratis.StratisPoolDevice.encrypted` + * :class:`~blivet.devices.stratis.StratisFilesystemDevice` (see :ref:`inherited public API `) diff --git a/doc/api/formats.rst b/doc/api/formats.rst index aa409e4a9..61437a495 100644 --- a/doc/api/formats.rst +++ b/doc/api/formats.rst @@ -110,5 +110,10 @@ formats * :mod:`~blivet.formats.prepboot` * :class:`~blivet.formats.prepboot.PPCPRePBoot` (see :ref:`inherited public API `) +* :mod:`~blivet.formats.stratis` + * :class:`~blivet.formats.stratis.StratisBlockdev` (see :ref:`inherited public API `) + * :attr:`~blivet.formats.stratis.StratisBlockdev.has_key` + * :meth:`~blivet.formats.stratis.StratisBlockdev.unlock_pool` + * :mod:`~blivet.formats.swap` * :class:`~blivet.formats.swap.SwapSpace` (see :ref:`inherited public API `) From 6c3958c615ba13de9629ab5fcde182b92f5e9871 Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Wed, 12 May 2021 17:56:01 +0200 Subject: [PATCH 22/26] Add MountClass for StratisXFS filesystem --- blivet/formats/fs.py | 1 + blivet/tasks/fsmount.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/blivet/formats/fs.py b/blivet/formats/fs.py index 749af0864..b9fb8988b 100644 --- a/blivet/formats/fs.py +++ b/blivet/formats/fs.py @@ -1137,6 +1137,7 @@ class StratisXFS(XFS): _type = "stratis xfs" + _mount_class = fsmount.StratisXFSMount _resize_class = fsresize.UnimplementedFSResize _size_info_class = fssize.UnimplementedFSSize _info_class = fsinfo.UnimplementedFSInfo diff --git a/blivet/tasks/fsmount.py b/blivet/tasks/fsmount.py index 15f79a3e5..65b2470ac 100644 --- a/blivet/tasks/fsmount.py +++ b/blivet/tasks/fsmount.py @@ -191,6 +191,10 @@ def _availability_errors(self): return errors +class StratisXFSMount(FSMount): + fstype = "xfs" + + class TmpFSMount(NoDevFSMount): def _modify_options(self, options): From 6dc9b6b16939134f14e0b5f2fb174810ff2d0af5 Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Fri, 14 May 2021 12:33:14 +0200 Subject: [PATCH 23/26] Add fstab options for Stratis Filesystem devices --- blivet/devices/stratis.py | 2 ++ blivet/formats/fs.py | 14 ++++++++++++++ blivet/populator/helpers/stratis.py | 1 + 3 files changed, 17 insertions(+) diff --git a/blivet/devices/stratis.py b/blivet/devices/stratis.py index 446d4ac86..ecca32c05 100644 --- a/blivet/devices/stratis.py +++ b/blivet/devices/stratis.py @@ -193,6 +193,8 @@ def _post_create(self): raise DeviceError("Failed to get information about newly created filesystem %s" % self.name) self.uuid = fs_info.uuid + self.format.pool_uuid = fs_info.pool_uuid + def _destroy(self): """ Destroy the device. """ log_method_call(self, self.name, status=self.status) diff --git a/blivet/formats/fs.py b/blivet/formats/fs.py index b9fb8988b..14d7c858f 100644 --- a/blivet/formats/fs.py +++ b/blivet/formats/fs.py @@ -1145,6 +1145,20 @@ class StratisXFS(XFS): _writelabel_class = fswritelabel.UnimplementedFSWriteLabel _writeuuid_class = fswriteuuid.UnimplementedFSWriteUUID + def __init__(self, **kwargs): + super(StratisXFS, self).__init__(**kwargs) + self.pool_uuid = kwargs.pop("pool_uuid", None) + + def _get_options(self): + opts = super(StratisXFS, self)._get_options() + if self.mountpoint != "/": + stratis_opts = "x-systemd.requires=stratis-fstab-setup@%s," \ + "x-systemd.after=stratis-fstab-setup@%s" % (self.pool_uuid, + self.pool_uuid) + else: + stratis_opts = None + return ",".join(o for o in (opts, stratis_opts) if o) + def _create(self, **kwargs): # pylint: disable=unused-argument # format is created together with the stratis filesystem device pass diff --git a/blivet/populator/helpers/stratis.py b/blivet/populator/helpers/stratis.py index 410f1b26f..ddcd8ec53 100644 --- a/blivet/populator/helpers/stratis.py +++ b/blivet/populator/helpers/stratis.py @@ -186,5 +186,6 @@ def match(cls, data, device): # pylint: disable=arguments-differ,unused-argumen def run(self): """ Create a format instance and associate it with the device instance. """ kwargs = self._get_kwargs() + kwargs["pool_uuid"] = self.device.pool.uuid log.info("type detected on '%s' is '%s'", self.device.name, self.type_spec) self.device.format = get_format(self.type_spec, **kwargs) From f3bbdd77e8395c58e2860bc43f4ad76f55704f7f Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Wed, 1 Sep 2021 12:51:47 +0200 Subject: [PATCH 24/26] Fix/unify importing mock module in stratis tests See ad34dfa2f983bb3159af8b5780193e0427b505e9 for details. --- tests/devices_test/stratis_test.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/devices_test/stratis_test.py b/tests/devices_test/stratis_test.py index 9594a27be..12901b3b7 100644 --- a/tests/devices_test/stratis_test.py +++ b/tests/devices_test/stratis_test.py @@ -1,7 +1,9 @@ -import test_compat # pylint: disable=unused-import - import unittest -from six.moves.mock import patch # pylint: disable=no-name-in-module,import-error + +try: + from unittest.mock import patch +except ImportError: + from mock import patch import blivet From 3b642056d6761133bb27676993a83650c91b4057 Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Wed, 1 Sep 2021 13:47:30 +0200 Subject: [PATCH 25/26] Fix parameters differ from overridden in StratisPoolDevice --- blivet/devices/stratis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/blivet/devices/stratis.py b/blivet/devices/stratis.py index ecca32c05..17eb04fa9 100644 --- a/blivet/devices/stratis.py +++ b/blivet/devices/stratis.py @@ -91,8 +91,8 @@ def has_key(self): return ((self.__passphrase not in ["", None]) or (self._key_file and os.access(self._key_file, os.R_OK))) - def _pre_create(self, **kwargs): - super(StratisPoolDevice, self)._pre_create(**kwargs) + def _pre_create(self): + super(StratisPoolDevice, self)._pre_create() if self.encrypted and not self.has_key: raise StratisError("cannot create encrypted stratis pool without key") From 4a0ed7b2c92f776b5c8c5c16ceaefcf49f7677d7 Mon Sep 17 00:00:00 2001 From: Vojtech Trefny Date: Thu, 2 Sep 2021 11:21:16 +0200 Subject: [PATCH 26/26] Add stratis filesystem metadata size and pool free space When creating a new filesystem we need to check whether the pool has enough free space for it. --- blivet/devicefactory.py | 4 -- blivet/devicelibs/stratis.py | 31 +++++++++++++++ blivet/devices/stratis.py | 63 ++++++++++++++++++++++++++++++ blivet/static_data/stratis_info.py | 34 ++++++++++++++-- tests/devices_test/stratis_test.py | 8 +++- 5 files changed, 131 insertions(+), 9 deletions(-) diff --git a/blivet/devicefactory.py b/blivet/devicefactory.py index dfaf6d083..e6b8c1fad 100644 --- a/blivet/devicefactory.py +++ b/blivet/devicefactory.py @@ -2079,10 +2079,6 @@ def _get_total_space(self): else: return self.container_size - def _get_device_space(self): - """ The total disk space required for the factory device. """ - return Size(0) # FIXME - def _normalize_size(self): pass diff --git a/blivet/devicelibs/stratis.py b/blivet/devicelibs/stratis.py index 60587eb8a..c494020fe 100644 --- a/blivet/devicelibs/stratis.py +++ b/blivet/devicelibs/stratis.py @@ -26,12 +26,15 @@ from gi.repository import GLib, Gio +import json import os +import shutil from ..errors import StratisError from ..size import Size from ..static_data import stratis_info from .. import safe_dbus +from .. import util STRATIS_SERVICE = "org.storage.stratis2" @@ -44,11 +47,39 @@ STRATIS_FS_SIZE = Size("1 TiB") +STRATIS_FS_MD_SIZE = Size("600 MiB") + +STRATIS_BD_MD_SIZE = Size("4 MiB") +STRATIS_BD_ENC_MD_SIZE = Size("16 MiB") + + +STRATIS_PREDICT_USAGE = "stratis-predict-usage" safe_name_characters = "0-9a-zA-Z._-" +def pool_used(pool_name, blockdevs, encrypted=False): + if not shutil.which(STRATIS_PREDICT_USAGE): + raise StratisError("Utility for predicting stratis pool usage '%s' not available" % STRATIS_PREDICT_USAGE) + + cmd = [STRATIS_PREDICT_USAGE] + cmd.extend(blockdevs) + if encrypted: + cmd.append("--encrypted") + + rc, out = util.run_program_and_capture_output(cmd) + if rc: + raise StratisError("Failed to predict usage for stratis pool %s" % pool_name) + + try: + pred = json.loads(out) + except json.JSONDecodeError as e: + raise StratisError("Failed to get stratis pool usage") from e + + return Size(pred["used"]) + + def remove_pool(pool_uuid): if not safe_dbus.check_object_available(STRATIS_SERVICE, STRATIS_PATH): raise StratisError("Stratis DBus service not available") diff --git a/blivet/devices/stratis.py b/blivet/devices/stratis.py index 17eb04fa9..baa41f257 100644 --- a/blivet/devices/stratis.py +++ b/blivet/devices/stratis.py @@ -28,6 +28,7 @@ from ..static_data import stratis_info from ..storage_log import log_method_call from ..errors import DeviceError, StratisError +from ..size import Size from .. import devicelibs @@ -60,12 +61,60 @@ def blockdevs(self): """ A list of this pool block devices """ return self.parents[:] + @property + def filesystems(self): + """ A list of this pool block filesystems """ + return self.children[:] + @property def size(self): """ The size of this pool """ # sum up the sizes of the block devices return sum(parent.size for parent in self.parents) + @property + def _physical_size(self): + if self.exists: + pool_info = stratis_info.get_pool_info(self.name) + if not pool_info: + raise DeviceError("Failed to get information about pool %s" % self.name) + return pool_info.physical_size + else: + return self.size + + @property + def _physical_used(self): + physical_used = Size(0) + + # filesystems + for filesystem in self.filesystems: + physical_used += filesystem.used_size + + if self.exists: + # pool metadata using stratis-predict-usage which will give us used size + # for an empty pool with blockdevs we are using, i.e. metadata size for this pool + physical_used += devicelibs.stratis.pool_used(self.name, + [bd.path for bd in self.blockdevs], + self.encrypted) + else: + for bd in self.blockdevs: + if bd.exists: + # for existing blockdevs we can also use the stratis-predict-usage tool + physical_used += devicelibs.stratis.pool_used(self.name, + [bd.path], + self.encrypted) + else: + physical_used += devicelibs.stratis.STRATIS_BD_MD_SIZE + if self.encrypted: + physical_used += devicelibs.stratis.STRATIS_BD_ENC_MD_SIZE + + return physical_used + + @property + def free_space(self): + """ Free space in the pool usable for new filesystems """ + return self._physical_size - self._physical_used + @property def encrypted(self): """ True if this device is encrypted. """ @@ -160,6 +209,9 @@ def __init__(self, *args, **kwargs): super(StratisFilesystemDevice, self).__init__(*args, **kwargs) + if not self.exists and self.pool.free_space <= Size(0): + raise StratisError("cannot create new stratis filesystem, not enough free space in the pool") + def _get_name(self): """ This device's name. """ if self.pool is not None: @@ -180,6 +232,17 @@ def pool(self): return self.parents[0] + @property + def used_size(self): + """ Size used by this filesystem in the pool """ + if not self.exists: + return devicelibs.stratis.STRATIS_FS_MD_SIZE + else: + fs_info = stratis_info.get_filesystem_info(self.pool.name, self.fsname) + if not fs_info: + raise DeviceError("Failed to get information about filesystem %s" % self.name) + return fs_info.used_size + def _create(self): """ Create the device. """ log_method_call(self, self.name, status=self.status) diff --git a/blivet/static_data/stratis_info.py b/blivet/static_data/stratis_info.py index b58d90824..981e4afd2 100644 --- a/blivet/static_data/stratis_info.py +++ b/blivet/static_data/stratis_info.py @@ -46,8 +46,9 @@ STRATIS_MANAGER_INTF = STRATIS_SERVICE + ".Manager.r2" -StratisPoolInfo = namedtuple("StratisPoolInfo", ["name", "uuid", "physical_size", "object_path", "encrypted"]) -StratisFilesystemInfo = namedtuple("StratisFilesystemInfo", ["name", "uuid", "pool_name", "pool_uuid", "object_path"]) +StratisPoolInfo = namedtuple("StratisPoolInfo", ["name", "uuid", "physical_size", "physical_used", "object_path", "encrypted"]) +StratisFilesystemInfo = namedtuple("StratisFilesystemInfo", ["name", "uuid", "used_size", "pool_name", + "pool_uuid", "object_path"]) StratisBlockdevInfo = namedtuple("StratisBlockdevInfo", ["path", "uuid", "pool_name", "pool_uuid", "object_path"]) StratisLockedPoolInfo = namedtuple("StratisLockedPoolInfo", ["uuid", "key_desc", "devices"]) @@ -85,13 +86,21 @@ def _get_pool_info(self, pool_path): log.warning("Failed to get Stratis pool physical size for %s: %s", properties["Name"], pool_size) pool_size = 0 + + valid, pool_used = all_props.get("TotalPhysicalUsed", + (False, "TotalPhysicalUsed not available")) + if not valid: + log.warning("Failed to get Stratis pool physical used for %s: %s", + properties["Name"], pool_used) + pool_used = 0 else: log.error("Failed to get Stratis pool properties for %s.", properties["Name"]) pool_size = 0 + pool_used = 0 return StratisPoolInfo(name=properties["Name"], uuid=properties["Uuid"], - physical_size=Size(pool_size), object_path=pool_path, - encrypted=properties["Encrypted"]) + physical_size=Size(pool_size), physical_used=Size(pool_used), + object_path=pool_path, encrypted=properties["Encrypted"]) def _get_filesystem_info(self, filesystem_path): try: @@ -110,7 +119,24 @@ def _get_filesystem_info(self, filesystem_path): if not pool_info: return None + all_props = safe_dbus.call_sync(STRATIS_SERVICE, + filesystem_path, + STRATIS_PROPS_INTF, + "GetAllProperties", + None)[0] + if all_props: + valid, used_size = all_props.get("Used", + (False, "Used not available")) + if not valid: + log.warning("Failed to get Stratis filesystem used size for %s: %s", + properties["Name"], used_size) + used_size = 0 + else: + log.error("Failed to get Stratis filesystem properties for %s.", properties["Name"]) + used_size = 0 + return StratisFilesystemInfo(name=properties["Name"], uuid=properties["Uuid"], + used_size=Size(used_size), pool_name=pool_info.name, pool_uuid=pool_info.uuid, object_path=filesystem_path) diff --git a/tests/devices_test/stratis_test.py b/tests/devices_test/stratis_test.py index 12901b3b7..fe814e5bc 100644 --- a/tests/devices_test/stratis_test.py +++ b/tests/devices_test/stratis_test.py @@ -1,3 +1,4 @@ +import six import unittest try: @@ -10,6 +11,7 @@ from blivet.devices import StorageDevice from blivet.devices import StratisPoolDevice from blivet.devices import StratisFilesystemDevice +from blivet.errors import StratisError from blivet.size import Size @@ -25,7 +27,7 @@ class BlivetNewStratisDeviceTest(unittest.TestCase): def test_new_stratis(self): b = blivet.Blivet() bd = StorageDevice("bd1", fmt=blivet.formats.get_format("stratis"), - size=Size("1 GiB"), exists=True) + size=Size("1 GiB"), exists=False) b.devicetree._add_device(bd) @@ -41,6 +43,10 @@ def test_new_stratis(self): self.assertEqual(fs.pool, pool) self.assertEqual(fs.format.type, "stratis xfs") + with six.assertRaisesRegex(self, StratisError, "not enough free space in the pool"): + # only 1 GiB pool, not enough space for second fs + b.new_stratis_filesystem(name="testfs2", parents=[pool]) + b.create_device(pool) b.create_device(fs)