Merge branch '3.8-devel'

README.md

@@ -4,7 +4,7 @@ Blivet is a python module for system storage configuration.
 
 <img alt="CI status" src="https://fedorapeople.org/groups/storage_apis/statuses/blivet-master.svg" width="100%" height="300ex" />
 
-### Licence
+### License
 
 See [COPYING](https://github.com/storaged-project/blivet/blob/master/COPYING)
 
@@ -53,7 +53,7 @@ To install these dependencies use following commands:
        # dnf install python3-blockdev libblockdev-plugins-all python3-bytesize libbytesize python3-pyparted parted libselinux-python3
  * On Debian and Ubuntu based distributions:
 
-       # apt-get install python3-blockdev python3-bytesize python3-parted python3-selinux gir1.2-blockdev-2.0 libblockdev-lvm2 libblockdev-btrfs2 libblockdev-swap2 libblockdev-loop2 libblockdev-crypto2 libblockdev-mpath2 libblockdev-dm2 libblockdev-mdraid2 libblockdev-nvdimm2
+       # apt-get install python3-blockdev python3-bytesize python3-parted python3-selinux gir1.2-blockdev-3.0 libblockdev-lvm3 libblockdev-btrfs3 libblockdev-swap3 libblockdev-loop3 libblockdev-crypto3 libblockdev-mpath3 libblockdev-dm3 libblockdev-mdraid3 libblockdev-nvdimm3
 
 ### Development
 

blivet/__init__.py

@@ -63,15 +63,20 @@ def log_bd_message(level, msg):
 from gi.repository import GLib
 from gi.repository import BlockDev as blockdev
 if arch.is_s390():
-    _REQUESTED_PLUGIN_NAMES = set(("lvm", "btrfs", "swap", "crypto", "loop", "mdraid", "mpath", "dm", "s390", "nvdimm"))
+    _REQUESTED_PLUGIN_NAMES = set(("btrfs", "swap", "crypto", "loop", "mdraid", "mpath", "dm", "s390", "nvdimm"))
 else:
-    _REQUESTED_PLUGIN_NAMES = set(("lvm", "btrfs", "swap", "crypto", "loop", "mdraid", "mpath", "dm", "nvdimm"))
+    _REQUESTED_PLUGIN_NAMES = set(("btrfs", "swap", "crypto", "loop", "mdraid", "mpath", "dm", "nvdimm"))
 
 # nvme plugin is not generally available
 if hasattr(blockdev.Plugin, "NVME"):
     _REQUESTED_PLUGIN_NAMES.add("nvme")
 
 _requested_plugins = blockdev.plugin_specs_from_names(_REQUESTED_PLUGIN_NAMES)
+# XXX force non-dbus LVM plugin
+lvm_plugin = blockdev.PluginSpec()
+lvm_plugin.name = blockdev.Plugin.LVM
+lvm_plugin.so_name = "libbd_lvm.so.3"
+_requested_plugins.append(lvm_plugin)
 try:
     succ_, avail_plugs = blockdev.try_reinit(require_plugins=_requested_plugins, reload=False, log_func=log_bd_message)
 except GLib.GError as err:

blivet/devices/disk.py

@@ -725,7 +725,8 @@ class NVMeFabricsNamespaceDevice(NVMeNamespaceDevice, NetworkStorageDevice):
 
     """ NVMe fabrics namespace """
     _type = "nvme-fabrics"
-    _packages = ["nvme-cli"]
+    # dracut '95nvmf' module dependencies
+    _packages = ["nvme-cli", "dracut-network"]
 
     def __init__(self, device, **kwargs):
         """

blivet/devices/lvm.py

@@ -97,7 +97,8 @@ def get_supported_pe_sizes():
 
     def __init__(self, name, parents=None, size=None, free=None,
                  pe_size=None, pe_count=None, pe_free=None, pv_count=None,
-                 uuid=None, exists=False, sysfs_path='', exported=False):
+                 uuid=None, exists=False, sysfs_path='', exported=False,
+                 shared=False):
         """
             :param name: the device name (generally a device node's basename)
             :type name: str
@@ -124,6 +125,11 @@ def __init__(self, name, parents=None, size=None, free=None,
             :type pv_count: int
             :keyword uuid: the VG UUID
             :type uuid: str
+
+            For non-existing VGs only:
+
+            :keyword shared: whether to create this VG as shared
+            :type shared: bool
         """
         # These attributes are used by _add_parent, so they must be initialized
         # prior to instantiating the superclass.
@@ -137,6 +143,7 @@ def __init__(self, name, parents=None, size=None, free=None,
         self.pe_count = util.numeric_type(pe_count)
         self.pe_free = util.numeric_type(pe_free)
         self.exported = exported
+        self._shared = shared
 
         # TODO: validate pe_size if given
         if not self.pe_size:
@@ -157,6 +164,10 @@ def __init__(self, name, parents=None, size=None, free=None,
         # >0 is fixed
         self.size_policy = self.size
 
+        if self._shared:
+            for pv in self.parents:
+                pv.format._vg_shared = True
+
     def __repr__(self):
         s = super(LVMVolumeGroupDevice, self).__repr__()
         s += ("  free = %(free)s  PE Size = %(pe_size)s  PE Count = %(pe_count)s\n"
@@ -257,11 +268,23 @@ def _create(self):
         """ Create the device. """
         log_method_call(self, self.name, status=self.status)
         pv_list = [pv.path for pv in self.parents]
+        extra = dict()
+        if self._shared:
+            extra["shared"] = ""
         try:
-            blockdev.lvm.vgcreate(self.name, pv_list, self.pe_size)
+            blockdev.lvm.vgcreate(self.name, pv_list, self.pe_size, **extra)
         except blockdev.LVMError as err:
             raise errors.LVMError(err)
 
+        if self._shared:
+            if availability.BLOCKDEV_LVM_PLUGIN_SHARED.available:
+                try:
+                    blockdev.lvm.vglock_start(self.name)
+                except blockdev.LVMError as err:
+                    raise errors.LVMError(err)
+            else:
+                raise errors.LVMError("Shared LVM is not fully supported: %s" % ",".join(availability.BLOCKDEV_LVM_PLUGIN_SHARED.availability_errors))
+
     def _post_create(self):
         self._complete = True
         super(LVMVolumeGroupDevice, self)._post_create()
@@ -682,7 +705,7 @@ class LVMLogicalVolumeBase(DMDevice, RaidDevice):
     def __init__(self, name, parents=None, size=None, uuid=None, seg_type=None,
                  fmt=None, exists=False, sysfs_path='', grow=None, maxsize=None,
                  percent=None, cache_request=None, pvs=None, from_lvs=None,
-                 stripe_size=0):
+                 stripe_size=0, shared=False):
 
         if not exists:
             if seg_type not in [None, "linear", "thin", "thin-pool", "cache", "vdo-pool", "vdo", "cache-pool"] + lvm.raid_seg_types:
@@ -711,6 +734,7 @@ def __init__(self, name, parents=None, size=None, uuid=None, seg_type=None,
         self.seg_type = seg_type or "linear"
         self._raid_level = None
         self.ignore_skip_activation = 0
+        self._shared = shared
 
         self.req_grow = None
         self.req_max_size = Size(0)
@@ -2375,7 +2399,8 @@ def __init__(self, name, parents=None, size=None, uuid=None, seg_type=None,
                  parent_lv=None, int_type=None, origin=None, vorigin=False,
                  metadata_size=None, chunk_size=None, profile=None, from_lvs=None,
                  compression=False, deduplication=False, index_memory=0,
-                 write_policy=None, cache_mode=None, attach_to=None, stripe_size=0):
+                 write_policy=None, cache_mode=None, attach_to=None, stripe_size=0,
+                 shared=False):
         """
             :param name: the device name (generally a device node's basename)
             :type name: str
@@ -2406,6 +2431,8 @@ def __init__(self, name, parents=None, size=None, uuid=None, seg_type=None,
             :type cache_request: :class:`~.devices.lvm.LVMCacheRequest`
             :keyword pvs: list of PVs to allocate extents from (size could be specified for each PV)
             :type pvs: list of :class:`~.devices.StorageDevice` or :class:`LVPVSpec` objects (tuples)
+            :keyword shared: whether to activate the newly create LV in shared mode
+            :type shared: bool
 
             For internal LVs only:
 
@@ -2481,7 +2508,7 @@ def __init__(self, name, parents=None, size=None, uuid=None, seg_type=None,
         LVMLogicalVolumeBase.__init__(self, name, parents, size, uuid, seg_type,
                                       fmt, exists, sysfs_path, grow, maxsize,
                                       percent, cache_request, pvs, from_lvs,
-                                      stripe_size)
+                                      stripe_size, shared)
         LVMVDOPoolMixin.__init__(self, compression, deduplication, index_memory,
                                  write_policy)
         LVMVDOLogicalVolumeMixin.__init__(self)
@@ -2714,7 +2741,13 @@ def _setup(self, orig=False):
                         controllable=self.controllable)
         ignore_skip_activation = self.is_snapshot_lv or self.ignore_skip_activation > 0
         try:
-            blockdev.lvm.lvactivate(self.vg.name, self._name, ignore_skip=ignore_skip_activation)
+            if self._shared:
+                if availability.BLOCKDEV_LVM_PLUGIN_SHARED.available:
+                    blockdev.lvm.lvactivate(self.vg.name, self._name, ignore_skip=ignore_skip_activation, shared=True)
+                else:
+                    raise errors.LVMError("Shared LVM is not fully supported: %s" % ",".join(availability.BLOCKDEV_LVM_PLUGIN_SHARED.availability_errors))
+            else:
+                blockdev.lvm.lvactivate(self.vg.name, self._name, ignore_skip=ignore_skip_activation)
         except blockdev.LVMError as err:
             raise errors.LVMError(err)
 
@@ -2754,6 +2787,9 @@ def _create(self):
             if self._stripe_size:
                 extra["stripesize"] = str(int(self._stripe_size.convert_to("KiB")))
 
+            if self._shared:
+                extra["activate"] = "sy"
+
             try:
                 blockdev.lvm.lvcreate(self.vg.name, self._name, self.size,
                                       type=self.seg_type, pv_list=pvs, **extra)

blivet/formats/lvmpv.py

@@ -95,6 +95,7 @@ def __init__(self, **kwargs):
         self.pe_start = kwargs.get("pe_start", lvm.LVM_PE_START)
         self.data_alignment = kwargs.get("data_alignment", Size(0))
         self._free = kwargs.get("free")  # None means unknown
+        self._vg_shared = False
 
         self.inconsistent_vg = False
 
@@ -155,6 +156,10 @@ def _create(self, **kwargs):
                         type=self.type, status=self.status)
         lvm.lvm_devices_add(self.device)
 
+        if self._vg_shared:
+            log.info("Shared VG: skipping pvcreate, PV format will be created by vgcreate")
+            return
+
         ea_yes = blockdev.ExtraArg.new("-y", "")
 
         if lvm.HAVE_LVMDEVICES:

blivet/nvme.py

@@ -18,16 +18,20 @@
 #
 
 import os
-import shutil
 
 from . import errors
-from . import util
+
+import gi
+gi.require_version("BlockDev", "3.0")
+
+from gi.repository import BlockDev as blockdev
 
 import logging
 log = logging.getLogger("blivet")
 
-HOSTNQN_FILE = "/etc/nvme/hostnqn"
-HOSTID_FILE = "/etc/nvme/hostid"
+ETC_NVME_PATH = "/etc/nvme/"
+HOSTNQN_FILE = ETC_NVME_PATH + "hostnqn"
+HOSTID_FILE = ETC_NVME_PATH + "hostid"
 
 
 class NVMe(object):
@@ -40,6 +44,8 @@ class NVMe(object):
 
     def __init__(self):
         self.started = False
+        self._hostnqn = None
+        self._hostid = None
 
     # So that users can write nvme() to get the singleton instance
     def __call__(self):
@@ -52,28 +58,38 @@ def startup(self):
         if self.started:
             return
 
-        rc, nqn = util.run_program_and_capture_output(["nvme", "gen-hostnqn"])
-        if rc != 0:
-            raise errors.NVMeError("Failed to generate hostnqn")
-
-        with open(HOSTNQN_FILE, "w") as f:
-            f.write(nqn)
-
-        rc, hid = util.run_program_and_capture_output(["dmidecode", "-s", "system-uuid"])
-        if rc != 0:
-            raise errors.NVMeError("Failed to generate host ID")
-
-        with open(HOSTID_FILE, "w") as f:
-            f.write(hid)
+        self._hostnqn = blockdev.nvme_get_host_nqn()
+        self._hostid = blockdev.nvme_get_host_id()
+        if not self._hostnqn:
+            self._hostnqn = blockdev.nvme_generate_host_nqn()
+        if not self._hostnqn:
+            raise errors.NVMeError("Failed to generate HostNQN")
+        if not self._hostid:
+            if 'uuid:' not in self._hostnqn:
+                raise errors.NVMeError("Missing UUID part in the HostNQN string '%s'" % self._hostnqn)
+            # derive HostID from HostNQN's UUID part
+            self._hostid = self._hostnqn.split('uuid:')[1]
+
+        # do not overwrite existing files, taken e.g. from initramfs
+        self.write("/", overwrite=False)
 
         self.started = True
 
-    def write(self, root):  # pylint: disable=unused-argument
-        # copy the hostnqn and hostid files
-        if not os.path.isdir(root + "/etc/nvme"):
-            os.makedirs(root + "/etc/nvme", 0o755)
-        shutil.copyfile(HOSTNQN_FILE, root + HOSTNQN_FILE)
-        shutil.copyfile(HOSTID_FILE, root + HOSTID_FILE)
+    def write(self, root, overwrite=True):  # pylint: disable=unused-argument
+        # write down the hostnqn and hostid files
+        p = root + ETC_NVME_PATH
+        if not os.path.isdir(p):
+            os.makedirs(p, 0o755)
+        p = root + HOSTNQN_FILE
+        if overwrite or not os.path.isfile(p):
+            with open(p, "w") as f:
+                f.write(self._hostnqn)
+                f.write("\n")
+        p = root + HOSTID_FILE
+        if overwrite or not os.path.isfile(p):
+            with open(p, "w") as f:
+                f.write(self._hostid)
+                f.write("\n")
 
 
 # Create nvme singleton

blivet/tasks/availability.py

@@ -429,6 +429,14 @@ def available_resource(name):
 else:
     BLOCKDEV_LVM_TECH_VDO = _UnavailableMethod(error_msg="Installed version of libblockdev doesn't support LVM VDO technology")
 
+if hasattr(blockdev.LVMTech, "SHARED"):
+    BLOCKDEV_LVM_SHARED = BlockDevTechInfo(plugin_name="lvm",
+                                           check_fn=blockdev.lvm_is_tech_avail,
+                                           technologies={blockdev.LVMTech.SHARED: blockdev.LVMTechMode.MODIFY})  # pylint: disable=no-member
+    BLOCKDEV_LVM_TECH_SHARED = BlockDevMethod(BLOCKDEV_LVM_SHARED)
+else:
+    BLOCKDEV_LVM_TECH_SHARED = _UnavailableMethod(error_msg="Installed version of libblockdev doesn't support shared LVM technology")
+
 # libblockdev mdraid plugin required technologies and modes
 BLOCKDEV_MD_ALL_MODES = (blockdev.MDTechMode.CREATE |
                          blockdev.MDTechMode.DELETE |
@@ -469,6 +477,7 @@ def available_resource(name):
 BLOCKDEV_LOOP_PLUGIN = blockdev_plugin("libblockdev loop plugin", BLOCKDEV_LOOP_TECH)
 BLOCKDEV_LVM_PLUGIN = blockdev_plugin("libblockdev lvm plugin", BLOCKDEV_LVM_TECH)
 BLOCKDEV_LVM_PLUGIN_VDO = blockdev_plugin("libblockdev lvm plugin (vdo technology)", BLOCKDEV_LVM_TECH_VDO)
+BLOCKDEV_LVM_PLUGIN_SHARED = blockdev_plugin("libblockdev lvm plugin (shared LVM technology)", BLOCKDEV_LVM_TECH_SHARED)
 BLOCKDEV_MDRAID_PLUGIN = blockdev_plugin("libblockdev mdraid plugin", BLOCKDEV_MD_TECH)
 BLOCKDEV_MPATH_PLUGIN = blockdev_plugin("libblockdev mpath plugin", BLOCKDEV_MPATH_TECH)
 BLOCKDEV_SWAP_PLUGIN = blockdev_plugin("libblockdev swap plugin", BLOCKDEV_SWAP_TECH)

tests/unit_tests/devices_test/lvm_test.py

@@ -477,6 +477,36 @@ def test_skip_activate(self):
                 lv.setup()
                 lvm.lvactivate.assert_called_with(vg.name, lv.lvname, ignore_skip=False)
 
+    @patch("blivet.tasks.availability.BLOCKDEV_LVM_PLUGIN_SHARED",
+           new=blivet.tasks.availability.ExternalResource(blivet.tasks.availability.AvailableMethod, ""))
+    def test_lv_activate_shared(self):
+        pv = StorageDevice("pv1", fmt=blivet.formats.get_format("lvmpv"),
+                           size=Size("1 GiB"), exists=True)
+        vg = LVMVolumeGroupDevice("testvg", parents=[pv], exists=True)
+        lv = LVMLogicalVolumeDevice("data_lv", parents=[vg], size=Size("500 MiB"), exists=True, shared=True)
+
+        with patch("blivet.devices.lvm.blockdev.lvm") as lvm:
+            with patch.object(lv, "_pre_setup"):
+                lv.setup()
+                lvm.lvactivate.assert_called_with(vg.name, lv.lvname, ignore_skip=False, shared=True)
+
+    @patch("blivet.tasks.availability.BLOCKDEV_LVM_PLUGIN_SHARED",
+           new=blivet.tasks.availability.ExternalResource(blivet.tasks.availability.AvailableMethod, ""))
+    def test_vg_create_shared(self):
+        pv = StorageDevice("pv1", fmt=blivet.formats.get_format("lvmpv"),
+                           size=Size("1 GiB"), exists=True)
+        vg = LVMVolumeGroupDevice("testvg", parents=[pv], shared=True)
+
+        self.assertTrue(pv.format._vg_shared)
+        with patch("blivet.devices.lvm.blockdev.lvm") as lvm:
+            pv.format._create()
+            lvm.pvcreate.assert_not_called()
+
+        with patch("blivet.devices.lvm.blockdev.lvm") as lvm:
+            vg._create()
+            lvm.vgcreate.assert_called_with(vg.name, [pv.path], Size("4 MiB"), shared="")
+            lvm.vglock_start.assert_called_with(vg.name)
+
     def test_vg_is_empty(self):
         pv = StorageDevice("pv1", fmt=blivet.formats.get_format("lvmpv"),
                            size=Size("1024 MiB"))