LVMVDO support (#181)

* LVMVDO support - added support for LVM VDO - new accepted pool options: - vdo_compression - vdo_deduplication - vdo_size - added tests for LVM VDO - added support for COPR repositories * remove extraneous blank lines at end of file remove extraneous blank lines at end of file * use pipefail; reduce line length use pipefail; reduce line length * fix line length issues; use multiline when fix line length issues; use multiline `when` * make sure all tasks are named make sure all tasks are named * fix line length using multiline when; remove extra blank lines fix line length using multiline `when`; remove extra blank lines * wrap lines to shorten line length wrap lines to shorten line length Co-authored-by: Richard Megginson <[email protected]>
linux-system-roles · Jun 23, 2021 · ec5f2ab · ec5f2ab
1 parent b3b4561
commit ec5f2ab
Show file tree

Hide file tree

Showing 13 changed files with 301 additions and 6 deletions.
diff --git a/README.md b/README.md
@@ -94,6 +94,8 @@ must contain only a single item.
 ##### `size`
 The `size` specifies the size of the file system. The format for this is intended to
 be human-readable, e.g.: "10g", "50 GiB".
+When using `compression` or `deduplication`, `size` can be set higher than actual available space,
+e.g.: 3 times the size of the volume, based on duplicity and/or compressibility of stored data.
 
 __NOTE__: The requested volume size may be reduced as necessary so the volume can
           fit in the available pool space, but only if the required reduction is
@@ -157,6 +159,30 @@ This integer specifies the LUKS key size (in bits).
 ##### `encryption_luks_version`
 This integer specifies the LUKS version to use.
 
+##### `deduplication`
+This specifies whether or not the Virtual Data Optimizer (VDO) will be used.
+When set, duplicate data stored on storage volume will be
+deduplicated resulting in more storage capacity.
+Can be used together with `compression` and `vdo_pool_size`.
+Volume has to be part of the LVM `storage_pool`.
+Limit one VDO `storage_volume` per `storage_pool`.
+Underlying volume has to be at least 9 GB (bare minimum is around 5 GiB).
+
+##### `compression`
+This specifies whether or not the Virtual Data Optimizer (VDO) will be used.
+When set, data stored on storage volume will be
+compressed resulting in more storage capacity.
+Volume has to be part of the LVM `storage_pool`.
+Can be used together with `deduplication` and `vdo_pool_size`.
+Limit one VDO `storage_volume` per `storage_pool`.
+
+##### `vdo_pool_size`
+When Virtual Data Optimizer (VDO) is used, this specifies the actual size the volume
+will take on the device. Virtual size of VDO volume is set by `size` parameter.
+`vdo_pool_size` format is intended to be human-readable,
+e.g.: "30g", "50GiB".
+Default value is equal to the size of the volume.
+
 #### `storage_safe_mode`
 When true (the default), an error will occur instead of automatically removing existing devices and/or formatting.
 

diff --git a/defaults/main.yml b/defaults/main.yml
@@ -56,3 +56,7 @@ storage_volume_defaults:
   encryption_cipher: null
   encryption_key_size: null
   encryption_luks_version: null
+
+  compression: null
+  deduplication: null
+  vdo_pool_size: null
diff --git a/library/blivet.py b/library/blivet.py
@@ -320,6 +320,8 @@ def required_packages(self):
         packages.extend(fmt.packages)
         if self._volume.get('encryption'):
             packages.extend(get_format('luks').packages)
+        if self._volume.get('compression') or self._volume.get('deduplication'):
+            packages.extend(['vdo', 'kmod-kvdo'])
         return packages
 
     @property
@@ -645,10 +647,41 @@ def _create(self):
         fmt = self._get_format()
         trim_percent = (1.0 - float(parent.free_space / size)) * 100
         log.debug("size: %s ; %s", size, trim_percent)
-        if size > parent.free_space:
+
+        create_vdo = self._volume['deduplication'] or self._volume['compression']
+
+        if create_vdo:
+            pool_size = parent.free_space if size > parent.free_space else size
+            if self._volume['vdo_pool_size']:
+                pool_size = Size(self._volume['vdo_pool_size'])
+                if pool_size > parent.free_space:
+                    if trim_percent > MAX_TRIM_PERCENT:
+                        raise BlivetAnsibleError("specified 'vdo_pool_size' for volume '%s' "
+                                                 "exceeds available space in pool '%s' (%s)" % (pool_size,
+                                                                                                parent.name,
+                                                                                                parent.free_space))
+                    else:
+                        log.info("adjusting %s size from %s to %s to fit in %s free space",
+                                 self._volume['name'],
+                                 size,
+                                 parent.free_space,
+                                 parent.name)
+                        pool_size = parent.free_space
+
+            try:
+                vdopool = self._blivet.new_lv(name='vdopool', vdo_pool=True,
+                                              parents=[parent], compression=self._volume['compression'],
+                                              deduplication=self._volume['deduplication'],
+                                              size=pool_size)
+            except Exception as e:
+                raise BlivetAnsibleError("failed to set up VDO pool '%s': %s" % (self._volume['name'], str(e)))
+
+            self._blivet.create_device(vdopool)
+        elif size > parent.free_space:
             if trim_percent > MAX_TRIM_PERCENT:
-                raise BlivetAnsibleError("specified size for volume '%s' exceeds available space in pool '%s' (%s)"
-                                         % (size, parent.name, parent.free_space))
+                raise BlivetAnsibleError("specified size for volume '%s' exceeds available space in pool '%s' (%s)" % (size,
+                                                                                                                       parent.name,
+                                                                                                                       parent.free_space))
             else:
                 log.info("adjusting %s size from %s to %s to fit in %s free space",
                          self._volume['name'],
@@ -658,8 +691,16 @@ def _create(self):
                 size = parent.free_space
 
         try:
-            device = self._blivet.new_lv(name=self._volume['name'],
-                                         parents=[parent], size=size, fmt=fmt)
+            if create_vdo:
+                device = self._blivet.new_lv(name=self._volume['name'], vdo_lv=create_vdo,
+                                             parents=[vdopool if create_vdo else parent],
+                                             size=size, fmt=fmt)
+            else:
+                # This is here for backwards compatibility. Until 8.4 blivet does not support
+                # vdo_lv optional parameter
+                device = self._blivet.new_lv(name=self._volume['name'],
+                                             parents=[vdopool if create_vdo else parent],
+                                             size=size, fmt=fmt)
         except Exception as e:
             raise BlivetAnsibleError("failed to set up volume '%s': %s" % (self._volume['name'], str(e)))
 
@@ -818,7 +859,6 @@ def required_packages(self):
 
         if self._pool.get('encryption'):
             packages.extend(get_format('luks').packages)
-
         return packages
 
     @property

diff --git a/tasks/enable_copr.yml b/tasks/enable_copr.yml
@@ -0,0 +1,16 @@
+---
+- name: make sure COPR is not already enabled
+  shell: |
+    set -euo pipefail
+    {{ ansible_pkg_mgr }} repolist | \
+    grep -c `echo {{ repo.repository }} | tr / :` || true
+  args:
+    warn: false
+  register: copr_present
+  changed_when: false
+
+- name: get list of COPRs to be enabled
+  command:
+    cmd: "{{ ansible_pkg_mgr }} -y copr enable {{ repo.repository }}"
+    warn: false
+  when: copr_present.stdout == "0"
diff --git a/tasks/enable_coprs.yml b/tasks/enable_coprs.yml
@@ -0,0 +1,26 @@
+---
+- name: check if the COPR support packages should be installed
+  set_fact:
+    install_copr: yes
+  loop: "{{ _storage_copr_packages|default([]) }}"
+  loop_control:
+    loop_var: repo
+  when:
+    - _storage_copr_packages is defined
+    - copr_packages is defined
+    - repo.packages|intersect(copr_packages)|length
+
+- name: make sure COPR support packages are present
+  package:
+    name: "{{ _storage_copr_support_packages }}"
+  when: install_copr is defined and install_copr | bool
+
+- name: enable COPRs
+  include_tasks: "enable_copr.yml"
+  loop: "{{ _storage_copr_packages|default([]) }}"
+  loop_control:
+    loop_var: repo
+  when:
+    - _storage_copr_packages is defined
+    - copr_packages is defined
+    - repo.packages|intersect(copr_packages)|length
diff --git a/tasks/main-blivet.yml b/tasks/main-blivet.yml
@@ -26,6 +26,11 @@
     packages_only: true
   register: package_info
 
+- name: enable copr repositories if needed
+  include_tasks: "enable_coprs.yml"
+  vars:
+    copr_packages: "{{ package_info.packages }}"
+
 - name: make sure required packages are installed
   package:
     name: "{{ package_info.packages }}"

diff --git a/tests/test-verify-pool-members.yml b/tests/test-verify-pool-members.yml
@@ -57,6 +57,9 @@
 - name: Check member encryption
   include_tasks: verify-pool-members-encryption.yml
 
+- name: Check VDO
+  include_tasks: verify-pool-members-vdo.yml
+
 - set_fact:
     _storage_test_expected_pv_type: null
     _storage_test_expected_pv_count: null

diff --git a/tests/tests_create_lvmvdo_then_remove.yml b/tests/tests_create_lvmvdo_then_remove.yml
@@ -0,0 +1,118 @@
+---
+- hosts: all
+  become: true
+  vars:
+    storage_safe_mode: false
+    mount_location: '/opt/test1'
+    volume_group_size: '10g'
+    volume_size: '12g'
+    pool_size: '9g'
+
+  tasks:
+    - include_role:
+        name: linux-system-roles.storage
+
+    - name: Gather package facts
+      package_facts:
+        # gather information about packages
+
+    - name: Set blivet package name
+      set_fact:
+        blivet_pkg_name: "{{ ansible_facts.packages | select('search','blivet') | select('search', 'python') | list }}"
+
+    - name: Set blivet package version
+      set_fact:
+        blivet_pkg_version: "{{ ansible_facts.packages[blivet_pkg_name[0]][0]['version'] + '-' + ansible_facts.packages[blivet_pkg_name[0]][0]['release'] }}"
+
+    - block:
+        - include_tasks: get_unused_disk.yml
+          vars:
+            min_size: "{{ volume_group_size }}"
+            max_return: 1
+
+        - name: Create LVM VDO volume under volume group 'pool1'
+          include_role:
+            name: linux-system-roles.storage
+          vars:
+            storage_pools:
+              - name: pool1
+                disks: "{{ unused_disks }}"
+                volumes:
+                  - name: volume1
+                    compression: true
+                    deduplication: true
+                    vdo_pool_size: "{{ pool_size }}"
+                    size: "{{ volume_size }}"
+                    mount_point: "{{ mount_location }}"
+
+        - include_tasks: verify-role-results.yml
+
+        - name: Repeat the previous invocation to verify idempotence
+          include_role:
+            name: linux-system-roles.storage
+          vars:
+            storage_pools:
+              - name: pool1
+                disks: "{{ unused_disks }}"
+                volumes:
+                  - name: volume1
+                    compression: true
+                    deduplication: true
+                    vdo_pool_size: "{{ pool_size }}"
+                    size: "{{ volume_size }}"
+                    mount_point: "{{ mount_location }}"
+
+        - include_tasks: verify-role-results.yml
+
+        - name: Remove LVM VDO volume in 'pool1' created above
+          include_role:
+            name: linux-system-roles.storage
+          vars:
+            storage_pools:
+              - name: pool1
+                disks: "{{ unused_disks }}"
+                state: "absent"
+                volumes:
+                  - name: volume1
+                    compression: true
+                    deduplication: true
+                    vdo_pool_size: "{{ pool_size }}"
+                    size: "{{ volume_size }}"
+                    mount_point: "{{ mount_location }}"
+
+        - include_tasks: verify-role-results.yml
+
+        - name: Create LVM VDO volume under volume group 'pool1' (this time default size)
+          include_role:
+            name: linux-system-roles.storage
+          vars:
+            storage_pools:
+              - name: pool1
+                disks: "{{ unused_disks }}"
+                volumes:
+                  - name: volume1
+                    compression: true
+                    deduplication: true
+                    size: "{{ volume_size }}"
+                    mount_point: "{{ mount_location }}"
+
+        - include_tasks: verify-role-results.yml
+
+        - name: Remove LVM VDO volume in 'pool1' created above
+          include_role:
+            name: linux-system-roles.storage
+          vars:
+            storage_pools:
+              - name: pool1
+                disks: "{{ unused_disks }}"
+                state: "absent"
+                volumes:
+                  - name: volume1
+                    compression: true
+                    deduplication: true
+                    size: "{{ volume_size }}"
+                    mount_point: "{{ mount_location }}"
+
+        - include_tasks: verify-role-results.yml
+
+      when: blivet_pkg_version is version("3.2.2-10", ">=") and ansible_facts["distribution"] != "Fedora"
diff --git a/tests/verify-pool-member-vdo.yml b/tests/verify-pool-member-vdo.yml
@@ -0,0 +1,40 @@
+- name: check VDO
+  block:
+    - name: get information about VDO deduplication
+      command: "vdostats --verbose {{ storage_test_pool.name }}-vdopool-vpool"
+      register: storage_test_vdo_status
+      changed_when: false
+
+    - set_fact:
+        storage_test_vdo_dedupe_re: "{{ ('maximum dedupe queries +: +0$') }}"
+
+    - assert:
+        that: "storage_test_vdo_status.stdout is regex(storage_test_vdo_dedupe_re)"
+        msg: "VDO deduplication is on but it should not"
+      when: not storage_test_vdo_volume.deduplication
+
+    - assert:
+        that: "storage_test_vdo_status.stdout is not regex(storage_test_vdo_dedupe_re)"
+        msg: "VDO deduplication is off but it should not"
+      when: storage_test_vdo_volume.deduplication | bool
+
+    - set_fact:
+        storage_test_vdo_compress_re: "{{ ('compressed fragments written +: +0$') }}"
+
+    - assert:
+        that: "storage_test_vdo_status.stdout is regex(storage_test_vdo_compress_re)"
+        msg: "VDO compression is on but it should not"
+      when: not storage_test_vdo_volume.compression
+
+    - assert:
+        that: "storage_test_vdo_status.stdout is not regex(storage_test_vdo_compress_re)"
+        msg: "VDO compression is off but it should not"
+      when: storage_test_vdo_volume.compression | bool
+
+  when:
+    - storage_test_vdo_volume.deduplication != none or storage_test_vdo_volume.compression != none
+    - storage_test_pool.state != "absent"
+    - storage_test_vdo_volume.state != "absent"
+
+- set_fact:
+    storage_test_vdo_status: null
diff --git a/tests/verify-pool-members-vdo.yml b/tests/verify-pool-members-vdo.yml
@@ -0,0 +1,6 @@
+- name: Validate pool member VDO settings
+  include_tasks: verify-pool-member-vdo.yml
+  loop: "{{ storage_test_pool.volumes }}"
+  loop_control:
+    loop_var: storage_test_vdo_volume
+  when: storage_test_pool.type == 'lvm'
diff --git a/vars/CentOS_8.yml b/vars/CentOS_8.yml
@@ -5,3 +5,5 @@ blivet_package_list:
   - libblockdev-lvm
   - libblockdev-mdraid
   - libblockdev-swap
+  - vdo
+  - kmod-kvdo
diff --git a/vars/Fedora.yml b/vars/Fedora.yml
@@ -5,3 +5,10 @@ blivet_package_list:
   - libblockdev-lvm
   - libblockdev-mdraid
   - libblockdev-swap
+_storage_copr_packages:
+  - repository: "rhawalsh/dm-vdo"
+    packages:
+      - vdo
+      - kmod-vdo
+_storage_copr_support_packages:
+  - dnf-plugins-core
diff --git a/vars/RedHat_8.yml b/vars/RedHat_8.yml
@@ -5,3 +5,5 @@ blivet_package_list:
   - libblockdev-lvm
   - libblockdev-mdraid
   - libblockdev-swap
+  - vdo
+  - kmod-kvdo