Feat(cv_device_v3): Add "inventory_mode" to allow ignoring missing de…

…vices (#594)

* Refactor: Avoid redundant API calls

* Feat: Add "inventory_mode" to cv_device_v3 to allow ignoring missing devices

* Update docs

* Run black

* fix indentation

* pep8 fix

---------

Co-authored-by: Sugetha Chandhrasekar <[email protected]>

ansible_collections/arista/cvp/docs/how-to/v3/cv_device_v3.md

@@ -39,6 +39,9 @@ CVP_DEVICES:
 - `apply_mode`: Define how configlets configured to the devices are managed by ansible:
   - `loose` (default): Configure new configlets to device and **ignore** configlet already configured but not listed.
   - `strict`: Configure new configlets to device and **remove** configlet already configured but not listed.
+- `inventory_mode`: Define how missing devices are handled.
+  - `loose`: Ignore missing devices.
+  - `strict` (default): Fail on any missing device.
 - `search_key`: Define key to use to search for devices.
   - `hostname`: Use Hostname to get devices.
   - `fqdn`: Use Hostname + DNS to get devices.
@@ -54,6 +57,13 @@ CVP_DEVICES:
     devices: "{{CVP_DEVICES}}"
   register: CVP_DEVICES_RESULTS
 
+# Use loose inventory_mode
+- name: "Configure devices on {{inventory_hostname}}"
+  arista.cvp.cv_device_v3:
+    devices: "{{CVP_DEVICES}}"
+    inventory_mode: "loose"
+  register: CVP_DEVICES_RESULTS
+
 # Use strict apply_mode
 - name: "Configure devices on {{inventory_hostname}}"
   arista.cvp.cv_device_v3:

ansible_collections/arista/cvp/docs/schema/cv_device_v3.md

@@ -3,6 +3,7 @@
 | Variable | Type | Required | Default | Choices | Description |
 | -------- | ---- | -------- | ------- | ------------------ | ----------- |
 | apply_mode | str | No | loose | loose<br>strict | Set how configlets are attached/detached on device. If set to strict all configlets not listed in your vars are detached |
+| inventory_mode | str | No | strict | loose<br>strict | Define how missing devices are handled. "loose" will ignore missing devices. "strict" will fail on any missing device |
 | search_key | str | No | hostname | fqdn<br>hostname<br>serialNumber | Key name to use to look for device in CloudVision |
 | state | str| No | present | present<br>factory_reset<br>provisioning_reset<br>absent | Set if Ansible should build, remove devices from provisioning, fully decommission or factory reset devices on CloudVision |
 | devices | List | Yes |  |  | List of devices with their container and configlets information |

ansible_collections/arista/cvp/plugins/module_utils/device_tools.py

@@ -627,11 +627,56 @@ def __check_devices_exist(self, user_inventory: DeviceInventory):
             self.__ansible.fail_json(msg=error_message)
         return True
 
-    def __state_present(
-        self,
-        user_inventory: DeviceInventory,
-        apply_mode: str = ModuleOptionValues.APPLY_MODE_LOOSE,
-    ):
+    def __remove_missing_devices(self, user_inventory: DeviceInventory):
+        """
+        __remove_missing_devices Check if devices are present on Cloudvision and remove missing devices from the user_inventory
+
+        Parameters
+        ----------
+        user_inventory : DeviceInventory
+            Inventory provided by user
+
+        Returns
+        -------
+        DeviceInventory
+            Device inventory where missing devices are removed
+        """
+
+        # Use dict to map the relevant attribute of the device to use for search
+        DEVICE_SEARCH_ATTRIBUTE = {
+            Api.device.HOSTNAME: "fqdn",
+            Api.device.FQDN: "fqdn",
+            Api.device.SYSMAC: "system_mac",
+            Api.device.SERIAL: "serial_number",
+        }
+
+        MODULE_LOGGER.debug("Remove missing devices from user_inventory")
+
+        # Loop through devices and record index of missing ones for later removal
+        missing_devices_indexes: list = []
+        for index, device in enumerate(user_inventory.devices):
+            device_search_key = getattr(
+                device, DEVICE_SEARCH_ATTRIBUTE[self.__search_by], "fqdn"
+            )
+            if (
+                self.is_device_exist(device_search_key, search_mode=self.__search_by)
+                is False
+            ):
+                missing_devices_indexes.append(index)
+                MODULE_LOGGER.debug(
+                    "Device not present in CVP, removing from user_inventory: %s",
+                    device_search_key,
+                )
+
+        # Remove missing devices from list - reversed to not change indexes.
+        missing_devices_indexes.reverse()
+        for index in missing_devices_indexes:
+            user_inventory.devices.pop(index)
+
+        return user_inventory
+
+    def __state_present(self, user_inventory: DeviceInventory, apply_mode: str = ModuleOptionValues.APPLY_MODE_LOOSE,
+                        inventory_mode: str = ModuleOptionValues.INVENTORY_MODE_STRICT):
         """
         __state_present Execute actions when user configures state=present
 
@@ -669,8 +714,14 @@ def __state_present(
         )
         response = CvAnsibleResponse()
 
-        # Check if all devices are present on CV
-        self.__check_devices_exist(user_inventory=user_inventory)
+        if inventory_mode == ModuleOptionValues.INVENTORY_MODE_LOOSE:
+            # Remove missing devices on CV from inventory (ignore missing)
+            user_inventory = self.__remove_missing_devices(
+                user_inventory=user_inventory
+            )
+        else:
+            # Check if all devices are present on CV (fail on missing)
+            self.__check_devices_exist(user_inventory=user_inventory)
 
         # Refresh UserInventory data with data from Cloudvision
         user_inventory = self.__refresh_user_inventory(user_inventory=user_inventory)
@@ -745,7 +796,11 @@ def __state_present(
 
         return response
 
-    def __state_factory_reset(self, user_inventory: DeviceInventory):
+    def __state_factory_reset(
+        self,
+        user_inventory: DeviceInventory,
+        inventory_mode: str = ModuleOptionValues.INVENTORY_MODE_STRICT,
+    ):
         """
         __state_factory_reset Execute actions when user configures state=factory_reset
 
@@ -765,8 +820,14 @@ def __state_factory_reset(self, user_inventory: DeviceInventory):
         response = CvAnsibleResponse()
         cv_reset = CvManagerResult(builder_name=DeviceResponseFields.DEVICE_RESET)
 
-        # Check if all user defined devices are present in CV
-        self.__check_devices_exist(user_inventory=user_inventory)
+        if inventory_mode == ModuleOptionValues.INVENTORY_MODE_LOOSE:
+            # Remove missing devices on CV from inventory (ignore missing)
+            user_inventory = self.__remove_missing_devices(
+                user_inventory=user_inventory
+            )
+        else:
+            # Check if all devices are present on CV (fail on missing)
+            self.__check_devices_exist(user_inventory=user_inventory)
 
         user_inventory = self.__refresh_user_inventory(user_inventory=user_inventory)
 
@@ -779,7 +840,11 @@ def __state_factory_reset(self, user_inventory: DeviceInventory):
         response.add_manager(cv_reset)
         return response
 
-    def __state_provisioning_reset(self, user_inventory: DeviceInventory):
+    def __state_provisioning_reset(
+        self,
+        user_inventory: DeviceInventory,
+        inventory_mode: str = ModuleOptionValues.INVENTORY_MODE_STRICT,
+    ):
         """
         __state_provisioning_reset Execute actions when user configures state=provisioning_reset
 
@@ -800,8 +865,14 @@ def __state_provisioning_reset(self, user_inventory: DeviceInventory):
         response = CvAnsibleResponse()
         cv_reset = CvManagerResult(builder_name=DeviceResponseFields.DEVICE_REMOVED)
 
-        # Check if all user defined devices are present in CV
-        self.__check_devices_exist(user_inventory=user_inventory)
+        if inventory_mode == ModuleOptionValues.INVENTORY_MODE_LOOSE:
+            # Remove missing devices on CV from inventory (ignore missing)
+            user_inventory = self.__remove_missing_devices(
+                user_inventory=user_inventory
+            )
+        else:
+            # Check if all devices are present on CV (fail on missing)
+            self.__check_devices_exist(user_inventory=user_inventory)
 
         user_inventory = self.__refresh_user_inventory(user_inventory=user_inventory)
 
@@ -814,7 +885,11 @@ def __state_provisioning_reset(self, user_inventory: DeviceInventory):
         response.add_manager(cv_reset)
         return response
 
-    def __state_absent(self, user_inventory: DeviceInventory):
+    def __state_absent(
+        self,
+        user_inventory: DeviceInventory,
+        inventory_mode: str = ModuleOptionValues.INVENTORY_MODE_STRICT,
+    ):
         """
         __state_absent Execute actions when user configures state=absent
 
@@ -836,8 +911,14 @@ def __state_absent(self, user_inventory: DeviceInventory):
             builder_name=DeviceResponseFields.DEVICE_DECOMMISSIONED
         )
 
-        # Check if all user defined devices are present in CV
-        self.__check_devices_exist(user_inventory=user_inventory)
+        if inventory_mode == ModuleOptionValues.INVENTORY_MODE_LOOSE:
+            # Remove missing devices on CV from inventory (ignore missing)
+            user_inventory = self.__remove_missing_devices(
+                user_inventory=user_inventory
+            )
+        else:
+            # Check if all devices are present on CV (fail on missing)
+            self.__check_devices_exist(user_inventory=user_inventory)
 
         user_inventory = self.__refresh_user_inventory(user_inventory=user_inventory)
 
@@ -1326,6 +1407,7 @@ def manager(
         search_mode: str = Api.device.HOSTNAME,
         apply_mode: str = ModuleOptionValues.APPLY_MODE_LOOSE,
         state: str = ModuleOptionValues.STATE_MODE_PRESENT,
+        inventory_mode: str = ModuleOptionValues.INVENTORY_MODE_STRICT,
     ):
         """
         manager Main entry point to support all device
@@ -1345,6 +1427,8 @@ def manager(
             Define how manager will apply configlets to device: loose (only attach listed configlet) or strict (attach listed configlet, remove others)
         state: str, optional
             Define if devices must be provisioned or reset to default configuration.
+        inventory_mode: str, optional
+            Define how manager will handle missing devices: loose (ignore missing devices) or strict (fail on missing devices)
 
         Returns
         -------
@@ -1358,20 +1442,28 @@ def manager(
         if state == ModuleOptionValues.STATE_MODE_PRESENT:
             MODULE_LOGGER.info("Processing data to create/update devices")
             response = self.__state_present(
-                user_inventory=user_inventory, apply_mode=apply_mode
+                user_inventory=user_inventory,
+                apply_mode=apply_mode,
+                inventory_mode=inventory_mode,
             )
 
         elif state == ModuleOptionValues.STATE_MODE_ABSENT:
             MODULE_LOGGER.info("Processing data to reset devices")
-            response = self.__state_factory_reset(user_inventory=user_inventory)
+            response = self.__state_factory_reset(
+                user_inventory=user_inventory, inventory_mode=inventory_mode
+            )
 
         elif state == ModuleOptionValues.STATE_MODE_REMOVE:
             MODULE_LOGGER.info("Processing data to reset devices")
-            response = self.__state_provisioning_reset(user_inventory=user_inventory)
+            response = self.__state_provisioning_reset(
+                user_inventory=user_inventory, inventory_mode=inventory_mode
+            )
 
         elif state == ModuleOptionValues.STATE_MODE_DECOMM:
             MODULE_LOGGER.info("Processing data to decommission devices")
-            response = self.__state_absent(user_inventory=user_inventory)
+            response = self.__state_absent(
+                user_inventory=user_inventory, inventory_mode=inventory_mode
+            )
 
         return response.content
 
@@ -2009,6 +2101,7 @@ def deploy_device(self, user_inventory: DeviceInventory):
                 current_container_info = self.get_container_current(
                     device_lookup=device.info[self.__search_by]
                 )
+
                 MODULE_LOGGER.debug(
                     "Device {0} is currently under {1}".format(
                         device.fqdn, current_container_info[Api.generic.NAME]

ansible_collections/arista/cvp/plugins/module_utils/resources/modules/fields.py

@@ -29,6 +29,8 @@ class ModuleOptionValues():
     VALIDATE_MODE_IGNORE: str = 'ignore'
     VALIDATE_MODE_STOP_ON_WARNING: str = 'stop_on_warning'
     VALIDATE_MODE_STOP_ON_ERROR: str = 'stop_on_error'
+    INVENTORY_MODE_LOOSE: str = 'loose'
+    INVENTORY_MODE_STRICT: str = 'strict'
     STATE_MODE_ABSENT: str = 'factory_reset'
     STATE_MODE_PRESENT: str = 'present'
     STATE_MODE_DECOMM: str = 'absent'

ansible_collections/arista/cvp/plugins/modules/cv_device_v3.py

@@ -50,6 +50,12 @@
     default: 'loose'
     choices: ['loose', 'strict']
     type: str
+  inventory_mode:
+    description: Define how missing devices are handled. "loose" will ignore missing devices. "strict" will fail on any missing device.
+    required: false
+    default: 'strict'
+    choices: ['loose', 'strict']
+    type: str
   search_key:
     description: Key name to use to look for device in CloudVision.
     required: false
@@ -59,7 +65,7 @@
 '''
 
 EXAMPLES = r'''
-# task in loose mode using fqdn (default)
+# task in loose apply_mode using fqdn (default)
 - name: Device Management in CloudVision
   hosts: cv_server
   connection: local
@@ -80,7 +86,30 @@
         state: present
         search_key: fqdn
 
-# task in loose mode using serial
+# task in loose apply_mode and loose inventory_mode using fqdn (default)
+- name: Device Management in CloudVision
+  hosts: cv_server
+  connection: local
+  gather_facts: false
+  collections:
+    - arista.cvp
+  vars:
+    CVP_DEVICES:
+      - fqdn: NON-EXISTING-DEVICE
+        parentContainerName: ANSIBLE
+        configlets:
+            - 'CV-EOS-ANSIBLE01'
+        imageBundle: leaf_image_bundle
+  tasks:
+    - name: "Configure devices on {{inventory_hostname}}"
+      arista.cvp.cv_device_v3:
+        devices: '{{CVP_DEVICES}}'
+        state: present
+        search_key: fqdn
+        inventory_mode: loose
+
+
+# task in loose apply_mode using serial
 - name: Device Management in CloudVision
   hosts: cv_server
   connection: local
@@ -100,7 +129,7 @@
         state: present
         search_key: serialNumber
 
-# task in strict mode
+# task in strict apply_mode
 - name: Device Management in CloudVision
   hosts: cv_server
   connection: local
@@ -221,6 +250,10 @@ def main():
                         required=False,
                         default='loose',
                         choices=['loose', 'strict']),
+        inventory_mode=dict(type='str',
+                            required=False,
+                            default='strict',
+                            choices=['loose', 'strict']),
         search_key=dict(type='str',
                         required=False,
                         default='hostname',
@@ -259,7 +292,9 @@ def main():
         user_inventory=user_topology,
         apply_mode=ansible_module.params['apply_mode'],
         search_mode=ansible_module.params['search_key'],
-        state=ansible_module.params['state'])
+        state=ansible_module.params['state'],
+        inventory_mode=ansible_module.params['inventory_mode'],
+    )
 
     ansible_module.exit_json(**result)