feat: xlate openconfig-system-ext:track-interface when sending to tes… (

#31) * feat: xlate openconfig-system-ext:track-interface when sending to test devices * feat: remove VTY ACLs when sending configs to test * feat: reformat code * feat: reformat code * Updates to make config truncate more generic * Improvements and bug fixes * Update galaxy.yml * Added truncate config to update playbook * Fixed indentation error * feat: prune harvested configs * Updated config truncate to accept a list of key paths * feat: better format for yaml files * fix: README.md typos * feat: now compare weights at the same hierarchy level * feat: now compare weights at the same hierarchy level * Added openconfig-acl interfaces to truncate --------- Co-authored-by: Jason King <[email protected]>
model-driven-devops · Sep 29, 2023 · 6b2460f · 6b2460f
1 parent 74186a2
commit 6b2460f
Show file tree

Hide file tree

Showing 11 changed files with 122 additions and 31 deletions.
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# Cisco Model Driven Devops Ansible Collection
+# Cisco Model-Driven Devops Ansible Collection
 
 This Ansible Collection is an implementation of Model-Driven DevOps and includes the following roles:
 * [oc](https://github.com/model-driven-devops/ansible-mdd/blob/main/roles/oc/README.md)
@@ -8,14 +8,14 @@ This Ansible Collection is an implementation of Model-Driven DevOps and includes
 * [netbox](https://github.com/model-driven-devops/ansible-mdd/blob/main/roles/netbox/README.md)
 * [nso](https://github.com/model-driven-devops/ansible-mdd/blob/main/roles/nso/README.md)
 
-## Dependancies
-### Environmnet Variables
+## Dependencies
+### Environment Variables
 If using NetBox:
 - `NETBOX_API`
 - `NETBOX_TOKEN`
 
-To use an external NSO (ie. not deployed in CML):
-- `NSO_URL` Protocol, FQDN or IP address and port (ex. `http://192.168.1.100:8080`)
+To use an external NSO (i.e. not deployed in CML):
+- `NSO_URL` Protocol, FQDN or IP address and port (e.g. `http://192.168.1.100:8080`)
 - `NSO_USERNAME`
 - `NSO_PASSWORD`
 

diff --git a/galaxy.yml b/galaxy.yml
@@ -8,7 +8,7 @@ namespace: ciscops
 name: mdd
 
 # The version of the collection. Must be compatible with semantic versioning
-version: 1.2.7
+version: 1.2.8
 
 # The path to the Markdown (.md) readme file. This path is relative to the root of the collection
 readme: README.md

diff --git a/playbooks/cml_update_lab.yml b/playbooks/cml_update_lab.yml
@@ -93,13 +93,13 @@
 
     - name: Create topology file
       copy:
-        content: "{{ topology | to_nice_yaml(indent=2,sort_keys=False) }}"
+        content: "{{ topology | ciscops.mdd.to_even_nicer_yaml }}"
         dest: "{{ lookup('env', 'PWD') }}/files/cml_lab.yaml"
       run_once: yes
       no_log: yes
 
     - name: Create mapping file
       copy:
-        content: "{{ {'all': {'hosts': results.mappings}} | to_nice_yaml(indent=2,sort_keys=False) }}"
+        content: "{{ {'all': {'hosts': results.mappings}} | ciscops.mdd.to_even_nicer_yaml }}"
         dest: "{{ lookup('env', 'PWD') }}/{{ inventory_dir }}/cml_intf_map.yml"
       run_once: yes
diff --git a/playbooks/nso_dump_config.yml b/playbooks/nso_dump_config.yml
@@ -20,5 +20,5 @@
         nso_device_config: "{ 'mdd_data': { 'config': {{ nso_device_config['config'] }} } }"
 
     - copy:
-        content: "{{ nso_device_config | to_nice_yaml(indent=2,sort_keys=False) }}"
+        content: "{{ nso_device_config | ciscops.mdd.to_even_nicer_yaml }}"
         dest: "{{ current_dir }}/device_config_data/{{ inventory_hostname }}.yml"
diff --git a/playbooks/nso_update_data.yml b/playbooks/nso_update_data.yml
@@ -17,10 +17,10 @@
       when: ('oc' in mdd_data_types)
       throttle: "{{ workers }}"
       block:
-        - name: Translate and truncate interface names
+        - name: Translate/truncate interface names and truncate config
           set_fact:
-            mdd_data: "{{ mdd_data | ciscops.mdd.intf_xform(cml_intf_xlate) }}"
-          when: (cml_group is defined and cml_group in group_names) and (cml_intf_xlate is defined and cml_intf_xlate)
+            mdd_data: "{{ mdd_data | ciscops.mdd.config_xform(cml_intf_xlate | default(None), cml_truncate_list | default(None)) }}"
+          when: (cml_intf_xlate is defined and cml_intf_xlate) or (cml_truncate_list is defined and cml_truncate_list)
 
         - name: Update MDD Data
           ansible.builtin.include_role:

diff --git a/playbooks/show.yml b/playbooks/show.yml
@@ -4,10 +4,10 @@
   roles:
     - ciscops.mdd.data
   tasks:
-    - name: Translate and truncate interface names
+    - name: Translate/truncate interface names and truncate config
       set_fact:
-        mdd_data: "{{ mdd_data | ciscops.mdd.intf_xform(cml_intf_xlate) }}"
-      when: (cml_group is defined and cml_group in group_names) and (cml_intf_xlate is defined and cml_intf_xlate)
-        
+        mdd_data: "{{ mdd_data | ciscops.mdd.config_xform(cml_intf_xlate | default(None), cml_truncate_list | default(None)) }}"
+      when: (cml_intf_xlate is defined and cml_intf_xlate) or (cml_truncate_list is defined and cml_truncate_list)
+
     - debug:
         var: mdd_data
diff --git a/plugins/filter/intf.py b/plugins/filter/intf.py
@@ -15,7 +15,8 @@
     "openconfig-system-ext:ssh-source-interface",
     "openconfig-network-instance:interface-id",
     "openconfig-network-instance:index",
-    "openconfig-network-instance:local-address"
+    "openconfig-network-instance:local-address",
+    "openconfig-system-ext:track-interface"
 ]
 
 
@@ -51,20 +52,26 @@ def xlate_value(data, intf_dict):
         return
 
 
-def intf_xlate(data, intf_dict):
+def intf_xlate(data, intf_dict=None):
     if not data:
         return {}
 
+    if intf_dict is None:
+        return data
+
     data_out = data.copy()
     xlate_value(data_out, intf_dict)
 
     return data_out
 
 
-def intf_truncate(data, intf_dict):
+def intf_truncate(data, intf_dict=None):
     if not data:
         return {}
 
+    if intf_dict is None:
+        return data
+
     regex_list = intf_dict.keys()
     temp_interface_list = []
     temp_stp_interface_list = []
@@ -145,12 +152,48 @@ def intf_truncate(data, intf_dict):
                      [instance_index]["openconfig-network-instance:mpls"]["openconfig-network-instance:global"]
                      ["openconfig-network-instance:interface-attributes"]["openconfig-network-instance:interface"]) = temp_mpls_interface_list
 
+        # Truncate openconfig-acl interfaces
+        if "openconfig-acl:acl" in oc_data and "openconfig-acl:interfaces" in oc_data["openconfig-acl:acl"]:
+            temp_acl_interface_list = []
+            for interface in oc_data["openconfig-acl:acl"]["openconfig-acl:interfaces"]["openconfig-acl:interface"]:
+                if found_full_match(interface["openconfig-acl:id"].split(".")[0], intf_dict):
+                    temp_acl_interface_list.append(interface)
+
+            data_out["mdd:openconfig"]["openconfig-acl:acl"]["openconfig-acl:interfaces"]["openconfig-acl:interface"] = temp_acl_interface_list
+
+    return data_out
+
+
+def delete_key(data, key_list):
+    if isinstance(data, dict):
+        if key_list[0] in list(data):
+            if len(key_list) == 1:
+                del data[key_list[0]]
+            else:
+                key = key_list.pop(0)
+                delete_key(data[key], key_list)
+
+
+def config_truncate(data, truncate_list=None):
+    """Find all values from a nested dictionary for a given key."""
+
+    if not data:
+        return {}
+
+    if truncate_list is None:
+        return data
+
+    data_out = data.copy()
+
+    for path in truncate_list:
+        delete_key(data_out, path)
     return data_out
 
 
-def intf_xform(data, intf_dict):
+def config_xform(data, intf_dict=None, truncate_list=None):
     data = intf_truncate(data, intf_dict)
     data = intf_xlate(data, intf_dict)
+    data = config_truncate(data, truncate_list)
     return data
 
 
@@ -160,5 +203,5 @@ def filters(self):
         return {
             'intf_xlate': intf_xlate,
             'intf_truncate': intf_truncate,
-            'intf_xform': intf_xform
+            'config_xform': config_xform
         }
diff --git a/plugins/filter/nso_oc.py b/plugins/filter/nso_oc.py
@@ -26,7 +26,10 @@ def nso_oc(config_data):
     }
     native_dict = copy.deepcopy(config_data)
     translation_notes = []
-    main_xe.build_xe_to_oc(config_data, native_dict, oc_dict, translation_notes)
+    oc_dict_clean = main_xe.build_xe_to_oc(config_data, native_dict, oc_dict, translation_notes)
+
+    if oc_dict_clean:
+        oc_dict = oc_dict_clean
 
     mdd_dict['mdd_data'] = {
         "mdd:openconfig": oc_dict['mdd:openconfig'],

diff --git a/plugins/filter/to_even_nicer_yaml.py b/plugins/filter/to_even_nicer_yaml.py
@@ -0,0 +1,36 @@
+from __future__ import absolute_import, division, print_function
+
+__metaclass__ = type
+
+from ansible.utils.unsafe_proxy import AnsibleUnsafeText
+import yaml
+
+
+class MyDumper(yaml.Dumper):
+    def increase_indent(self, flow=False, indentless=False):
+        return super(MyDumper, self).increase_indent(flow, False)
+
+
+def convert_ansible_unsafe_text_to_safe(data):
+    if isinstance(data, dict):
+        return {convert_ansible_unsafe_text_to_safe(k): convert_ansible_unsafe_text_to_safe(v) for k, v in data.items()}
+    elif isinstance(data, list):
+        return [convert_ansible_unsafe_text_to_safe(item) for item in data]
+    elif isinstance(data, AnsibleUnsafeText):
+        return str(data)
+    else:
+        return data
+
+
+def to_even_nicer_yaml(config_data):
+    ansible_safe_config_data = convert_ansible_unsafe_text_to_safe(config_data)
+    return yaml.dump(ansible_safe_config_data, Dumper=MyDumper, default_flow_style=False, explicit_start=True,
+                     sort_keys=False)
+
+
+class FilterModule(object):
+
+    def filters(self):
+        return {
+            'to_even_nicer_yaml': to_even_nicer_yaml
+        }
diff --git a/plugins/modules/mdd_combine.py b/plugins/modules/mdd_combine.py
@@ -175,18 +175,27 @@ def _merge(result_cfgs, v, path=None, filepath=None, hierarchy_level=None, playb
                 else:
                     result_cfgs[k] = _merge({}, v, path + [str(k)], filepath, hierarchy_level, playbook_tags, weight)
             else:
-                # if key not there, add
+                # if key not there, add it
                 if k not in result_cfgs:
                     result_cfgs[k] = (v, filepath, playbook_tags, hierarchy_level, weight)
-                # if key found multiple places at same hierarchy level, error
-                elif k in result_cfgs and hierarchy_level == result_cfgs[k][3] and result_cfgs[k][0]:
+                # if key found multiple places at same hierarchy level and the new key's weight is higher, go with the highest weight.
+                elif k in result_cfgs and hierarchy_level == result_cfgs[k][3] and result_cfgs[k][0] and weight > \
+                        result_cfgs[k][4]:
+                    result_cfgs[k] = (v, filepath, playbook_tags, hierarchy_level, weight)
+                # if key found multiple places at same hierarchy level and the new key's weight is lower, skip.
+                elif k in result_cfgs and hierarchy_level == result_cfgs[k][3] and result_cfgs[k][0] and weight < \
+                        result_cfgs[k][4]:
+                    continue
+                # if key found multiple places at same hierarchy level and if the weight is the same, error.
+                elif k in result_cfgs and hierarchy_level == result_cfgs[k][3] and result_cfgs[k][0] and weight == \
+                        result_cfgs[k][4]:
                     if filepath == result_cfgs[k][1]:
                         module.fail_json(
-                            msg="Merge Error: key {1} was found multiple times at the same hierarchy level (level: {2}) in file {3}.".format(
+                            msg="Merge Error: key {0} was found multiple times at the same hierarchy level (level: {1}) in file {2}.".format(
                                 k, result_cfgs[k][3], filepath))
                     else:
                         module.fail_json(
-                            msg="Merge Error: key {1} was found multiple times at the same hierarchy level (level: {2}) in files {3} and {4}.".format(
+                            msg="Merge Error: key {0} was found multiple times at the same hierarchy level (level: {1}) in files {2} and {3}.".format(
                                 k, result_cfgs[k][3], filepath, result_cfgs[k][1]))
                     module.exit_json(changed=False, failed=True)
                 # if key exists but weight is higher, replace
@@ -378,7 +387,7 @@ def find_and_read_configs(top_dir, device_name, filespec_list, default_weight, t
                                         )
                             except yaml.YAMLError:
                                 module.fail_json(
-                                    msg="An error occurred loading file {1}".format(os.path.join(current_dir, filename)))
+                                    msg="An error occurred loading file {0}".format(os.path.join(current_dir, filename)))
                                 module.exit_json(changed=False, failed=True)
 
                 hierarchy_level += 1

diff --git a/roles/nso/tasks/update_yaml_data.yml b/roles/nso/tasks/update_yaml_data.yml
@@ -4,7 +4,7 @@
 
 - name: Write Config Data to Device Directory
   copy:
-    content: "---\n{{ data | to_nice_yaml(indent=2,sort_keys=False) }}"
+    content: "{{ data | ciscops.mdd.to_even_nicer_yaml }}"
     dest: "{{ mdd_device_dir }}/config-data.yml"
   vars:
     data: "{{ { 'mdd_data': { 'config': nso_device_config['config'] } } }}"
@@ -17,15 +17,15 @@
 
   - name: Write OC Data to Device Directory
     copy:
-      content: "---\n{{ data | to_nice_yaml(indent=2,sort_keys=False) }}"
+      content: "{{ data | ciscops.mdd.to_even_nicer_yaml }}"
       dest: "{{ mdd_device_dir }}/oc-{{ (item.key | ansible.builtin.split(':'))[1] }}.yml"
     with_dict: "{{ oc_data['mdd_data']['mdd:openconfig'] }}"
     vars:
       data: "{{ { 'mdd_data': { 'mdd:openconfig' : { item.key: item.value } } } }}"
 
   - name: Write Native Data to Device Directory
     copy:
-      content: "---\n{{ data | to_nice_yaml(indent=2,sort_keys=False) }}"
+      content: "{{ data | ciscops.mdd.to_even_nicer_yaml }}"
       dest: "{{ mdd_device_dir }}/config-remaining.yml"
     vars:
       data: "{{ { 'mdd_data': { 'config': oc_data['mdd_data']['config'] } } }}"