Updates/improvements for DevNet NSO sandbox and customer environments (…

…#26)

* Fix for top level == mdd:openconfig

* Fixes for sandbox NetBox version

* Update yamllint settings

* Added interface translation for OC data

* Updated for full key names and support for STP

* Added option to set the parser command

* Add devices to NetBox based on ansible inventory

* Added support for auto-generating CML lab and interface mapping

* Updated collection version

* Added inventory_dir variable to control where interface mapping file is placed

* Added var for setting scale when graphing

* Fixing error in scale var

* Setting explicit type of scale parameter

* Fixes for interface mapping

* Moved device and interface mapping dicts to ansible vars

* CML lab generation fixes

* Fixed syntax error in regex

* Cat9kv requires 24 ports (for 24-port version)

* Improved support for NSO global settings

* Added truncation of OSPF interfaces

* feat: add checks for MPLS and network instance interfaces

Simplify search and replace logic

* Added document start to YAML files

* Fixed formatting to pass ansible-test

* Updated 'all' target

* Updated galaxy version to 1.2.6

* refactor: use regex for interface search and replace

* Use single quotes for data since data from NSO may contain double quotes

* Fixed error in l3switch day0 config

* Fixed ansible-test errors

* Updated to Draft202012Validator

* Renamed throttle variable to avoid conflict with Ansible built-in

* Fixes for mdd_combine

* Use localhost for nso_init

* Fixes to NSO role

* Update galaxy.yml

* Fixes for mdd_combine

---------

Co-authored-by: m0lass3s <[email protected]>

.yamllint.yml

@@ -0,0 +1,67 @@
+# -*- mode: yaml -*-
+# vim:ts=2:sw=2:ai:si:syntax=yaml
+#
+# yamllint configuration directives
+# Project Homepage: https://github.com/adrienverge/yamllint
+#
+# Overriding rules in files:
+# http://yamllint.readthedocs.io/en/latest/disable_with_comments.html
+---
+extends: default
+
+# Rules documentation: http://yamllint.readthedocs.io/en/latest/rules.html
+rules:
+
+  braces:
+    # Defaults
+    # min-spaces-inside: 0
+    # max-spaces-inside: 0
+
+    # Keeping 0 min-spaces to not error on empty collection definitions
+    min-spaces-inside: 0
+    # Allowing one space inside braces to improve code readability
+    max-spaces-inside: 1
+
+  brackets:
+    # Defaults
+    # min-spaces-inside: 0
+    # max-spaces-inside: 0
+
+    # Keeping 0 min-spaces to not error on empty collection definitions
+    min-spaces-inside: 0
+    # Allowing one space inside braces to improve code readability
+    max-spaces-inside: 1
+
+  comments:
+    # Defaults
+    # level: warning
+    # require-starting-space: true
+    # min-spaces-from-content: 2
+
+    # Disabling to allow for code comment blocks and #!/usr/bin/ansible-playbook
+    require-starting-space: false
+
+  indentation:
+    # Defaults
+    # spaces: consistent
+    # indent-sequences: true
+    # check-multi-line-strings: false
+
+    # Requiring 2 space indentation
+    spaces: 2
+    # Requiring consistent indentation within a file, either indented or not
+    indent-sequences: consistent
+
+  # Disabling due to copious amounts of long lines in the code which would
+  # require a code style change to resolve
+  line-length: disable
+    # Defaults
+    # max: 80
+    # allow-non-breakable-words: true
+    # allow-non-breakable-inline-mappings: false
+
+  # Disabling due to copious amounts of truthy warnings in the code which would
+  # require a code style change to resolve
+  truthy: disable
+    # Defaults
+    # level: warning

Makefile

@@ -13,7 +13,7 @@ help: ## Display help
 	printf "\033[36m%-30s\033[0m %s\n", $$1, $$NF \
 	}' $(MAKEFILE_LIST)
 
-all: test build publish ## Setup python-viptela env and run tests
+all: clean build test ## Setup python-viptela env and run tests
 
 $(VENV): $(VENV_BIN)/activate ## Build virtual environment
 

galaxy.yml

@@ -8,7 +8,7 @@ namespace: ciscops
 name: mdd
 
 # The version of the collection. Must be compatible with semantic versioning
-version: 1.2.4
+version: 1.2.5
 
 # The path to the Markdown (.md) readme file. This path is relative to the root of the collection
 readme: README.md

playbooks/cml_update_lab.yml

@@ -0,0 +1,101 @@
+---
+- name: Get CDP Data for Devices
+  hosts: network
+  connection: local
+  gather_facts: no
+  roles:
+    - ciscops.mdd.nso
+  vars:
+    start_from: 2
+    layout: kamada_kawai
+    inventory_dir: inventory
+    scale: 500
+    use_cat9kv: False
+    default_cml_device_template:
+      switch:
+        node_definition: iosvl2
+        ram: 768
+        tags:
+          - switch
+        type: switch
+      router:
+        node_definition: csr1000v
+        ram: 3072
+        tags:
+          - router
+        type: router
+      # l3switch:
+      #   node_definition: Cat9000v
+      #   image_definition: Cat9k
+      #   ram: 18432
+      #   cpus: 4
+      #   tags:
+      #     - l3switch
+      #   type: l3switch
+      l3switch:
+        node_definition: iosvl2
+        ram: 768
+        tags:
+          - l3switch
+        type: l3switch
+      ext_conn:
+        node_definition: external_connector
+        ram: 0
+        tags: []
+    # Default interface mapping for CML
+    default_cml_default_mappings:
+      Loopback(\d+): Loopback\1
+      Vlan(\d+): Vlan\1
+  tasks:
+
+    - name: Get CDP data
+      include_role:
+        name: ciscops.mdd.nso
+        tasks_from: exec_command
+      vars:
+        nso_exec_command: show cdp neighbors    
+
+    - set_fact:
+        cdp_data: "{{ {'hostname': inventory_hostname, 'tags': tags if tags is defined else {}, 'cdp': nso_command_output} }}"
+
+    - name: Create device list with CDP data
+      set_fact:
+        devices: "{{ groups['network'] | map('extract', hostvars, 'cdp_data') | list }}"
+      run_once: yes
+
+    - set_fact:
+        cml_device_template: "{{ default_cml_device_template }}"
+      when: cml_device_template is not defined
+
+    - set_fact:
+        cml_default_mappings: "{{ default_cml_default_mappings }}"
+      when: cml_default_mappings is not defined
+
+    - name: Generate topology
+      ciscops.mdd.cml_lab:
+        devices: "{{ devices }}"
+        start_from: "{{ start_from }}"
+        device_template: "{{ cml_device_template }}"
+        default_mappings: "{{ cml_default_mappings }}"
+        use_cat9kv: "{{ use_cat9kv | bool }}"
+      register: results
+      run_once: yes
+
+    - name: "Layout graph using the {{ layout }} layout"
+      set_fact:
+        topology: "{{ results.topology | ciscops.mdd.graph(layout=layout,scale=scale) }}"
+      run_once: yes
+      when: not layout == 'none'
+
+    - name: Create topology file
+      copy:
+        content: "{{ topology | to_nice_yaml(indent=2,sort_keys=False) }}"
+        dest: "{{ lookup('env', 'PWD') }}/files/cml_lab.yaml"
+      run_once: yes
+      no_log: yes
+
+    - name: Create mapping file
+      copy:
+        content: "{{ {'all': {'hosts': results.mappings}} | to_nice_yaml(indent=2,sort_keys=False) }}"
+        dest: "{{ lookup('env', 'PWD') }}/{{ inventory_dir }}/cml_intf_map.yml"
+      run_once: yes

playbooks/inventory_update_netbox.yml

@@ -0,0 +1,35 @@
+- hosts: localhost
+  gather_facts: no
+  roles:
+    - ciscops.mdd.netbox
+  tasks:
+    - include_role:
+        name: ciscops.mdd.netbox
+        tasks_from: netbox_inventory
+
+    - name: Add new device
+      include_role:
+        name: ciscops.mdd.netbox
+        tasks_from: add_device
+      when: inventory_hostname not in netbox_host_list
+      with_items: "{{ groups['network'] }}"
+      vars:
+        netbox_device_name: "{{ hostvars[item].inventory_hostname }}"
+        netbox_device_interface_name: "{{ hostvars[item].mgmt_interface }}"
+        netbox_device_ipv4_address: "{{ hostvars[item].ansible_host }}"
+        netbox_device_type: unknown
+        netbox_device_role: unknown
+
+    - include_role:
+        name: ciscops.mdd.netbox
+        tasks_from: netbox_inventory
+
+    - include_role:
+        name: ciscops.mdd.netbox
+        tasks_from: netbox_mgmt
+      when: hostvars[item].mgmt_interface is defined and hostvars[item].inventory_hostname in netbox_host_list
+      with_items: "{{ groups['network'] }}"
+      vars:
+        netbox_device_name: "{{ hostvars[item].inventory_hostname }}"
+        netbox_device_interface_name: "{{ hostvars[item].mgmt_interface }}"
+        netbox_device_ipv4_address: "{{ hostvars[item].ansible_host }}"

playbooks/nso_init.yml

@@ -3,6 +3,9 @@
   gather_facts: no
   roles:
     - ciscops.mdd.nso
+  vars:
+    ansible_python_interpreter: "{{ hostvars['localhost'].ansible_python_interpreter}}"
+
   tasks:
     - name: Add NSO Auth Groups
       include_role:

playbooks/nso_update_data.yml

@@ -7,15 +7,15 @@
     - ciscops.mdd.data
   vars:
     dry_run: true
-    throttle: 10
+    workers: 10
   tasks:
     - include_role:
         name: ciscops.mdd.nso
         tasks_from: check_sync
 
     - name: Update OC Data
       when: ('oc' in mdd_data_types)
-      throttle: "{{ throttle }}"
+      throttle: "{{ workers }}"
       block:
         # - name: Combine Device Data with OC Data
         #   ansible.builtin.set_fact:
@@ -26,6 +26,11 @@
         #     nso_device_config: "{{ nso_device_config | ansible.builtin.combine( { 'config': mdd_data['config'] }, recursive=true) }}"
         #   when: mdd_data['config'] is defined and mdd_data['mdd:openconfig'] is defined
 
+        - name: Translate and truncate interface names
+          set_fact:
+            mdd_data: "{{ mdd_data | ciscops.mdd.intf_xform(cml_intf_xlate) }}"
+          when: (cml_group is defined and cml_group in group_names) and (cml_intf_xlate is defined and cml_intf_xlate)
+
         - name: Update MDD Data
           ansible.builtin.include_role:
             name: ciscops.mdd.nso
@@ -37,7 +42,7 @@
 
     - name: Update Config Data
       when: ('config' in mdd_data_types) and not ('oc' in mdd_data_types)
-      throttle: "{{ throttle }}"
+      throttle: "{{ workers }}"
       block:
         - name: Get Device Data
           ansible.builtin.include_role:

plugins/filter/data.py

@@ -15,9 +15,9 @@
 # the list to a hash, then merge the hash.  If there is not match, the list
 # is replaced.
 list_key_map = {
-    "openconfig-network-instance:network-instance": "openconfig-network-instance:name",
+    "openconfig-network-instance:network-instance$": "openconfig-network-instance:name",
     "openconfig-network-instance:vlan": "openconfig-network-instance:vlan-id",
-    "^openconfig-interfaces:interfaces:openconfig-interfaces:interface": "openconfig-interfaces:name",
+    "^mdd:openconfig:openconfig-interfaces:interfaces:openconfig-interfaces:interface$": "openconfig-interfaces:name",
     ":openconfig-interfaces:interface$": "openconfig-network-instance:id",
     "openconfig-network-instance:static": "openconfig-network-instance:prefix",
     "openconfig-network-instance:protocol": "openconfig-network-instance:name",
@@ -150,7 +150,7 @@ def merge_hash(x, y, path, recursive=True, list_merge='replace'):
         if path == '':
             path = key
         else:
-            path = ":".join([path, key])
+            path = ":".join([str(item) for item in [path, key]])
 
         # if both x's element and y's element are dicts
         # recursively "combine" them or override x's with y's element

plugins/filter/graph.py

@@ -0,0 +1,47 @@
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+from ansible.module_utils.six import raise_from
+from ansible.errors import AnsibleError
+try:
+    import networkx as nx
+except ImportError as imp_exc:
+    NETWORKX_IMPORT_ERROR = imp_exc
+else:
+    NETWORKX_IMPORT_ERROR = None
+
+
+def graph(topology_data, layout='kamada_kawai', scale=500):
+    if NETWORKX_IMPORT_ERROR:
+        raise_from(AnsibleError('networkx must be installed to use this plugin'), NETWORKX_IMPORT_ERROR)
+
+    pos = {}
+    g = nx.Graph()
+
+    for link in topology_data['links']:
+        g.add_edge(link['n1'], link['n2'])
+
+    if layout == 'spring':
+        pos = nx.layout.spring_layout(g, scale=int(scale))
+    elif layout == 'planar':
+        pos = nx.layout.planar_layout(g, scale=int(scale))
+    elif layout == 'spectral':
+        pos = nx.layout.spectral_layout(g, scale=int(scale))
+    elif layout == 'kamada_kawai':
+        pos = nx.layout.kamada_kawai_layout(g, scale=int(scale))
+
+    for key, value in pos.items():
+        for node in topology_data['nodes']:
+            if node['id'] == key:
+                node['x'] = int(value[0])
+                node['y'] = int(value[1])
+
+    return topology_data
+
+
+class FilterModule(object):
+
+    def filters(self):
+        return {
+            'graph': graph,
+        }