Fetch interface data from devices and save the interfaces as entities…

… to Grafolean

snmpbot.py

@@ -37,6 +37,10 @@ class InvalidOutputPath(Exception):
 r = redis.Redis(host=REDIS_HOST)
 
 
+OID_IF_DESCR = '1.3.6.1.2.1.2.2.1.2'
+OID_IF_SPEED = '1.3.6.1.2.1.2.2.1.5'
+
+
 def _get_previous_counter_value(counter_ident):
  prev_value = r.hgetall(counter_ident)
  if not prev_value: # empty dict
@@ -195,6 +199,33 @@ def send_results_to_grafolean(backend_url, bot_token, account_id, values):
 
 class SNMPBot(Collector):
 
+ @staticmethod
+ def _create_snmp_sesssion(job_info):
+ # initialize SNMP session:
+ session_kwargs = {
+ "hostname": job_info["details"]["ipv4"],
+ "use_numeric": True,
+ }
+ cred = job_info["credential_details"]
+ snmp_version = int(cred["version"][5:6])
+ session_kwargs["version"] = snmp_version
+ if snmp_version in [1, 2]:
+ session_kwargs["community"] = cred["snmpv12_community"]
+ elif snmp_version == 3:
+ session_kwargs = {
+ **session_kwargs,
+ "security_username": cred["snmpv3_securityName"],
+ "security_level": cred["snmpv3_securityLevel"], # easysnmp supports camelCase level names too
+ "privacy_protocol": cred.get("snmpv3_privProtocol", 'DEFAULT'),
+ "privacy_password": cred.get("snmpv3_privKey", ''),
+ "auth_protocol": cred.get("snmpv3_authProtocol", 'DEFAULT'),
+ "auth_password": cred.get("snmpv3_authKey", ''),
+ }
+ else:
+ raise Exception("Invalid SNMP version")
+ session = Session(**session_kwargs)
+ return session
+
  @staticmethod
  def do_snmp(*args, **job_info):
  """
@@ -249,29 +280,7 @@ def do_snmp(*args, **job_info):
  ipv4=job_info["details"]["ipv4"],
  ))
 
- # initialize SNMP session:
- session_kwargs = {
- "hostname": job_info["details"]["ipv4"],
- "use_numeric": True,
- }
- cred = job_info["credential_details"]
- snmp_version = int(cred["version"][5:6])
- session_kwargs["version"] = snmp_version
- if snmp_version in [1, 2]:
- session_kwargs["community"] = cred["snmpv12_community"]
- elif snmp_version == 3:
- session_kwargs = {
- **session_kwargs,
- "security_username": cred["snmpv3_securityName"],
- "security_level": cred["snmpv3_securityLevel"], # easysnmp supports camelCase level names too
- "privacy_protocol": cred.get("snmpv3_privProtocol", 'DEFAULT'),
- "privacy_password": cred.get("snmpv3_privKey", ''),
- "auth_protocol": cred.get("snmpv3_authProtocol", 'DEFAULT'),
- "auth_password": cred.get("snmpv3_authKey", ''),
- }
- else:
- raise Exception("Invalid SNMP version")
- session = Session(**session_kwargs)
+ session = SNMPBot._create_snmp_sesssion(job_info)
 
  # filter out only those sensors that are supposed to run at this interval:
  affecting_intervals, = args
@@ -308,6 +317,88 @@ def do_snmp(*args, **job_info):
  send_results_to_grafolean(job_info['backend_url'], job_info['bot_token'], job_info['account_id'], values)
 
 
+ @staticmethod
+ def update_if_entities(*args, **job_info):
+ log.info("Running interfaces job for account [{account_id}], IP [{ipv4}]".format(
+ account_id=job_info["account_id"],
+ ipv4=job_info["details"]["ipv4"],
+ ))
+
+ session = SNMPBot._create_snmp_sesssion(job_info)
+
+ parent_entity_id = job_info["entity_id"]
+ account_id = job_info["account_id"]
+ backend_url = job_info['backend_url']
+ bot_token = job_info['bot_token']
+ # fetch interfaces and update the interface entities:
+ result_descr = session.walk(OID_IF_DESCR)
+ result_speed = session.walk(OID_IF_SPEED)
+
+ # make sure that indexes of results are aligned - we don't want to have incorrect data:
+ if any([if_speed.oid_index != if_descr.oid_index for if_descr, if_speed in zip(result_descr, result_speed)]):
+ log.warning(f"Out-of-order results for interface names on entity {parent_entity_id}, sorting not yet implemented, bailing out!")
+ return
+
+ # - get those entities on this account, which have this entity as their parent and filter them by type ('interface')
+ requests_session = requests.Session()
+ url = f'{backend_url}/accounts/{account_id}/entities/?parent={parent_entity_id}&entity_type=interface&b={bot_token}'
+ r = requests_session.get(url)
+ r.raise_for_status()
+ # existing_entities = {x['details']['snmp_index']: (x['name'], x['details']['speed_bps'], x['id'],) for x in r.json()['list']}
+ # Temporary, until we implement filtering in API:
+ existing_entities = {x['details']['snmp_index']: (x['name'], x['details']['speed_bps'], x['id'],) for x in r.json()['list'] if x["entity_type"] == 'interface' and x["parent"] == parent_entity_id}
+
+ for if_descr_snmpvalue, if_speed_snmpvalue in zip(result_descr, result_speed):
+ oid_index = if_descr_snmpvalue.oid_index
+ descr = if_descr_snmpvalue.value
+ speed_bps = if_speed_snmpvalue.value
+ # - for each new entity:
+ # - make sure it exists (if not, create it - POST)
+ if oid_index not in existing_entities:
+ log.debug(f"Entity with OID index {oid_index} is new, inserting.")
+ url = f'{backend_url}/accounts/{account_id}/entities/?b={bot_token}'
+ payload = {
+ "name": descr,
+ "entity_type": "interface",
+ "parent": parent_entity_id,
+ "details":{
+ "snmp_index": oid_index,
+ "speed_bps": speed_bps,
+ },
+ }
+ r = requests_session.post(url, json=payload)
+ continue
+
+ # - make sure the description and speed are correct (if not, update them - PUT)
+ existing_descr, existing_speed, existing_id = existing_entities[oid_index]
+ if existing_descr != descr or existing_speed != speed_bps:
+ log.debug(f"Entity with OID index {oid_index} changed data, updating.")
+ url = f'{backend_url}/accounts/{account_id}/entities/{existing_id}/?b={bot_token}'
+ payload = {
+ "name": descr,
+ "entity_type": "interface",
+ # "parent": parent_entity_id, # changing entity parent is not possible
+ "details":{
+ "snmp_index": oid_index,
+ "speed_bps": speed_bps,
+ },
+ }
+ r = requests_session.put(url, json=payload)
+ del existing_entities[oid_index]
+ continue
+
+ # - mark it as processed
+ log.debug(f"Entity with OID index {oid_index} didn't change.")
+ del existing_entities[oid_index]
+
+ # - for every existing entity that is not among the new ones, remove it (no point in keeping it - we don't keep old versions of enities data either)
+ for oid_index in existing_entities:
+ _, _, existing_id = existing_entities[oid_index]
+ log.debug(f"Entity with OID index {oid_index} no longer exists, removing.")
+ url = f'{backend_url}/accounts/{account_id}/entities/{existing_id}/?b={bot_token}'
+ r = requests_session.delete(url)
+
+
  def jobs(self):
  """
  Each entity (device) is a single job, no matter how many sensors it has. The reason is
@@ -316,9 +407,15 @@ def jobs(self):
  for entity_info in self.fetch_job_configs('snmp'):
  intervals = list(set([sensor_info["interval"] for sensor_info in entity_info["sensors"]]))
  job_info = { **entity_info, "backend_url": self.backend_url, "bot_token": self.bot_token }
- job_id = str(entity_info["entity_id"])
+ job_id = f'{entity_info["entity_id"]}'
  yield job_id, intervals, SNMPBot.do_snmp, job_info
 
+ # We also collect interface data from each entity; the assumption is that everyone who wants
+ # to use SNMP also wants to know about network interfaces.
+ # Since `job_info` has all the necessary data, we simply pass it along:
+ job_id = f'{entity_info["entity_id"]}-interfaces'
+ yield job_id, [5*60], SNMPBot.update_if_entities, job_info
+
 
 def wait_for_grafolean(backend_url):
  while True: