feat: improve logging and add logger

Signed-off-by: djerfy <[email protected]>

src/config.yaml

@@ -70,3 +70,7 @@ monitoring:
     labels:
       include: []
       exclude: []
+
+metrics:
+  output: DEBUG
+  memory: True

src/modules/kubernetes/base/cronjobs.py

@@ -2,9 +2,10 @@
 from datetime import datetime
 from pyzabbix import ZabbixMetric
 from modules.common.functions import *
-import json, urllib3
+import json, urllib3, logging
 
 urllib3.disable_warnings()
+logging = logging.getLogger("kubernetes.base.cronjobs")
 
 def kubernetesGetCronjobs(config=None):
     """
@@ -114,6 +115,7 @@ def baseCronjobs(mode=None, config=None):
     description: monitoring cronjobs
     return: class ZabbixMetric
     """
+    logging.info(f"Function baseCronjobs() executed: {mode}")
     if mode == "discovery":
         return zabbixDiscoveryCronjobs(config['kubernetes']['name'], kubernetesGetCronjobs(config['monitoring']['cronjobs']))
     if mode == "item":

src/modules/kubernetes/base/daemonsets.py

@@ -1,9 +1,10 @@
 from kubernetes import client
 from pyzabbix import ZabbixMetric
 from modules.common.functions import *
-import json, urllib3
+import json, urllib3, logging
 
 urllib3.disable_warnings()
+logging = logging.getLogger("kubernetes.base.daemonsets")
 
 def kubernetesGetDaemonsets(config=None):
     """
@@ -82,6 +83,7 @@ def baseDaemonsets(mode=None, config=None):
     description: monitoring daemonsets
     return: class ZabbixMetric
     """
+    logging.info(f"Function baseDaemonsets() executed: {mode}")
     if mode == "discovery":
         return zabbixDiscoveryDaemonsets(config['kubernetes']['name'], kubernetesGetDaemonsets(config['monitoring']['daemonsets']))
     if mode == "item":

src/modules/kubernetes/base/deployments.py

@@ -1,9 +1,10 @@
 from kubernetes import client
 from pyzabbix import ZabbixMetric
 from modules.common.functions import *
-import json, urllib3
+import json, urllib3, logging
 
 urllib3.disable_warnings()
+logging = logging.getLogger("kubernetes.base.deployments")
 
 def kubernetesGetDeployments(config=None):
     """
@@ -80,6 +81,7 @@ def baseDeployments(mode=None, config=None):
     description: monitoring deployments
     return: class ZabbixMetric
     """
+    logging.info(f"Function baseDeployments() executed: {mode}")
     if mode == "discovery":
         return zabbixDiscoveryDeployments(config['kubernetes']['name'], kubernetesGetDeployments(config['monitoring']['deployments']))
     if mode == "item":

src/modules/kubernetes/base/nodes.py

@@ -1,9 +1,10 @@
 from kubernetes import client
 from pyzabbix import ZabbixMetric
 from modules.common.functions import *
-import json, urllib3
+import json, urllib3, logging
 
 urllib3.disable_warnings()
+logging = logging.getLogger("kubernetes.base.nodes")
 
 def kubernetesGetNodes(config=None):
     """
@@ -87,6 +88,7 @@ def baseNodes(mode=None, config=None):
     description: monitoring nodes
     return: class ZabbixMetric
     """
+    logging.info(f"Function baseNodes() executed: {mode}")
     if mode == "discovery":
         return zabbixDiscoveryNodes(config['kubernetes']['name'], kubernetesGetNodes(config['monitoring']['nodes']))
     if mode == "item":

src/modules/kubernetes/base/statefulsets.py

@@ -1,9 +1,10 @@
 from kubernetes import client
 from pyzabbix import ZabbixMetric
 from modules.common.functions import *
-import json, urllib3
+import json, urllib3, logging
 
 urllib3.disable_warnings()
+logging = logging.getLogger("kubernetes.base.statefulsets")
 
 def kubernetesGetStatefulsets(config=None):
     """
@@ -80,6 +81,7 @@ def baseStatefulsets(mode=None, config=None):
     description: monitoring statefulsets
     return: class ZabbixMetric
     """
+    logging.info(f"Function baseStatefulsets() executed: {mode}")
     if mode == "discovery":
         return zabbixDiscoveryStatefulsets(config['kubernetes']['name'], kubernetesGetStatefulsets(config['monitoring']['statefulsets']))
     if mode == "item":

src/modules/kubernetes/base/volumes.py

@@ -4,6 +4,7 @@
 import json, urllib3, re, logging
 
 urllib3.disable_warnings()
+logging = logging.getLogger("kubernetes.base.volumes")
 
 def kubernetesGetVolumes(config=None):
     """
@@ -93,6 +94,7 @@ def baseVolumes(mode=None, config=None):
     description: monitoring volumes
     return: class ZabbixMetric
     """
+    logging.info(f"Function baseVolumes() executed: {mode}")
     if mode == "discovery":
         return zabbixDiscoveryVolumes(config['kubernetes']['name'], kubernetesGetVolumes(config['monitoring']['volumes']))
     if mode == "item":

src/modules/kubernetes/openebs/cstorpoolclusters.py

@@ -1,9 +1,10 @@
 from kubernetes import client
 from pyzabbix import ZabbixMetric
 from modules.common.functions import *
-import json, urllib3
+import json, urllib3, logging
 
 urllib3.disable_warnings()
+logging = logging.getLogger("kubernetes.openebs.cstorpoolclusters")
 
 def openebsGetCstorpoolclusters(config=None):
     """
@@ -85,6 +86,7 @@ def baseOpenebsCstorpoolclusters(mode=None, config=None):
     description: monitoring openebs cstorpoolclusters
     return: class ZabbixMetric
     """
+    logging.info(f"Function baseOpenebsCstorpoolclusters() executed: {mode}")
     if mode == "discovery":
         return ZabbixDiscoveryCstorpoolclusters(config['kubernetes']['name'], openebsGetCstorpoolclusters(config['monitoring']['openebs']))
     if mode == "item":

src/modules/kubernetes/openebs/cstorpoolinstances.py

@@ -1,9 +1,10 @@
 from kubernetes import client
 from pyzabbix import ZabbixMetric
 from modules.common.functions import *
-import json, urllib3
+import json, urllib3, logging
 
 urllib3.disable_warnings()
+logging = logging.getLogger("kubernetes.openebs.cstorpoolinstances")
 
 def openebsGetCstorpoolinstances(config=None):
     """
@@ -82,6 +83,7 @@ def baseOpenebsCstorpoolinstances(mode=None, config=None):
     description: monitoring openebs cstorpoolinstances
     return: class ZabbixMetric
     """
+    logging.info(f"Function baseOpenebsCstorpoolinstances() executed: {mode}")
     if mode == "discovery":
         return ZabbixDiscoveryCstorpoolinstances(config['kubernetes']['name'], openebsGetCstorpoolinstances(config['monitoring']['openebs']))
     if mode == "item":

src/modules/kubernetes/trivy/vulnerabilityreports.py

@@ -1,9 +1,10 @@
 from kubernetes import client
 from pyzabbix import ZabbixMetric
 from modules.common.functions import *
-import json, urllib3
+import json, urllib3, logging
 
 urllib3.disable_warnings()
+logging = logging.getLogger("kubernetes.trivy.vulnerabilityreports")
 
 def trivyGetVulnerabilityreports(config=None):
     """

src/requirements.txt

@@ -3,3 +3,4 @@ kubernetes==29.0.0
 py-zabbix==1.1.7
 urllib3==2.1.0
 schedule==1.2.1
+psutil==5.9.7

src/zabbix-kubernetes-discovery.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse, sys, os, yaml, queue
-import logging, schedule, threading
+import logging, schedule, threading, psutil
 from time import sleep
 from kubernetes import config as kube_config
 from pyzabbix import ZabbixSender
@@ -16,8 +16,8 @@
 logging.basicConfig(
     datefmt="%d/%m/%Y %H:%M:%S",
     format="[%(asctime)s] (%(levelname)s) %(name)s.%(funcName)s():%(lineno)d - %(message)s",
-    level=getattr(logging, args.log_level)
-)
+    level=getattr(logging, args.log_level))
+logging = logging.getLogger("main")
 
 if os.path.exists("/var/run/secrets/kubernetes.io/serviceaccount/token") and not os.getenv('KUBECONFIG'):
     kube_config.load_incluster_config()
@@ -48,13 +48,17 @@ def executeSender(data):
         logging.debug(e)
 
 def executeJobs():
+    p = psutil.Process(os.getpid())
+    logging.debug(f"Program memory used (rss): {p.memory_info().rss / 1024 / 1024} MiB")
+
     while True:
+        logging.debug(f"{jobs_queue.qsize()} job(s) in queue")
         jobs = jobs_queue.get()
         if jobs is not None:
             jobs()
             jobs_queue.task_done()
         else:
-            logging.info("No job in queue")
+            logging.debug("0 job in queue")
 
 if __name__ == "__main__":
     logging.info("Application zabbix-kubernetes-discovery started")
@@ -107,5 +111,6 @@ def executeJobs():
     # tasks
     while True:
         schedule.run_pending()
+        logging
         sleep(1)