add error descripiton to crd

pixelfederation · Nov 26, 2023 · 918658d · 918658d
2 parents 7f0ef77 + 272b42e
commit 918658d
Show file tree

Hide file tree

Showing 2 changed files with 46 additions and 17 deletions.
diff --git a/operator-crds/es-template-crd.yml b/operator-crds/es-template-crd.yml
@@ -76,6 +76,12 @@ spec:
                 lastRollover:
                   type: string
                   format: date-time
+                lastError:
+                  type: string
+                  format: date-time
+                lastErrorDescription:
+                  type: string
+
       subresources:
         status: {}
   scope: Namespaced

diff --git a/prometheus_es_exporter/__init__.py b/prometheus_es_exporter/__init__.py
@@ -511,6 +511,7 @@ def updateStatus(item, merge_patch):
                 log.error(f"Updating template {name} k8s ElasticQueryExporterTemplate Status Exception: {str(e)}")
 
         def updateEsTemplate(item, template):
+            err = None
             templateName = item['spec']['templateName']
             namespace = item['metadata']['namespace']
             log.debug(f"Updating template {templateName} on ES cluster")
@@ -522,30 +523,38 @@ def updateEsTemplate(item, template):
                 )
             except Exception as e:
                 log.error(f"Updating estemplate {template} Exception: {str(e)}")
+                err = e
             else:
-                if es_request.ok:
-                    return True
-                else:
-                    log.error(f"Updating estemplate {template} returned error -> {json.dumps(r.json())}")
-            return False
+                if not es_request.ok:
+                    log.error(f"Updating estemplate {template} returned error -> {json.dumps(es_request.json())}")
+                    if not err:
+                        err = json.dumps(es_request.json())
+            return err
 
         def deleteEsTemplate(templateName):
+            err = None
             log.debug(f"Deleting template {templateName}")
             try:
                 es_request = requests.delete(f"{es_cluster[0]}/_template/{templateName}", timeout=15)
             except Exception as e:
                 log.error(f"Deleting estemplate {template} Exception: {str(e)}")
-            if not es_request.ok:
-                log.error(f"Deleting template {template} returned error {json.dumps(es_request.json())}")
-
+                err = str(e)
+            else:
+                if not es_request.ok:
+                    log.error(f"Deleting template {template} returned error {json.dumps(es_request.json())}")
+                    if not err:
+                        err = json.dumps(es_request.json())
+            return err
 
         def buildTemplate(resource):
+            err = None
             try:
                 mappings = json.loads(item['spec']['mappings'])
                 idxPatterns = item['spec']['indexPatterns']
                 idxPatterns = [pattern + "-*" for pattern in idxPatterns]
             except ValueError as e:
                 log.error(f"Mapping error in {item['spec']['templateName']} custom resource")
+                err = str(e)
             else:
                 template = {
                     "index_patterns": idxPatterns,
@@ -563,8 +572,8 @@ def buildTemplate(resource):
                 if item['spec']['order']:
                     template['order'] = item['spec']['order']
 
-                return template
-            return None
+                return template, None
+            return None, err
 
         def rolloverAlias(item):
             idxPatterns = item['spec']['indexPatterns']
@@ -597,14 +606,21 @@ def rolloverAlias(item):
                     name = item['metadata']['name']
                     log.debug(f"Template event {ev['type']}, name: {name}, resource version: {item_resource_version}")
 
+                    status_patch = {}
+
                     if ev['type'] == 'DELETED':
                         if item['spec']['deleteTemplate']:
-                            deleteEsTemplate(templateName)
+                            err = deleteEsTemplate(templateName)
                             log.info(f"Deleted template {templateName} from ES")
+                            if err:
+                                status_patch = {"status": {"lastError": now, "lastErrorDescription": err}}
+                                updateStatus(item, status_patch)
                     if ev['type'] == 'ADDED' or ev['type'] == 'MODIFIED':
-                        template = buildTemplate(item)
+                        template, err = buildTemplate(item)
                         if not template:
                             log.error(f"Bad template {name}")
+                            status_patch = {"status": {"lastError": now, "lastErrorDescription": err}}
+                            updateStatus(item, status_patch)
                             continue
 
                         template_hash = hashlib.md5(json.dumps(template).encode()).hexdigest()
@@ -620,24 +636,31 @@ def rolloverAlias(item):
                                                 "provisioned": "yes",
                                                 "lastProvision": TEMPLATES_PROVISION_STATUS[templateName]["last_provision"],
                                                 "lastProvisionedConfigHash": TEMPLATES_PROVISION_STATUS[templateName]["hash"],
-                                                "lastRollover": TEMPLATES_PROVISION_STATUS[templateName]["last_rollover"]
+                                                "lastRollover": TEMPLATES_PROVISION_STATUS[templateName]["last_rollover"],
+                                                "lastErrorDescription": ""
                                                 }
                             })
                             continue
 
                         # We need just one status update in this cycle, because it will generate modiefied event
-                        status_patch = {}
                         now = datetime.datetime.now(datetime.timezone.utc)
-                        update_ok = updateEsTemplate(item, template)
-                        if update_ok:
+                        err = updateEsTemplate(item, template)
+                        if not err:
                             TEMPLATES_PROVISION_STATUS[templateName] = {"hash": template_hash, "last_provision": now, "last_rollover": None }
-                            status_patch = {"status": {"provisioned": "yes", "lastProvision": now, "lastProvisionedConfigHash": template_hash}}
+                            status_patch = { "status": {
+                                             "provisioned": "yes",
+                                             "lastProvision": now,
+                                             "lastProvisionedConfigHash": template_hash,
+                                             "lastErrorDescription": ""
+                                            }}
                             log.info(f"Updating ES {template}, {name} sucessfull")
                             if item['spec']['rolloverIndexAfterUpdate']:
                                 rollover_ok = rolloverAlias(item)
                                 if rollover_ok:
                                     status_patch["status"]["lastRollover"] = now
                                     TEMPLATES_PROVISION_STATUS[templateName]["last_rollover"] = now
+                        else:
+                            status_patch = {"status": {"lastError": now, "lastErrorDescription": err}}
                         if 'status' in status_patch:
                             updateStatus(item, status_patch)
             except Exception as e: