Fix sentry error on cronjobs

add header for icrc
fix None error on climate data

api/management/commands/ingest_icrc.py

@@ -14,7 +14,13 @@ class Command(BaseCommand):
 
     def handle(self, *args, **kwargs):
         logger.info("Strating ICRC data ingest")
-        response = requests.get(url="https://www.icrc.org/en/where-we-work", headers={"User-Agent": ""})
+        HEADERS = {
+            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",  # noqa
+        }
+        response = requests.get(
+            url="https://www.icrc.org/en/where-we-work",
+            headers=HEADERS,
+        )
         if response.status_code != 200:
             text_to_log = "Error querying ICRC feed at https://www.icrc.org/en/where-we-work"
             logger.error(text_to_log)

api/management/commands/ingest_ns_directory.py

@@ -15,6 +15,11 @@ class Command(BaseCommand):
     help = "Add ns contact details"
 
     def handle(self, *args, **kwargs):
+        def postprocessor(path, key, value):
+            if key == "@i:nil":
+                return None
+            return key, value
+
         logger.info("Starting NS Contacts")
         url = "https://go-api.ifrc.org/"
         headers = {"accept": "application/xml;q=0.9, */*;q=0.8"}
@@ -36,7 +41,7 @@ def handle(self, *args, **kwargs):
             raise Exception("Error querying NationalSocietiesContacts")
 
         added = 0
-        dict_data = xmltodict.parse(response.content)
+        dict_data = xmltodict.parse(response.content, postprocessor=postprocessor)
         for data in dict_data["ArrayOfNationalSocietiesContacts"]["NationalSocietiesContacts"]:
             country_name = data["CON_country"] if isinstance(data["CON_country"], str) else None
             if country_name is not None:

api/management/commands/ingest_ns_document.py

@@ -103,17 +103,26 @@ def save_documents_to_database(self, result):
             if country is None:
                 continue
 
-            country_key_document, _ = CountryKeyDocument.objects.get_or_create(
+            country_key_document, created = CountryKeyDocument.objects.get_or_create(
                 country=country,
                 url=document["url"],
+                defaults={
+                    "name": document["name"],
+                    "thumbnail": document["thumbnail"],
+                    "document_type": document["document_type"],
+                    "year": document["year"],
+                    "end_year": document["end_year"],
+                    "year_text": document["year_text"],
+                },
             )
-            country_key_document.name = document["name"]
-            country_key_document.thumbnail = document["thumbnail"]
-            country_key_document.document_type = document["document_type"]
-            country_key_document.year = document["year"]
-            country_key_document.end_year = document["end_year"]
-            country_key_document.year_text = document["year_text"]
-            country_key_document.save()
+            if not created:
+                country_key_document.name = document["name"]
+                country_key_document.thumbnail = document["thumbnail"]
+                country_key_document.document_type = document["document_type"]
+                country_key_document.year = document["year"]
+                country_key_document.end_year = document["end_year"]
+                country_key_document.year_text = document["year_text"]
+                country_key_document.save(update_fields=["name", "thumbnail", "document_type", "year", "end_year", "year_text"])
             added += 1
         return added
 

api/management/commands/ingest_ns_initiatives.py

@@ -47,16 +47,23 @@ def handle(self, *args, **kwargs):
             # TODO: Filter not by society name
             country = Country.objects.filter(society_name__iexact=data[0]).first()
             if country:
-                nsd_initiatives, _ = NSDInitiatives.objects.get_or_create(
+                nsd_initiatives, created = NSDInitiatives.objects.get_or_create(
                     country=country,
                     year=data[1],
                     fund_type=data[2],
+                    defaults={
+                        "title": data[3],
+                        "categories": data[4],
+                        "allocation": data[5],
+                        "funding_period": data[6],
+                    },
                 )
-                nsd_initiatives.title = data[3]
-                nsd_initiatives.allocation = data[5]
-                nsd_initiatives.funding_period = data[6]
-                nsd_initiatives.categories = data[4]
-                nsd_initiatives.save()
+                if not created:
+                    nsd_initiatives.title = data[3]
+                    nsd_initiatives.allocation = data[5]
+                    nsd_initiatives.funding_period = data[6]
+                    nsd_initiatives.categories = data[4]
+                    nsd_initiatives.save(update_fields=["title", "allocation", "funding_period", "categories"])
                 added += 1
 
         text_to_log = "%s Ns initiatives added" % added

databank/management/commands/ingest_climate.py

@@ -59,25 +59,34 @@ def handle(self, *args, **options):
 
                 for value in merged_data.values():
                     for k, v in value.items():
+                        if v[0] is None or v[1] is None or v[2] is None or v[3] is None:
+                            continue
                         year_month = k.split("-")
-                        country_key_climate, _ = CountryKeyClimate.objects.get_or_create(
+                        country_key_climate, created = CountryKeyClimate.objects.get_or_create(
                             overview=overview,
                             year=year_month[0],
                             month=year_month[1],
+                            defaults={
+                                "precipitation": v[0],
+                                "avg_temp": v[1],
+                                "min_temp": v[2],
+                                "max_temp": v[3],
+                            },
                         )
-                        country_key_climate.max_temp = v[3]
-                        country_key_climate.min_temp = v[2]
-                        country_key_climate.avg_temp = v[1]
-                        country_key_climate.precipitation = v[0]
-                        # TODO: Use bulk manager
-                        country_key_climate.save(
-                            update_fields=(
-                                "max_temp",
-                                "min_temp",
-                                "avg_temp",
-                                "precipitation",
+                        if not created:
+                            country_key_climate.max_temp = v[3]
+                            country_key_climate.min_temp = v[2]
+                            country_key_climate.avg_temp = v[1]
+                            country_key_climate.precipitation = v[0]
+                            # TODO: Use bulk manager
+                            country_key_climate.save(
+                                update_fields=(
+                                    "max_temp",
+                                    "min_temp",
+                                    "avg_temp",
+                                    "precipitation",
+                                )
                             )
-                        )
             except Exception:
                 logger.error("Error in ingesting climate data", exc_info=True)
                 continue

deploy/helm/ifrcgo-helm/values.yaml

@@ -127,7 +127,7 @@ cronjobs:
   - command: 'FDRS_INCOME'
     schedule: '0 0 * * 0'
   - command: 'ingest_acaps'
-    schedule: '0 0 * * 0'
+    schedule: '0 1 * * 0'
   - command: 'ingest_climate'
     schedule: '0 0 * * 0'
   - command: 'ingest_databank'
@@ -137,7 +137,7 @@ cronjobs:
   - command: 'ingest_unicef'
     schedule: '0 0 * * 0'
   - command: 'ingest_worldbank'
-    schedule: '0 0 * * 0'
+    schedule: '0 2 * * 0'
   - command: 'ingest_disaster_law'
     schedule: '0 0 * * 0'
   - command: 'ingest_ns_contact'
@@ -151,7 +151,7 @@ cronjobs:
   - command: 'ingest_ns_initiatives'
     schedule: '0 0 * * 0'
   - command: 'ingest_icrc'
-    schedule: '0 0 * * 0'
+    schedule: '0 3 * * 0'
 
 
 elasticsearch:

main/sentry.py

@@ -116,19 +116,19 @@ class SentryMonitor(models.TextChoices):
     UPDATE_SURGE_ALERT_STATUS = "update_surge_alert_status", "1 */12 * * *"
     FDRS_ANNUAL_INCOME = "fdrs_annual_income", "0 0 * * 0"
     FDRS_INCOME = "FDRS_INCOME", "0 0 * * 0"
-    INGEST_ACAPS = "ingest_acaps", "0 0 * * 0"
+    INGEST_ACAPS = "ingest_acaps", "0 1 * * 0"
     INGEST_CLIMATE = "ingest_climate", "0 0 * * 0"
     INGEST_DATABANK = "ingest_databank", "0 0 * * 0"
     INGEST_HDR = "ingest_hdr", "0 0 * * 0"
     INGEST_UNICEF = "ingest_unicef", "0 0 * * 0"
-    INGEST_WORLDBANK = "ingest_worldbank", "0 0 * * 0"
+    INGEST_WORLDBANK = "ingest_worldbank", "0 2 * * 0"
     INGEST_DISASTER_LAW = "ingest_disaster_law", "0 0 * * 0"
     INGEST_NS_CONTACT = "ingest_ns_contact", "0 0 * * 0"
     INGEST_NS_CAPACITY = "ingest_ns_capacity", "0 0 * * 0"
     INGEST_NS_DIRECTORY = "ingest_ns_directory", "0 0 * * 0"
     INGEST_NS_DOCUMENT = "ingest_ns_document", "0 0 * * 0"
     INGEST_NS_INITIATIVES = "ingest_ns_initiatives", "0 0 * * 0"
-    INGEST_ICRC = "ingest_icrc", "0 0 * * 0"
+    INGEST_ICRC = "ingest_icrc", "0 3 * * 0"
 
     @staticmethod
     def load_cron_data() -> typing.List[typing.Tuple[str, str]]: