diff --git a/api/management/commands/ingest_icrc.py b/api/management/commands/ingest_icrc.py index 0368b4419..bf3f47663 100644 --- a/api/management/commands/ingest_icrc.py +++ b/api/management/commands/ingest_icrc.py @@ -14,7 +14,13 @@ class Command(BaseCommand): def handle(self, *args, **kwargs): logger.info("Strating ICRC data ingest") - response = requests.get(url="https://www.icrc.org/en/where-we-work", headers={"User-Agent": ""}) + HEADERS = { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36", # noqa + } + response = requests.get( + url="https://www.icrc.org/en/where-we-work", + headers=HEADERS, + ) if response.status_code != 200: text_to_log = "Error querying ICRC feed at https://www.icrc.org/en/where-we-work" logger.error(text_to_log) diff --git a/api/management/commands/ingest_ns_directory.py b/api/management/commands/ingest_ns_directory.py index 0fb3a1cd7..0a5de0bbe 100644 --- a/api/management/commands/ingest_ns_directory.py +++ b/api/management/commands/ingest_ns_directory.py @@ -15,6 +15,11 @@ class Command(BaseCommand): help = "Add ns contact details" def handle(self, *args, **kwargs): + def postprocessor(path, key, value): + if key == "@i:nil": + return None + return key, value + logger.info("Starting NS Contacts") url = "https://go-api.ifrc.org/" headers = {"accept": "application/xml;q=0.9, */*;q=0.8"} @@ -36,7 +41,7 @@ def handle(self, *args, **kwargs): raise Exception("Error querying NationalSocietiesContacts") added = 0 - dict_data = xmltodict.parse(response.content) + dict_data = xmltodict.parse(response.content, postprocessor=postprocessor) for data in dict_data["ArrayOfNationalSocietiesContacts"]["NationalSocietiesContacts"]: country_name = data["CON_country"] if isinstance(data["CON_country"], str) else None if country_name is not None: diff --git a/api/management/commands/ingest_ns_document.py b/api/management/commands/ingest_ns_document.py index 92f5fb3c3..fca3d3280 100644 --- a/api/management/commands/ingest_ns_document.py +++ b/api/management/commands/ingest_ns_document.py @@ -103,17 +103,26 @@ def save_documents_to_database(self, result): if country is None: continue - country_key_document, _ = CountryKeyDocument.objects.get_or_create( + country_key_document, created = CountryKeyDocument.objects.get_or_create( country=country, url=document["url"], + defaults={ + "name": document["name"], + "thumbnail": document["thumbnail"], + "document_type": document["document_type"], + "year": document["year"], + "end_year": document["end_year"], + "year_text": document["year_text"], + }, ) - country_key_document.name = document["name"] - country_key_document.thumbnail = document["thumbnail"] - country_key_document.document_type = document["document_type"] - country_key_document.year = document["year"] - country_key_document.end_year = document["end_year"] - country_key_document.year_text = document["year_text"] - country_key_document.save() + if not created: + country_key_document.name = document["name"] + country_key_document.thumbnail = document["thumbnail"] + country_key_document.document_type = document["document_type"] + country_key_document.year = document["year"] + country_key_document.end_year = document["end_year"] + country_key_document.year_text = document["year_text"] + country_key_document.save(update_fields=["name", "thumbnail", "document_type", "year", "end_year", "year_text"]) added += 1 return added diff --git a/api/management/commands/ingest_ns_initiatives.py b/api/management/commands/ingest_ns_initiatives.py index 40227e3e3..47dd00523 100644 --- a/api/management/commands/ingest_ns_initiatives.py +++ b/api/management/commands/ingest_ns_initiatives.py @@ -47,16 +47,23 @@ def handle(self, *args, **kwargs): # TODO: Filter not by society name country = Country.objects.filter(society_name__iexact=data[0]).first() if country: - nsd_initiatives, _ = NSDInitiatives.objects.get_or_create( + nsd_initiatives, created = NSDInitiatives.objects.get_or_create( country=country, year=data[1], fund_type=data[2], + defaults={ + "title": data[3], + "categories": data[4], + "allocation": data[5], + "funding_period": data[6], + }, ) - nsd_initiatives.title = data[3] - nsd_initiatives.allocation = data[5] - nsd_initiatives.funding_period = data[6] - nsd_initiatives.categories = data[4] - nsd_initiatives.save() + if not created: + nsd_initiatives.title = data[3] + nsd_initiatives.allocation = data[5] + nsd_initiatives.funding_period = data[6] + nsd_initiatives.categories = data[4] + nsd_initiatives.save(update_fields=["title", "allocation", "funding_period", "categories"]) added += 1 text_to_log = "%s Ns initiatives added" % added diff --git a/databank/management/commands/ingest_climate.py b/databank/management/commands/ingest_climate.py index 3fd13cd01..bf99e844e 100644 --- a/databank/management/commands/ingest_climate.py +++ b/databank/management/commands/ingest_climate.py @@ -59,25 +59,34 @@ def handle(self, *args, **options): for value in merged_data.values(): for k, v in value.items(): + if v[0] is None or v[1] is None or v[2] is None or v[3] is None: + continue year_month = k.split("-") - country_key_climate, _ = CountryKeyClimate.objects.get_or_create( + country_key_climate, created = CountryKeyClimate.objects.get_or_create( overview=overview, year=year_month[0], month=year_month[1], + defaults={ + "precipitation": v[0], + "avg_temp": v[1], + "min_temp": v[2], + "max_temp": v[3], + }, ) - country_key_climate.max_temp = v[3] - country_key_climate.min_temp = v[2] - country_key_climate.avg_temp = v[1] - country_key_climate.precipitation = v[0] - # TODO: Use bulk manager - country_key_climate.save( - update_fields=( - "max_temp", - "min_temp", - "avg_temp", - "precipitation", + if not created: + country_key_climate.max_temp = v[3] + country_key_climate.min_temp = v[2] + country_key_climate.avg_temp = v[1] + country_key_climate.precipitation = v[0] + # TODO: Use bulk manager + country_key_climate.save( + update_fields=( + "max_temp", + "min_temp", + "avg_temp", + "precipitation", + ) ) - ) except Exception: logger.error("Error in ingesting climate data", exc_info=True) continue diff --git a/deploy/helm/ifrcgo-helm/values.yaml b/deploy/helm/ifrcgo-helm/values.yaml index d0f0f0014..913e04806 100644 --- a/deploy/helm/ifrcgo-helm/values.yaml +++ b/deploy/helm/ifrcgo-helm/values.yaml @@ -127,7 +127,7 @@ cronjobs: - command: 'FDRS_INCOME' schedule: '0 0 * * 0' - command: 'ingest_acaps' - schedule: '0 0 * * 0' + schedule: '0 1 * * 0' - command: 'ingest_climate' schedule: '0 0 * * 0' - command: 'ingest_databank' @@ -137,7 +137,7 @@ cronjobs: - command: 'ingest_unicef' schedule: '0 0 * * 0' - command: 'ingest_worldbank' - schedule: '0 0 * * 0' + schedule: '0 2 * * 0' - command: 'ingest_disaster_law' schedule: '0 0 * * 0' - command: 'ingest_ns_contact' @@ -151,7 +151,7 @@ cronjobs: - command: 'ingest_ns_initiatives' schedule: '0 0 * * 0' - command: 'ingest_icrc' - schedule: '0 0 * * 0' + schedule: '0 3 * * 0' elasticsearch: diff --git a/main/sentry.py b/main/sentry.py index 5ae70114d..c69ee55a6 100644 --- a/main/sentry.py +++ b/main/sentry.py @@ -116,19 +116,19 @@ class SentryMonitor(models.TextChoices): UPDATE_SURGE_ALERT_STATUS = "update_surge_alert_status", "1 */12 * * *" FDRS_ANNUAL_INCOME = "fdrs_annual_income", "0 0 * * 0" FDRS_INCOME = "FDRS_INCOME", "0 0 * * 0" - INGEST_ACAPS = "ingest_acaps", "0 0 * * 0" + INGEST_ACAPS = "ingest_acaps", "0 1 * * 0" INGEST_CLIMATE = "ingest_climate", "0 0 * * 0" INGEST_DATABANK = "ingest_databank", "0 0 * * 0" INGEST_HDR = "ingest_hdr", "0 0 * * 0" INGEST_UNICEF = "ingest_unicef", "0 0 * * 0" - INGEST_WORLDBANK = "ingest_worldbank", "0 0 * * 0" + INGEST_WORLDBANK = "ingest_worldbank", "0 2 * * 0" INGEST_DISASTER_LAW = "ingest_disaster_law", "0 0 * * 0" INGEST_NS_CONTACT = "ingest_ns_contact", "0 0 * * 0" INGEST_NS_CAPACITY = "ingest_ns_capacity", "0 0 * * 0" INGEST_NS_DIRECTORY = "ingest_ns_directory", "0 0 * * 0" INGEST_NS_DOCUMENT = "ingest_ns_document", "0 0 * * 0" INGEST_NS_INITIATIVES = "ingest_ns_initiatives", "0 0 * * 0" - INGEST_ICRC = "ingest_icrc", "0 0 * * 0" + INGEST_ICRC = "ingest_icrc", "0 3 * * 0" @staticmethod def load_cron_data() -> typing.List[typing.Tuple[str, str]]: