diff --git a/api/management/commands/cron_job_monitor.py b/api/management/commands/cron_job_monitor.py index 0a11d97a0..9c540b34d 100644 --- a/api/management/commands/cron_job_monitor.py +++ b/api/management/commands/cron_job_monitor.py @@ -5,7 +5,7 @@ from django.conf import settings from django.core.management.base import BaseCommand -from main.sentry import SentryMonitor +from main.sentry import SentryMonitor, SentryMonitorConfig from main.settings import SENTRY_DSN logger = logging.getLogger(__name__) @@ -48,6 +48,10 @@ def handle(self, *args, **options): "type": "crontab", "value": str(schedule), }, + "tz": settings.TIME_ZONE, + "checkin_margin": SentryMonitorConfig.get_checkin_margin(cronjob), + "failure_issue_threshold": SentryMonitorConfig.get_failure_issue_threshold(cronjob), + "recovery_threshold": SentryMonitorConfig.get_recovery_threshold(cronjob), }, "environment": settings.GO_ENVIRONMENT, "status": "ok", diff --git a/api/management/commands/index_and_notify.py b/api/management/commands/index_and_notify.py index 246d9199c..241f6e22a 100644 --- a/api/management/commands/index_and_notify.py +++ b/api/management/commands/index_and_notify.py @@ -67,7 +67,6 @@ } -@monitor(monitor_slug=SentryMonitor.INDEX_AND_NOTIFY) class Command(BaseCommand): help = "Index and send notifications about new/changed records" @@ -920,6 +919,7 @@ def check_ingest_issues(self, having_ingest_issue): + ", notification sent to IM team" ) + @monitor(monitor_slug=SentryMonitor.INDEX_AND_NOTIFY) def handle(self, *args, **options): if self.is_digest_mode(): time_diff = self.diff_1_week() # in digest mode (once a week, for new_entities only) we use a bigger interval diff --git a/api/management/commands/ingest_appeals.py b/api/management/commands/ingest_appeals.py index c032f8e75..53d7189be 100644 --- a/api/management/commands/ingest_appeals.py +++ b/api/management/commands/ingest_appeals.py @@ -33,7 +33,6 @@ GEC_CODES = GECCode.objects.select_related("country").all() -@monitor(monitor_slug=SentryMonitor.INGEST_APPEALS) class Command(BaseCommand): help = "Add new entries from Access database file" @@ -292,6 +291,7 @@ def parse_appeal_record(self, r, **options): return fields + @monitor(monitor_slug=SentryMonitor.INGEST_APPEALS) def handle(self, *args, **options): logger.info("Starting appeals ingest") start_appeals_count = Appeal.objects.all().count() diff --git a/api/management/commands/ingest_disaster_law.py b/api/management/commands/ingest_disaster_law.py index 313f7e72f..d114aeca7 100644 --- a/api/management/commands/ingest_disaster_law.py +++ b/api/management/commands/ingest_disaster_law.py @@ -1,14 +1,17 @@ import requests from bs4 import BeautifulSoup from django.core.management.base import BaseCommand +from sentry_sdk.crons import monitor from api.logger import logger from api.models import Country, CronJob, CronJobStatus +from main.sentry import SentryMonitor class Command(BaseCommand): help = "Add ICRC data" + @monitor(monitor_slug=SentryMonitor.INGEST_DISASTER_LAW) def handle(self, *args, **kwargs): logger.info("Starting Disaster Law data") home_url = "https://disasterlaw.ifrc.org/" diff --git a/api/management/commands/ingest_icrc.py b/api/management/commands/ingest_icrc.py index f6e65d90c..55f71939c 100644 --- a/api/management/commands/ingest_icrc.py +++ b/api/management/commands/ingest_icrc.py @@ -1,17 +1,26 @@ import requests from bs4 import BeautifulSoup from django.core.management.base import BaseCommand +from sentry_sdk.crons import monitor from api.logger import logger from api.models import Country, CountryICRCPresence, CronJob, CronJobStatus +from main.sentry import SentryMonitor class Command(BaseCommand): help = "Add ICRC data" + @monitor(monitor_slug=SentryMonitor.INGEST_ICRC) def handle(self, *args, **kwargs): logger.info("Strating ICRC data ingest") - response = requests.get(url="https://www.icrc.org/en/where-we-work", headers={"User-Agent": ""}) + HEADERS = { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36", # noqa + } + response = requests.get( + url="https://www.icrc.org/en/where-we-work", + headers=HEADERS, + ) if response.status_code != 200: text_to_log = "Error querying ICRC feed at https://www.icrc.org/en/where-we-work" logger.error(text_to_log) @@ -57,19 +66,19 @@ def handle(self, *args, **kwargs): "Description": description, } ) + added = 0 for data in country_list: - country = Country.objects.filter(name__exact=data["Country"]) - if country.exists(): - dict_data = { - "country": country.first(), - "icrc_presence": data["ICRC presence"], - "url": data["URL"], - "key_operation": data["Key operation"], - "description": data["Description"], - } + country = Country.objects.filter(name__exact=data["Country"]).first() + if country: + country_icrc_presence, _ = CountryICRCPresence.objects.get_or_create(country=country) + + country_icrc_presence.icrc_presence = data["ICRC presence"] + country_icrc_presence.url = data["URL"] + country_icrc_presence.key_operation = data["Key operation"] + country_icrc_presence.description = data["Description"] + country_icrc_presence.save() added += 1 - CountryICRCPresence.objects.create(**dict_data) text_to_log = "%s ICRC added" % added logger.info(text_to_log) diff --git a/api/management/commands/ingest_ns_capacity.py b/api/management/commands/ingest_ns_capacity.py index 69b7d80d4..edffdfcb5 100644 --- a/api/management/commands/ingest_ns_capacity.py +++ b/api/management/commands/ingest_ns_capacity.py @@ -1,14 +1,17 @@ import requests from django.conf import settings from django.core.management.base import BaseCommand +from sentry_sdk.crons import monitor from api.logger import logger from api.models import Country, CountryCapacityStrengthening, CronJob, CronJobStatus +from main.sentry import SentryMonitor class Command(BaseCommand): help = "Add ns contact details" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_CAPACITY) def handle(self, *args, **kwargs): logger.info("Starting NS Contacts") @@ -44,7 +47,7 @@ def handle(self, *args, **kwargs): text_to_log = "%s Ns capacity added" % ocaa_count logger.info(text_to_log) body = { - "name": "ingest_ns_capaciity", + "name": "ingest_ns_capacity", "message": text_to_log, "num_result": ocaa_count, "status": CronJobStatus.SUCCESSFUL, diff --git a/api/management/commands/ingest_ns_contact.py b/api/management/commands/ingest_ns_contact.py index 1d01dd759..380ecffcf 100644 --- a/api/management/commands/ingest_ns_contact.py +++ b/api/management/commands/ingest_ns_contact.py @@ -5,15 +5,20 @@ import xmltodict from django.conf import settings from django.core.management.base import BaseCommand +from django.db import transaction from requests.auth import HTTPBasicAuth +from sentry_sdk.crons import monitor from api.logger import logger from api.models import Country, CronJob, CronJobStatus +from main.sentry import SentryMonitor class Command(BaseCommand): help = "Add ns contact details" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_CONTACT) + @transaction.atomic def handle(self, *args, **kwargs): logger.info("Starting NS Contacts") url = "https://go-api.ifrc.org/" diff --git a/api/management/commands/ingest_ns_directory.py b/api/management/commands/ingest_ns_directory.py index f42421a11..e7f2b55b1 100644 --- a/api/management/commands/ingest_ns_directory.py +++ b/api/management/commands/ingest_ns_directory.py @@ -2,16 +2,26 @@ import xmltodict from django.conf import settings from django.core.management.base import BaseCommand +from django.db import transaction from requests.auth import HTTPBasicAuth +from sentry_sdk.crons import monitor from api.logger import logger from api.models import Country, CountryDirectory, CronJob, CronJobStatus +from main.sentry import SentryMonitor class Command(BaseCommand): help = "Add ns contact details" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_DIRECTORY) + @transaction.atomic def handle(self, *args, **kwargs): + def postprocessor(path, key, value): + if key == "@i:nil": + return None + return key, value + logger.info("Starting NS Contacts") url = "https://go-api.ifrc.org/" headers = {"accept": "application/xml;q=0.9, */*;q=0.8"} @@ -33,7 +43,8 @@ def handle(self, *args, **kwargs): raise Exception("Error querying NationalSocietiesContacts") added = 0 - dict_data = xmltodict.parse(response.content) + created_country_directory_ids = [] + dict_data = xmltodict.parse(response.content, postprocessor=postprocessor) for data in dict_data["ArrayOfNationalSocietiesContacts"]["NationalSocietiesContacts"]: country_name = data["CON_country"] if isinstance(data["CON_country"], str) else None if country_name is not None: @@ -54,7 +65,15 @@ def handle(self, *args, **kwargs): "position": data["CON_title"], "country": country, } - CountryDirectory.objects.create(**data) + country_directory, _ = CountryDirectory.objects.get_or_create( + country=country, + first_name__iexact=data["first_name"], + last_name__iexact=data["last_name"], + position__iexact=data["position"], + ) + created_country_directory_ids.append(country_directory.pk) + # NOTE: Deleting the country directory which are not available in the source + CountryDirectory.objects.exclude(id__in=created_country_directory_ids).delete() text_to_log = "%s Ns Directory added" % added logger.info(text_to_log) body = {"name": "ingest_ns_directory", "message": text_to_log, "num_result": added, "status": CronJobStatus.SUCCESSFUL} diff --git a/api/management/commands/ingest_ns_document.py b/api/management/commands/ingest_ns_document.py index 0864b02b7..fb8c47dba 100644 --- a/api/management/commands/ingest_ns_document.py +++ b/api/management/commands/ingest_ns_document.py @@ -5,14 +5,19 @@ import requests from django.conf import settings from django.core.management.base import BaseCommand +from django.db import transaction +from sentry_sdk.crons import monitor from api.logger import logger from api.models import Country, CountryKeyDocument, CronJob, CronJobStatus +from main.sentry import SentryMonitor class Command(BaseCommand): help = "Add ns documents" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_DOCUMENT) + @transaction.atomic def handle(self, *args, **kwargs): logger.info("Starting NS Key Documents") @@ -95,21 +100,45 @@ def fetch_all_country_documents(self, api_key, country_table): def save_documents_to_database(self, result): added = 0 + created_country_key_document_ids = [] for document in result: country = Country.objects.filter(fdrs=document["country_code"]).first() - if country: - added += 1 - data = { - "country": country, + if country is None: + continue + + country_key_document, created = CountryKeyDocument.objects.get_or_create( + country=country, + url=document["url"], + defaults={ "name": document["name"], - "url": document["url"], "thumbnail": document["thumbnail"], "document_type": document["document_type"], "year": document["year"], "end_year": document["end_year"], "year_text": document["year_text"], - } - CountryKeyDocument.objects.create(**data) + }, + ) + if not created: + country_key_document.name = document["name"] + country_key_document.thumbnail = document["thumbnail"] + country_key_document.document_type = document["document_type"] + country_key_document.year = document["year"] + country_key_document.end_year = document["end_year"] + country_key_document.year_text = document["year_text"] + country_key_document.save( + update_fields=[ + "name", + "thumbnail", + "document_type", + "year", + "end_year", + "year_text", + ] + ) + created_country_key_document_ids.append(country_key_document.pk) + added += 1 + # NOTE: Deleting the CountryKeyDocument that are not in the source + CountryKeyDocument.objects.exclude(id__in=created_country_key_document_ids).delete() return added def sync_cron_success(self, text_to_log, added): diff --git a/api/management/commands/ingest_ns_initiatives.py b/api/management/commands/ingest_ns_initiatives.py index d759702e5..1207f567d 100644 --- a/api/management/commands/ingest_ns_initiatives.py +++ b/api/management/commands/ingest_ns_initiatives.py @@ -3,14 +3,19 @@ import requests from django.conf import settings from django.core.management.base import BaseCommand +from django.db import transaction +from sentry_sdk.crons import monitor from api.logger import logger from api.models import Country, CronJob, CronJobStatus, NSDInitiatives +from main.sentry import SentryMonitor class Command(BaseCommand): help = "Add ns initiatives" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_INITIATIVES) + @transaction.atomic def handle(self, *args, **kwargs): logger.info("Starting NS Inititatives") api_key = settings.NS_INITIATIVES_API_KEY @@ -40,21 +45,33 @@ def handle(self, *args, **kwargs): ], ) funding_data = funding_data.replace({np.nan: None}) + created_ns_initiatives_pk = [] for data in funding_data.values.tolist(): # TODO: Filter not by society name country = Country.objects.filter(society_name__iexact=data[0]).first() if country: - dict_data = { - "country": country, - "title": data[3], - "fund_type": data[2], - "allocation": data[5], - "year": data[1], - "funding_period": data[6], - "categories": data[4], - } + nsd_initiatives, created = NSDInitiatives.objects.get_or_create( + country=country, + year=data[1], + fund_type=data[2], + defaults={ + "title": data[3], + "categories": data[4], + "allocation": data[5], + "funding_period": data[6], + }, + ) + if not created: + nsd_initiatives.title = data[3] + nsd_initiatives.categories = data[4] + nsd_initiatives.allocation = data[5] + nsd_initiatives.funding_period = data[6] + nsd_initiatives.save(update_fields=["title", "categories", "allocation", "funding_period"]) + created_ns_initiatives_pk.append(nsd_initiatives.pk) added += 1 - NSDInitiatives.objects.create(**dict_data) + # NOTE: Delete the NSDInitiatives that are not in the source + NSDInitiatives.objects.exclude(id__in=created_ns_initiatives_pk).delete() + text_to_log = "%s Ns initiatives added" % added logger.info(text_to_log) body = {"name": "ingest_ns_initiatives", "message": text_to_log, "num_result": added, "status": CronJobStatus.SUCCESSFUL} diff --git a/api/management/commands/revoke_staff_status.py b/api/management/commands/revoke_staff_status.py index 5c965ca4d..db4725e0b 100644 --- a/api/management/commands/revoke_staff_status.py +++ b/api/management/commands/revoke_staff_status.py @@ -8,7 +8,6 @@ # from registrations.views import is_valid_domain -@monitor(monitor_slug=SentryMonitor.REVOKE_STAFF_STATUS) class Command(BaseCommand): help = 'Update staff status in auth_user table according to "Read only" group' @@ -53,6 +52,7 @@ def get_ifrc_domain_users(self): # # return editors + @monitor(monitor_slug=SentryMonitor.REVOKE_STAFF_STATUS) def handle(self, *args, **options): logger.info("Moving Read only users out of staff status...") diff --git a/api/management/commands/sync_appealdocs.py b/api/management/commands/sync_appealdocs.py index 889805c49..12a27e059 100644 --- a/api/management/commands/sync_appealdocs.py +++ b/api/management/commands/sync_appealdocs.py @@ -21,7 +21,6 @@ FEDNET_SOURCE = "https://go-api.ifrc.org/Api/FedNetAppeals?Hidden=false&BaseAppealnumber=" -@monitor(monitor_slug=SentryMonitor.SYNC_APPEALDOCS) class Command(BaseCommand): help = "Ingest existing appeal documents" @@ -37,6 +36,7 @@ def parse_date(self, date_string): timeformat = "%Y-%m-%dT%H:%M:%S" return datetime.strptime(date_string[:18], timeformat).replace(tzinfo=timezone.utc) + @monitor(monitor_slug=SentryMonitor.SYNC_APPEALDOCS) def handle(self, *args, **options): logger.info("Starting appeal document ingest") diff --git a/api/management/commands/sync_molnix.py b/api/management/commands/sync_molnix.py index 9941871c2..60b483a18 100644 --- a/api/management/commands/sync_molnix.py +++ b/api/management/commands/sync_molnix.py @@ -519,10 +519,10 @@ def sync_open_positions(molnix_positions, molnix_api, countries): return messages, warnings, successful_creates -@monitor(monitor_slug=SentryMonitor.SYNC_MOLNIX) class Command(BaseCommand): help = "Sync data from Molnix API to GO db" + @monitor(monitor_slug=SentryMonitor.SYNC_MOLNIX) @transaction.atomic def handle(self, *args, **options): logger.info("Starting Sync Molnix job") diff --git a/api/management/commands/user_registration_reminder.py b/api/management/commands/user_registration_reminder.py index 6bcfb7dc7..c6ceaadae 100644 --- a/api/management/commands/user_registration_reminder.py +++ b/api/management/commands/user_registration_reminder.py @@ -10,13 +10,13 @@ from registrations.models import Pending -@monitor(monitor_slug=SentryMonitor.USER_REGISTRATION_REMINDER) class Command(BaseCommand): help = "Send reminder about the pending registrations" def diff_3_day(self): return datetime.utcnow().replace(tzinfo=timezone.utc) - timedelta(days=3) + @monitor(monitor_slug=SentryMonitor.USER_REGISTRATION_REMINDER) def handle(self, *args, **options): region_ids = Region.objects.all().values_list("id", flat=True) time_diff_3_day = self.diff_3_day() diff --git a/api/models.py b/api/models.py index 084cf2744..50a5daade 100644 --- a/api/models.py +++ b/api/models.py @@ -349,6 +349,7 @@ class CountryKeyDocument(models.Model): end_year = models.DateField(verbose_name=_("End Year"), null=True, blank=True) year_text = models.CharField(verbose_name=_("Year Text"), max_length=255, null=True, blank=True) + # TODO: Add unique_together country, url def __str__(self): return f"{self.country.name} - {self.name}" @@ -391,6 +392,7 @@ class NSDInitiatives(models.Model): funding_period = models.IntegerField(verbose_name=_("Funding Period in Month")) categories = ArrayField(models.CharField(max_length=255), verbose_name=_("Funding categories"), default=list, null=True) + # TODO: Add unique_together country, year, fund_type def __str__(self): return f"{self.country.name} - {self.title}" diff --git a/country_plan/management/commands/ingest_country_plan_file.py b/country_plan/management/commands/ingest_country_plan_file.py index a911c505a..ad594539a 100644 --- a/country_plan/management/commands/ingest_country_plan_file.py +++ b/country_plan/management/commands/ingest_country_plan_file.py @@ -55,7 +55,6 @@ def _get_filename_from_headers(resp): return resp.headers.get("Content-Type"), _get_filename_from_headers(resp) -@monitor(monitor_slug=SentryMonitor.INGEST_COUNTRY_PLAN_FILE) class Command(BaseCommand): @staticmethod def load_file_to_country_plan(country_plan: CountryPlan, url: str, filename: str, field_name: str): @@ -121,6 +120,7 @@ def load(self, url: str, file_field: str, field_inserted_date_field: str): except Exception: logger.error("Could not update countries plan", exc_info=True) + @monitor(monitor_slug=SentryMonitor.INGEST_COUNTRY_PLAN_FILE) def handle(self, **_): # Public self.stdout.write("Fetching data for country plans:: PUBLIC") diff --git a/databank/management/commands/FDRS_INCOME.py b/databank/management/commands/FDRS_INCOME.py index e103c3b63..d5f4d2505 100644 --- a/databank/management/commands/FDRS_INCOME.py +++ b/databank/management/commands/FDRS_INCOME.py @@ -2,9 +2,13 @@ import requests from django.conf import settings +from django.core.management import call_command from django.core.management.base import BaseCommand +from django.db import transaction +from sentry_sdk.crons import monitor from databank.models import CountryOverview, FDRSIncome, FDRSIndicator +from main.sentry import SentryMonitor logger = logging.getLogger(__name__) @@ -12,7 +16,11 @@ class Command(BaseCommand): help = "Import FDRS income data" + @monitor(monitor_slug=SentryMonitor.FDRS_INCOME) def handle(self, *args, **kwargs): + # NOTE: Loading FDRS indicators + call_command("loaddata", "fdrs_indicator.json", verbosity=2) + fdrs_indicator_enum_data = { "h_gov_CHF": "Home Government", "f_gov_CHF": "Foreign Government", @@ -42,15 +50,24 @@ def handle(self, *args, **kwargs): return fdrs_entities.raise_for_status() fdrs_entities = fdrs_entities.json() - for d in fdrs_entities["data"]: - indicator = next(iter(d.values())) - income_list = d["data"][0]["data"] - if len(income_list): - for income in income_list: - data = { - "date": str(income["year"]) + "-01-01", - "value": income["value"], - "indicator": map_indicators.get(fdrs_indicator_enum_data.get(indicator)), - "overview": overview, - } - FDRSIncome.objects.create(**data) + created_fdrs_income_ids = [] + with transaction.atomic(): + for d in fdrs_entities["data"]: + indicator = next(iter(d.values())) + fdrs_indicator = map_indicators[fdrs_indicator_enum_data[indicator]] + income_list = d["data"][0]["data"] + if len(income_list): + for income in income_list: + income_value = income["value"] + fdrs_income, created = FDRSIncome.objects.get_or_create( + overview=overview, + indicator=fdrs_indicator, + date=str(income["year"]) + "-01-01", + defaults={"value": income_value}, + ) + if not created: + fdrs_income.value = income_value + fdrs_income.save(update_fields=["value"]) + created_fdrs_income_ids.append(fdrs_income.pk) + # NOTE: Delete the FDRSIncome that are not in the source + FDRSIncome.objects.filter(overview=overview).exclude(id__in=created_fdrs_income_ids).delete() diff --git a/databank/management/commands/fdrs_annual_income.py b/databank/management/commands/fdrs_annual_income.py index a41d962cb..52ce73a08 100644 --- a/databank/management/commands/fdrs_annual_income.py +++ b/databank/management/commands/fdrs_annual_income.py @@ -3,8 +3,11 @@ import requests from django.conf import settings from django.core.management.base import BaseCommand +from django.db import models, transaction +from sentry_sdk.crons import monitor from databank.models import CountryOverview, FDRSAnnualIncome +from main.sentry import SentryMonitor logger = logging.getLogger(__name__) @@ -12,21 +15,49 @@ class Command(BaseCommand): help = "Import FDRS income data" + @monitor(monitor_slug=SentryMonitor.FDRS_ANNUAL_INCOME) def handle(self, *args, **kwargs): - for overview in CountryOverview.objects.all(): - country_fdrs_code = overview.country.fdrs - fdrs_api = f"https://data-api.ifrc.org/api/data?indicator=KPI_IncomeLC_CHF&KPI_Don_Code={country_fdrs_code}&apiKey={settings.FDRS_APIKEY}" # noqa: E501 - fdrs_entities = requests.get(fdrs_api) + overview_qs = CountryOverview.objects.annotate( + country_fdrd=models.F("country__fdrs"), + ) + fdrs_data_count = 0 + for overview in overview_qs.all(): + country_fdrs_code = overview.country_fdrd + fdrs_entities = requests.get( + "https://data-api.ifrc.org/api/data", + params={ + "apiKey": settings.FDRS_APIKEY, + "indicator": "KPI_IncomeLC_CHF", + "KPI_Don_Code": country_fdrs_code, + }, + ) if fdrs_entities.status_code != 200: - return + continue + fdrs_entities.raise_for_status() fdrs_entities = fdrs_entities.json() - fdrs_data_count = 0 - if len(fdrs_entities["data"]): - income_list = fdrs_entities["data"][0]["data"][0]["data"] - if len(income_list): - for income in income_list: - data = {"date": str(income["year"]) + "-01-01", "value": income["value"], "overview": overview} - fdrs_data_count += 1 - FDRSAnnualIncome.objects.get_or_create(**data) + + if len(fdrs_entities["data"]) == 0: + continue + + income_list = fdrs_entities["data"][0]["data"][0]["data"] + if len(income_list) == 0: + continue + + created_fdrs_income_ids = [] + with transaction.atomic(): + for income in income_list: + income_value = income["value"] + fdrs_annual_income, created = FDRSAnnualIncome.objects.get_or_create( + overview=overview, + date=str(income["year"]) + "-01-01", + defaults={"value": income_value}, + ) + if not created: + fdrs_annual_income.value = income_value + fdrs_annual_income.save(update_fields=["value"]) + created_fdrs_income_ids.append(fdrs_annual_income.pk) + fdrs_data_count += 1 + # NOTE: Deleting the FDRSAnnualIncome objects that are not in the source + FDRSAnnualIncome.objects.filter(overview=overview).exclude(id__in=created_fdrs_income_ids).delete() logger.info(f"Successfully created {fdrs_data_count} country data") diff --git a/databank/management/commands/ingest_acaps.py b/databank/management/commands/ingest_acaps.py index 28ab823ef..1789f5a70 100644 --- a/databank/management/commands/ingest_acaps.py +++ b/databank/management/commands/ingest_acaps.py @@ -4,40 +4,57 @@ import requests from django.conf import settings from django.core.management.base import BaseCommand +from django.db import models, transaction +from sentry_sdk.crons import monitor from api.logger import logger from api.models import CountryType from databank.models import AcapsSeasonalCalender, CountryOverview +from main.sentry import SentryMonitor class Command(BaseCommand): help = "Add Acaps seasonal calender data" + @monitor(monitor_slug=SentryMonitor.INGEST_ACAPS) + @transaction.atomic + def load_country(self, overview): + # Remove all existing Seasonal Calendar data for this country + AcapsSeasonalCalender.objects.filter(overview=overview).all().delete() + + name = overview.country_name + if "," in name: + name = name.split(",")[0] + response = requests.get( + "https://api.acaps.org/api/v1/seasonal-events-calendar/seasonal-calendar/", + params={"country": name}, + headers={"Authorization": "Token %s" % settings.ACAPS_API_TOKEN}, + ) + logger.info(f"Importing for country {name}") + response_data = response.json() + if "results" in response_data and len(response_data["results"]): + df = pd.DataFrame.from_records(response_data["results"]) + for df_data in df.values.tolist(): + df_country = df_data[2] + if name.lower() == df_country[0].lower(): + dict_data = { + "overview": overview, + "month": df_data[6], + "event": df_data[7], + "event_type": df_data[8], + "label": df_data[9], + "source": df_data[11], + "source_date": df_data[12], + } + # Use bulk manager + AcapsSeasonalCalender.objects.create(**dict_data) + # NOTE: Acaps throttles our requests + time.sleep(5) + def handle(self, *args, **kwargs): logger.info("Importing Acaps Data") - country_name = CountryOverview.objects.filter(country__record_type=CountryType.COUNTRY).values_list( - "country__name", flat=True + country_overview_qs = CountryOverview.objects.filter(country__record_type=CountryType.COUNTRY).annotate( + country_name=models.F("country__name"), ) - for name in country_name: - if "," in name: - name = name.split(",")[0] - SEASONAL_EVENTS_API = f"https://api.acaps.org/api/v1/seasonal-events-calendar/seasonal-calendar/?country={name}" - response = requests.get(SEASONAL_EVENTS_API, headers={"Authorization": "Token %s" % settings.ACAPS_API_TOKEN}) - logger.info(f"Importing for country {name}") - response_data = response.json() - if "results" in response_data and len(response_data["results"]): - df = pd.DataFrame.from_records(response_data["results"]) - for df_data in df.values.tolist(): - df_country = df_data[2] - if name.lower() == df_country[0].lower(): - dict_data = { - "overview": CountryOverview.objects.filter(country__name__icontains=name).first(), - "month": df_data[6], - "event": df_data[7], - "event_type": df_data[8], - "label": df_data[9], - "source": df_data[11], - "source_date": df_data[12], - } - AcapsSeasonalCalender.objects.create(**dict_data) - time.sleep(5) + for overview in country_overview_qs: + self.load_country(overview) diff --git a/databank/management/commands/ingest_climate.py b/databank/management/commands/ingest_climate.py index f90914864..3cb01cbe4 100644 --- a/databank/management/commands/ingest_climate.py +++ b/databank/management/commands/ingest_climate.py @@ -2,9 +2,12 @@ import requests from django.core.management.base import BaseCommand +from django.db import models, transaction +from sentry_sdk.crons import monitor from api.models import CountryType from databank.models import CountryKeyClimate, CountryOverview +from main.sentry import SentryMonitor logger = logging.getLogger(__name__) @@ -12,46 +15,75 @@ class Command(BaseCommand): help = "Add minimum, maximum and Average temperature of country temperature data from source api" + @monitor(monitor_slug=SentryMonitor.INGEST_CLIMATE) def handle(self, *args, **options): - for co in CountryOverview.objects.filter(country__record_type=CountryType.COUNTRY, country__iso3__isnull=False).all(): - country_iso3 = co.country.iso3 - if country_iso3: - response = requests.get( - f"https://climateknowledgeportal.worldbank.org/api/v1/cru-x0.5_climatology_tasmin,tas,tasmax,pr_climatology_monthly_1991-2020_mean_historical_cru_ts4.07_mean/{country_iso3}?_format=json" # noqa: E501 - ) - response.raise_for_status() - try: - response_data = response.json() - data = response_data.get("data", {}) - if data: - precipation = data.get("pr", {}) - average_temp = data.get("tas", {}) - min_temp = data.get("tasmin", {}) - max_temp = data.get("tasmax", {}) - merged_data = { - country: { - date: ( - precipation[country][date], - average_temp[country][date], - min_temp[country][date], - max_temp[country][date], - ) - for date in precipation[country] - } - for country in precipation - } - for key, value in merged_data.items(): - for k, v in value.items(): - year_month = k.split("-") - data = { - "year": year_month[0], - "month": year_month[1], - "max_temp": v[3], - "min_temp": v[2], - "avg_temp": v[1], - "precipitation": v[0], - } - CountryKeyClimate.objects.create(overview=co, **data) - except Exception: - logger.error("Error in ingesting climate data", exc_info=True) + overview_qs = CountryOverview.objects.filter( + country__record_type=CountryType.COUNTRY, + country__iso3__isnull=False, + ).annotate( + country_iso3=models.F("country__iso3"), + ) + + for overview in overview_qs.all(): + country_iso3 = overview.country_iso3 + if not country_iso3: continue + + response = requests.get( + f"https://climateknowledgeportal.worldbank.org/api/v1/cru-x0.5_climatology_tasmin,tas,tasmax,pr_climatology_monthly_1991-2020_mean_historical_cru_ts4.07_mean/{country_iso3}?_format=json" # noqa: E501 + ) + + response.raise_for_status() + response_data = response.json() + data = response_data.get("data", {}) + if not data: + continue + + precipitation = data.get("pr", {}) + average_temp = data.get("tas", {}) + min_temp = data.get("tasmin", {}) + max_temp = data.get("tasmax", {}) + merged_data = { + country: { + date: { + "precipitation": precipitation[country][date], + "average_temp": average_temp[country][date], + "min_temp": min_temp[country][date], + "max_temp": max_temp[country][date], + } + for date in precipitation[country] + } + for country in precipitation + } + created_country_key_climate_ids = [] + with transaction.atomic(): + for value in merged_data.values(): + for k, v in value.items(): + if ( + v["precipitation"] is None + or v["average_temp"] is None + or v["min_temp"] is None + or v["max_temp"] is None + ): + continue + year_month = k.split("-") + country_key_climate, created = CountryKeyClimate.objects.get_or_create( + overview=overview, + year=year_month[0], + month=year_month[1], + defaults={ + "precipitation": v["precipitation"], + "avg_temp": v["average_temp"], + "min_temp": v["min_temp"], + "max_temp": v["max_temp"], + }, + ) + if not created: + country_key_climate.precipitation = v.get("precipitation") + country_key_climate.avg_temp = v.get("average_temp") + country_key_climate.min_temp = v.get("min_temp") + country_key_climate.max_temp = v.get("max_temp") + country_key_climate.save(update_fields=["precipitation", "avg_temp", "min_temp", "max_temp"]) + created_country_key_climate_ids.append(country_key_climate.pk) + # NOTE: Deleting the CountryKeyclimate that are not in the source + CountryKeyClimate.objects.filter(overview=overview).exclude(id__in=created_country_key_climate_ids).delete() diff --git a/databank/management/commands/ingest_databank.py b/databank/management/commands/ingest_databank.py index ea404bbf4..245ab1674 100644 --- a/databank/management/commands/ingest_databank.py +++ b/databank/management/commands/ingest_databank.py @@ -3,9 +3,11 @@ from django.core.management.base import BaseCommand from django.utils import timezone +from sentry_sdk.crons import monitor from api.models import Country, CronJob, CronJobStatus from databank.models import CountryOverview +from main.sentry import SentryMonitor from .sources import FDRS, FTS_HPC, INFORM, RELIEFWEB, START_NETWORK, WB @@ -108,6 +110,7 @@ def load(self): } ) + @monitor(monitor_slug=SentryMonitor.INGEST_DATABANK) def handle(self, *args, **kwargs): start = datetime.datetime.now() self.load() diff --git a/databank/management/commands/ingest_hdr.py b/databank/management/commands/ingest_hdr.py index 16da085b2..97f5ce60a 100644 --- a/databank/management/commands/ingest_hdr.py +++ b/databank/management/commands/ingest_hdr.py @@ -2,8 +2,11 @@ import requests from django.core.management.base import BaseCommand +from django.db import models, transaction +from sentry_sdk.crons import monitor -from databank.models import CountryOverview as CO +from databank.models import CountryOverview +from main.sentry import SentryMonitor logger = logging.getLogger(__name__) @@ -11,15 +14,29 @@ class Command(BaseCommand): help = "Add HDR GII data" + @monitor(monitor_slug=SentryMonitor.INGEST_HDR) + @transaction.atomic def handle(self, *args, **kwargs): - for overview in CO.objects.all(): - HDR_API = f"https://api.hdrdata.org/CountryIndicators/filter?country={overview.country.iso3}&year=2021&indicator=gii" - hdr_entities = requests.get(HDR_API) + overview_qs = CountryOverview.objects.annotate( + country_iso3=models.F("country__iso3"), + ) + for overview in overview_qs.all(): + hdr_entities = requests.get( + "https://api.hdrdata.org/CountryIndicators/filter", + params={ + "country": overview.country_iso3, + "year": 2021, + "indicator": "gii", + }, + ) if hdr_entities.status_code != 200: continue hdr_entities.raise_for_status() hdr_entities = hdr_entities.json() - if len(hdr_entities): - hdr_gii = hdr_entities[0]["value"] - overview.hdr_gii = hdr_gii - overview.save(update_fields=["hdr_gii"]) + + if len(hdr_entities) == 0: + continue + + hdr_gii = hdr_entities[0]["value"] + overview.hdr_gii = hdr_gii + overview.save(update_fields=["hdr_gii"]) diff --git a/databank/management/commands/ingest_unicef.py b/databank/management/commands/ingest_unicef.py index 786b6f0b7..f90396efd 100644 --- a/databank/management/commands/ingest_unicef.py +++ b/databank/management/commands/ingest_unicef.py @@ -2,8 +2,10 @@ import requests from django.core.management.base import BaseCommand +from sentry_sdk.crons import monitor from databank.models import CountryOverview as CO +from main.sentry import SentryMonitor logger = logging.getLogger(__name__) @@ -11,6 +13,7 @@ class Command(BaseCommand): help = "Add Unicef population data" + @monitor(monitor_slug=SentryMonitor.INGEST_UNICEF) def handle(self, *args, **kwargs): for overview in CO.objects.all(): UNICEF_API = f"https://sdmx.data.unicef.org/ws/public/sdmxapi/rest/data/UNICEF,DM,1.0/{overview.country.iso3}.DM_POP_U18._T._T.?format=sdmx-json" # noqa: E501 diff --git a/databank/management/commands/ingest_worldbank.py b/databank/management/commands/ingest_worldbank.py index 9a6eef5f2..ba2422dc0 100644 --- a/databank/management/commands/ingest_worldbank.py +++ b/databank/management/commands/ingest_worldbank.py @@ -4,9 +4,11 @@ import requests from django.core.management.base import BaseCommand +from sentry_sdk.crons import monitor from api.models import Country, CountryType from databank.models import CountryOverview as CO +from main.sentry import SentryMonitor from .sources.utils import get_country_by_iso3 @@ -16,6 +18,7 @@ class Command(BaseCommand): help = "Add Acaps seasonal calendar data" + @monitor(monitor_slug=SentryMonitor.INGEST_WORLDBANK) def handle(self, *args, **kwargs): world_bank_indicator_map = ( ("SP.POP.TOTL", CO.world_bank_population), diff --git a/databank/models.py b/databank/models.py index 2bcd3e1cf..19f9fdbf0 100644 --- a/databank/models.py +++ b/databank/models.py @@ -435,6 +435,7 @@ class FDRSIncome(models.Model): indicator = models.ForeignKey(FDRSIndicator, on_delete=models.CASCADE, verbose_name=_("FDRS Indicator")) value = models.FloatField(verbose_name=_("value"), null=True, blank=True) + # TODO: Add unique_together (overview, date, indicator) def __str__(self): return f"{self.overview.country.name} - {self.date} - {self.indicator.title} - {self.value}" @@ -446,6 +447,7 @@ class FDRSAnnualIncome(models.Model): date = models.DateField(verbose_name=_("date")) value = models.FloatField(verbose_name=_("value"), null=True, blank=True) + # TODO: Add unique_together (overview, date) def __str__(self): return f"{self.overview.country.name} - {self.date} - {self.value}" diff --git a/deploy/helm/ifrcgo-helm/values.yaml b/deploy/helm/ifrcgo-helm/values.yaml index 4185846b1..2b1107fde 100644 --- a/deploy/helm/ifrcgo-helm/values.yaml +++ b/deploy/helm/ifrcgo-helm/values.yaml @@ -122,6 +122,36 @@ cronjobs: schedule: '1 0 * * *' - command: 'update_surge_alert_status' schedule: '1 */12 * * *' + - command: 'fdrs_annual_income' + schedule: '0 0 * * 0' + - command: 'FDRS_INCOME' + schedule: '0 0 * * 0' + - command: 'ingest_acaps' + schedule: '0 1 * * 0' + - command: 'ingest_climate' + schedule: '0 0 * * 0' + - command: 'ingest_databank' + schedule: '0 0 * * 0' + - command: 'ingest_hdr' + schedule: '0 0 * * 0' + - command: 'ingest_unicef' + schedule: '0 0 * * 0' + - command: 'ingest_worldbank' + schedule: '0 2 * * 0' + - command: 'ingest_disaster_law' + schedule: '0 0 * * 0' + - command: 'ingest_ns_contact' + schedule: '0 0 * * 0' + - command: 'ingest_ns_capacity' + schedule: '0 0 * * 0' + - command: 'ingest_ns_directory' + schedule: '0 0 * * 0' + - command: 'ingest_ns_document' + schedule: '0 0 * * 0' + - command: 'ingest_ns_initiatives' + schedule: '0 0 * * 0' + - command: 'ingest_icrc' + schedule: '0 3 * * 0' elasticsearch: diff --git a/deployments/management/commands/update_project_status.py b/deployments/management/commands/update_project_status.py index 171ffe665..d3c047a36 100644 --- a/deployments/management/commands/update_project_status.py +++ b/deployments/management/commands/update_project_status.py @@ -21,11 +21,11 @@ ) -@monitor(monitor_slug=SentryMonitor.UPDATE_PROJECT_STATUS) class Command(BaseCommand): help = "Update project status using start/end date" + @monitor(monitor_slug=SentryMonitor.UPDATE_PROJECT_STATUS) def handle(self, *args, **options): now = timezone.now().date() diff --git a/main/sentry.py b/main/sentry.py index b567032be..d393c9133 100644 --- a/main/sentry.py +++ b/main/sentry.py @@ -114,6 +114,21 @@ class SentryMonitor(models.TextChoices): USER_REGISTRATION_REMINDER = "user_registration_reminder", "0 9 * * *" INGEST_COUNTRY_PLAN_FILE = "ingest_country_plan_file", "1 0 * * *" UPDATE_SURGE_ALERT_STATUS = "update_surge_alert_status", "1 */12 * * *" + FDRS_ANNUAL_INCOME = "fdrs_annual_income", "0 0 * * 0" + FDRS_INCOME = "FDRS_INCOME", "0 0 * * 0" + INGEST_ACAPS = "ingest_acaps", "0 1 * * 0" + INGEST_CLIMATE = "ingest_climate", "0 0 * * 0" + INGEST_DATABANK = "ingest_databank", "0 0 * * 0" + INGEST_HDR = "ingest_hdr", "0 0 * * 0" + INGEST_UNICEF = "ingest_unicef", "0 0 * * 0" + INGEST_WORLDBANK = "ingest_worldbank", "0 2 * * 0" + INGEST_DISASTER_LAW = "ingest_disaster_law", "0 0 * * 0" + INGEST_NS_CONTACT = "ingest_ns_contact", "0 0 * * 0" + INGEST_NS_CAPACITY = "ingest_ns_capacity", "0 0 * * 0" + INGEST_NS_DIRECTORY = "ingest_ns_directory", "0 0 * * 0" + INGEST_NS_DOCUMENT = "ingest_ns_document", "0 0 * * 0" + INGEST_NS_INITIATIVES = "ingest_ns_initiatives", "0 0 * * 0" + INGEST_ICRC = "ingest_icrc", "0 3 * * 0" @staticmethod def load_cron_data() -> typing.List[typing.Tuple[str, str]]: @@ -146,3 +161,41 @@ def validate_config(cls): ) ) ) + + +class SentryMonitorConfig: + """ + Custom config for SentryMonitor + https://docs.sentry.io/product/crons/getting-started/http/#creating-or-updating-a-monitor-through-a-check-in-optional + """ + + MAX_RUNTIME_DEFAULT = 30 # Our default is 30 min + + FAILURE_THRESHOLD_DEFAULT = 1 + FAILURE_THRESHOLD = { + # NOTE: INDEX_AND_NOTIFY runs every 5 minutes; we allow up to 6 consecutive failures + SentryMonitor.INDEX_AND_NOTIFY: 6, + } + + @classmethod + def get_checkin_margin(cls, _: SentryMonitor) -> int: + """ + The amount of time (in minutes) [Sentry Default 1 min] + Sentry should wait for your checkin before it's considered missed ("grace period") + """ + return cls.MAX_RUNTIME_DEFAULT + + @classmethod + def get_failure_issue_threshold(cls, enum: SentryMonitor) -> int: + """ + The number of consecutive failed check-ins it takes before an issue is created. Optional. + """ + return cls.FAILURE_THRESHOLD.get(enum, cls.FAILURE_THRESHOLD_DEFAULT) + + @classmethod + def get_recovery_threshold(cls, _: SentryMonitor) -> int: + """ + [Sentry Default 1] + The number of consecutive OK check-ins it takes before an issue is resolved. Optional. + """ + return 1 diff --git a/notifications/management/commands/update_surge_alert_status.py b/notifications/management/commands/update_surge_alert_status.py index d42397b80..0fe04dbcb 100644 --- a/notifications/management/commands/update_surge_alert_status.py +++ b/notifications/management/commands/update_surge_alert_status.py @@ -12,7 +12,6 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.UPDATE_SURGE_ALERT_STATUS) class Command(BaseCommand): """ Updating the Surge Alert Status according: @@ -23,6 +22,7 @@ class Command(BaseCommand): help = "Update surge alert status" + @monitor(monitor_slug=SentryMonitor.UPDATE_SURGE_ALERT_STATUS) def handle(self, *args, **options): now = timezone.now() try: