diff --git a/api/management/commands/cron_job_monitor.py b/api/management/commands/cron_job_monitor.py index 0a11d97a0e..9c540b34d3 100644 --- a/api/management/commands/cron_job_monitor.py +++ b/api/management/commands/cron_job_monitor.py @@ -5,7 +5,7 @@ from django.conf import settings from django.core.management.base import BaseCommand -from main.sentry import SentryMonitor +from main.sentry import SentryMonitor, SentryMonitorConfig from main.settings import SENTRY_DSN logger = logging.getLogger(__name__) @@ -48,6 +48,10 @@ def handle(self, *args, **options): "type": "crontab", "value": str(schedule), }, + "tz": settings.TIME_ZONE, + "checkin_margin": SentryMonitorConfig.get_checkin_margin(cronjob), + "failure_issue_threshold": SentryMonitorConfig.get_failure_issue_threshold(cronjob), + "recovery_threshold": SentryMonitorConfig.get_recovery_threshold(cronjob), }, "environment": settings.GO_ENVIRONMENT, "status": "ok", diff --git a/api/management/commands/index_and_notify.py b/api/management/commands/index_and_notify.py index 246d9199ce..241f6e22af 100644 --- a/api/management/commands/index_and_notify.py +++ b/api/management/commands/index_and_notify.py @@ -67,7 +67,6 @@ } -@monitor(monitor_slug=SentryMonitor.INDEX_AND_NOTIFY) class Command(BaseCommand): help = "Index and send notifications about new/changed records" @@ -920,6 +919,7 @@ def check_ingest_issues(self, having_ingest_issue): + ", notification sent to IM team" ) + @monitor(monitor_slug=SentryMonitor.INDEX_AND_NOTIFY) def handle(self, *args, **options): if self.is_digest_mode(): time_diff = self.diff_1_week() # in digest mode (once a week, for new_entities only) we use a bigger interval diff --git a/api/management/commands/ingest_appeals.py b/api/management/commands/ingest_appeals.py index c032f8e75b..53d7189be9 100644 --- a/api/management/commands/ingest_appeals.py +++ b/api/management/commands/ingest_appeals.py @@ -33,7 +33,6 @@ GEC_CODES = GECCode.objects.select_related("country").all() -@monitor(monitor_slug=SentryMonitor.INGEST_APPEALS) class Command(BaseCommand): help = "Add new entries from Access database file" @@ -292,6 +291,7 @@ def parse_appeal_record(self, r, **options): return fields + @monitor(monitor_slug=SentryMonitor.INGEST_APPEALS) def handle(self, *args, **options): logger.info("Starting appeals ingest") start_appeals_count = Appeal.objects.all().count() diff --git a/api/management/commands/ingest_disaster_law.py b/api/management/commands/ingest_disaster_law.py index c900b5f4bc..d114aeca79 100644 --- a/api/management/commands/ingest_disaster_law.py +++ b/api/management/commands/ingest_disaster_law.py @@ -8,10 +8,10 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_DISASTER_LAW) class Command(BaseCommand): help = "Add ICRC data" + @monitor(monitor_slug=SentryMonitor.INGEST_DISASTER_LAW) def handle(self, *args, **kwargs): logger.info("Starting Disaster Law data") home_url = "https://disasterlaw.ifrc.org/" diff --git a/api/management/commands/ingest_icrc.py b/api/management/commands/ingest_icrc.py index bf3f47663d..55f71939c2 100644 --- a/api/management/commands/ingest_icrc.py +++ b/api/management/commands/ingest_icrc.py @@ -8,10 +8,10 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_ICRC) class Command(BaseCommand): help = "Add ICRC data" + @monitor(monitor_slug=SentryMonitor.INGEST_ICRC) def handle(self, *args, **kwargs): logger.info("Strating ICRC data ingest") HEADERS = { diff --git a/api/management/commands/ingest_ns_capacity.py b/api/management/commands/ingest_ns_capacity.py index c0c65853b2..edffdfcb5d 100644 --- a/api/management/commands/ingest_ns_capacity.py +++ b/api/management/commands/ingest_ns_capacity.py @@ -8,10 +8,10 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_NS_CAPACITY) class Command(BaseCommand): help = "Add ns contact details" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_CAPACITY) def handle(self, *args, **kwargs): logger.info("Starting NS Contacts") diff --git a/api/management/commands/ingest_ns_contact.py b/api/management/commands/ingest_ns_contact.py index 5af14f47b0..380ecffcf2 100644 --- a/api/management/commands/ingest_ns_contact.py +++ b/api/management/commands/ingest_ns_contact.py @@ -5,6 +5,7 @@ import xmltodict from django.conf import settings from django.core.management.base import BaseCommand +from django.db import transaction from requests.auth import HTTPBasicAuth from sentry_sdk.crons import monitor @@ -13,10 +14,11 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_NS_CONTACT) class Command(BaseCommand): help = "Add ns contact details" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_CONTACT) + @transaction.atomic def handle(self, *args, **kwargs): logger.info("Starting NS Contacts") url = "https://go-api.ifrc.org/" diff --git a/api/management/commands/ingest_ns_directory.py b/api/management/commands/ingest_ns_directory.py index 0a5de0bbe8..e7f2b55b1b 100644 --- a/api/management/commands/ingest_ns_directory.py +++ b/api/management/commands/ingest_ns_directory.py @@ -2,6 +2,7 @@ import xmltodict from django.conf import settings from django.core.management.base import BaseCommand +from django.db import transaction from requests.auth import HTTPBasicAuth from sentry_sdk.crons import monitor @@ -10,10 +11,11 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_NS_DIRECTORY) class Command(BaseCommand): help = "Add ns contact details" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_DIRECTORY) + @transaction.atomic def handle(self, *args, **kwargs): def postprocessor(path, key, value): if key == "@i:nil": @@ -41,6 +43,7 @@ def postprocessor(path, key, value): raise Exception("Error querying NationalSocietiesContacts") added = 0 + created_country_directory_ids = [] dict_data = xmltodict.parse(response.content, postprocessor=postprocessor) for data in dict_data["ArrayOfNationalSocietiesContacts"]["NationalSocietiesContacts"]: country_name = data["CON_country"] if isinstance(data["CON_country"], str) else None @@ -62,14 +65,15 @@ def postprocessor(path, key, value): "position": data["CON_title"], "country": country, } - existing_qs = CountryDirectory.objects.filter( + country_directory, _ = CountryDirectory.objects.get_or_create( country=country, first_name__iexact=data["first_name"], last_name__iexact=data["last_name"], position__iexact=data["position"], ) - if not existing_qs.exists(): - CountryDirectory.objects.create(**data) + created_country_directory_ids.append(country_directory.pk) + # NOTE: Deleting the country directory which are not available in the source + CountryDirectory.objects.exclude(id__in=created_country_directory_ids).delete() text_to_log = "%s Ns Directory added" % added logger.info(text_to_log) body = {"name": "ingest_ns_directory", "message": text_to_log, "num_result": added, "status": CronJobStatus.SUCCESSFUL} diff --git a/api/management/commands/ingest_ns_document.py b/api/management/commands/ingest_ns_document.py index fca3d32809..02be542a40 100644 --- a/api/management/commands/ingest_ns_document.py +++ b/api/management/commands/ingest_ns_document.py @@ -5,6 +5,7 @@ import requests from django.conf import settings from django.core.management.base import BaseCommand +from django.db import transaction from sentry_sdk.crons import monitor from api.logger import logger @@ -12,10 +13,10 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_NS_DOCUMENT) class Command(BaseCommand): help = "Add ns documents" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_DOCUMENT) def handle(self, *args, **kwargs): logger.info("Starting NS Key Documents") @@ -98,32 +99,46 @@ def fetch_all_country_documents(self, api_key, country_table): def save_documents_to_database(self, result): added = 0 - for document in result: - country = Country.objects.filter(fdrs=document["country_code"]).first() - if country is None: - continue - - country_key_document, created = CountryKeyDocument.objects.get_or_create( - country=country, - url=document["url"], - defaults={ - "name": document["name"], - "thumbnail": document["thumbnail"], - "document_type": document["document_type"], - "year": document["year"], - "end_year": document["end_year"], - "year_text": document["year_text"], - }, - ) - if not created: - country_key_document.name = document["name"] - country_key_document.thumbnail = document["thumbnail"] - country_key_document.document_type = document["document_type"] - country_key_document.year = document["year"] - country_key_document.end_year = document["end_year"] - country_key_document.year_text = document["year_text"] - country_key_document.save(update_fields=["name", "thumbnail", "document_type", "year", "end_year", "year_text"]) - added += 1 + created_country_key_document_ids = [] + with transaction.atomic(): + for document in result: + country = Country.objects.filter(fdrs=document["country_code"]).first() + if country is None: + continue + + country_key_document, created = CountryKeyDocument.objects.get_or_create( + country=country, + url=document["url"], + defaults={ + "name": document["name"], + "thumbnail": document["thumbnail"], + "document_type": document["document_type"], + "year": document["year"], + "end_year": document["end_year"], + "year_text": document["year_text"], + }, + ) + if not created: + country_key_document.name = document["name"] + country_key_document.thumbnail = document["thumbnail"] + country_key_document.document_type = document["document_type"] + country_key_document.year = document["year"] + country_key_document.end_year = document["end_year"] + country_key_document.year_text = document["year_text"] + country_key_document.save( + update_fields=[ + "name", + "thumbnail", + "document_type", + "year", + "end_year", + "year_text", + ] + ) + created_country_key_document_ids.append(country_key_document.pk) + added += 1 + # NOTE: Deleting the CountryKeyDocument that are not in the source + CountryKeyDocument.objects.exclude(id__in=created_country_key_document_ids).delete() return added def sync_cron_success(self, text_to_log, added): diff --git a/api/management/commands/ingest_ns_initiatives.py b/api/management/commands/ingest_ns_initiatives.py index 47dd005232..926e4d0634 100644 --- a/api/management/commands/ingest_ns_initiatives.py +++ b/api/management/commands/ingest_ns_initiatives.py @@ -3,6 +3,7 @@ import requests from django.conf import settings from django.core.management.base import BaseCommand +from django.db import transaction from sentry_sdk.crons import monitor from api.logger import logger @@ -10,10 +11,10 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_NS_INITIATIVES) class Command(BaseCommand): help = "Add ns initiatives" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_INITIATIVES) def handle(self, *args, **kwargs): logger.info("Starting NS Inititatives") api_key = settings.NS_INITIATIVES_API_KEY @@ -43,28 +44,33 @@ def handle(self, *args, **kwargs): ], ) funding_data = funding_data.replace({np.nan: None}) - for data in funding_data.values.tolist(): - # TODO: Filter not by society name - country = Country.objects.filter(society_name__iexact=data[0]).first() - if country: - nsd_initiatives, created = NSDInitiatives.objects.get_or_create( - country=country, - year=data[1], - fund_type=data[2], - defaults={ - "title": data[3], - "categories": data[4], - "allocation": data[5], - "funding_period": data[6], - }, - ) - if not created: - nsd_initiatives.title = data[3] - nsd_initiatives.allocation = data[5] - nsd_initiatives.funding_period = data[6] - nsd_initiatives.categories = data[4] - nsd_initiatives.save(update_fields=["title", "allocation", "funding_period", "categories"]) - added += 1 + created_ns_initiatives_pk = [] + with transaction.atomic(): + for data in funding_data.values.tolist(): + # TODO: Filter not by society name + country = Country.objects.filter(society_name__iexact=data[0]).first() + if country: + nsd_initiatives, created = NSDInitiatives.objects.get_or_create( + country=country, + year=data[1], + fund_type=data[2], + defaults={ + "title": data[3], + "categories": data[4], + "allocation": data[5], + "funding_period": data[6], + }, + ) + if not created: + nsd_initiatives.title = data[3] + nsd_initiatives.categories = data[4] + nsd_initiatives.allocation = data[5] + nsd_initiatives.funding_period = data[6] + nsd_initiatives.save(update_fields=["title", "categories", "allocation", "funding_period"]) + created_ns_initiatives_pk.append(nsd_initiatives.pk) + added += 1 + # NOTE: Delete the NSDInitiatives that are not in the source + NSDInitiatives.objects.exclude(id__in=created_ns_initiatives_pk).delete() text_to_log = "%s Ns initiatives added" % added logger.info(text_to_log) diff --git a/api/management/commands/revoke_staff_status.py b/api/management/commands/revoke_staff_status.py index 5c965ca4d4..db4725e0b0 100644 --- a/api/management/commands/revoke_staff_status.py +++ b/api/management/commands/revoke_staff_status.py @@ -8,7 +8,6 @@ # from registrations.views import is_valid_domain -@monitor(monitor_slug=SentryMonitor.REVOKE_STAFF_STATUS) class Command(BaseCommand): help = 'Update staff status in auth_user table according to "Read only" group' @@ -53,6 +52,7 @@ def get_ifrc_domain_users(self): # # return editors + @monitor(monitor_slug=SentryMonitor.REVOKE_STAFF_STATUS) def handle(self, *args, **options): logger.info("Moving Read only users out of staff status...") diff --git a/api/management/commands/sync_appealdocs.py b/api/management/commands/sync_appealdocs.py index 889805c49f..12a27e0599 100644 --- a/api/management/commands/sync_appealdocs.py +++ b/api/management/commands/sync_appealdocs.py @@ -21,7 +21,6 @@ FEDNET_SOURCE = "https://go-api.ifrc.org/Api/FedNetAppeals?Hidden=false&BaseAppealnumber=" -@monitor(monitor_slug=SentryMonitor.SYNC_APPEALDOCS) class Command(BaseCommand): help = "Ingest existing appeal documents" @@ -37,6 +36,7 @@ def parse_date(self, date_string): timeformat = "%Y-%m-%dT%H:%M:%S" return datetime.strptime(date_string[:18], timeformat).replace(tzinfo=timezone.utc) + @monitor(monitor_slug=SentryMonitor.SYNC_APPEALDOCS) def handle(self, *args, **options): logger.info("Starting appeal document ingest") diff --git a/api/management/commands/sync_molnix.py b/api/management/commands/sync_molnix.py index 9941871c23..60b483a18d 100644 --- a/api/management/commands/sync_molnix.py +++ b/api/management/commands/sync_molnix.py @@ -519,10 +519,10 @@ def sync_open_positions(molnix_positions, molnix_api, countries): return messages, warnings, successful_creates -@monitor(monitor_slug=SentryMonitor.SYNC_MOLNIX) class Command(BaseCommand): help = "Sync data from Molnix API to GO db" + @monitor(monitor_slug=SentryMonitor.SYNC_MOLNIX) @transaction.atomic def handle(self, *args, **options): logger.info("Starting Sync Molnix job") diff --git a/api/management/commands/user_registration_reminder.py b/api/management/commands/user_registration_reminder.py index 6bcfb7dc7b..c6ceaadae0 100644 --- a/api/management/commands/user_registration_reminder.py +++ b/api/management/commands/user_registration_reminder.py @@ -10,13 +10,13 @@ from registrations.models import Pending -@monitor(monitor_slug=SentryMonitor.USER_REGISTRATION_REMINDER) class Command(BaseCommand): help = "Send reminder about the pending registrations" def diff_3_day(self): return datetime.utcnow().replace(tzinfo=timezone.utc) - timedelta(days=3) + @monitor(monitor_slug=SentryMonitor.USER_REGISTRATION_REMINDER) def handle(self, *args, **options): region_ids = Region.objects.all().values_list("id", flat=True) time_diff_3_day = self.diff_3_day() diff --git a/country_plan/management/commands/ingest_country_plan_file.py b/country_plan/management/commands/ingest_country_plan_file.py index a911c505aa..ad594539ae 100644 --- a/country_plan/management/commands/ingest_country_plan_file.py +++ b/country_plan/management/commands/ingest_country_plan_file.py @@ -55,7 +55,6 @@ def _get_filename_from_headers(resp): return resp.headers.get("Content-Type"), _get_filename_from_headers(resp) -@monitor(monitor_slug=SentryMonitor.INGEST_COUNTRY_PLAN_FILE) class Command(BaseCommand): @staticmethod def load_file_to_country_plan(country_plan: CountryPlan, url: str, filename: str, field_name: str): @@ -121,6 +120,7 @@ def load(self, url: str, file_field: str, field_inserted_date_field: str): except Exception: logger.error("Could not update countries plan", exc_info=True) + @monitor(monitor_slug=SentryMonitor.INGEST_COUNTRY_PLAN_FILE) def handle(self, **_): # Public self.stdout.write("Fetching data for country plans:: PUBLIC") diff --git a/databank/management/commands/FDRS_INCOME.py b/databank/management/commands/FDRS_INCOME.py index 69d8a84271..d5f4d25052 100644 --- a/databank/management/commands/FDRS_INCOME.py +++ b/databank/management/commands/FDRS_INCOME.py @@ -2,7 +2,9 @@ import requests from django.conf import settings +from django.core.management import call_command from django.core.management.base import BaseCommand +from django.db import transaction from sentry_sdk.crons import monitor from databank.models import CountryOverview, FDRSIncome, FDRSIndicator @@ -11,11 +13,14 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.FDRS_INCOME) class Command(BaseCommand): help = "Import FDRS income data" + @monitor(monitor_slug=SentryMonitor.FDRS_INCOME) def handle(self, *args, **kwargs): + # NOTE: Loading FDRS indicators + call_command("loaddata", "fdrs_indicator.json", verbosity=2) + fdrs_indicator_enum_data = { "h_gov_CHF": "Home Government", "f_gov_CHF": "Foreign Government", @@ -45,18 +50,24 @@ def handle(self, *args, **kwargs): return fdrs_entities.raise_for_status() fdrs_entities = fdrs_entities.json() - for d in fdrs_entities["data"]: - indicator = next(iter(d.values())) - fdrs_indicator = map_indicators[fdrs_indicator_enum_data[indicator]] - income_list = d["data"][0]["data"] - if len(income_list): - for income in income_list: - income_value = income["value"] - fdrs_income, _ = FDRSIncome.objects.get_or_create( - overview=overview, - indicator=fdrs_indicator, - date=str(income["year"]) + "-01-01", - ) - fdrs_income.value = income_value - # TODO: Use bulk - fdrs_income.save(update_fields=("value",)) + created_fdrs_income_ids = [] + with transaction.atomic(): + for d in fdrs_entities["data"]: + indicator = next(iter(d.values())) + fdrs_indicator = map_indicators[fdrs_indicator_enum_data[indicator]] + income_list = d["data"][0]["data"] + if len(income_list): + for income in income_list: + income_value = income["value"] + fdrs_income, created = FDRSIncome.objects.get_or_create( + overview=overview, + indicator=fdrs_indicator, + date=str(income["year"]) + "-01-01", + defaults={"value": income_value}, + ) + if not created: + fdrs_income.value = income_value + fdrs_income.save(update_fields=["value"]) + created_fdrs_income_ids.append(fdrs_income.pk) + # NOTE: Delete the FDRSIncome that are not in the source + FDRSIncome.objects.filter(overview=overview).exclude(id__in=created_fdrs_income_ids).delete() diff --git a/databank/management/commands/fdrs_annual_income.py b/databank/management/commands/fdrs_annual_income.py index fca1cf7caf..52ce73a08e 100644 --- a/databank/management/commands/fdrs_annual_income.py +++ b/databank/management/commands/fdrs_annual_income.py @@ -3,7 +3,7 @@ import requests from django.conf import settings from django.core.management.base import BaseCommand -from django.db import models +from django.db import models, transaction from sentry_sdk.crons import monitor from databank.models import CountryOverview, FDRSAnnualIncome @@ -12,10 +12,10 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.FDRS_ANNUAL_INCOME) class Command(BaseCommand): help = "Import FDRS income data" + @monitor(monitor_slug=SentryMonitor.FDRS_ANNUAL_INCOME) def handle(self, *args, **kwargs): overview_qs = CountryOverview.objects.annotate( country_fdrd=models.F("country__fdrs"), @@ -44,14 +44,20 @@ def handle(self, *args, **kwargs): if len(income_list) == 0: continue - for income in income_list: - income_value = income["value"] - fdrs_annual_income, _ = FDRSAnnualIncome.objects.get_or_create( - overview=overview, - date=str(income["year"]) + "-01-01", - ) - fdrs_annual_income.value = income_value - fdrs_annual_income.save(update_fields=("value",)) - fdrs_data_count += 1 - + created_fdrs_income_ids = [] + with transaction.atomic(): + for income in income_list: + income_value = income["value"] + fdrs_annual_income, created = FDRSAnnualIncome.objects.get_or_create( + overview=overview, + date=str(income["year"]) + "-01-01", + defaults={"value": income_value}, + ) + if not created: + fdrs_annual_income.value = income_value + fdrs_annual_income.save(update_fields=["value"]) + created_fdrs_income_ids.append(fdrs_annual_income.pk) + fdrs_data_count += 1 + # NOTE: Deleting the FDRSAnnualIncome objects that are not in the source + FDRSAnnualIncome.objects.filter(overview=overview).exclude(id__in=created_fdrs_income_ids).delete() logger.info(f"Successfully created {fdrs_data_count} country data") diff --git a/databank/management/commands/ingest_acaps.py b/databank/management/commands/ingest_acaps.py index d764f9737e..1789f5a707 100644 --- a/databank/management/commands/ingest_acaps.py +++ b/databank/management/commands/ingest_acaps.py @@ -13,10 +13,10 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_ACAPS) class Command(BaseCommand): help = "Add Acaps seasonal calender data" + @monitor(monitor_slug=SentryMonitor.INGEST_ACAPS) @transaction.atomic def load_country(self, overview): # Remove all existing Seasonal Calendar data for this country diff --git a/databank/management/commands/ingest_climate.py b/databank/management/commands/ingest_climate.py index bf99e844ea..a2fbe2c9a5 100644 --- a/databank/management/commands/ingest_climate.py +++ b/databank/management/commands/ingest_climate.py @@ -2,7 +2,7 @@ import requests from django.core.management.base import BaseCommand -from django.db import models +from django.db import models, transaction from sentry_sdk.crons import monitor from api.models import CountryType @@ -12,10 +12,10 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.INGEST_CLIMATE) class Command(BaseCommand): help = "Add minimum, maximum and Average temperature of country temperature data from source api" + @monitor(monitor_slug=SentryMonitor.INGEST_CLIMATE) def handle(self, *args, **options): overview_qs = CountryOverview.objects.filter( country__record_type=CountryType.COUNTRY, @@ -33,33 +33,38 @@ def handle(self, *args, **options): f"https://climateknowledgeportal.worldbank.org/api/v1/cru-x0.5_climatology_tasmin,tas,tasmax,pr_climatology_monthly_1991-2020_mean_historical_cru_ts4.07_mean/{country_iso3}?_format=json" # noqa: E501 ) - try: - response.raise_for_status() - response_data = response.json() - data = response_data.get("data", {}) - if not data: - continue + response.raise_for_status() + response_data = response.json() + data = response_data.get("data", {}) + if not data: + continue - precipitation = data.get("pr", {}) - average_temp = data.get("tas", {}) - min_temp = data.get("tasmin", {}) - max_temp = data.get("tasmax", {}) - merged_data = { - country: { - date: ( - precipitation[country][date], - average_temp[country][date], - min_temp[country][date], - max_temp[country][date], - ) - for date in precipitation[country] + precipitation = data.get("pr", {}) + average_temp = data.get("tas", {}) + min_temp = data.get("tasmin", {}) + max_temp = data.get("tasmax", {}) + merged_data = { + country: { + date: { + "precipitation": precipitation[country][date], + "average_temp": average_temp[country][date], + "min_temp": min_temp[country][date], + "max_temp": max_temp[country][date], } - for country in precipitation + for date in precipitation[country] } - + for country in precipitation + } + created_country_key_climate_ids = [] + with transaction.atomic(): for value in merged_data.values(): for k, v in value.items(): - if v[0] is None or v[1] is None or v[2] is None or v[3] is None: + if ( + v.get("precipitation") is None + or v.get("average_temp") is None + or v.get("min_temp") is None + or v.get("max_temp") is None + ): continue year_month = k.split("-") country_key_climate, created = CountryKeyClimate.objects.get_or_create( @@ -67,26 +72,18 @@ def handle(self, *args, **options): year=year_month[0], month=year_month[1], defaults={ - "precipitation": v[0], - "avg_temp": v[1], - "min_temp": v[2], - "max_temp": v[3], + "precipitation": v.get("precipitation"), + "avg_temp": v.get("average_temp"), + "min_temp": v.get("min_temp"), + "max_temp": v.get("max_temp"), }, ) if not created: - country_key_climate.max_temp = v[3] - country_key_climate.min_temp = v[2] - country_key_climate.avg_temp = v[1] - country_key_climate.precipitation = v[0] - # TODO: Use bulk manager - country_key_climate.save( - update_fields=( - "max_temp", - "min_temp", - "avg_temp", - "precipitation", - ) - ) - except Exception: - logger.error("Error in ingesting climate data", exc_info=True) - continue + country_key_climate.precipitation = v.get("precipitation") + country_key_climate.avg_temp = v.get("average_temp") + country_key_climate.min_temp = v.get("min_temp") + country_key_climate.max_temp = v.get("max_temp") + country_key_climate.save(update_fields=["precipitation", "avg_temp", "min_temp", "max_temp"]) + created_country_key_climate_ids.append(country_key_climate.pk) + # NOTE: Deleting the CountryKeyclimate that are not in the source + CountryKeyClimate.objects.filter(overview=overview).exclude(id__in=created_country_key_climate_ids).delete() diff --git a/databank/management/commands/ingest_databank.py b/databank/management/commands/ingest_databank.py index 66be701fb5..245ab1674b 100644 --- a/databank/management/commands/ingest_databank.py +++ b/databank/management/commands/ingest_databank.py @@ -26,7 +26,6 @@ ] -@monitor(monitor_slug=SentryMonitor.INGEST_DATABANK) class Command(BaseCommand): def load(self): """ @@ -111,6 +110,7 @@ def load(self): } ) + @monitor(monitor_slug=SentryMonitor.INGEST_DATABANK) def handle(self, *args, **kwargs): start = datetime.datetime.now() self.load() diff --git a/databank/management/commands/ingest_hdr.py b/databank/management/commands/ingest_hdr.py index 28ec9e6cf7..97f5ce60a9 100644 --- a/databank/management/commands/ingest_hdr.py +++ b/databank/management/commands/ingest_hdr.py @@ -2,7 +2,7 @@ import requests from django.core.management.base import BaseCommand -from django.db import models +from django.db import models, transaction from sentry_sdk.crons import monitor from databank.models import CountryOverview @@ -11,10 +11,11 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.INGEST_HDR) class Command(BaseCommand): help = "Add HDR GII data" + @monitor(monitor_slug=SentryMonitor.INGEST_HDR) + @transaction.atomic def handle(self, *args, **kwargs): overview_qs = CountryOverview.objects.annotate( country_iso3=models.F("country__iso3"), diff --git a/databank/management/commands/ingest_unicef.py b/databank/management/commands/ingest_unicef.py index 0ce0c3602e..f90396efd0 100644 --- a/databank/management/commands/ingest_unicef.py +++ b/databank/management/commands/ingest_unicef.py @@ -10,10 +10,10 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.INGEST_UNICEF) class Command(BaseCommand): help = "Add Unicef population data" + @monitor(monitor_slug=SentryMonitor.INGEST_UNICEF) def handle(self, *args, **kwargs): for overview in CO.objects.all(): UNICEF_API = f"https://sdmx.data.unicef.org/ws/public/sdmxapi/rest/data/UNICEF,DM,1.0/{overview.country.iso3}.DM_POP_U18._T._T.?format=sdmx-json" # noqa: E501 diff --git a/databank/management/commands/ingest_worldbank.py b/databank/management/commands/ingest_worldbank.py index fa0b881c22..ba2422dc05 100644 --- a/databank/management/commands/ingest_worldbank.py +++ b/databank/management/commands/ingest_worldbank.py @@ -15,10 +15,10 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.INGEST_WORLDBANK) class Command(BaseCommand): help = "Add Acaps seasonal calendar data" + @monitor(monitor_slug=SentryMonitor.INGEST_WORLDBANK) def handle(self, *args, **kwargs): world_bank_indicator_map = ( ("SP.POP.TOTL", CO.world_bank_population), diff --git a/deployments/management/commands/update_project_status.py b/deployments/management/commands/update_project_status.py index 171ffe6654..d3c047a369 100644 --- a/deployments/management/commands/update_project_status.py +++ b/deployments/management/commands/update_project_status.py @@ -21,11 +21,11 @@ ) -@monitor(monitor_slug=SentryMonitor.UPDATE_PROJECT_STATUS) class Command(BaseCommand): help = "Update project status using start/end date" + @monitor(monitor_slug=SentryMonitor.UPDATE_PROJECT_STATUS) def handle(self, *args, **options): now = timezone.now().date() diff --git a/main/sentry.py b/main/sentry.py index ce4146bfa4..d393c9133e 100644 --- a/main/sentry.py +++ b/main/sentry.py @@ -161,3 +161,41 @@ def validate_config(cls): ) ) ) + + +class SentryMonitorConfig: + """ + Custom config for SentryMonitor + https://docs.sentry.io/product/crons/getting-started/http/#creating-or-updating-a-monitor-through-a-check-in-optional + """ + + MAX_RUNTIME_DEFAULT = 30 # Our default is 30 min + + FAILURE_THRESHOLD_DEFAULT = 1 + FAILURE_THRESHOLD = { + # NOTE: INDEX_AND_NOTIFY runs every 5 minutes; we allow up to 6 consecutive failures + SentryMonitor.INDEX_AND_NOTIFY: 6, + } + + @classmethod + def get_checkin_margin(cls, _: SentryMonitor) -> int: + """ + The amount of time (in minutes) [Sentry Default 1 min] + Sentry should wait for your checkin before it's considered missed ("grace period") + """ + return cls.MAX_RUNTIME_DEFAULT + + @classmethod + def get_failure_issue_threshold(cls, enum: SentryMonitor) -> int: + """ + The number of consecutive failed check-ins it takes before an issue is created. Optional. + """ + return cls.FAILURE_THRESHOLD.get(enum, cls.FAILURE_THRESHOLD_DEFAULT) + + @classmethod + def get_recovery_threshold(cls, _: SentryMonitor) -> int: + """ + [Sentry Default 1] + The number of consecutive OK check-ins it takes before an issue is resolved. Optional. + """ + return 1 diff --git a/notifications/management/commands/update_surge_alert_status.py b/notifications/management/commands/update_surge_alert_status.py index d42397b800..0fe04dbcb1 100644 --- a/notifications/management/commands/update_surge_alert_status.py +++ b/notifications/management/commands/update_surge_alert_status.py @@ -12,7 +12,6 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.UPDATE_SURGE_ALERT_STATUS) class Command(BaseCommand): """ Updating the Surge Alert Status according: @@ -23,6 +22,7 @@ class Command(BaseCommand): help = "Update surge alert status" + @monitor(monitor_slug=SentryMonitor.UPDATE_SURGE_ALERT_STATUS) def handle(self, *args, **options): now = timezone.now() try: