From f93510aeb2533a025f3c79f58e62c22227777b98 Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Wed, 3 Jul 2024 17:21:22 +0545 Subject: [PATCH] Add configuration for thresholds and runtime Add PR changes --- api/management/commands/cron_job_monitor.py | 6 ++- api/management/commands/index_and_notify.py | 2 +- api/management/commands/ingest_appeals.py | 2 +- .../commands/ingest_disaster_law.py | 2 +- api/management/commands/ingest_icrc.py | 2 +- api/management/commands/ingest_ns_capacity.py | 2 +- api/management/commands/ingest_ns_contact.py | 4 +- .../commands/ingest_ns_directory.py | 12 +++-- api/management/commands/ingest_ns_document.py | 18 +++---- .../commands/ingest_ns_initiatives.py | 16 +++--- .../commands/revoke_staff_status.py | 2 +- api/management/commands/sync_appealdocs.py | 2 +- api/management/commands/sync_molnix.py | 2 +- .../commands/user_registration_reminder.py | 2 +- .../commands/ingest_country_plan_file.py | 2 +- databank/management/commands/FDRS_INCOME.py | 16 ++++-- .../management/commands/fdrs_annual_income.py | 13 +++-- databank/management/commands/ingest_acaps.py | 2 +- .../management/commands/ingest_climate.py | 53 +++++++++---------- .../management/commands/ingest_databank.py | 2 +- databank/management/commands/ingest_hdr.py | 5 +- databank/management/commands/ingest_unicef.py | 2 +- .../management/commands/ingest_worldbank.py | 2 +- .../commands/update_project_status.py | 2 +- main/sentry.py | 38 +++++++++++++ .../commands/update_surge_alert_status.py | 2 +- 26 files changed, 133 insertions(+), 80 deletions(-) diff --git a/api/management/commands/cron_job_monitor.py b/api/management/commands/cron_job_monitor.py index 0a11d97a0..9c540b34d 100644 --- a/api/management/commands/cron_job_monitor.py +++ b/api/management/commands/cron_job_monitor.py @@ -5,7 +5,7 @@ from django.conf import settings from django.core.management.base import BaseCommand -from main.sentry import SentryMonitor +from main.sentry import SentryMonitor, SentryMonitorConfig from main.settings import SENTRY_DSN logger = logging.getLogger(__name__) @@ -48,6 +48,10 @@ def handle(self, *args, **options): "type": "crontab", "value": str(schedule), }, + "tz": settings.TIME_ZONE, + "checkin_margin": SentryMonitorConfig.get_checkin_margin(cronjob), + "failure_issue_threshold": SentryMonitorConfig.get_failure_issue_threshold(cronjob), + "recovery_threshold": SentryMonitorConfig.get_recovery_threshold(cronjob), }, "environment": settings.GO_ENVIRONMENT, "status": "ok", diff --git a/api/management/commands/index_and_notify.py b/api/management/commands/index_and_notify.py index 246d9199c..241f6e22a 100644 --- a/api/management/commands/index_and_notify.py +++ b/api/management/commands/index_and_notify.py @@ -67,7 +67,6 @@ } -@monitor(monitor_slug=SentryMonitor.INDEX_AND_NOTIFY) class Command(BaseCommand): help = "Index and send notifications about new/changed records" @@ -920,6 +919,7 @@ def check_ingest_issues(self, having_ingest_issue): + ", notification sent to IM team" ) + @monitor(monitor_slug=SentryMonitor.INDEX_AND_NOTIFY) def handle(self, *args, **options): if self.is_digest_mode(): time_diff = self.diff_1_week() # in digest mode (once a week, for new_entities only) we use a bigger interval diff --git a/api/management/commands/ingest_appeals.py b/api/management/commands/ingest_appeals.py index c032f8e75..53d7189be 100644 --- a/api/management/commands/ingest_appeals.py +++ b/api/management/commands/ingest_appeals.py @@ -33,7 +33,6 @@ GEC_CODES = GECCode.objects.select_related("country").all() -@monitor(monitor_slug=SentryMonitor.INGEST_APPEALS) class Command(BaseCommand): help = "Add new entries from Access database file" @@ -292,6 +291,7 @@ def parse_appeal_record(self, r, **options): return fields + @monitor(monitor_slug=SentryMonitor.INGEST_APPEALS) def handle(self, *args, **options): logger.info("Starting appeals ingest") start_appeals_count = Appeal.objects.all().count() diff --git a/api/management/commands/ingest_disaster_law.py b/api/management/commands/ingest_disaster_law.py index c900b5f4b..d114aeca7 100644 --- a/api/management/commands/ingest_disaster_law.py +++ b/api/management/commands/ingest_disaster_law.py @@ -8,10 +8,10 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_DISASTER_LAW) class Command(BaseCommand): help = "Add ICRC data" + @monitor(monitor_slug=SentryMonitor.INGEST_DISASTER_LAW) def handle(self, *args, **kwargs): logger.info("Starting Disaster Law data") home_url = "https://disasterlaw.ifrc.org/" diff --git a/api/management/commands/ingest_icrc.py b/api/management/commands/ingest_icrc.py index bf3f47663..55f71939c 100644 --- a/api/management/commands/ingest_icrc.py +++ b/api/management/commands/ingest_icrc.py @@ -8,10 +8,10 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_ICRC) class Command(BaseCommand): help = "Add ICRC data" + @monitor(monitor_slug=SentryMonitor.INGEST_ICRC) def handle(self, *args, **kwargs): logger.info("Strating ICRC data ingest") HEADERS = { diff --git a/api/management/commands/ingest_ns_capacity.py b/api/management/commands/ingest_ns_capacity.py index c0c65853b..edffdfcb5 100644 --- a/api/management/commands/ingest_ns_capacity.py +++ b/api/management/commands/ingest_ns_capacity.py @@ -8,10 +8,10 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_NS_CAPACITY) class Command(BaseCommand): help = "Add ns contact details" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_CAPACITY) def handle(self, *args, **kwargs): logger.info("Starting NS Contacts") diff --git a/api/management/commands/ingest_ns_contact.py b/api/management/commands/ingest_ns_contact.py index 5af14f47b..380ecffcf 100644 --- a/api/management/commands/ingest_ns_contact.py +++ b/api/management/commands/ingest_ns_contact.py @@ -5,6 +5,7 @@ import xmltodict from django.conf import settings from django.core.management.base import BaseCommand +from django.db import transaction from requests.auth import HTTPBasicAuth from sentry_sdk.crons import monitor @@ -13,10 +14,11 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_NS_CONTACT) class Command(BaseCommand): help = "Add ns contact details" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_CONTACT) + @transaction.atomic def handle(self, *args, **kwargs): logger.info("Starting NS Contacts") url = "https://go-api.ifrc.org/" diff --git a/api/management/commands/ingest_ns_directory.py b/api/management/commands/ingest_ns_directory.py index 0a5de0bbe..e7f2b55b1 100644 --- a/api/management/commands/ingest_ns_directory.py +++ b/api/management/commands/ingest_ns_directory.py @@ -2,6 +2,7 @@ import xmltodict from django.conf import settings from django.core.management.base import BaseCommand +from django.db import transaction from requests.auth import HTTPBasicAuth from sentry_sdk.crons import monitor @@ -10,10 +11,11 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_NS_DIRECTORY) class Command(BaseCommand): help = "Add ns contact details" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_DIRECTORY) + @transaction.atomic def handle(self, *args, **kwargs): def postprocessor(path, key, value): if key == "@i:nil": @@ -41,6 +43,7 @@ def postprocessor(path, key, value): raise Exception("Error querying NationalSocietiesContacts") added = 0 + created_country_directory_ids = [] dict_data = xmltodict.parse(response.content, postprocessor=postprocessor) for data in dict_data["ArrayOfNationalSocietiesContacts"]["NationalSocietiesContacts"]: country_name = data["CON_country"] if isinstance(data["CON_country"], str) else None @@ -62,14 +65,15 @@ def postprocessor(path, key, value): "position": data["CON_title"], "country": country, } - existing_qs = CountryDirectory.objects.filter( + country_directory, _ = CountryDirectory.objects.get_or_create( country=country, first_name__iexact=data["first_name"], last_name__iexact=data["last_name"], position__iexact=data["position"], ) - if not existing_qs.exists(): - CountryDirectory.objects.create(**data) + created_country_directory_ids.append(country_directory.pk) + # NOTE: Deleting the country directory which are not available in the source + CountryDirectory.objects.exclude(id__in=created_country_directory_ids).delete() text_to_log = "%s Ns Directory added" % added logger.info(text_to_log) body = {"name": "ingest_ns_directory", "message": text_to_log, "num_result": added, "status": CronJobStatus.SUCCESSFUL} diff --git a/api/management/commands/ingest_ns_document.py b/api/management/commands/ingest_ns_document.py index fca3d3280..56aaced58 100644 --- a/api/management/commands/ingest_ns_document.py +++ b/api/management/commands/ingest_ns_document.py @@ -5,6 +5,7 @@ import requests from django.conf import settings from django.core.management.base import BaseCommand +from django.db import transaction from sentry_sdk.crons import monitor from api.logger import logger @@ -12,10 +13,11 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_NS_DOCUMENT) class Command(BaseCommand): help = "Add ns documents" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_DOCUMENT) + @transaction.atomic def handle(self, *args, **kwargs): logger.info("Starting NS Key Documents") @@ -98,12 +100,13 @@ def fetch_all_country_documents(self, api_key, country_table): def save_documents_to_database(self, result): added = 0 + created_country_key_document_ids = [] for document in result: country = Country.objects.filter(fdrs=document["country_code"]).first() if country is None: continue - country_key_document, created = CountryKeyDocument.objects.get_or_create( + country_key_document, _ = CountryKeyDocument.objects.get_or_create( country=country, url=document["url"], defaults={ @@ -115,15 +118,10 @@ def save_documents_to_database(self, result): "year_text": document["year_text"], }, ) - if not created: - country_key_document.name = document["name"] - country_key_document.thumbnail = document["thumbnail"] - country_key_document.document_type = document["document_type"] - country_key_document.year = document["year"] - country_key_document.end_year = document["end_year"] - country_key_document.year_text = document["year_text"] - country_key_document.save(update_fields=["name", "thumbnail", "document_type", "year", "end_year", "year_text"]) + created_country_key_document_ids.append(country_key_document.pk) added += 1 + # NOTE: Deleting the CountryKeyDocument that are not in the source + CountryKeyDocument.objects.exclude(id__in=created_country_key_document_ids).delete() return added def sync_cron_success(self, text_to_log, added): diff --git a/api/management/commands/ingest_ns_initiatives.py b/api/management/commands/ingest_ns_initiatives.py index 47dd00523..0767b9e3b 100644 --- a/api/management/commands/ingest_ns_initiatives.py +++ b/api/management/commands/ingest_ns_initiatives.py @@ -3,6 +3,7 @@ import requests from django.conf import settings from django.core.management.base import BaseCommand +from django.db import transaction from sentry_sdk.crons import monitor from api.logger import logger @@ -10,10 +11,11 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_NS_INITIATIVES) class Command(BaseCommand): help = "Add ns initiatives" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_INITIATIVES) + @transaction.atomic def handle(self, *args, **kwargs): logger.info("Starting NS Inititatives") api_key = settings.NS_INITIATIVES_API_KEY @@ -43,11 +45,12 @@ def handle(self, *args, **kwargs): ], ) funding_data = funding_data.replace({np.nan: None}) + created_ns_initiatives_pk = [] for data in funding_data.values.tolist(): # TODO: Filter not by society name country = Country.objects.filter(society_name__iexact=data[0]).first() if country: - nsd_initiatives, created = NSDInitiatives.objects.get_or_create( + nsd_initiatives, _ = NSDInitiatives.objects.get_or_create( country=country, year=data[1], fund_type=data[2], @@ -58,13 +61,10 @@ def handle(self, *args, **kwargs): "funding_period": data[6], }, ) - if not created: - nsd_initiatives.title = data[3] - nsd_initiatives.allocation = data[5] - nsd_initiatives.funding_period = data[6] - nsd_initiatives.categories = data[4] - nsd_initiatives.save(update_fields=["title", "allocation", "funding_period", "categories"]) + created_ns_initiatives_pk.append(nsd_initiatives.pk) added += 1 + # NOTE: Delete the NSDInitiatives that are not in the source + NSDInitiatives.objects.exclude(id__in=created_ns_initiatives_pk).delete() text_to_log = "%s Ns initiatives added" % added logger.info(text_to_log) diff --git a/api/management/commands/revoke_staff_status.py b/api/management/commands/revoke_staff_status.py index 5c965ca4d..db4725e0b 100644 --- a/api/management/commands/revoke_staff_status.py +++ b/api/management/commands/revoke_staff_status.py @@ -8,7 +8,6 @@ # from registrations.views import is_valid_domain -@monitor(monitor_slug=SentryMonitor.REVOKE_STAFF_STATUS) class Command(BaseCommand): help = 'Update staff status in auth_user table according to "Read only" group' @@ -53,6 +52,7 @@ def get_ifrc_domain_users(self): # # return editors + @monitor(monitor_slug=SentryMonitor.REVOKE_STAFF_STATUS) def handle(self, *args, **options): logger.info("Moving Read only users out of staff status...") diff --git a/api/management/commands/sync_appealdocs.py b/api/management/commands/sync_appealdocs.py index 889805c49..12a27e059 100644 --- a/api/management/commands/sync_appealdocs.py +++ b/api/management/commands/sync_appealdocs.py @@ -21,7 +21,6 @@ FEDNET_SOURCE = "https://go-api.ifrc.org/Api/FedNetAppeals?Hidden=false&BaseAppealnumber=" -@monitor(monitor_slug=SentryMonitor.SYNC_APPEALDOCS) class Command(BaseCommand): help = "Ingest existing appeal documents" @@ -37,6 +36,7 @@ def parse_date(self, date_string): timeformat = "%Y-%m-%dT%H:%M:%S" return datetime.strptime(date_string[:18], timeformat).replace(tzinfo=timezone.utc) + @monitor(monitor_slug=SentryMonitor.SYNC_APPEALDOCS) def handle(self, *args, **options): logger.info("Starting appeal document ingest") diff --git a/api/management/commands/sync_molnix.py b/api/management/commands/sync_molnix.py index 9941871c2..60b483a18 100644 --- a/api/management/commands/sync_molnix.py +++ b/api/management/commands/sync_molnix.py @@ -519,10 +519,10 @@ def sync_open_positions(molnix_positions, molnix_api, countries): return messages, warnings, successful_creates -@monitor(monitor_slug=SentryMonitor.SYNC_MOLNIX) class Command(BaseCommand): help = "Sync data from Molnix API to GO db" + @monitor(monitor_slug=SentryMonitor.SYNC_MOLNIX) @transaction.atomic def handle(self, *args, **options): logger.info("Starting Sync Molnix job") diff --git a/api/management/commands/user_registration_reminder.py b/api/management/commands/user_registration_reminder.py index 6bcfb7dc7..c6ceaadae 100644 --- a/api/management/commands/user_registration_reminder.py +++ b/api/management/commands/user_registration_reminder.py @@ -10,13 +10,13 @@ from registrations.models import Pending -@monitor(monitor_slug=SentryMonitor.USER_REGISTRATION_REMINDER) class Command(BaseCommand): help = "Send reminder about the pending registrations" def diff_3_day(self): return datetime.utcnow().replace(tzinfo=timezone.utc) - timedelta(days=3) + @monitor(monitor_slug=SentryMonitor.USER_REGISTRATION_REMINDER) def handle(self, *args, **options): region_ids = Region.objects.all().values_list("id", flat=True) time_diff_3_day = self.diff_3_day() diff --git a/country_plan/management/commands/ingest_country_plan_file.py b/country_plan/management/commands/ingest_country_plan_file.py index a911c505a..ad594539a 100644 --- a/country_plan/management/commands/ingest_country_plan_file.py +++ b/country_plan/management/commands/ingest_country_plan_file.py @@ -55,7 +55,6 @@ def _get_filename_from_headers(resp): return resp.headers.get("Content-Type"), _get_filename_from_headers(resp) -@monitor(monitor_slug=SentryMonitor.INGEST_COUNTRY_PLAN_FILE) class Command(BaseCommand): @staticmethod def load_file_to_country_plan(country_plan: CountryPlan, url: str, filename: str, field_name: str): @@ -121,6 +120,7 @@ def load(self, url: str, file_field: str, field_inserted_date_field: str): except Exception: logger.error("Could not update countries plan", exc_info=True) + @monitor(monitor_slug=SentryMonitor.INGEST_COUNTRY_PLAN_FILE) def handle(self, **_): # Public self.stdout.write("Fetching data for country plans:: PUBLIC") diff --git a/databank/management/commands/FDRS_INCOME.py b/databank/management/commands/FDRS_INCOME.py index 69d8a8427..523d30a93 100644 --- a/databank/management/commands/FDRS_INCOME.py +++ b/databank/management/commands/FDRS_INCOME.py @@ -2,7 +2,9 @@ import requests from django.conf import settings +from django.core.management import call_command from django.core.management.base import BaseCommand +from django.db import transaction from sentry_sdk.crons import monitor from databank.models import CountryOverview, FDRSIncome, FDRSIndicator @@ -11,11 +13,15 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.FDRS_INCOME) class Command(BaseCommand): help = "Import FDRS income data" + @monitor(monitor_slug=SentryMonitor.FDRS_INCOME) + @transaction.atomic def handle(self, *args, **kwargs): + # NOTE: Loading FDRS indicators + call_command("loaddata", "fdrs_indicator.json", verbosity=2) + fdrs_indicator_enum_data = { "h_gov_CHF": "Home Government", "f_gov_CHF": "Foreign Government", @@ -45,6 +51,7 @@ def handle(self, *args, **kwargs): return fdrs_entities.raise_for_status() fdrs_entities = fdrs_entities.json() + created_fdrs_income_ids = [] for d in fdrs_entities["data"]: indicator = next(iter(d.values())) fdrs_indicator = map_indicators[fdrs_indicator_enum_data[indicator]] @@ -56,7 +63,8 @@ def handle(self, *args, **kwargs): overview=overview, indicator=fdrs_indicator, date=str(income["year"]) + "-01-01", + defaults={"value": income_value}, ) - fdrs_income.value = income_value - # TODO: Use bulk - fdrs_income.save(update_fields=("value",)) + created_fdrs_income_ids.append(fdrs_income.pk) + # NOTE: Delete the FDRSIncome that are not in the source + FDRSIncome.objects.filter(overview=overview).exclude(id__in=created_fdrs_income_ids).delete() diff --git a/databank/management/commands/fdrs_annual_income.py b/databank/management/commands/fdrs_annual_income.py index fca1cf7ca..cb3541c6d 100644 --- a/databank/management/commands/fdrs_annual_income.py +++ b/databank/management/commands/fdrs_annual_income.py @@ -3,7 +3,7 @@ import requests from django.conf import settings from django.core.management.base import BaseCommand -from django.db import models +from django.db import models, transaction from sentry_sdk.crons import monitor from databank.models import CountryOverview, FDRSAnnualIncome @@ -12,10 +12,11 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.FDRS_ANNUAL_INCOME) class Command(BaseCommand): help = "Import FDRS income data" + @monitor(monitor_slug=SentryMonitor.FDRS_ANNUAL_INCOME) + @transaction.atomic def handle(self, *args, **kwargs): overview_qs = CountryOverview.objects.annotate( country_fdrd=models.F("country__fdrs"), @@ -44,14 +45,16 @@ def handle(self, *args, **kwargs): if len(income_list) == 0: continue + created_fdrs_income_ids = [] for income in income_list: income_value = income["value"] fdrs_annual_income, _ = FDRSAnnualIncome.objects.get_or_create( overview=overview, date=str(income["year"]) + "-01-01", + defaults={"value": income_value}, ) - fdrs_annual_income.value = income_value - fdrs_annual_income.save(update_fields=("value",)) + created_fdrs_income_ids.append(fdrs_annual_income.pk) fdrs_data_count += 1 - + # NOTE: Deleting the FDRSAnnualIncome objects that are not in the source + FDRSAnnualIncome.objects.filter(overview=overview).exclude(id__in=created_fdrs_income_ids).delete() logger.info(f"Successfully created {fdrs_data_count} country data") diff --git a/databank/management/commands/ingest_acaps.py b/databank/management/commands/ingest_acaps.py index d764f9737..1789f5a70 100644 --- a/databank/management/commands/ingest_acaps.py +++ b/databank/management/commands/ingest_acaps.py @@ -13,10 +13,10 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_ACAPS) class Command(BaseCommand): help = "Add Acaps seasonal calender data" + @monitor(monitor_slug=SentryMonitor.INGEST_ACAPS) @transaction.atomic def load_country(self, overview): # Remove all existing Seasonal Calendar data for this country diff --git a/databank/management/commands/ingest_climate.py b/databank/management/commands/ingest_climate.py index bf99e844e..56dccb840 100644 --- a/databank/management/commands/ingest_climate.py +++ b/databank/management/commands/ingest_climate.py @@ -2,7 +2,7 @@ import requests from django.core.management.base import BaseCommand -from django.db import models +from django.db import models, transaction from sentry_sdk.crons import monitor from api.models import CountryType @@ -12,10 +12,11 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.INGEST_CLIMATE) class Command(BaseCommand): help = "Add minimum, maximum and Average temperature of country temperature data from source api" + @monitor(monitor_slug=SentryMonitor.INGEST_CLIMATE) + @transaction.atomic def handle(self, *args, **options): overview_qs = CountryOverview.objects.filter( country__record_type=CountryType.COUNTRY, @@ -46,47 +47,41 @@ def handle(self, *args, **options): max_temp = data.get("tasmax", {}) merged_data = { country: { - date: ( - precipitation[country][date], - average_temp[country][date], - min_temp[country][date], - max_temp[country][date], - ) + date: { + "precipitation": precipitation[country][date], + "average_temp": average_temp[country][date], + "min_temp": min_temp[country][date], + "max_temp": max_temp[country][date], + } for date in precipitation[country] } for country in precipitation } - + created_country_key_climate_ids = [] for value in merged_data.values(): for k, v in value.items(): - if v[0] is None or v[1] is None or v[2] is None or v[3] is None: + if ( + v.get("precipitation") is None + or v.get("average_temp") is None + or v.get("min_temp") is None + or v.get("max_temp") is None + ): continue year_month = k.split("-") - country_key_climate, created = CountryKeyClimate.objects.get_or_create( + country_key_climate, _ = CountryKeyClimate.objects.get_or_create( overview=overview, year=year_month[0], month=year_month[1], defaults={ - "precipitation": v[0], - "avg_temp": v[1], - "min_temp": v[2], - "max_temp": v[3], + "precipitation": v.get("precipitation"), + "avg_temp": v.get("average_temp"), + "min_temp": v.get("min_temp"), + "max_temp": v.get("max_temp"), }, ) - if not created: - country_key_climate.max_temp = v[3] - country_key_climate.min_temp = v[2] - country_key_climate.avg_temp = v[1] - country_key_climate.precipitation = v[0] - # TODO: Use bulk manager - country_key_climate.save( - update_fields=( - "max_temp", - "min_temp", - "avg_temp", - "precipitation", - ) - ) + created_country_key_climate_ids.append(country_key_climate.pk) + # NOTE: Deleting the CountryKeyclimate that are not in the source + CountryKeyClimate.objects.filter(overview=overview).exclude(id__in=created_country_key_climate_ids).delete() except Exception: logger.error("Error in ingesting climate data", exc_info=True) continue diff --git a/databank/management/commands/ingest_databank.py b/databank/management/commands/ingest_databank.py index 66be701fb..245ab1674 100644 --- a/databank/management/commands/ingest_databank.py +++ b/databank/management/commands/ingest_databank.py @@ -26,7 +26,6 @@ ] -@monitor(monitor_slug=SentryMonitor.INGEST_DATABANK) class Command(BaseCommand): def load(self): """ @@ -111,6 +110,7 @@ def load(self): } ) + @monitor(monitor_slug=SentryMonitor.INGEST_DATABANK) def handle(self, *args, **kwargs): start = datetime.datetime.now() self.load() diff --git a/databank/management/commands/ingest_hdr.py b/databank/management/commands/ingest_hdr.py index 28ec9e6cf..97f5ce60a 100644 --- a/databank/management/commands/ingest_hdr.py +++ b/databank/management/commands/ingest_hdr.py @@ -2,7 +2,7 @@ import requests from django.core.management.base import BaseCommand -from django.db import models +from django.db import models, transaction from sentry_sdk.crons import monitor from databank.models import CountryOverview @@ -11,10 +11,11 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.INGEST_HDR) class Command(BaseCommand): help = "Add HDR GII data" + @monitor(monitor_slug=SentryMonitor.INGEST_HDR) + @transaction.atomic def handle(self, *args, **kwargs): overview_qs = CountryOverview.objects.annotate( country_iso3=models.F("country__iso3"), diff --git a/databank/management/commands/ingest_unicef.py b/databank/management/commands/ingest_unicef.py index 0ce0c3602..f90396efd 100644 --- a/databank/management/commands/ingest_unicef.py +++ b/databank/management/commands/ingest_unicef.py @@ -10,10 +10,10 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.INGEST_UNICEF) class Command(BaseCommand): help = "Add Unicef population data" + @monitor(monitor_slug=SentryMonitor.INGEST_UNICEF) def handle(self, *args, **kwargs): for overview in CO.objects.all(): UNICEF_API = f"https://sdmx.data.unicef.org/ws/public/sdmxapi/rest/data/UNICEF,DM,1.0/{overview.country.iso3}.DM_POP_U18._T._T.?format=sdmx-json" # noqa: E501 diff --git a/databank/management/commands/ingest_worldbank.py b/databank/management/commands/ingest_worldbank.py index fa0b881c2..ba2422dc0 100644 --- a/databank/management/commands/ingest_worldbank.py +++ b/databank/management/commands/ingest_worldbank.py @@ -15,10 +15,10 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.INGEST_WORLDBANK) class Command(BaseCommand): help = "Add Acaps seasonal calendar data" + @monitor(monitor_slug=SentryMonitor.INGEST_WORLDBANK) def handle(self, *args, **kwargs): world_bank_indicator_map = ( ("SP.POP.TOTL", CO.world_bank_population), diff --git a/deployments/management/commands/update_project_status.py b/deployments/management/commands/update_project_status.py index 171ffe665..d3c047a36 100644 --- a/deployments/management/commands/update_project_status.py +++ b/deployments/management/commands/update_project_status.py @@ -21,11 +21,11 @@ ) -@monitor(monitor_slug=SentryMonitor.UPDATE_PROJECT_STATUS) class Command(BaseCommand): help = "Update project status using start/end date" + @monitor(monitor_slug=SentryMonitor.UPDATE_PROJECT_STATUS) def handle(self, *args, **options): now = timezone.now().date() diff --git a/main/sentry.py b/main/sentry.py index ce4146bfa..d393c9133 100644 --- a/main/sentry.py +++ b/main/sentry.py @@ -161,3 +161,41 @@ def validate_config(cls): ) ) ) + + +class SentryMonitorConfig: + """ + Custom config for SentryMonitor + https://docs.sentry.io/product/crons/getting-started/http/#creating-or-updating-a-monitor-through-a-check-in-optional + """ + + MAX_RUNTIME_DEFAULT = 30 # Our default is 30 min + + FAILURE_THRESHOLD_DEFAULT = 1 + FAILURE_THRESHOLD = { + # NOTE: INDEX_AND_NOTIFY runs every 5 minutes; we allow up to 6 consecutive failures + SentryMonitor.INDEX_AND_NOTIFY: 6, + } + + @classmethod + def get_checkin_margin(cls, _: SentryMonitor) -> int: + """ + The amount of time (in minutes) [Sentry Default 1 min] + Sentry should wait for your checkin before it's considered missed ("grace period") + """ + return cls.MAX_RUNTIME_DEFAULT + + @classmethod + def get_failure_issue_threshold(cls, enum: SentryMonitor) -> int: + """ + The number of consecutive failed check-ins it takes before an issue is created. Optional. + """ + return cls.FAILURE_THRESHOLD.get(enum, cls.FAILURE_THRESHOLD_DEFAULT) + + @classmethod + def get_recovery_threshold(cls, _: SentryMonitor) -> int: + """ + [Sentry Default 1] + The number of consecutive OK check-ins it takes before an issue is resolved. Optional. + """ + return 1 diff --git a/notifications/management/commands/update_surge_alert_status.py b/notifications/management/commands/update_surge_alert_status.py index d42397b80..0fe04dbcb 100644 --- a/notifications/management/commands/update_surge_alert_status.py +++ b/notifications/management/commands/update_surge_alert_status.py @@ -12,7 +12,6 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.UPDATE_SURGE_ALERT_STATUS) class Command(BaseCommand): """ Updating the Surge Alert Status according: @@ -23,6 +22,7 @@ class Command(BaseCommand): help = "Update surge alert status" + @monitor(monitor_slug=SentryMonitor.UPDATE_SURGE_ALERT_STATUS) def handle(self, *args, **options): now = timezone.now() try: