diff --git a/api/management/commands/index_and_notify.py b/api/management/commands/index_and_notify.py index 246d9199c..241f6e22a 100644 --- a/api/management/commands/index_and_notify.py +++ b/api/management/commands/index_and_notify.py @@ -67,7 +67,6 @@ } -@monitor(monitor_slug=SentryMonitor.INDEX_AND_NOTIFY) class Command(BaseCommand): help = "Index and send notifications about new/changed records" @@ -920,6 +919,7 @@ def check_ingest_issues(self, having_ingest_issue): + ", notification sent to IM team" ) + @monitor(monitor_slug=SentryMonitor.INDEX_AND_NOTIFY) def handle(self, *args, **options): if self.is_digest_mode(): time_diff = self.diff_1_week() # in digest mode (once a week, for new_entities only) we use a bigger interval diff --git a/api/management/commands/ingest_appeals.py b/api/management/commands/ingest_appeals.py index c032f8e75..53d7189be 100644 --- a/api/management/commands/ingest_appeals.py +++ b/api/management/commands/ingest_appeals.py @@ -33,7 +33,6 @@ GEC_CODES = GECCode.objects.select_related("country").all() -@monitor(monitor_slug=SentryMonitor.INGEST_APPEALS) class Command(BaseCommand): help = "Add new entries from Access database file" @@ -292,6 +291,7 @@ def parse_appeal_record(self, r, **options): return fields + @monitor(monitor_slug=SentryMonitor.INGEST_APPEALS) def handle(self, *args, **options): logger.info("Starting appeals ingest") start_appeals_count = Appeal.objects.all().count() diff --git a/api/management/commands/ingest_disaster_law.py b/api/management/commands/ingest_disaster_law.py index c900b5f4b..d114aeca7 100644 --- a/api/management/commands/ingest_disaster_law.py +++ b/api/management/commands/ingest_disaster_law.py @@ -8,10 +8,10 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_DISASTER_LAW) class Command(BaseCommand): help = "Add ICRC data" + @monitor(monitor_slug=SentryMonitor.INGEST_DISASTER_LAW) def handle(self, *args, **kwargs): logger.info("Starting Disaster Law data") home_url = "https://disasterlaw.ifrc.org/" diff --git a/api/management/commands/ingest_icrc.py b/api/management/commands/ingest_icrc.py index bf3f47663..55f71939c 100644 --- a/api/management/commands/ingest_icrc.py +++ b/api/management/commands/ingest_icrc.py @@ -8,10 +8,10 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_ICRC) class Command(BaseCommand): help = "Add ICRC data" + @monitor(monitor_slug=SentryMonitor.INGEST_ICRC) def handle(self, *args, **kwargs): logger.info("Strating ICRC data ingest") HEADERS = { diff --git a/api/management/commands/ingest_ns_capacity.py b/api/management/commands/ingest_ns_capacity.py index c0c65853b..edffdfcb5 100644 --- a/api/management/commands/ingest_ns_capacity.py +++ b/api/management/commands/ingest_ns_capacity.py @@ -8,10 +8,10 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_NS_CAPACITY) class Command(BaseCommand): help = "Add ns contact details" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_CAPACITY) def handle(self, *args, **kwargs): logger.info("Starting NS Contacts") diff --git a/api/management/commands/ingest_ns_contact.py b/api/management/commands/ingest_ns_contact.py index 5af14f47b..f95374292 100644 --- a/api/management/commands/ingest_ns_contact.py +++ b/api/management/commands/ingest_ns_contact.py @@ -13,10 +13,10 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_NS_CONTACT) class Command(BaseCommand): help = "Add ns contact details" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_CONTACT) def handle(self, *args, **kwargs): logger.info("Starting NS Contacts") url = "https://go-api.ifrc.org/" diff --git a/api/management/commands/ingest_ns_directory.py b/api/management/commands/ingest_ns_directory.py index 0a5de0bbe..798cfec9d 100644 --- a/api/management/commands/ingest_ns_directory.py +++ b/api/management/commands/ingest_ns_directory.py @@ -10,10 +10,10 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_NS_DIRECTORY) class Command(BaseCommand): help = "Add ns contact details" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_DIRECTORY) def handle(self, *args, **kwargs): def postprocessor(path, key, value): if key == "@i:nil": diff --git a/api/management/commands/ingest_ns_document.py b/api/management/commands/ingest_ns_document.py index fca3d3280..a850c2e4e 100644 --- a/api/management/commands/ingest_ns_document.py +++ b/api/management/commands/ingest_ns_document.py @@ -12,10 +12,10 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_NS_DOCUMENT) class Command(BaseCommand): help = "Add ns documents" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_DOCUMENT) def handle(self, *args, **kwargs): logger.info("Starting NS Key Documents") diff --git a/api/management/commands/ingest_ns_initiatives.py b/api/management/commands/ingest_ns_initiatives.py index 47dd00523..a48e3730e 100644 --- a/api/management/commands/ingest_ns_initiatives.py +++ b/api/management/commands/ingest_ns_initiatives.py @@ -10,10 +10,10 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_NS_INITIATIVES) class Command(BaseCommand): help = "Add ns initiatives" + @monitor(monitor_slug=SentryMonitor.INGEST_NS_INITIATIVES) def handle(self, *args, **kwargs): logger.info("Starting NS Inititatives") api_key = settings.NS_INITIATIVES_API_KEY diff --git a/api/management/commands/revoke_staff_status.py b/api/management/commands/revoke_staff_status.py index 5c965ca4d..db4725e0b 100644 --- a/api/management/commands/revoke_staff_status.py +++ b/api/management/commands/revoke_staff_status.py @@ -8,7 +8,6 @@ # from registrations.views import is_valid_domain -@monitor(monitor_slug=SentryMonitor.REVOKE_STAFF_STATUS) class Command(BaseCommand): help = 'Update staff status in auth_user table according to "Read only" group' @@ -53,6 +52,7 @@ def get_ifrc_domain_users(self): # # return editors + @monitor(monitor_slug=SentryMonitor.REVOKE_STAFF_STATUS) def handle(self, *args, **options): logger.info("Moving Read only users out of staff status...") diff --git a/api/management/commands/sync_appealdocs.py b/api/management/commands/sync_appealdocs.py index 889805c49..12a27e059 100644 --- a/api/management/commands/sync_appealdocs.py +++ b/api/management/commands/sync_appealdocs.py @@ -21,7 +21,6 @@ FEDNET_SOURCE = "https://go-api.ifrc.org/Api/FedNetAppeals?Hidden=false&BaseAppealnumber=" -@monitor(monitor_slug=SentryMonitor.SYNC_APPEALDOCS) class Command(BaseCommand): help = "Ingest existing appeal documents" @@ -37,6 +36,7 @@ def parse_date(self, date_string): timeformat = "%Y-%m-%dT%H:%M:%S" return datetime.strptime(date_string[:18], timeformat).replace(tzinfo=timezone.utc) + @monitor(monitor_slug=SentryMonitor.SYNC_APPEALDOCS) def handle(self, *args, **options): logger.info("Starting appeal document ingest") diff --git a/api/management/commands/sync_molnix.py b/api/management/commands/sync_molnix.py index 9929263f5..aeebb305f 100644 --- a/api/management/commands/sync_molnix.py +++ b/api/management/commands/sync_molnix.py @@ -519,10 +519,10 @@ def sync_open_positions(molnix_positions, molnix_api, countries): return messages, warnings, successful_creates -@monitor(monitor_slug=SentryMonitor.SYNC_MOLNIX) class Command(BaseCommand): help = "Sync data from Molnix API to GO db" + @monitor(monitor_slug=SentryMonitor.SYNC_MOLNIX) @transaction.atomic def handle(self, *args, **options): logger.info("Starting Sync Molnix job") diff --git a/api/management/commands/user_registration_reminder.py b/api/management/commands/user_registration_reminder.py index 6bcfb7dc7..c6ceaadae 100644 --- a/api/management/commands/user_registration_reminder.py +++ b/api/management/commands/user_registration_reminder.py @@ -10,13 +10,13 @@ from registrations.models import Pending -@monitor(monitor_slug=SentryMonitor.USER_REGISTRATION_REMINDER) class Command(BaseCommand): help = "Send reminder about the pending registrations" def diff_3_day(self): return datetime.utcnow().replace(tzinfo=timezone.utc) - timedelta(days=3) + @monitor(monitor_slug=SentryMonitor.USER_REGISTRATION_REMINDER) def handle(self, *args, **options): region_ids = Region.objects.all().values_list("id", flat=True) time_diff_3_day = self.diff_3_day() diff --git a/country_plan/management/commands/ingest_country_plan_file.py b/country_plan/management/commands/ingest_country_plan_file.py index a911c505a..ad594539a 100644 --- a/country_plan/management/commands/ingest_country_plan_file.py +++ b/country_plan/management/commands/ingest_country_plan_file.py @@ -55,7 +55,6 @@ def _get_filename_from_headers(resp): return resp.headers.get("Content-Type"), _get_filename_from_headers(resp) -@monitor(monitor_slug=SentryMonitor.INGEST_COUNTRY_PLAN_FILE) class Command(BaseCommand): @staticmethod def load_file_to_country_plan(country_plan: CountryPlan, url: str, filename: str, field_name: str): @@ -121,6 +120,7 @@ def load(self, url: str, file_field: str, field_inserted_date_field: str): except Exception: logger.error("Could not update countries plan", exc_info=True) + @monitor(monitor_slug=SentryMonitor.INGEST_COUNTRY_PLAN_FILE) def handle(self, **_): # Public self.stdout.write("Fetching data for country plans:: PUBLIC") diff --git a/databank/management/commands/FDRS_INCOME.py b/databank/management/commands/FDRS_INCOME.py index 69d8a8427..a6159293c 100644 --- a/databank/management/commands/FDRS_INCOME.py +++ b/databank/management/commands/FDRS_INCOME.py @@ -11,10 +11,10 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.FDRS_INCOME) class Command(BaseCommand): help = "Import FDRS income data" + @monitor(monitor_slug=SentryMonitor.FDRS_INCOME) def handle(self, *args, **kwargs): fdrs_indicator_enum_data = { "h_gov_CHF": "Home Government", diff --git a/databank/management/commands/fdrs_annual_income.py b/databank/management/commands/fdrs_annual_income.py index fca1cf7ca..fa779a218 100644 --- a/databank/management/commands/fdrs_annual_income.py +++ b/databank/management/commands/fdrs_annual_income.py @@ -12,10 +12,10 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.FDRS_ANNUAL_INCOME) class Command(BaseCommand): help = "Import FDRS income data" + @monitor(monitor_slug=SentryMonitor.FDRS_ANNUAL_INCOME) def handle(self, *args, **kwargs): overview_qs = CountryOverview.objects.annotate( country_fdrd=models.F("country__fdrs"), diff --git a/databank/management/commands/ingest_acaps.py b/databank/management/commands/ingest_acaps.py index d764f9737..1789f5a70 100644 --- a/databank/management/commands/ingest_acaps.py +++ b/databank/management/commands/ingest_acaps.py @@ -13,10 +13,10 @@ from main.sentry import SentryMonitor -@monitor(monitor_slug=SentryMonitor.INGEST_ACAPS) class Command(BaseCommand): help = "Add Acaps seasonal calender data" + @monitor(monitor_slug=SentryMonitor.INGEST_ACAPS) @transaction.atomic def load_country(self, overview): # Remove all existing Seasonal Calendar data for this country diff --git a/databank/management/commands/ingest_climate.py b/databank/management/commands/ingest_climate.py index bf99e844e..a0f5dd37f 100644 --- a/databank/management/commands/ingest_climate.py +++ b/databank/management/commands/ingest_climate.py @@ -12,10 +12,10 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.INGEST_CLIMATE) class Command(BaseCommand): help = "Add minimum, maximum and Average temperature of country temperature data from source api" + @monitor(monitor_slug=SentryMonitor.INGEST_CLIMATE) def handle(self, *args, **options): overview_qs = CountryOverview.objects.filter( country__record_type=CountryType.COUNTRY, diff --git a/databank/management/commands/ingest_databank.py b/databank/management/commands/ingest_databank.py index 66be701fb..245ab1674 100644 --- a/databank/management/commands/ingest_databank.py +++ b/databank/management/commands/ingest_databank.py @@ -26,7 +26,6 @@ ] -@monitor(monitor_slug=SentryMonitor.INGEST_DATABANK) class Command(BaseCommand): def load(self): """ @@ -111,6 +110,7 @@ def load(self): } ) + @monitor(monitor_slug=SentryMonitor.INGEST_DATABANK) def handle(self, *args, **kwargs): start = datetime.datetime.now() self.load() diff --git a/databank/management/commands/ingest_hdr.py b/databank/management/commands/ingest_hdr.py index 28ec9e6cf..6951fd1ad 100644 --- a/databank/management/commands/ingest_hdr.py +++ b/databank/management/commands/ingest_hdr.py @@ -11,10 +11,10 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.INGEST_HDR) class Command(BaseCommand): help = "Add HDR GII data" + @monitor(monitor_slug=SentryMonitor.INGEST_HDR) def handle(self, *args, **kwargs): overview_qs = CountryOverview.objects.annotate( country_iso3=models.F("country__iso3"), diff --git a/databank/management/commands/ingest_unicef.py b/databank/management/commands/ingest_unicef.py index 0ce0c3602..f90396efd 100644 --- a/databank/management/commands/ingest_unicef.py +++ b/databank/management/commands/ingest_unicef.py @@ -10,10 +10,10 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.INGEST_UNICEF) class Command(BaseCommand): help = "Add Unicef population data" + @monitor(monitor_slug=SentryMonitor.INGEST_UNICEF) def handle(self, *args, **kwargs): for overview in CO.objects.all(): UNICEF_API = f"https://sdmx.data.unicef.org/ws/public/sdmxapi/rest/data/UNICEF,DM,1.0/{overview.country.iso3}.DM_POP_U18._T._T.?format=sdmx-json" # noqa: E501 diff --git a/databank/management/commands/ingest_worldbank.py b/databank/management/commands/ingest_worldbank.py index fa0b881c2..ba2422dc0 100644 --- a/databank/management/commands/ingest_worldbank.py +++ b/databank/management/commands/ingest_worldbank.py @@ -15,10 +15,10 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.INGEST_WORLDBANK) class Command(BaseCommand): help = "Add Acaps seasonal calendar data" + @monitor(monitor_slug=SentryMonitor.INGEST_WORLDBANK) def handle(self, *args, **kwargs): world_bank_indicator_map = ( ("SP.POP.TOTL", CO.world_bank_population), diff --git a/deployments/management/commands/update_project_status.py b/deployments/management/commands/update_project_status.py index 171ffe665..d3c047a36 100644 --- a/deployments/management/commands/update_project_status.py +++ b/deployments/management/commands/update_project_status.py @@ -21,11 +21,11 @@ ) -@monitor(monitor_slug=SentryMonitor.UPDATE_PROJECT_STATUS) class Command(BaseCommand): help = "Update project status using start/end date" + @monitor(monitor_slug=SentryMonitor.UPDATE_PROJECT_STATUS) def handle(self, *args, **options): now = timezone.now().date() diff --git a/main/sentry.py b/main/sentry.py index c69ee55a6..30fa518a9 100644 --- a/main/sentry.py +++ b/main/sentry.py @@ -161,3 +161,49 @@ def validate_config(cls): ) ) ) + + +class SentryMonitorConfig: + """ + Custom config for SentryMonitor + https://docs.sentry.io/product/crons/getting-started/http/#creating-or-updating-a-monitor-through-a-check-in-optional + """ + + MAX_RUNTIME_DEFAULT = 30 # Our default is 30 min + FAILURE_THRESHOLD_DEFAULT = 1 + FAILURE_THRESHOLD = { + SentryMonitor.INDEX_AND_NOTIFY: 6, + } + + @classmethod + def get_checkin_margin(cls, _: SentryMonitor) -> int: + """ + The amount of time (in minutes) [Sentry Default 1 min] + Sentry should wait for your checkin before it's considered missed ("grace period") + """ + # We have low number of CPU in worker instance, which might cause tasks to not started on the specified time + # So using the MAX_RUNTIME_DEFAULT + return cls.MAX_RUNTIME_DEFAULT + + @classmethod + def get_failure_issue_threshold(cls, _: SentryMonitor) -> int: + """ + [Sentry Default 1] + The number of consecutive failed check-ins it takes before an issue is created. Optional. + """ + return cls.FAILURE_THRESHOLD_DEFAULT + + @classmethod + def get_recovery_threshold(cls, _: SentryMonitor) -> int: + """ + [Sentry Default 1] + The number of consecutive OK check-ins it takes before an issue is resolved. Optional. + """ + return 1 + + @classmethod + def get_failure_threshold(cls, enum: SentryMonitor) -> int: + """ + The number of consecutive failed check-ins it takes before an issue is created. Optional. + """ + return cls.FAILURE_THRESHOLD.get(enum, cls.FAILURE_THRESHOLD_DEFAULT) diff --git a/notifications/management/commands/update_surge_alert_status.py b/notifications/management/commands/update_surge_alert_status.py index d42397b80..0fe04dbcb 100644 --- a/notifications/management/commands/update_surge_alert_status.py +++ b/notifications/management/commands/update_surge_alert_status.py @@ -12,7 +12,6 @@ logger = logging.getLogger(__name__) -@monitor(monitor_slug=SentryMonitor.UPDATE_SURGE_ALERT_STATUS) class Command(BaseCommand): """ Updating the Surge Alert Status according: @@ -23,6 +22,7 @@ class Command(BaseCommand): help = "Update surge alert status" + @monitor(monitor_slug=SentryMonitor.UPDATE_SURGE_ALERT_STATUS) def handle(self, *args, **options): now = timezone.now() try: