Add configuration for thresholds and runtime

api/management/commands/index_and_notify.py

@@ -67,7 +67,6 @@
 }
 
 
-@monitor(monitor_slug=SentryMonitor.INDEX_AND_NOTIFY)
 class Command(BaseCommand):
     help = "Index and send notifications about new/changed records"
 
@@ -920,6 +919,7 @@ def check_ingest_issues(self, having_ingest_issue):
                 + ", notification sent to IM team"
             )
 
+    @monitor(monitor_slug=SentryMonitor.INDEX_AND_NOTIFY)
     def handle(self, *args, **options):
         if self.is_digest_mode():
             time_diff = self.diff_1_week()  # in digest mode (once a week, for new_entities only) we use a bigger interval

api/management/commands/ingest_appeals.py

@@ -33,7 +33,6 @@
 GEC_CODES = GECCode.objects.select_related("country").all()
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_APPEALS)
 class Command(BaseCommand):
     help = "Add new entries from Access database file"
 
@@ -292,6 +291,7 @@ def parse_appeal_record(self, r, **options):
 
         return fields
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_APPEALS)
     def handle(self, *args, **options):
         logger.info("Starting appeals ingest")
         start_appeals_count = Appeal.objects.all().count()

api/management/commands/ingest_disaster_law.py

@@ -8,10 +8,10 @@
 from main.sentry import SentryMonitor
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_DISASTER_LAW)
 class Command(BaseCommand):
     help = "Add ICRC data"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_DISASTER_LAW)
     def handle(self, *args, **kwargs):
         logger.info("Starting Disaster Law data")
         home_url = "https://disasterlaw.ifrc.org/"

api/management/commands/ingest_icrc.py

@@ -8,10 +8,10 @@
 from main.sentry import SentryMonitor
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_ICRC)
 class Command(BaseCommand):
     help = "Add ICRC data"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_ICRC)
     def handle(self, *args, **kwargs):
         logger.info("Strating ICRC data ingest")
         HEADERS = {

api/management/commands/ingest_ns_capacity.py

@@ -8,10 +8,10 @@
 from main.sentry import SentryMonitor
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_NS_CAPACITY)
 class Command(BaseCommand):
     help = "Add ns contact details"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_NS_CAPACITY)
     def handle(self, *args, **kwargs):
         logger.info("Starting NS Contacts")
 

api/management/commands/ingest_ns_contact.py

@@ -13,10 +13,10 @@
 from main.sentry import SentryMonitor
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_NS_CONTACT)
 class Command(BaseCommand):
     help = "Add ns contact details"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_NS_CONTACT)
     def handle(self, *args, **kwargs):
         logger.info("Starting NS Contacts")
         url = "https://go-api.ifrc.org/"

api/management/commands/ingest_ns_directory.py

@@ -10,10 +10,10 @@
 from main.sentry import SentryMonitor
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_NS_DIRECTORY)
 class Command(BaseCommand):
     help = "Add ns contact details"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_NS_DIRECTORY)
     def handle(self, *args, **kwargs):
         def postprocessor(path, key, value):
             if key == "@i:nil":

api/management/commands/ingest_ns_document.py

@@ -12,10 +12,10 @@
 from main.sentry import SentryMonitor
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_NS_DOCUMENT)
 class Command(BaseCommand):
     help = "Add ns documents"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_NS_DOCUMENT)
     def handle(self, *args, **kwargs):
         logger.info("Starting NS Key Documents")
 

api/management/commands/ingest_ns_initiatives.py

@@ -10,10 +10,10 @@
 from main.sentry import SentryMonitor
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_NS_INITIATIVES)
 class Command(BaseCommand):
     help = "Add ns initiatives"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_NS_INITIATIVES)
     def handle(self, *args, **kwargs):
         logger.info("Starting NS Inititatives")
         api_key = settings.NS_INITIATIVES_API_KEY

api/management/commands/revoke_staff_status.py

@@ -8,7 +8,6 @@
 # from registrations.views import is_valid_domain
 
 
-@monitor(monitor_slug=SentryMonitor.REVOKE_STAFF_STATUS)
 class Command(BaseCommand):
     help = 'Update staff status in auth_user table according to "Read only" group'
 
@@ -53,6 +52,7 @@ def get_ifrc_domain_users(self):
     #
     #       return editors
 
+    @monitor(monitor_slug=SentryMonitor.REVOKE_STAFF_STATUS)
     def handle(self, *args, **options):
         logger.info("Moving Read only users out of staff status...")
 

api/management/commands/sync_appealdocs.py

@@ -21,7 +21,6 @@
 FEDNET_SOURCE = "https://go-api.ifrc.org/Api/FedNetAppeals?Hidden=false&BaseAppealnumber="
 
 
-@monitor(monitor_slug=SentryMonitor.SYNC_APPEALDOCS)
 class Command(BaseCommand):
     help = "Ingest existing appeal documents"
 
@@ -37,6 +36,7 @@ def parse_date(self, date_string):
         timeformat = "%Y-%m-%dT%H:%M:%S"
         return datetime.strptime(date_string[:18], timeformat).replace(tzinfo=timezone.utc)
 
+    @monitor(monitor_slug=SentryMonitor.SYNC_APPEALDOCS)
     def handle(self, *args, **options):
         logger.info("Starting appeal document ingest")
 

api/management/commands/sync_molnix.py

@@ -519,10 +519,10 @@ def sync_open_positions(molnix_positions, molnix_api, countries):
     return messages, warnings, successful_creates
 
 
-@monitor(monitor_slug=SentryMonitor.SYNC_MOLNIX)
 class Command(BaseCommand):
     help = "Sync data from Molnix API to GO db"
 
+    @monitor(monitor_slug=SentryMonitor.SYNC_MOLNIX)
     @transaction.atomic
     def handle(self, *args, **options):
         logger.info("Starting Sync Molnix job")

api/management/commands/user_registration_reminder.py

@@ -10,13 +10,13 @@
 from registrations.models import Pending
 
 
-@monitor(monitor_slug=SentryMonitor.USER_REGISTRATION_REMINDER)
 class Command(BaseCommand):
     help = "Send reminder about the pending registrations"
 
     def diff_3_day(self):
         return datetime.utcnow().replace(tzinfo=timezone.utc) - timedelta(days=3)
 
+    @monitor(monitor_slug=SentryMonitor.USER_REGISTRATION_REMINDER)
     def handle(self, *args, **options):
         region_ids = Region.objects.all().values_list("id", flat=True)
         time_diff_3_day = self.diff_3_day()

country_plan/management/commands/ingest_country_plan_file.py

@@ -55,7 +55,6 @@ def _get_filename_from_headers(resp):
     return resp.headers.get("Content-Type"), _get_filename_from_headers(resp)
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_COUNTRY_PLAN_FILE)
 class Command(BaseCommand):
     @staticmethod
     def load_file_to_country_plan(country_plan: CountryPlan, url: str, filename: str, field_name: str):
@@ -121,6 +120,7 @@ def load(self, url: str, file_field: str, field_inserted_date_field: str):
             except Exception:
                 logger.error("Could not update countries plan", exc_info=True)
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_COUNTRY_PLAN_FILE)
     def handle(self, **_):
         # Public
         self.stdout.write("Fetching data for country plans:: PUBLIC")

databank/management/commands/FDRS_INCOME.py

@@ -11,10 +11,10 @@
 logger = logging.getLogger(__name__)
 
 
-@monitor(monitor_slug=SentryMonitor.FDRS_INCOME)
 class Command(BaseCommand):
     help = "Import FDRS income data"
 
+    @monitor(monitor_slug=SentryMonitor.FDRS_INCOME)
     def handle(self, *args, **kwargs):
         fdrs_indicator_enum_data = {
             "h_gov_CHF": "Home Government",

databank/management/commands/fdrs_annual_income.py

@@ -12,10 +12,10 @@
 logger = logging.getLogger(__name__)
 
 
-@monitor(monitor_slug=SentryMonitor.FDRS_ANNUAL_INCOME)
 class Command(BaseCommand):
     help = "Import FDRS income data"
 
+    @monitor(monitor_slug=SentryMonitor.FDRS_ANNUAL_INCOME)
     def handle(self, *args, **kwargs):
         overview_qs = CountryOverview.objects.annotate(
             country_fdrd=models.F("country__fdrs"),

databank/management/commands/ingest_acaps.py

@@ -13,10 +13,10 @@
 from main.sentry import SentryMonitor
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_ACAPS)
 class Command(BaseCommand):
     help = "Add Acaps seasonal calender data"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_ACAPS)
     @transaction.atomic
     def load_country(self, overview):
         # Remove all existing Seasonal Calendar data for this country

databank/management/commands/ingest_climate.py

@@ -12,10 +12,10 @@
 logger = logging.getLogger(__name__)
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_CLIMATE)
 class Command(BaseCommand):
     help = "Add minimum, maximum and Average temperature of country temperature data from source api"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_CLIMATE)
     def handle(self, *args, **options):
         overview_qs = CountryOverview.objects.filter(
             country__record_type=CountryType.COUNTRY,

databank/management/commands/ingest_databank.py

@@ -26,7 +26,6 @@
 ]
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_DATABANK)
 class Command(BaseCommand):
     def load(self):
         """
@@ -111,6 +110,7 @@ def load(self):
                 }
             )
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_DATABANK)
     def handle(self, *args, **kwargs):
         start = datetime.datetime.now()
         self.load()

databank/management/commands/ingest_hdr.py

@@ -11,10 +11,10 @@
 logger = logging.getLogger(__name__)
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_HDR)
 class Command(BaseCommand):
     help = "Add HDR GII data"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_HDR)
     def handle(self, *args, **kwargs):
         overview_qs = CountryOverview.objects.annotate(
             country_iso3=models.F("country__iso3"),

databank/management/commands/ingest_unicef.py

@@ -10,10 +10,10 @@
 logger = logging.getLogger(__name__)
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_UNICEF)
 class Command(BaseCommand):
     help = "Add Unicef population data"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_UNICEF)
     def handle(self, *args, **kwargs):
         for overview in CO.objects.all():
             UNICEF_API = f"https://sdmx.data.unicef.org/ws/public/sdmxapi/rest/data/UNICEF,DM,1.0/{overview.country.iso3}.DM_POP_U18._T._T.?format=sdmx-json"  # noqa: E501

databank/management/commands/ingest_worldbank.py

@@ -15,10 +15,10 @@
 logger = logging.getLogger(__name__)
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_WORLDBANK)
 class Command(BaseCommand):
     help = "Add Acaps seasonal calendar data"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_WORLDBANK)
     def handle(self, *args, **kwargs):
         world_bank_indicator_map = (
             ("SP.POP.TOTL", CO.world_bank_population),

deployments/management/commands/update_project_status.py

@@ -21,11 +21,11 @@
 )
 
 
-@monitor(monitor_slug=SentryMonitor.UPDATE_PROJECT_STATUS)
 class Command(BaseCommand):
 
     help = "Update project status using start/end date"
 
+    @monitor(monitor_slug=SentryMonitor.UPDATE_PROJECT_STATUS)
     def handle(self, *args, **options):
         now = timezone.now().date()
 

main/sentry.py

@@ -161,3 +161,49 @@ def validate_config(cls):
                 )
             )
         )
+
+
+class SentryMonitorConfig:
+    """
+    Custom config for SentryMonitor
+    https://docs.sentry.io/product/crons/getting-started/http/#creating-or-updating-a-monitor-through-a-check-in-optional
+    """
+
+    MAX_RUNTIME_DEFAULT = 30  # Our default is 30 min
+    FAILURE_THRESHOLD_DEFAULT = 1
+    FAILURE_THRESHOLD = {
+        SentryMonitor.INDEX_AND_NOTIFY: 6,
+    }
+
+    @classmethod
+    def get_checkin_margin(cls, _: SentryMonitor) -> int:
+        """
+        The amount of time (in minutes) [Sentry Default 1 min]
+        Sentry should wait for your checkin before it's considered missed ("grace period")
+        """
+        # We have low number of CPU in worker instance, which might cause tasks to not started on the specified time
+        # So using the MAX_RUNTIME_DEFAULT
+        return cls.MAX_RUNTIME_DEFAULT
+
+    @classmethod
+    def get_failure_issue_threshold(cls, _: SentryMonitor) -> int:
+        """
+        [Sentry Default 1]
+        The number of consecutive failed check-ins it takes before an issue is created. Optional.
+        """
+        return cls.FAILURE_THRESHOLD_DEFAULT
+
+    @classmethod
+    def get_recovery_threshold(cls, _: SentryMonitor) -> int:
+        """
+        [Sentry Default 1]
+        The number of consecutive OK check-ins it takes before an issue is resolved. Optional.
+        """
+        return 1
+
+    @classmethod
+    def get_failure_threshold(cls, enum: SentryMonitor) -> int:
+        """
+        The number of consecutive failed check-ins it takes before an issue is created. Optional.
+        """
+        return cls.FAILURE_THRESHOLD.get(enum, cls.FAILURE_THRESHOLD_DEFAULT)

notifications/management/commands/update_surge_alert_status.py

@@ -12,7 +12,6 @@
 logger = logging.getLogger(__name__)
 
 
-@monitor(monitor_slug=SentryMonitor.UPDATE_SURGE_ALERT_STATUS)
 class Command(BaseCommand):
     """
     Updating the Surge Alert Status according:
@@ -23,6 +22,7 @@ class Command(BaseCommand):
 
     help = "Update surge alert status"
 
+    @monitor(monitor_slug=SentryMonitor.UPDATE_SURGE_ALERT_STATUS)
     def handle(self, *args, **options):
         now = timezone.now()
         try: