Merge pull request #2174 from IFRCGo/feature/countrypage-cronjon

Country page cronjobs scheduled to every week

api/management/commands/cron_job_monitor.py

@@ -5,7 +5,7 @@
 from django.conf import settings
 from django.core.management.base import BaseCommand
 
-from main.sentry import SentryMonitor
+from main.sentry import SentryMonitor, SentryMonitorConfig
 from main.settings import SENTRY_DSN
 
 logger = logging.getLogger(__name__)
@@ -48,6 +48,10 @@ def handle(self, *args, **options):
                         "type": "crontab",
                         "value": str(schedule),
                     },
+                    "tz": settings.TIME_ZONE,
+                    "checkin_margin": SentryMonitorConfig.get_checkin_margin(cronjob),
+                    "failure_issue_threshold": SentryMonitorConfig.get_failure_issue_threshold(cronjob),
+                    "recovery_threshold": SentryMonitorConfig.get_recovery_threshold(cronjob),
                 },
                 "environment": settings.GO_ENVIRONMENT,
                 "status": "ok",

api/management/commands/index_and_notify.py

@@ -67,7 +67,6 @@
 }
 
 
-@monitor(monitor_slug=SentryMonitor.INDEX_AND_NOTIFY)
 class Command(BaseCommand):
     help = "Index and send notifications about new/changed records"
 
@@ -920,6 +919,7 @@ def check_ingest_issues(self, having_ingest_issue):
                 + ", notification sent to IM team"
             )
 
+    @monitor(monitor_slug=SentryMonitor.INDEX_AND_NOTIFY)
     def handle(self, *args, **options):
         if self.is_digest_mode():
             time_diff = self.diff_1_week()  # in digest mode (once a week, for new_entities only) we use a bigger interval

api/management/commands/ingest_appeals.py

@@ -33,7 +33,6 @@
 GEC_CODES = GECCode.objects.select_related("country").all()
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_APPEALS)
 class Command(BaseCommand):
     help = "Add new entries from Access database file"
 
@@ -292,6 +291,7 @@ def parse_appeal_record(self, r, **options):
 
         return fields
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_APPEALS)
     def handle(self, *args, **options):
         logger.info("Starting appeals ingest")
         start_appeals_count = Appeal.objects.all().count()

api/management/commands/ingest_disaster_law.py

@@ -1,14 +1,17 @@
 import requests
 from bs4 import BeautifulSoup
 from django.core.management.base import BaseCommand
+from sentry_sdk.crons import monitor
 
 from api.logger import logger
 from api.models import Country, CronJob, CronJobStatus
+from main.sentry import SentryMonitor
 
 
 class Command(BaseCommand):
     help = "Add ICRC data"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_DISASTER_LAW)
     def handle(self, *args, **kwargs):
         logger.info("Starting Disaster Law data")
         home_url = "https://disasterlaw.ifrc.org/"

api/management/commands/ingest_icrc.py

@@ -1,17 +1,26 @@
 import requests
 from bs4 import BeautifulSoup
 from django.core.management.base import BaseCommand
+from sentry_sdk.crons import monitor
 
 from api.logger import logger
 from api.models import Country, CountryICRCPresence, CronJob, CronJobStatus
+from main.sentry import SentryMonitor
 
 
 class Command(BaseCommand):
     help = "Add ICRC data"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_ICRC)
     def handle(self, *args, **kwargs):
         logger.info("Strating ICRC data ingest")
-        response = requests.get(url="https://www.icrc.org/en/where-we-work", headers={"User-Agent": ""})
+        HEADERS = {
+            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",  # noqa
+        }
+        response = requests.get(
+            url="https://www.icrc.org/en/where-we-work",
+            headers=HEADERS,
+        )
         if response.status_code != 200:
             text_to_log = "Error querying ICRC feed at https://www.icrc.org/en/where-we-work"
             logger.error(text_to_log)
@@ -57,19 +66,19 @@ def handle(self, *args, **kwargs):
                         "Description": description,
                     }
                 )
+
         added = 0
         for data in country_list:
-            country = Country.objects.filter(name__exact=data["Country"])
-            if country.exists():
-                dict_data = {
-                    "country": country.first(),
-                    "icrc_presence": data["ICRC presence"],
-                    "url": data["URL"],
-                    "key_operation": data["Key operation"],
-                    "description": data["Description"],
-                }
+            country = Country.objects.filter(name__exact=data["Country"]).first()
+            if country:
+                country_icrc_presence, _ = CountryICRCPresence.objects.get_or_create(country=country)
+
+                country_icrc_presence.icrc_presence = data["ICRC presence"]
+                country_icrc_presence.url = data["URL"]
+                country_icrc_presence.key_operation = data["Key operation"]
+                country_icrc_presence.description = data["Description"]
+                country_icrc_presence.save()
                 added += 1
-                CountryICRCPresence.objects.create(**dict_data)
 
         text_to_log = "%s ICRC added" % added
         logger.info(text_to_log)

api/management/commands/ingest_ns_capacity.py

@@ -1,14 +1,17 @@
 import requests
 from django.conf import settings
 from django.core.management.base import BaseCommand
+from sentry_sdk.crons import monitor
 
 from api.logger import logger
 from api.models import Country, CountryCapacityStrengthening, CronJob, CronJobStatus
+from main.sentry import SentryMonitor
 
 
 class Command(BaseCommand):
     help = "Add ns contact details"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_NS_CAPACITY)
     def handle(self, *args, **kwargs):
         logger.info("Starting NS Contacts")
 
@@ -44,7 +47,7 @@ def handle(self, *args, **kwargs):
         text_to_log = "%s Ns capacity added" % ocaa_count
         logger.info(text_to_log)
         body = {
-            "name": "ingest_ns_capaciity",
+            "name": "ingest_ns_capacity",
             "message": text_to_log,
             "num_result": ocaa_count,
             "status": CronJobStatus.SUCCESSFUL,

api/management/commands/ingest_ns_contact.py

@@ -5,15 +5,20 @@
 import xmltodict
 from django.conf import settings
 from django.core.management.base import BaseCommand
+from django.db import transaction
 from requests.auth import HTTPBasicAuth
+from sentry_sdk.crons import monitor
 
 from api.logger import logger
 from api.models import Country, CronJob, CronJobStatus
+from main.sentry import SentryMonitor
 
 
 class Command(BaseCommand):
     help = "Add ns contact details"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_NS_CONTACT)
+    @transaction.atomic
     def handle(self, *args, **kwargs):
         logger.info("Starting NS Contacts")
         url = "https://go-api.ifrc.org/"

api/management/commands/ingest_ns_directory.py

@@ -2,16 +2,26 @@
 import xmltodict
 from django.conf import settings
 from django.core.management.base import BaseCommand
+from django.db import transaction
 from requests.auth import HTTPBasicAuth
+from sentry_sdk.crons import monitor
 
 from api.logger import logger
 from api.models import Country, CountryDirectory, CronJob, CronJobStatus
+from main.sentry import SentryMonitor
 
 
 class Command(BaseCommand):
     help = "Add ns contact details"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_NS_DIRECTORY)
+    @transaction.atomic
     def handle(self, *args, **kwargs):
+        def postprocessor(path, key, value):
+            if key == "@i:nil":
+                return None
+            return key, value
+
         logger.info("Starting NS Contacts")
         url = "https://go-api.ifrc.org/"
         headers = {"accept": "application/xml;q=0.9, */*;q=0.8"}
@@ -33,7 +43,8 @@ def handle(self, *args, **kwargs):
             raise Exception("Error querying NationalSocietiesContacts")
 
         added = 0
-        dict_data = xmltodict.parse(response.content)
+        created_country_directory_ids = []
+        dict_data = xmltodict.parse(response.content, postprocessor=postprocessor)
         for data in dict_data["ArrayOfNationalSocietiesContacts"]["NationalSocietiesContacts"]:
             country_name = data["CON_country"] if isinstance(data["CON_country"], str) else None
             if country_name is not None:
@@ -54,7 +65,15 @@ def handle(self, *args, **kwargs):
                         "position": data["CON_title"],
                         "country": country,
                     }
-                    CountryDirectory.objects.create(**data)
+                    country_directory, _ = CountryDirectory.objects.get_or_create(
+                        country=country,
+                        first_name__iexact=data["first_name"],
+                        last_name__iexact=data["last_name"],
+                        position__iexact=data["position"],
+                    )
+                    created_country_directory_ids.append(country_directory.pk)
+        # NOTE: Deleting the country directory which are not available in the source
+        CountryDirectory.objects.exclude(id__in=created_country_directory_ids).delete()
         text_to_log = "%s Ns Directory added" % added
         logger.info(text_to_log)
         body = {"name": "ingest_ns_directory", "message": text_to_log, "num_result": added, "status": CronJobStatus.SUCCESSFUL}

api/management/commands/ingest_ns_document.py

@@ -5,14 +5,19 @@
 import requests
 from django.conf import settings
 from django.core.management.base import BaseCommand
+from django.db import transaction
+from sentry_sdk.crons import monitor
 
 from api.logger import logger
 from api.models import Country, CountryKeyDocument, CronJob, CronJobStatus
+from main.sentry import SentryMonitor
 
 
 class Command(BaseCommand):
     help = "Add ns documents"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_NS_DOCUMENT)
+    @transaction.atomic
     def handle(self, *args, **kwargs):
         logger.info("Starting NS Key Documents")
 
@@ -95,21 +100,45 @@ def fetch_all_country_documents(self, api_key, country_table):
 
     def save_documents_to_database(self, result):
         added = 0
+        created_country_key_document_ids = []
         for document in result:
             country = Country.objects.filter(fdrs=document["country_code"]).first()
-            if country:
-                added += 1
-                data = {
-                    "country": country,
+            if country is None:
+                continue
+
+            country_key_document, created = CountryKeyDocument.objects.get_or_create(
+                country=country,
+                url=document["url"],
+                defaults={
                     "name": document["name"],
-                    "url": document["url"],
                     "thumbnail": document["thumbnail"],
                     "document_type": document["document_type"],
                     "year": document["year"],
                     "end_year": document["end_year"],
                     "year_text": document["year_text"],
-                }
-                CountryKeyDocument.objects.create(**data)
+                },
+            )
+            if not created:
+                country_key_document.name = document["name"]
+                country_key_document.thumbnail = document["thumbnail"]
+                country_key_document.document_type = document["document_type"]
+                country_key_document.year = document["year"]
+                country_key_document.end_year = document["end_year"]
+                country_key_document.year_text = document["year_text"]
+                country_key_document.save(
+                    update_fields=[
+                        "name",
+                        "thumbnail",
+                        "document_type",
+                        "year",
+                        "end_year",
+                        "year_text",
+                    ]
+                )
+            created_country_key_document_ids.append(country_key_document.pk)
+            added += 1
+        # NOTE: Deleting the CountryKeyDocument that are not in the source
+        CountryKeyDocument.objects.exclude(id__in=created_country_key_document_ids).delete()
         return added
 
     def sync_cron_success(self, text_to_log, added):

api/management/commands/ingest_ns_initiatives.py

@@ -3,14 +3,19 @@
 import requests
 from django.conf import settings
 from django.core.management.base import BaseCommand
+from django.db import transaction
+from sentry_sdk.crons import monitor
 
 from api.logger import logger
 from api.models import Country, CronJob, CronJobStatus, NSDInitiatives
+from main.sentry import SentryMonitor
 
 
 class Command(BaseCommand):
     help = "Add ns initiatives"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_NS_INITIATIVES)
+    @transaction.atomic
     def handle(self, *args, **kwargs):
         logger.info("Starting NS Inititatives")
         api_key = settings.NS_INITIATIVES_API_KEY
@@ -40,21 +45,33 @@ def handle(self, *args, **kwargs):
             ],
         )
         funding_data = funding_data.replace({np.nan: None})
+        created_ns_initiatives_pk = []
         for data in funding_data.values.tolist():
             # TODO: Filter not by society name
             country = Country.objects.filter(society_name__iexact=data[0]).first()
             if country:
-                dict_data = {
-                    "country": country,
-                    "title": data[3],
-                    "fund_type": data[2],
-                    "allocation": data[5],
-                    "year": data[1],
-                    "funding_period": data[6],
-                    "categories": data[4],
-                }
+                nsd_initiatives, created = NSDInitiatives.objects.get_or_create(
+                    country=country,
+                    year=data[1],
+                    fund_type=data[2],
+                    defaults={
+                        "title": data[3],
+                        "categories": data[4],
+                        "allocation": data[5],
+                        "funding_period": data[6],
+                    },
+                )
+                if not created:
+                    nsd_initiatives.title = data[3]
+                    nsd_initiatives.categories = data[4]
+                    nsd_initiatives.allocation = data[5]
+                    nsd_initiatives.funding_period = data[6]
+                    nsd_initiatives.save(update_fields=["title", "categories", "allocation", "funding_period"])
+                created_ns_initiatives_pk.append(nsd_initiatives.pk)
                 added += 1
-                NSDInitiatives.objects.create(**dict_data)
+        # NOTE: Delete the NSDInitiatives that are not in the source
+        NSDInitiatives.objects.exclude(id__in=created_ns_initiatives_pk).delete()
+
         text_to_log = "%s Ns initiatives added" % added
         logger.info(text_to_log)
         body = {"name": "ingest_ns_initiatives", "message": text_to_log, "num_result": added, "status": CronJobStatus.SUCCESSFUL}

api/management/commands/revoke_staff_status.py

@@ -8,7 +8,6 @@
 # from registrations.views import is_valid_domain
 
 
-@monitor(monitor_slug=SentryMonitor.REVOKE_STAFF_STATUS)
 class Command(BaseCommand):
     help = 'Update staff status in auth_user table according to "Read only" group'
 
@@ -53,6 +52,7 @@ def get_ifrc_domain_users(self):
     #
     #       return editors
 
+    @monitor(monitor_slug=SentryMonitor.REVOKE_STAFF_STATUS)
     def handle(self, *args, **options):
         logger.info("Moving Read only users out of staff status...")
 

api/management/commands/sync_appealdocs.py

@@ -21,7 +21,6 @@
 FEDNET_SOURCE = "https://go-api.ifrc.org/Api/FedNetAppeals?Hidden=false&BaseAppealnumber="
 
 
-@monitor(monitor_slug=SentryMonitor.SYNC_APPEALDOCS)
 class Command(BaseCommand):
     help = "Ingest existing appeal documents"
 
@@ -37,6 +36,7 @@ def parse_date(self, date_string):
         timeformat = "%Y-%m-%dT%H:%M:%S"
         return datetime.strptime(date_string[:18], timeformat).replace(tzinfo=timezone.utc)
 
+    @monitor(monitor_slug=SentryMonitor.SYNC_APPEALDOCS)
     def handle(self, *args, **options):
         logger.info("Starting appeal document ingest")