diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3496f4b87..beb9a9e1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,6 +104,7 @@ jobs: DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres # pragma: allowlist secret WEBPACK_DISABLE_LOADER_STATS: "True" ELASTICSEARCH_URL: localhost:9200 + EMERITUS_API_KEY: fake_emeritus_api_key # pragma: allowlist secret MAILGUN_KEY: fake_mailgun_key MAILGUN_SENDER_DOMAIN: other.fake.site MITOL_DIGITAL_CREDENTIALS_VERIFY_SERVICE_BASE_URL: http://localhost:5000 diff --git a/RELEASE.rst b/RELEASE.rst index 1b92bfee4..1c28cdf6e 100644 --- a/RELEASE.rst +++ b/RELEASE.rst @@ -1,6 +1,11 @@ Release Notes ============= +Version 0.151.0 +--------------- + +- feat: ingest external course APIs (#2998) + Version 0.150.0 (Released June 24, 2024) --------------- diff --git a/app.json b/app.json index 4a345feab..acd8c1814 100644 --- a/app.json +++ b/app.json @@ -86,6 +86,14 @@ "description": "'hours' value for the 'generate-course-certificate' scheduled task (defaults to midnight)", "required": false }, + "CRON_EMERITUS_COURSERUN_SYNC_DAYS": { + "description": "'day_of_week' value for 'sync-emeritus-course-runs' scheduled task (default will run once a day).", + "required": false + }, + "CRON_EMERITUS_COURSERUN_SYNC_HOURS": { + "description": "'hours' value for the 'sync-emeritus-course-runs' scheduled task (defaults to midnight)", + "required": false + }, "CSRF_TRUSTED_ORIGINS": { "description": "Comma separated string of trusted domains that should be CSRF exempt", "required": false @@ -214,6 +222,18 @@ "description": "Timeout (in seconds) for requests made via the edX API client", "required": false }, + "EMERITUS_API_BASE_URL": { + "description": "Base API URL for Emeritus API", + "required": false + }, + "EMERITUS_API_KEY": { + "description": "The API Key for Emeritus API", + "required": true + }, + "EMERITUS_API_TIMEOUT": { + "description": "API request timeout for Emeritus APIs in seconds", + "required": false + }, "ENROLLMENT_CHANGE_SHEET_ID": { "description": "ID of the Google Sheet that contains the enrollment change request worksheets (refunds, transfers, etc)", "required": false diff --git a/courses/admin.py b/courses/admin.py index 590e0b8f0..3967650e8 100644 --- a/courses/admin.py +++ b/courses/admin.py @@ -52,9 +52,9 @@ class CourseAdmin(admin.ModelAdmin): """Admin for Course""" model = Course - search_fields = ["title", "readable_id", "platform__name"] + search_fields = ["title", "readable_id", "platform__name", "external_course_id"] list_display = ("id", "title", "get_program", "position_in_program", "platform") - list_filter = ["live", "program", "platform"] + list_filter = ["live", "platform", "program"] formfield_overrides = { models.CharField: {"widget": TextInput(attrs={"size": "80"})} } @@ -73,7 +73,12 @@ class CourseRunAdmin(TimestampedModelAdmin): """Admin for CourseRun""" model = CourseRun - search_fields = ["title", "courseware_id"] + search_fields = [ + "title", + "courseware_id", + "external_course_run_id", + "course__external_course_id", + ] list_display = ( "id", "title", diff --git a/courses/api.py b/courses/api.py index c4a84c02d..79e90f99a 100644 --- a/courses/api.py +++ b/courses/api.py @@ -345,3 +345,21 @@ def defer_enrollment( except EdxEnrollmentCreateError: # noqa: TRY302 raise return from_enrollment, first_or_none(to_enrollments) + + +def generate_course_readable_id(course_tag): + """ + Generates course readable ID using the course tag. + + Args: + course_tag (str): Course tag of course + + Returns: + str: Course readable id + """ + if not course_tag: + raise ValidationError( + "course_tag is required to generate a valid readable ID for a course." # noqa: EM101 + ) + + return f"course-v1:xPRO+{course_tag}" diff --git a/courses/api_test.py b/courses/api_test.py index fcfa93774..2bbbee708 100644 --- a/courses/api_test.py +++ b/courses/api_test.py @@ -16,6 +16,7 @@ deactivate_program_enrollment, deactivate_run_enrollment, defer_enrollment, + generate_course_readable_id, get_user_enrollments, ) from courses.constants import ( @@ -537,3 +538,24 @@ def test_defer_enrollment_validation(mocker, user): force=True, ) assert patched_create_enrollments.call_count == 2 + + +@pytest.mark.parametrize( + ("course_tag", "expected_readable_id", "raises_validation_error"), + [ + ("DBIP", "course-v1:xPRO+DBIP", False), + ("DBIP.ES", "course-v1:xPRO+DBIP.ES", False), + ("DBIP.SEPO", "course-v1:xPRO+DBIP.SEPO", False), + ("", "", True), + (None, None, True), + ], +) +def test_generate_course_readable_id( + course_tag, expected_readable_id, raises_validation_error +): + """Test that `generate_course_readable_id` returns expected course readable_id for course tags.""" + if raises_validation_error: + with pytest.raises(ValidationError): + generate_course_readable_id(course_tag) + else: + assert generate_course_readable_id(course_tag) == expected_readable_id diff --git a/courses/management/commands/sync_external_course_runs.py b/courses/management/commands/sync_external_course_runs.py new file mode 100644 index 000000000..820d5e724 --- /dev/null +++ b/courses/management/commands/sync_external_course_runs.py @@ -0,0 +1,129 @@ +"""Management command to sync external course runs""" + +from django.core.management.base import BaseCommand + +from courses.sync_external_courses.emeritus_api import ( + EmeritusKeyMap, + fetch_emeritus_courses, + update_emeritus_course_runs, +) +from mitxpro import settings + + +class Command(BaseCommand): + """Sync external course runs""" + + help = "Management command to sync external course runs from the vendor APIs." + + def add_arguments(self, parser): + parser.add_argument( + "--vendor-name", + type=str, + help="The name of the vendor i.e. `Emeritus`", + required=True, + ) + super().add_arguments(parser) + + def handle(self, *args, **options): # noqa: ARG002 + """Handle command execution""" + if not settings.FEATURES.get("ENABLE_EXTERNAL_COURSE_SYNC", False): + self.stdout.write( + self.style.ERROR( + "External Course Sync is disabled. You can enable it by turning on the feature flag " + "`ENABLE_EXTERNAL_COURSE_SYNC`" + ) + ) + return + + vendor_name = options["vendor_name"] + if vendor_name.lower() == EmeritusKeyMap.PLATFORM_NAME.value.lower(): + self.stdout.write(f"Starting course sync for {vendor_name}.") + emeritus_course_runs = fetch_emeritus_courses() + stats = update_emeritus_course_runs(emeritus_course_runs) + self.stdout.write( + self.style.SUCCESS( + f"Number of Courses Created {len(stats['courses_created'])}." + ) + ) + self.stdout.write( + self.style.SUCCESS( + f"External Course Codes: {stats['courses_created'] if stats['courses_created'] else None}.\n" + ) + ) + self.stdout.write( + self.style.SUCCESS( + f"Number of existing Courses {len(stats['existing_courses'])}." + ) + ) + self.stdout.write( + self.style.SUCCESS( + f"External Course Codes: {stats['existing_courses'] if stats['existing_courses'] else None}.\n" + ) + ) + self.stdout.write( + self.style.SUCCESS( + f"Number of Course Runs Created {len(stats['course_runs_created'])}." + ) + ) + self.stdout.write( + self.style.SUCCESS( + f"External Course Run Codes: {stats['course_runs_created'] if stats['course_runs_created'] else None}.\n" + ) + ) + self.stdout.write( + self.style.SUCCESS( + f"Number of Course Runs Updated {len(stats['course_runs_updated'])}." + ) + ) + self.stdout.write( + self.style.SUCCESS( + f"External Course Run Codes: {stats['course_runs_updated'] if stats['course_runs_updated'] else None}.\n" + ) + ) + self.stdout.write( + self.style.SUCCESS( + f"Number of Courses Pages Created {len(stats['course_pages_created'])}." + ) + ) + self.stdout.write( + self.style.SUCCESS( + f"External Course Codes: {stats['course_pages_created'] if stats['course_pages_created'] else None}.\n" + ) + ) + self.stdout.write( + self.style.SUCCESS( + f"Number of Courses Pages Updated {len(stats['course_pages_updated'])}." + ) + ) + self.stdout.write( + self.style.SUCCESS( + f"External Course Codes: {stats['course_pages_updated'] if stats['course_pages_updated'] else None}.\n" + ) + ) + self.stdout.write( + self.style.SUCCESS( + f"Number of Course Runs Skipped due to bad data {len(stats['course_runs_skipped'])}." + ) + ) + self.stdout.write( + self.style.SUCCESS( + f"External Course Codes: {stats['course_runs_skipped'] if stats['course_runs_skipped'] else None}.\n" + ) + ) + self.stdout.write( + self.style.SUCCESS( + f"Number of Expired Course Runs {len(stats['course_runs_expired'])}." + ) + ) + self.stdout.write( + self.style.SUCCESS( + f"External Course Codes: {stats['course_runs_expired'] if stats['course_runs_expired'] else None}.\n" + ) + ) + self.stdout.write( + self.style.SUCCESS( + f"External course sync successful for {vendor_name}." + ) + ) + else: + self.stdout.write(self.style.ERROR(f"Unknown vendor name {vendor_name}.")) diff --git a/courses/sync_external_courses/__init__.py b/courses/sync_external_courses/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/courses/sync_external_courses/emeritus_api.py b/courses/sync_external_courses/emeritus_api.py new file mode 100644 index 000000000..2211369a3 --- /dev/null +++ b/courses/sync_external_courses/emeritus_api.py @@ -0,0 +1,452 @@ +import json +import logging +import re +import time +from datetime import timedelta +from enum import Enum + +from django.db import transaction +from wagtail.models import Page + +from cms.models import ( + CourseIndexPage, + ExternalCoursePage, + LearningOutcomesPage, + WhoShouldEnrollPage, +) +from courses.api import generate_course_readable_id +from courses.models import Course, CourseRun, CourseTopic, Platform +from courses.sync_external_courses.emeritus_api_client import EmeritusAPIClient +from mitxpro.utils import clean_url, now_in_utc, strip_datetime + +log = logging.getLogger(__name__) + + +class EmeritusKeyMap(Enum): + """ + Emeritus course sync keys. + """ + + REPORT_NAMES = ["Batch"] + PLATFORM_NAME = "Emeritus" + DATE_FORMAT = "%Y-%m-%d" + COURSE_PAGE_SUBHEAD = "Delivered in collaboration with Emeritus." + WHO_SHOULD_ENROLL_PAGE_HEADING = "WHO SHOULD ENROLL" + LEARNING_OUTCOMES_PAGE_HEADING = "WHAT YOU WILL LEARN" + LEARNING_OUTCOMES_PAGE_SUBHEAD = ( + "MIT xPRO is collaborating with online education provider Emeritus to " + "deliver this online course. By clicking LEARN MORE, you will be taken to " + "a page where you can download the brochure and apply to the program via Emeritus." + ) + + +class EmeritusJobStatus(Enum): + """ + Status of an Emeritus Job. + """ + + READY = 3 + FAILED = 4 + CANCELLED = 5 + + +class EmeritusCourse: + """ + Emeritus course object. + + Parses an Emeritus course JSON to Python object. + """ + + def __init__(self, emeritus_course_json): + self.course_title = emeritus_course_json.get("program_name", None) + self.course_code = emeritus_course_json.get("course_code") + + # Emeritus course code format is `MO-`, where course tag can contain `.`, + # we will replace `.` with `_` to follow the internal readable id format. + self.course_readable_id = generate_course_readable_id( + self.course_code.split("-")[1].replace(".", "_") + ) + + self.course_run_code = emeritus_course_json.get("course_run_code") + self.course_run_tag = generate_emeritus_course_run_tag(self.course_run_code) + + self.start_date = strip_datetime( + emeritus_course_json.get("start_date"), EmeritusKeyMap.DATE_FORMAT.value + ) + end_datetime = strip_datetime( + emeritus_course_json.get("end_date"), EmeritusKeyMap.DATE_FORMAT.value + ) + self.end_date = ( + end_datetime.replace(hour=23, minute=59) if end_datetime else None + ) + + self.marketing_url = clean_url( + emeritus_course_json.get("landing_page_url"), remove_query_params=True + ) + total_weeks = int(emeritus_course_json.get("total_weeks")) + self.duration = f"{total_weeks} Weeks" if total_weeks != 0 else "" + + # Description can be null in Emeritus API data, we cannot store `None` as description is Non-Nullable + self.description = ( + emeritus_course_json.get("description") + if emeritus_course_json.get("description") + else "" + ) + self.format = emeritus_course_json.get("format") + self.category = emeritus_course_json.get("Category", None) + self.learning_outcomes_list = ( + parse_emeritus_data_str(emeritus_course_json.get("learning_outcomes")) + if emeritus_course_json.get("learning_outcomes") + else [] + ) + self.who_should_enroll_list = ( + parse_emeritus_data_str(emeritus_course_json.get("program_for")) + if emeritus_course_json.get("program_for") + else [] + ) + + +def fetch_emeritus_courses(): + """ + Fetches Emeritus courses data. + + Makes a request to get the list of available queries and then queries the required reports. + """ + end_date = now_in_utc() + start_date = end_date - timedelta(days=1) + + emeritus_api_client = EmeritusAPIClient() + queries = emeritus_api_client.get_queries_list() + + for query in queries: # noqa: RET503 + # Check if query is in list of desired reports + if query["name"] not in EmeritusKeyMap.REPORT_NAMES.value: + log.info( + "Report: {} not specified for extract...skipping".format(query["name"]) # noqa: G001 + ) + continue + + log.info("Requesting data for {}...".format(query["name"])) # noqa: G001 + query_response = emeritus_api_client.get_query_response( + query["id"], start_date, end_date + ) + if "job" in query_response: + # If a job is returned, we will poll until status = 3 (Success) + # Status values 1 and 2 correspond to in-progress, + # while 4 and 5 correspond to Failed, and Canceled, respectively. + job_id = query_response["job"]["id"] + log.info( + f"Job id: {job_id} found... waiting for completion..." # noqa: G004 + ) + while True: + job_status = emeritus_api_client.get_job_status(job_id) + if job_status["job"]["status"] == EmeritusJobStatus.READY.value: + # If true, the query_result is ready to be collected. + log.info("Job complete... requesting results...") + query_response = emeritus_api_client.get_query_result( + job_status["job"]["query_result_id"] + ) + break + elif job_status["job"]["status"] in [ + EmeritusJobStatus.FAILED.value, + EmeritusJobStatus.CANCELLED.value, + ]: + log.error("Job failed!") + break + else: + # Continue waiting until complete. + log.info("Job not yet complete... sleeping for 2 seconds...") + time.sleep(2) + + if "query_result" in query_response: + # Check that query_result is in the data payload. + # Return result as json + return dict(query_response["query_result"]["data"]).get("rows", []) + log.error("Something unexpected happened!") + + +def update_emeritus_course_runs(emeritus_courses): # noqa: C901 + """ + Updates or creates the required course data i.e. Course, CourseRun, + ExternalCoursePage, CourseTopic, WhoShouldEnrollPage, and LearningOutcomesPage + """ + platform, _ = Platform.objects.get_or_create( + name__iexact=EmeritusKeyMap.PLATFORM_NAME.value, + defaults={"name": EmeritusKeyMap.PLATFORM_NAME.value}, + ) + course_index_page = Page.objects.get(id=CourseIndexPage.objects.first().id).specific + stats = { + "courses_created": set(), + "existing_courses": set(), + "course_runs_created": set(), + "course_runs_updated": set(), + "course_pages_created": set(), + "course_pages_updated": set(), + "course_runs_skipped": set(), + "course_runs_expired": set(), + } + + for emeritus_course_json in emeritus_courses: + emeritus_course = EmeritusCourse(emeritus_course_json) + + log.info( + "Creating or updating course metadata for title: {}, course_code: {}, course_run_code: {}".format( # noqa: G001, UP032 + emeritus_course.course_title, + emeritus_course.course_code, + emeritus_course.course_run_code, + ) + ) + # If course_title, course_code, or course_run_code is missing, skip. + if not ( + emeritus_course.course_title + and emeritus_course.course_code + and emeritus_course.course_run_code + ): + log.info( + f"Missing required course data. Skipping... Course data: {json.dumps(emeritus_course_json)}" # noqa: G004 + ) + stats["course_runs_skipped"].add(emeritus_course.course_run_code) + continue + + if now_in_utc() > emeritus_course.end_date: + log.info( + f"Course run is expired, Skipping... Course data: {json.dumps(emeritus_course_json)}" # noqa: G004 + ) + stats["course_runs_expired"].add(emeritus_course.course_run_code) + continue + + with transaction.atomic(): + course, course_created = Course.objects.get_or_create( + external_course_id=emeritus_course.course_code, + platform=platform, + is_external=True, + defaults={ + "title": emeritus_course.course_title, + "readable_id": emeritus_course.course_readable_id, + # All new courses are live by default, we will change the status manually + "live": True, + }, + ) + log_msg = "Created course," if course_created else "Course already exists," + log.info( + f"{log_msg} title: {emeritus_course.course_title}, readable_id: {emeritus_course.course_readable_id}" # noqa: G004 + ) + + if course_created: + stats["courses_created"].add(emeritus_course.course_code) + else: + stats["existing_courses"].add(emeritus_course.course_code) + + log.info( + f"Creating or Updating course run, title: {emeritus_course.course_title}, course_run_code: {emeritus_course.course_run_code}" # noqa: G004 + ) + course_run, course_run_created = create_or_update_emeritus_course_run( + course, emeritus_course + ) + + if course_run_created: + stats["course_runs_created"].add(course_run.external_course_run_id) + else: + stats["course_runs_updated"].add(course_run.external_course_run_id) + + log.info( + f"Creating or Updating course page, title: {emeritus_course.course_title}, course_code: {emeritus_course.course_run_code}" # noqa: G004 + ) + course_page, course_page_created = create_or_update_emeritus_course_page( + course_index_page, course, emeritus_course + ) + + if course_page_created: + stats["course_pages_created"].add(emeritus_course.course_code) + else: + stats["course_pages_updated"].add(emeritus_course.course_code) + + if emeritus_course.category: + topic = CourseTopic.objects.filter( + name__iexact=emeritus_course.category + ).first() + if topic: + course_page.topics.add(topic) + course_page.save() + + if not course_page.outcomes and emeritus_course.learning_outcomes_list: + create_learning_outcomes_page( + course_page, emeritus_course.learning_outcomes_list + ) + + if ( + not course_page.who_should_enroll + and emeritus_course.who_should_enroll_list + ): + create_who_should_enroll_in_page( + course_page, emeritus_course.who_should_enroll_list + ) + + # As we get the API data for course runs, we can have duplicate course codes in course created and updated, + # so, we are removing the courses created from the updated courses list. + stats["existing_courses"] = stats["existing_courses"].difference( + stats["courses_created"] + ) + stats["course_pages_updated"] = stats["course_pages_updated"].difference( + stats["course_pages_created"] + ) + return stats + + +def generate_emeritus_course_run_tag(course_run_code): + """ + Returns the course run tag generated using the Emeritus Course run code. + + Emeritus course run codes follow a pattern `MO--`. This method returns the run tag. + """ + return re.search(r"[0-9]{2}-[0-9]{2}#[0-9]+$", course_run_code).group(0) + + +def generate_external_course_run_courseware_id(course_run_tag, course_readable_id): + """ + Returns course run courseware id using the course readable id and course run tag. + """ + return f"{course_readable_id}+{course_run_tag}" + + +def create_or_update_emeritus_course_page(course_index_page, course, emeritus_course): + """ + Creates or updates external course page for Emeritus course. + """ + course_page = ( + ExternalCoursePage.objects.select_for_update().filter(course=course).first() + ) + created = False + if not course_page: + course_page = ExternalCoursePage( + course=course, + title=emeritus_course.course_title, + external_marketing_url=emeritus_course.marketing_url, + subhead=EmeritusKeyMap.COURSE_PAGE_SUBHEAD.value, + duration=emeritus_course.duration, + format=emeritus_course.format, + description=emeritus_course.description, + ) + course_index_page.add_child(instance=course_page) + course_page.save() + log.info( + f"Created external course page for course title: {emeritus_course.course_title}" # noqa: G004 + ) + created = True + else: + # Only update course page fields with API if they are empty. + course_page_attrs_changed = False + if not course_page.external_marketing_url and emeritus_course.marketing_url: + course_page.external_marketing_url = emeritus_course.marketing_url + course_page_attrs_changed = True + if not course_page.duration and emeritus_course.duration: + course_page.duration = emeritus_course.duration + course_page_attrs_changed = True + if not course_page.description and emeritus_course.description: + course_page.description = emeritus_course.description + course_page_attrs_changed = True + + if course_page_attrs_changed: + course_page.save() + log.info( + f"Updated external course page for course title: {emeritus_course.course_title}" # noqa: G004 + ) + + return course_page, created + + +def create_or_update_emeritus_course_run(course, emeritus_course): + """ + Creates or updates the external emeritus course run. + + Args: + course (courses.Course): Course object + emeritus_course (EmeritusCourse): EmeritusCourse object + + Returns: + tuple: A tuple containing of course run and is course run created. + """ + course_run_courseware_id = generate_external_course_run_courseware_id( + emeritus_course.course_run_tag, course.readable_id + ) + course_run = ( + CourseRun.objects.select_for_update() + .filter(external_course_run_id=emeritus_course.course_run_code, course=course) + .first() + ) + + if not course_run: + course_run = CourseRun.objects.create( + external_course_run_id=emeritus_course.course_run_code, + course=course, + title=emeritus_course.course_title, + courseware_id=course_run_courseware_id, + run_tag=emeritus_course.course_run_tag, + start_date=emeritus_course.start_date, + end_date=emeritus_course.end_date, + live=True, + ) + log.info( + f"Created Course Run, title: {emeritus_course.course_title}, external_course_run_id: {emeritus_course.course_run_code}" # noqa: G004 + ) + return course_run, True + elif ( + course_run.start_date + and emeritus_course.start_date + and course_run.start_date.date() != emeritus_course.start_date.date() + ) or ( + course_run.end_date + and emeritus_course.end_date + and course_run.end_date.date() != emeritus_course.end_date.date() + ): + course_run.start_date = emeritus_course.start_date + course_run.end_date = emeritus_course.end_date + course_run.save() + log.info( + f"Updated Course Run, title: {emeritus_course.course_title}, external_course_run_id: {emeritus_course.course_run_code}" # noqa: G004 + ) + return course_run, False + + +def create_who_should_enroll_in_page(course_page, who_should_enroll_list): + """ + Creates `WhoShouldEnrollPage` for Emeritus course. + """ + content = json.dumps( + [ + {"type": "item", "value": who_should_enroll_item} + for who_should_enroll_item in who_should_enroll_list + ] + ) + + who_should_enroll_page = WhoShouldEnrollPage( + heading=EmeritusKeyMap.WHO_SHOULD_ENROLL_PAGE_HEADING.value, + content=content, + ) + course_page.add_child(instance=who_should_enroll_page) + who_should_enroll_page.save() + + +def create_learning_outcomes_page(course_page, outcomes_list): + """ + Creates `LearningOutcomesPage` for Emeritus course. + """ + outcome_items = json.dumps( + [{"type": "outcome", "value": outcome} for outcome in outcomes_list] + ) + + learning_outcome_page = LearningOutcomesPage( + heading=EmeritusKeyMap.LEARNING_OUTCOMES_PAGE_HEADING.value, + sub_heading=EmeritusKeyMap.LEARNING_OUTCOMES_PAGE_SUBHEAD.value, + outcome_items=outcome_items, + ) + course_page.add_child(instance=learning_outcome_page) + learning_outcome_page.save() + + +def parse_emeritus_data_str(items_str): + """ + Parses `WhoShouldEnrollPage` and `LearningOutcomesPage` items for the Emeritus API. + """ + items_list = items_str.strip().split("\r\n") + return [item.replace("●", "").strip() for item in items_list][1:] diff --git a/courses/sync_external_courses/emeritus_api_client.py b/courses/sync_external_courses/emeritus_api_client.py new file mode 100644 index 000000000..448fb275d --- /dev/null +++ b/courses/sync_external_courses/emeritus_api_client.py @@ -0,0 +1,74 @@ +""" +API client for Emeritus +""" + +import json + +import requests +from django.conf import settings + + +class EmeritusAPIClient: + """ + API client for Emeritus + """ + + def __init__(self): + self.api_key = settings.EMERITUS_API_KEY + self.base_url = settings.EMERITUS_API_BASE_URL + self.request_timeout = settings.EMERITUS_API_REQUEST_TIMEOUT + + def get_queries_list(self): + """ + Get a list of available queries + """ + queries = requests.get( + f"{self.base_url}/api/queries?api_key={self.api_key}", + timeout=self.request_timeout, + ) + queries.raise_for_status() + return queries.json()["results"] + + def get_query_response(self, query_id, start_date, end_date): + """ + Make a post request for the query. + + This will return either: + a) A query_result if one is cached for the parameters set, or + b) A Job object. + """ + query_response = requests.post( + f"{self.base_url}/api/queries/{query_id}/results?api_key={self.api_key}", + data=json.dumps( + { + "parameters": { + "date_range": {"start": f"{start_date}", "end": f"{end_date}"} + } + } + ), + timeout=self.request_timeout, + ) + query_response.raise_for_status() + return query_response.json() + + def get_job_status(self, job_id): + """ + Get the status of the job + """ + job_status = requests.get( + f"{self.base_url}/api/jobs/{job_id}?api_key={self.api_key}", + timeout=self.request_timeout, + ) + job_status.raise_for_status() + return job_status.json() + + def get_query_result(self, query_result_id): + """ + Get the query result + """ + query_result = requests.get( + f"{self.base_url}/api/query_results/{query_result_id}?api_key={self.api_key}", + timeout=self.request_timeout, + ) + query_result.raise_for_status() + return query_result.json() diff --git a/courses/sync_external_courses/emeritus_api_client_test.py b/courses/sync_external_courses/emeritus_api_client_test.py new file mode 100644 index 000000000..3cd6589d7 --- /dev/null +++ b/courses/sync_external_courses/emeritus_api_client_test.py @@ -0,0 +1,112 @@ +""" +Tests for emeritus_api_client +""" + +import json +from datetime import timedelta + +import pytest + +from courses.sync_external_courses.emeritus_api_client import EmeritusAPIClient +from mitxpro.test_utils import MockResponse +from mitxpro.utils import now_in_utc + + +@pytest.mark.parametrize( + ( + "patch_request_path", + "mock_response", + "client_method", + "args", + "expected_api_url", + ), + [ + ( + "courses.sync_external_courses.emeritus_api_client.requests.get", + MockResponse( + { + "results": [ + { + "id": 77, + "name": "Batch", + } + ] + } + ), + "get_queries_list", + [], + "https://test-emeritus-api.io/api/queries?api_key=test_emeritus_api_key", + ), + ( + "courses.sync_external_courses.emeritus_api_client.requests.get", + MockResponse({"job": {"status": 1}}), + "get_job_status", + [12], + "https://test-emeritus-api.io/api/jobs/12?api_key=test_emeritus_api_key", + ), + ( + "courses.sync_external_courses.emeritus_api_client.requests.get", + MockResponse({"query_result": {"data": {}}}), + "get_query_result", + [20], + "https://test-emeritus-api.io/api/query_results/20?api_key=test_emeritus_api_key", + ), + ], +) +def test_emeritus_api_client_get_requests( # noqa: PLR0913 + mocker, + settings, + patch_request_path, + mock_response, + client_method, + args, + expected_api_url, +): + settings.EMERITUS_API_KEY = "test_emeritus_api_key" + settings.EMERITUS_API_BASE_URL = "https://test-emeritus-api.io" + settings.EMERITUS_API_REQUEST_TIMEOUT = 60 + + mock_get = mocker.patch(patch_request_path) + mock_get.return_value = mock_response + + client = EmeritusAPIClient() + client_method_map = { + "get_queries_list": client.get_queries_list, + "get_job_status": client.get_job_status, + "get_query_result": client.get_query_result, + } + client_method_map[client_method](*args) + mock_get.assert_called_once_with( + expected_api_url, + timeout=60, + ) + + +def test_get_query_response(mocker, settings): + """ + Tests that `EmeritusAPIClient.get_query_response` makes the expected post request. + """ + end_date = now_in_utc() + start_date = end_date - timedelta(days=1) + + settings.EMERITUS_API_KEY = "test_emeritus_api_key" + settings.EMERITUS_API_BASE_URL = "https://test-emeritus-api.io" + + mock_post = mocker.patch( + "courses.sync_external_courses.emeritus_api_client.requests.post" + ) + mock_post.return_value = MockResponse({"job": {"id": 1}}) + + client = EmeritusAPIClient() + client.get_query_response(1, start_date, end_date) + mock_post.assert_called_once_with( + "https://test-emeritus-api.io/api/queries/1/results?api_key=test_emeritus_api_key", + data=json.dumps( + { + "parameters": { + "date_range": {"start": f"{start_date}", "end": f"{end_date}"} + } + } + ), + timeout=60, + ) diff --git a/courses/sync_external_courses/emeritus_api_test.py b/courses/sync_external_courses/emeritus_api_test.py new file mode 100644 index 000000000..7e6519623 --- /dev/null +++ b/courses/sync_external_courses/emeritus_api_test.py @@ -0,0 +1,405 @@ +""" +Sync external course API tests +""" + +import json +import logging +import random +from pathlib import Path + +import pytest + +from cms.factories import ( + CourseIndexPageFactory, + ExternalCoursePageFactory, + HomePageFactory, +) +from courses.factories import CourseFactory, CourseRunFactory, PlatformFactory +from courses.models import Course +from courses.sync_external_courses.emeritus_api import ( + EmeritusCourse, + EmeritusKeyMap, + create_learning_outcomes_page, + create_or_update_emeritus_course_page, + create_or_update_emeritus_course_run, + create_who_should_enroll_in_page, + fetch_emeritus_courses, + generate_emeritus_course_run_tag, + generate_external_course_run_courseware_id, + parse_emeritus_data_str, + update_emeritus_course_runs, +) +from mitxpro.test_utils import MockResponse +from mitxpro.utils import clean_url + + +@pytest.mark.parametrize( + ("emeritus_course_run_code", "expected_course_run_tag"), + [ + ("MO-EOB-18-01#1", "18-01#1"), + ("MO-EOB-08-01#1", "08-01#1"), + ("MO-EOB-08-12#1", "08-12#1"), + ("MO-EOB-18-01#12", "18-01#12"), + ("MO-EOB-18-01#212", "18-01#212"), + ], +) +def test_generate_emeritus_course_run_tag( + emeritus_course_run_code, expected_course_run_tag +): + """ + Tests that `generate_emeritus_course_run_tag` generates the expected course tag for Emeritus Course Run Codes. + """ + assert ( + generate_emeritus_course_run_tag(emeritus_course_run_code) + == expected_course_run_tag + ) + + +@pytest.mark.parametrize( + ("course_readable_id", "course_run_tag", "expected_course_run_courseware_id"), + [ + ("course-v1:xPRO+EOB", "18-01#1", "course-v1:xPRO+EOB+18-01#1"), + ("course-v1:xPRO+EOB", "08-01#1", "course-v1:xPRO+EOB+08-01#1"), + ("course-v1:xPRO+EOB", "18-01#12", "course-v1:xPRO+EOB+18-01#12"), + ("course-v1:xPRO+EOB", "18-01#212", "course-v1:xPRO+EOB+18-01#212"), + ], +) +def test_generate_external_course_run_courseware_id( + course_readable_id, course_run_tag, expected_course_run_courseware_id +): + """ + Test that `generate_external_course_run_courseware_id` returns the expected courseware_id for the given + course run tag and course readable id. + """ + assert ( + generate_external_course_run_courseware_id(course_run_tag, course_readable_id) + == expected_course_run_courseware_id + ) + + +@pytest.mark.parametrize("create_course_page", [True, False]) +@pytest.mark.django_db +def test_create_or_update_emeritus_course_page(create_course_page): + """ + Test that `create_or_update_emeritus_course_page` creates a new course or updates the existing. + """ + home_page = HomePageFactory.create(title="Home Page", subhead="

subhead

") + course_index_page = CourseIndexPageFactory.create(parent=home_page, title="Courses") + course = CourseFactory.create() + + emeritus_course_run = { + "program_name": "Internet of Things (IoT): Design and Applications", + "course_code": "MO-DBIP", + "course_run_code": "MO-DBIP.ELE-25-07#1", + "start_date": "2025-07-30", + "end_date": "2025-09-24", + "Category": "Technology", + "list_price": 2600, + "list_currency": "USD", + "total_weeks": 7, + "product_family": "Certificate", + "product_sub_type": "Short Form", + "format": "Online", + "suggested_duration": 49, + "language": "English", + "landing_page_url": "https://test-emeritus-api.io/Internet-of-things-iot-design-and-applications" + "?utm_medium=EmWebsite&utm_campaign=direct_EmWebsite?utm_campaign=school_website&utm_medium" + "=website&utm_source=MIT-web", + "Apply_now_url": "https://test-emeritus-api.io/?locale=en&program_sfid=01t2s000000OHA2AAO&source" + "=applynowlp&utm_campaign=school&utm_medium=MITWebsite&utm_source=MIT-web", + "description": "Test Description", + "learning_outcomes": None, + "program_for": None, + } + + if create_course_page: + ExternalCoursePageFactory.create( + course=course, + title=emeritus_course_run["program_name"], + external_marketing_url="", + duration="", + description="", + ) + + course_page, _ = create_or_update_emeritus_course_page( + course_index_page, course, EmeritusCourse(emeritus_course_run) + ) + assert course_page.title == emeritus_course_run["program_name"] + assert course_page.external_marketing_url == clean_url( + emeritus_course_run["landing_page_url"], remove_query_params=True + ) + assert course_page.course == course + assert course_page.duration == f"{emeritus_course_run['total_weeks']} Weeks" + assert course_page.description == emeritus_course_run["description"] + + +@pytest.mark.django_db +def test_create_who_should_enroll_in_page(): + """ + Tests that `create_who_should_enroll_in_page` creates the `WhoShouldEnrollPage`. + """ + course_page = ExternalCoursePageFactory.create() + who_should_enroll_str = ( + "The program is ideal for:\r\n● Early-career IT professionals, network engineers, " + "and system administrators wanting to gain a comprehensive overview of cybersecurity and " + "fast-track their career progression\r\n● IT project managers and engineers keen on " + "gaining the ability to think critically about the threat landscape, including " + "vulnerabilities in cybersecurity, and upgrading their resume for career " + "advancement\r\n● Mid- or later-career professionals seeking a career change and " + "looking to add critical cybersecurity knowledge and foundational lessons to their resume" + ) + create_who_should_enroll_in_page( + course_page, parse_emeritus_data_str(who_should_enroll_str) + ) + assert parse_emeritus_data_str(who_should_enroll_str) == [ + item.value.source for item in course_page.who_should_enroll.content + ] + assert course_page.who_should_enroll is not None + + +@pytest.mark.django_db +def test_create_learning_outcomes_page(): + """ + Tests that `create_learning_outcomes_page` creates the `LearningOutcomesPage`. + """ + course_page = ExternalCoursePageFactory.create() + learning_outcomes_str = ( + "This program will enable you to:\r\n● Gain an overview of cybersecurity risk " + "management, including its foundational concepts and relevant regulations\r\n● " + "Explore the domains covering various aspects of cloud technology\r\n● " + "Learn adversary tactics and techniques that are utilized as the foundational development " + "of specific threat models and methodologies\r\n● Understand the guidelines for " + "organizations to prepare themselves against cybersecurity attacks" + ) + create_learning_outcomes_page( + course_page, parse_emeritus_data_str(learning_outcomes_str) + ) + assert parse_emeritus_data_str(learning_outcomes_str) == [ + item.value for item in course_page.outcomes.outcome_items + ] + assert course_page.outcomes is not None + + +def test_parse_emeritus_data_str(): + """ + Tests that `parse_emeritus_data_str` parses who should enroll and learning outcomes strings as expected. + """ + data_str = ( + "This program will enable you to:\r\n● Gain an overview of cybersecurity risk " + "management, including its foundational concepts and relevant regulations\r\n● " + "Explore the domains covering various aspects of cloud technology\r\n● " + "Learn adversary tactics and techniques that are utilized as the foundational development " + "of specific threat models and methodologies\r\n● Understand the guidelines for " + "organizations to prepare themselves against cybersecurity attacks" + ) + assert parse_emeritus_data_str(data_str) == [ + "Gain an overview of cybersecurity risk management, including " + "its foundational concepts and relevant regulations", + "Explore the domains covering various aspects of cloud technology", + "Learn adversary tactics and techniques that are utilized as the foundational development " + "of specific threat models and methodologies", + "Understand the guidelines for organizations to prepare themselves against cybersecurity attacks", + ] + + +@pytest.mark.parametrize("create_existing_course_run", [True, False]) +@pytest.mark.django_db +def test_create_or_update_emeritus_course_run(create_existing_course_run): + """ + Tests that `create_or_update_emeritus_course_run` creates or updates a course run + """ + with Path( + "courses/sync_external_courses/test_data/batch_test.json" + ).open() as test_data_file: + emeritus_course = EmeritusCourse(json.load(test_data_file)["rows"][0]) + + course = CourseFactory.create() + if create_existing_course_run: + CourseRunFactory.create( + course=course, + external_course_run_id=emeritus_course.course_run_code, + enrollment_start=None, + enrollment_end=None, + expiration_date=None, + ) + + create_or_update_emeritus_course_run(course, emeritus_course) + course_runs = course.courseruns.all() + course_run_courseware_id = generate_external_course_run_courseware_id( + emeritus_course.course_run_tag, course.readable_id + ) + + assert len(course_runs) == 1 + if create_existing_course_run: + expected_data = { + "external_course_run_id": emeritus_course.course_run_code, + "start_date": emeritus_course.start_date, + "end_date": emeritus_course.end_date, + } + else: + expected_data = { + "title": emeritus_course.course_title, + "external_course_run_id": emeritus_course.course_run_code, + "courseware_id": course_run_courseware_id, + "run_tag": emeritus_course.course_run_tag, + "start_date": emeritus_course.start_date, + "end_date": emeritus_course.end_date, + "live": True, + } + for attr_name, expected_value in expected_data.items(): + assert getattr(course_runs[0], attr_name) == expected_value + + +@pytest.mark.parametrize("create_existing_course_runs", [True, False]) +@pytest.mark.django_db +def test_update_emeritus_course_runs(create_existing_course_runs): + """ + Tests that `update_emeritus_course_runs` creates new courses and updates existing. + """ + with Path( + "courses/sync_external_courses/test_data/batch_test.json" + ).open() as test_data_file: + emeritus_course_runs = json.load(test_data_file)["rows"] + + platform = PlatformFactory.create(name=EmeritusKeyMap.PLATFORM_NAME.value) + + if create_existing_course_runs: + for run in random.sample(emeritus_course_runs, len(emeritus_course_runs) // 2): + course = CourseFactory.create( + title=run["program_name"], + platform=platform, + external_course_id=run["course_code"], + is_external=True, + ) + CourseRunFactory.create( + course=course, + external_course_run_id=run["course_run_code"], + enrollment_start=None, + enrollment_end=None, + expiration_date=None, + ) + + home_page = HomePageFactory.create( + title="Home Page", subhead="

subhead

" + ) + CourseIndexPageFactory.create(parent=home_page, title="Courses") + ExternalCoursePageFactory.create( + course=course, + title=run["program_name"], + external_marketing_url="", + duration="", + description="", + ) + + update_emeritus_course_runs(emeritus_course_runs) + courses = Course.objects.filter(platform=platform) + assert len(courses) == len(emeritus_course_runs) + for emeritus_course_run in emeritus_course_runs: + course = Course.objects.filter( + platform=platform, + external_course_id=emeritus_course_run["course_code"], + is_external=True, + ).first() + assert course is not None + assert ( + course.courseruns.filter( + external_course_run_id=emeritus_course_run["course_run_code"] + ).count() + == 1 + ) + assert hasattr(course, "externalcoursepage") + + course_page = course.externalcoursepage + if emeritus_course_run["program_for"]: + assert course_page.who_should_enroll is not None + if emeritus_course_run["learning_outcomes"]: + assert course_page.outcomes is not None + + +def test_fetch_emeritus_courses_success(settings, mocker): + """ + Tests that `fetch_emeritus_courses` makes the required calls to the `Emeritus` API. Tests the success scenario. + + Here is the expected flow: + 1. Make a get request to get a list of reports. + 2. Make a post request for the `Batch` report. + 3. If the results are not ready, wait for the job to complete and make a get request to check the status. + 4. If the results are ready after the post request, return the results. + 5. If job status is 1 or 2, it is in progress. Wait for 2 seconds and make a get request for Job status. + 6. If job status is 3, the results are ready, make a get request to collect the results and return the data. + """ + settings.EMERITUS_API_BASE_URL = "https://test_emeritus_api.io" + settings.EMERITUS_API_KEY = "test_emeritus_api_key" + settings.EMERITUS_API_REQUEST_TIMEOUT = 60 + + mock_get = mocker.patch( + "courses.sync_external_courses.emeritus_api_client.requests.get" + ) + mock_post = mocker.patch( + "courses.sync_external_courses.emeritus_api_client.requests.post" + ) + + with Path( + "courses/sync_external_courses/test_data/batch_test.json" + ).open() as test_data_file: + emeritus_course_runs = json.load(test_data_file) + + batch_query = { + "id": 77, + "name": "Batch", + } + mock_get.side_effect = [ + MockResponse({"results": [batch_query]}), + MockResponse({"job": {"status": 1}}), + MockResponse({"job": {"status": 2}}), + MockResponse({"job": {"status": 3, "query_result_id": 1}}), + MockResponse({"query_result": {"data": emeritus_course_runs}}), + ] + mock_post.side_effect = [MockResponse({"job": {"id": 1}})] + + actual_course_runs = fetch_emeritus_courses() + + mock_get.assert_any_call( + "https://test_emeritus_api.io/api/queries?api_key=test_emeritus_api_key", + timeout=60, + ) + mock_post.assert_called_once() + mock_get.assert_any_call( + "https://test_emeritus_api.io/api/jobs/1?api_key=test_emeritus_api_key", + timeout=60, + ) + mock_get.assert_any_call( + "https://test_emeritus_api.io/api/query_results/1?api_key=test_emeritus_api_key", + timeout=60, + ) + assert actual_course_runs == emeritus_course_runs["rows"] + + +def test_fetch_emeritus_courses_error(settings, mocker, caplog): + """ + Tests that `fetch_emeritus_courses` specific calls to the Emeritus API and Fails for Job status 3 and 4. + """ + settings.EMERITUS_API_BASE_URL = "https://test_emeritus_api.com" + settings.EMERITUS_API_KEY = "test_emeritus_api_key" + mock_get = mocker.patch( + "courses.sync_external_courses.emeritus_api_client.requests.get" + ) + mock_post = mocker.patch( + "courses.sync_external_courses.emeritus_api_client.requests.post" + ) + + batch_query = { + "id": 77, + "name": "Batch", + } + mock_get.side_effect = [ + MockResponse({"results": [batch_query]}), + MockResponse({"job": {"status": 1}}), + MockResponse({"job": {"status": 2}}), + MockResponse({"job": {"status": 4}}), + ] + mock_post.side_effect = [MockResponse({"job": {"id": 1}})] + with caplog.at_level(logging.ERROR): + fetch_emeritus_courses() + assert "Job failed!" in caplog.text + assert "Something unexpected happened!" in caplog.text diff --git a/courses/sync_external_courses/test_data/batch_test.json b/courses/sync_external_courses/test_data/batch_test.json new file mode 100644 index 000000000..e0b5590c8 --- /dev/null +++ b/courses/sync_external_courses/test_data/batch_test.json @@ -0,0 +1,188 @@ +{ + "columns": [ + { + "name": "program_name", + "friendly_name": "program_name", + "type": "string" + }, + { + "name": "course_code", + "friendly_name": "course_code", + "type": "string" + }, + { + "name": "course_run_code", + "friendly_name": "course_run_code", + "type": "string" + }, + { + "name": "start_date", + "friendly_name": "start_date", + "type": "string" + }, + { + "name": "end_date", + "friendly_name": "end_date", + "type": "string" + }, + { + "name": "Category", + "friendly_name": "Category", + "type": "string" + }, + { + "name": "list_price", + "friendly_name": "list_price", + "type": "float" + }, + { + "name": "list_currency", + "friendly_name": "list_currency", + "type": "string" + }, + { + "name": "total_weeks", + "friendly_name": "total_weeks", + "type": "float" + }, + { + "name": "product_family", + "friendly_name": "product_family", + "type": "string" + }, + { + "name": "product_sub_type", + "friendly_name": "product_sub_type", + "type": "string" + }, + { + "name": "format", + "friendly_name": "format", + "type": "string" + }, + { + "name": "suggested_duration", + "friendly_name": "suggested_duration", + "type": "float" + }, + { + "name": "language", + "friendly_name": "language", + "type": "string" + }, + { + "name": "landing_page_url", + "friendly_name": "landing_page_url", + "type": "string" + }, + { + "name": "Apply_now_url", + "friendly_name": "Apply_now_url", + "type": "string" + }, + { + "name": "description", + "friendly_name": "description", + "type": "string" + }, + { + "name": "learning_outcomes", + "friendly_name": "learning_outcomes", + "type": "string" + }, + { + "name": "program_for", + "friendly_name": "program_for", + "type": "string" + } + ], + "rows": [ + { + "program_name": "Internet of Things (IoT): Design and Applications", + "course_code": "MO-DBIP", + "course_run_code": "MO-DBIP.ELE-25-07#1", + "start_date": "2025-07-30", + "end_date": "2025-09-24", + "Category": "Technology", + "list_price": 2600, + "list_currency": "USD", + "total_weeks": 7, + "product_family": "Certificate", + "product_sub_type": "Short Form", + "format": "Online", + "suggested_duration": 49, + "language": "English", + "landing_page_url": "https://test-emeritus-api.io/Internet-of-things-iot-design-and-applications?utm_medium=EmWebsite&utm_campaign=direct_EmWebsite?utm_campaign=school_website&utm_medium=website&utm_source=MIT-web", + "Apply_now_url": "https://test-emeritus-api.io/?locale=en&program_sfid=01t2s000000OHA2AAO&source=applynowlp&utm_campaign=school&utm_medium=MITWebsite&utm_source=MIT-web", + "description": "By 2030, McKinsey Digital research estimates that it could enable USD 5.5 trillion to USD 12.6 trillion in value globally, including the value captured by consumers and customers of Internet of Things ( IoT) products and services. This future dominated by IoT requires a paradigm shift in the way products and services are designed. MIT xPRO’s Internet of Things (IoT): Design and Applications program is designed to provide you with a strategic road map for creating user-centered, future-ready, differentiated products that drive business growth. Through activities, assignments, and industry examples, you will learn the fundamental principles of developing IoT-centric products and services.\r", + "learning_outcomes": "This program will enable you to:\r\n● Understand how IoT mindsets are contributing to a global shift in product and business strategy\r\n● Discover how IoT impacts the design of both products and businesses\r\n● Identify hardware, software, and data technologies that support IoT adoption\r\n● Explore examples of successful IoT product and business strategies\r\n● Examine legal, ethical, privacy, and security concerns related to IoT", + "program_for": "The program is ideal for:\r\n● Managers and functional leaders looking to learn the strategies for successfully leveraging IoT principles to drive business value\r\n● Product managers and designers who want to shift to an IoT-centric mindset to develop innovative products and services\r\n● Technology professionals keen on understanding the fundamental principles associated with the implementation of IoT and its wide array of applications\r\nNote: This is a nontechnical program for which there are no prerequisites.\r" + }, + { + "program_name": "MedTech Executive", + "course_code": "MO-MTE.SEPO", + "course_run_code": "MO-MTE.SEPO-25-06#1", + "start_date": "2025-06-30", + "end_date": "2025-12-29", + "Category": "Healthcare", + "list_price": null, + "list_currency": null, + "total_weeks": 28, + "product_family": "SEPO", + "product_sub_type": "Online", + "format": "Online", + "suggested_duration": 196, + "language": "English", + "landing_page_url": null, + "Apply_now_url": null, + "description": null, + "learning_outcomes": null, + "program_for": null + }, + { + "program_name": "Technology CEO Program", + "course_code": "MO-TCEO.SEPO", + "course_run_code": "MO-TCEO.SEPO-25-06#1", + "start_date": "2025-06-30", + "end_date": "2026-02-23", + "Category": "Technology", + "list_price": null, + "list_currency": null, + "total_weeks": 29, + "product_family": "SEPO", + "product_sub_type": "Online", + "format": "Online", + "suggested_duration": 203, + "language": "English", + "landing_page_url": null, + "Apply_now_url": null, + "description": null, + "learning_outcomes": null, + "program_for": null + }, + { + "program_name": "Professional Certificate in Cybersecurity", + "course_code": "MO-PCCY", + "course_run_code": "MO-PCCY-25-06#1", + "start_date": "2025-06-26", + "end_date": "2026-01-15", + "Category": "Cybersecurity", + "list_price": 7650, + "list_currency": "USD", + "total_weeks": 29, + "product_family": "Bootcamp", + "product_sub_type": null, + "format": "Online", + "suggested_duration": 203, + "language": "English", + "landing_page_url": "https://test-emeritus-api.io/professional-certificate-cybersecurity?utm_campaign=school_website&utm_medium=website&utm_source=MIT-web", + "Apply_now_url": "https://test-emeritus-api.io/?locale=en&program_sfid=01t2s000000ZdQKAA0&source=applynowlp&utm_campaign=school&utm_medium=MITWebsite&utm_source=MIT-web", + "description": "Cyberattacks are becoming more frequent, complex, and targeted, collectively costing organizations billions of dollars annually. It’s no wonder that cybersecurity is one of the fastest growing industries; by 2027, Forbes projects the value of the cybersecurity market to reach USD 403 billion. More and more companies and government agencies are seeking to hire cybersecurity professionals with the specialized technical skills needed to defend mission-critical computer systems, networks, and cloud applications against cyberattacks. If you’re keen to step into this high-growth field and advance your career, the MIT xPRO Professional Certificate in Cybersecurity is for you.\r", + "learning_outcomes": "This program will enable you to:\r\n● Gain an overview of cybersecurity risk management, including its foundational concepts and relevant regulations\r\n● Explore the domains covering various aspects of cloud technology\r\n● Learn adversary tactics and techniques that are utilized as the foundational development of specific threat models and methodologies\r\n● Understand the guidelines for organizations to prepare themselves against cybersecurity attacks", + "program_for": "The program is ideal for:\r\n● Early-career IT professionals, network engineers, and system administrators wanting to gain a comprehensive overview of cybersecurity and fast-track their career progression\r\n● IT project managers and engineers keen on gaining the ability to think critically about the threat landscape, including vulnerabilities in cybersecurity, and upgrading their resume for career advancement\r\n● Mid- or later-career professionals seeking a career change and looking to add critical cybersecurity knowledge and foundational lessons to their resume" + } + ], + "metadata": { + "data_scanned": 3811194 + } +} diff --git a/courses/tasks.py b/courses/tasks.py index b65db55eb..862b0b3c8 100644 --- a/courses/tasks.py +++ b/courses/tasks.py @@ -10,6 +10,10 @@ from requests.exceptions import HTTPError from courses.models import CourseRun, CourseRunCertificate +from courses.sync_external_courses.emeritus_api import ( + fetch_emeritus_courses, + update_emeritus_course_runs, +) from courses.utils import ( ensure_course_run_grade, process_course_run_grade_certificate, @@ -107,3 +111,14 @@ def sync_courseruns_data(): # `sync_course_runs` logs internally so no need to capture/output the returned values sync_course_runs(runs) + + +@app.task +def task_sync_emeritus_course_runs(): + """Task to sync Emeritus course runs""" + if not settings.FEATURES.get("ENABLE_EXTERNAL_COURSE_SYNC", False): + log.info("External Course sync is disabled.") + return + + emeritus_course_runs = fetch_emeritus_courses() + update_emeritus_course_runs(emeritus_course_runs) diff --git a/mitxpro/settings.py b/mitxpro/settings.py index ac54d4d43..e6e03e099 100644 --- a/mitxpro/settings.py +++ b/mitxpro/settings.py @@ -27,7 +27,7 @@ from mitxpro.celery_utils import OffsettingSchedule from mitxpro.sentry import init_sentry -VERSION = "0.150.0" +VERSION = "0.151.0" ENVIRONMENT = get_string( name="MITXPRO_ENVIRONMENT", @@ -751,6 +751,17 @@ description="'day_of_week' value for 'sync-courseruns-data' scheduled task (default will run once a day).", ) +CRON_EXTERNAL_COURSERUN_SYNC_HOURS = get_string( + name="CRON_EMERITUS_COURSERUN_SYNC_HOURS", + default="0", + description="'hours' value for the 'sync-emeritus-course-runs' scheduled task (defaults to midnight)", +) +CRON_EXTERNAL_COURSERUN_SYNC_DAYS = get_string( + name="CRON_EMERITUS_COURSERUN_SYNC_DAYS", + default=None, + description="'day_of_week' value for 'sync-emeritus-course-runs' scheduled task (default will run once a day).", +) + CELERY_TASK_SERIALIZER = "json" CELERY_RESULT_SERIALIZER = "json" @@ -847,6 +858,16 @@ month_of_year="*", ), }, + "sync-emeritus-course-runs": { + "task": "courses.tasks.task_sync_emeritus_course_runs", + "schedule": crontab( + minute="0", + hour=CRON_EXTERNAL_COURSERUN_SYNC_HOURS, + day_of_week=CRON_EXTERNAL_COURSERUN_SYNC_DAYS or "*", + day_of_month="*", + month_of_year="*", + ), + }, } if FEATURES.get("COUPON_SHEETS"): CELERY_BEAT_SCHEDULE["renew_all_file_watches"] = { @@ -1068,6 +1089,23 @@ description="Timeout (in seconds) for requests made via the edX API client", ) +EMERITUS_API_KEY = get_string( + name="EMERITUS_API_KEY", + default=None, + description="The API Key for Emeritus API", + required=True, +) +EMERITUS_API_BASE_URL = get_string( + name="EMERITUS_API_BASE_URL", + default="https://mit-xpro.emeritus-analytics.io/", + description="Base API URL for Emeritus API", +) +EMERITUS_API_REQUEST_TIMEOUT = get_int( + name="EMERITUS_API_TIMEOUT", + default=60, + description="API request timeout for Emeritus APIs in seconds", +) + # django debug toolbar only in debug mode if DEBUG: INSTALLED_APPS += ("debug_toolbar",) diff --git a/mitxpro/test_utils.py b/mitxpro/test_utils.py index f863b2f52..c1184e7cf 100644 --- a/mitxpro/test_utils.py +++ b/mitxpro/test_utils.py @@ -68,8 +68,13 @@ class MockResponse: Mock requests.Response """ - def __init__( - self, content, status_code=200, content_type="application/json", url=None + def __init__( # noqa: PLR0913 + self, + content, + status_code=200, + content_type="application/json", + url=None, + reason="", ): if isinstance(content, (dict, list)): self.content = json.dumps(content) @@ -78,6 +83,7 @@ def __init__( self.text = self.content self.status_code = status_code self.headers = {"Content-Type": content_type} + self.reason = reason if url: self.url = url @@ -85,6 +91,35 @@ def json(self): """Return json content""" return json.loads(self.content) + def raise_for_status(self): + """Raises :class:`HTTPError`, if one occurred.""" + + http_error_msg = "" + if isinstance(self.reason, bytes): + # We attempt to decode utf-8 first because some servers + # choose to localize their reason strings. If the string + # isn't utf-8, we fall back to iso-8859-1 for all other + # encodings. (See PR #3538) + try: + reason = self.reason.decode("utf-8") + except UnicodeDecodeError: + reason = self.reason.decode("iso-8859-1") + else: + reason = self.reason + + if 400 <= self.status_code < 500: + http_error_msg = ( + f"{self.status_code} Client Error: {reason} for url: {self.url}" + ) + + elif 500 <= self.status_code < 600: + http_error_msg = ( + f"{self.status_code} Server Error: {reason} for url: {self.url}" + ) + + if http_error_msg: + raise HTTPError(http_error_msg, response=self) + class MockHttpError(HTTPError): """Mocked requests.exceptions.HttpError""" diff --git a/mitxpro/utils.py b/mitxpro/utils.py index bc75ee6d5..73a050a33 100644 --- a/mitxpro/utils.py +++ b/mitxpro/utils.py @@ -603,3 +603,26 @@ def get_js_settings(request: HttpRequest): "enable_taxes_display": settings.FEATURES.get("ENABLE_TAXES_DISPLAY", False), "enable_enterprise": settings.FEATURES.get("ENABLE_ENTERPRISE", False), } + + +def clean_url(url, *, remove_query_params=False): + """ + Cleans a URL by removing the extra spaces and Optionally removes the query params to return the base URL. + """ + if not url: + return "" + + if remove_query_params: + url = url[: url.find("?")] + return url.strip() + + +def strip_datetime(date_str, date_format, date_timezone=None): + """ + Strip datetime from string using the format and set timezone. + """ + if not date_str or not date_format: + return None + + date_timezone = date_timezone if date_timezone else datetime.timezone.utc + return datetime.datetime.strptime(date_str, date_format).astimezone(date_timezone)