Update the ETL pipelines times (#1416)

mitodl · Aug 15, 2024 · c1e051d · c1e051d
1 parent a2d7b87
commit c1e051d
Showing 1 changed file with 12 additions and 14 deletions.
diff --git a/main/settings_celery.py b/main/settings_celery.py
@@ -23,35 +23,33 @@
 CELERY_BEAT_SCHEDULE = {
     "update_next-start-date-every-1-days": {
         "task": "learning_resources.tasks.update_next_start_date_and_prices",
-        "schedule": crontab(minute=0, hour=4),  # midnight EST
+        "schedule": crontab(minute=0, hour=7),  # 3:00am EST
     },
     "update_edx-courses-every-1-days": {
         "task": "learning_resources.tasks.get_mit_edx_data",
-        "schedule": crontab(minute=30, hour=15),  # 11:30am EST
+        "schedule": crontab(minute=0, hour=5),  # 1:00am EST
     },
     "update-edx-files-every-1-weeks": {
         "task": "learning_resources.tasks.import_all_mit_edx_files",
-        "schedule": crontab(
-            minute=0, hour=16, day_of_week=1
-        ),  # 12:00 PM EST on Mondays
+        "schedule": crontab(minute=0, hour=5, day_of_week=1),  # 12:00 PM EST on Mondays
     },
     "update-micromasters-programs-every-1-days": {
         "task": "learning_resources.tasks.get_micromasters_data",
-        "schedule": crontab(minute=30, hour=16),
+        "schedule": crontab(minute=0, hour=5),  # 1:00am EST
     },
     "update-mitxonline-courses-every-1-days": {
         "task": "learning_resources.tasks.get_mitxonline_data",
-        "schedule": crontab(minute=30, hour=19),  # 3:30pm EST
+        "schedule": crontab(minute=0, hour=5),  # 1:00am EST
     },
     "update-mitxonline-files-every-1-weeks": {
         "task": "learning_resources.tasks.import_all_mitxonline_files",
         "schedule": crontab(
-            minute=0, hour=16, day_of_week=3
+            minute=0, hour=5, day_of_week=3
         ),  # 12:00 PM EST on Wednesdays
     },
     "update-oll-courses-every-1-days": {
         "task": "learning_resources.tasks.get_oll_data",
-        "schedule": crontab(minute=30, hour=18),  # 2:30pm EST
+        "schedule": crontab(minute=0, hour=5),  # 1:00am EST
     },
     "update-podcasts": {
         "task": "learning_resources.tasks.get_podcast_data",
@@ -61,22 +59,22 @@
     },
     "update-prolearn-courses-every-1-days": {
         "task": "learning_resources.tasks.get_prolearn_data",
-        "schedule": crontab(minute=30, hour=21),  # 5:30pm EST
+        "schedule": crontab(minute=0, hour=5),  # 1:00am EST
     },
     "update-xpro-courses-every-1-days": {
         "task": "learning_resources.tasks.get_xpro_data",
-        "schedule": crontab(minute=30, hour=17),  # 1:30pm EST
+        "schedule": crontab(minute=0, hour=5),  # 1:00am EST
     },
     "update-xpro-files-every-1-weeks": {
         "task": "learning_resources.tasks.import_all_xpro_files",
         "schedule": crontab(
-            minute=0, hour=16, day_of_week=2
+            minute=0, hour=5, day_of_week=2
         ),  # 12:00 PM EST on Tuesdays
     },
     "update-oll-files-every-1-weeks": {
         "task": "learning_resources.tasks.import_all_oll_files",
         "schedule": crontab(
-            minute=0, hour=16, day_of_week=3
+            minute=0, hour=5, day_of_week=3
         ),  # 12:00 PM EST on Wednesdays
     },
     "update-youtube-videos": {
@@ -128,7 +126,7 @@
     },
     "update-search-featured-ranks-1-days": {
         "task": "learning_resources_search.tasks.update_featured_rank",
-        "schedule": crontab(minute=30, hour=5),  # 1:30am EST
+        "schedule": crontab(minute=30, hour=7),  # 3:30am EST
     },
 }