Shanbady/consolidate static fixtures (#1684)

* testing initial migration change

* testing initial migration change

* moving static fixtures into data_fixtures app

* restoring old management commands

* fixing import

* fixing imports and tests

* updating dev init script to run datamigrations

* restoring data migrations

* adding defaults to get or create

* adding defaults to get or create

* adding docstring

* removing duplicate backpopulate

.devcontainer/devcontainer.json

@@ -22,6 +22,6 @@
     "SEE_API_CLIENT_SECRET": {}
   },
   "features": {},
-  "postStartCommand": "while [ \"$(python manage.py showmigrations | grep \"\\[ \\]\" | wc -l)\" -ne \"0\" ]; do echo \"waiting for migrations\"; sleep 2; done && python manage.py update_offered_by && python manage.py update_platforms && python manage.py update_departments_schools && python manage.py backpopulate_resource_channels --overwrite --all && python manage.py update_course_number_departments && python manage.py backpopulate_mitxonline_data && python manage.py backpopulate_micromasters_data && python manage.py recreate_index --all",
+  "postStartCommand": "while [ \"$(python manage.py showmigrations | grep \"\\[ \\]\" | wc -l)\" -ne \"0\" ]; do echo \"waiting for migrations\"; sleep 2; done && python manage.py update_offered_by && python manage.py update_platforms && python manage.py update_departments_schools && python manage.py update_course_number_departments && python manage.py backpopulate_mitxonline_data && python manage.py backpopulate_micromasters_data && python manage.py backpopulate_resource_channels --overwrite --all && python manage.py recreate_index --all",
   "forwardPorts": [8062, 8063]
 }

data_fixtures/migrations/0001_add_testimonial_data.py

@@ -7,6 +7,16 @@
 from django.core.files import File
 from django.db import migrations
 
+from data_fixtures.utils import (
+    upsert_department_data,
+    upsert_offered_by_data,
+    upsert_platform_data,
+    upsert_school_data,
+    upsert_topic_data_file,
+)
+
+logger = logging.getLogger(__name__)
+
 """
 Fix an issue with PIL's logger when running in test
 https://github.com/camptocamp/pytest-odoo/issues/15
@@ -216,6 +226,27 @@
 ]
 
 
+def load_initial_fixtures(apps, schema_editor):
+    """
+    Load initial static fixtures required by
+    management commands further down
+    """
+    offerors = upsert_offered_by_data()
+    departments = upsert_department_data()
+    schools = upsert_school_data()
+    platforms = upsert_platform_data()
+    topics = upsert_topic_data_file()
+    logout = (
+        f"Updated:"
+        f"   {offerors} offerors"
+        f"   {departments} departments"
+        f"   {schools} schools"
+        f"   {platforms} platforms"
+        f"   {topics} topics"
+    )
+    logger.info(logout)
+
+
 def load_fixtures(apps, schema_editor):
     """
     Load fixtures for testimonials
@@ -252,5 +283,6 @@ class Migration(migrations.Migration):
     dependencies = []
 
     operations = [
+        migrations.RunPython(load_initial_fixtures, migrations.RunPython.noop),
         migrations.RunPython(load_fixtures, migrations.RunPython.noop),
     ]

data_fixtures/migrations/0004_upsert_initial_topic_data.py

@@ -2,7 +2,7 @@
 
 from django.db import migrations
 
-from learning_resources.utils import upsert_topic_data_file
+from data_fixtures.utils import upsert_topic_data_file
 
 
 def perform_topic_upsert(apps, schema_editor):

data_fixtures/migrations/0007_topic_mappings_edx_add_programming_coding_to_computer_science.py

@@ -5,7 +5,7 @@
 
 from django.db import migrations
 
-from learning_resources.utils import upsert_topic_data_string
+from data_fixtures.utils import upsert_topic_data_string
 
 map_changes = """
 ---

data_fixtures/migrations/0009_topics_update_engineering_add_manufacturing.py

@@ -8,7 +8,7 @@
 from django.db import migrations
 
 from channels.constants import ChannelType
-from learning_resources.utils import upsert_topic_data_string
+from data_fixtures.utils import upsert_topic_data_string
 
 map_changes = """
 ---

data_fixtures/migrations/0012_topic_mappings_adjust_sustainable_business_finance_accounting.py

@@ -7,7 +7,7 @@
 
 from django.db import migrations
 
-from learning_resources.utils import upsert_topic_data_string
+from data_fixtures.utils import upsert_topic_data_string
 
 forward_map_changes = """
 ---

data_fixtures/migrations/0014_add_department_SP.py

@@ -33,6 +33,10 @@ def add_sp(apps, schema_editor):
     department, _ = LearningResourceDepartment.objects.get_or_create(
         department_id="SP",
         name="Special Programs",
+        defaults={
+            "department_id": "SP",
+            "name": "Special Programs",
+        },
     )
     Channel = apps.get_model("channels", "Channel")
 
@@ -43,7 +47,9 @@ def add_sp(apps, schema_editor):
         title=department.name,
     )
     ChannelDepartmentDetail.objects.get_or_create(
-        channel=channel, department=department
+        channel=channel,
+        department=department,
+        defaults={"channel": channel, "department": department},
     )
 
 

data_fixtures/utils.py

@@ -0,0 +1,147 @@
+import json
+from pathlib import Path
+
+import yaml
+from django.db import transaction
+
+from learning_resources.models import (
+    LearningResourceDepartment,
+    LearningResourceOfferor,
+    LearningResourcePlatform,
+    LearningResourceSchool,
+)
+from learning_resources.utils import (
+    _walk_topic_map,
+    department_delete_actions,
+    offeror_delete_actions,
+    offeror_upserted_actions,
+)
+
+
+@transaction.atomic()
+def upsert_topic_data_file(
+    config_path: str = "learning_resources/data/topics.yaml",
+) -> None:
+    """
+    Load the topics from a yaml file.
+
+    The yaml file should have a root "topics" key, and then any number of topic
+    records beneath it. See _walk_topic_map for an explanation of the record
+    format.
+
+    Args:
+    - config_path (str): the path to the topics file.
+    Returns:
+    - None
+    """
+
+    with Path.open(Path(config_path)) as topic_file:
+        topic_file_yaml = topic_file.read()
+
+    topics = yaml.safe_load(topic_file_yaml)
+
+    _walk_topic_map(topics["topics"])
+
+
+@transaction.atomic()
+def upsert_offered_by_data():
+    """
+    Upsert LearningResourceOfferor data
+    """
+    offerors = []
+    with Path.open(Path(__file__).parent / "fixtures" / "offered_by.json") as inf:
+        offered_by_json = json.load(inf)
+        for offeror in offered_by_json:
+            offeror_fields = offeror["fields"]
+            offered_by, _ = LearningResourceOfferor.objects.update_or_create(
+                code=offeror_fields["code"],
+                defaults=offeror_fields,
+            )
+            offeror_upserted_actions(offered_by, overwrite=True)
+            offerors.append(offeror_fields["name"])
+        invalid_offerors = LearningResourceOfferor.objects.exclude(name__in=offerors)
+        for offeror in invalid_offerors:
+            offeror_delete_actions(offeror)
+    return offerors
+
+
+@transaction.atomic()
+def upsert_department_data():
+    """
+    Upsert LearningResourceDepartment data
+    """
+    departments = []
+    with Path.open(Path(__file__).parent / "fixtures" / "departments.json") as inf:
+        departments_json = json.load(inf)
+        for dept in departments_json:
+            department_fields = dept["fields"]
+            LearningResourceDepartment.objects.update_or_create(
+                department_id=department_fields["department_id"],
+                defaults=department_fields,
+            )
+            departments.append(department_fields["name"])
+        invalid_departments = LearningResourceDepartment.objects.exclude(
+            name__in=departments
+        ).all()
+        for invalid_department in invalid_departments:
+            department_delete_actions(invalid_department)
+    return departments
+
+
+@transaction.atomic()
+def upsert_school_data():
+    """
+    Upsert LearningResourceSchool data
+    """
+    schools = []
+    with Path.open(Path(__file__).parent / "fixtures" / "schools.json") as inf:
+        schools_json = json.load(inf)
+        for school in schools_json:
+            school_fields = school["fields"]
+            LearningResourceSchool.objects.update_or_create(
+                id=school_fields["id"],
+                defaults=school_fields,
+            )
+            schools.append(school_fields["name"])
+        LearningResourceSchool.objects.exclude(name__in=schools).delete()
+    return schools
+
+
+@transaction.atomic()
+def upsert_platform_data():
+    """
+    Upsert LearningResourcePlatform data
+    """
+    platforms = []
+    with Path.open(Path(__file__).parent / "fixtures" / "platforms.json") as inf:
+        platform_json = json.load(inf)
+        for platform in platform_json:
+            platform_fields = platform["fields"]
+            LearningResourcePlatform.objects.update_or_create(
+                code=platform_fields["code"],
+                defaults=platform_fields,
+            )
+            platforms.append(platform_fields["code"])
+        LearningResourcePlatform.objects.exclude(code__in=platforms).delete()
+    return platforms
+
+
+@transaction.atomic()
+def upsert_topic_data_string(yaml_data: str) -> None:
+    """
+    Load the topics from a yaml string.
+
+    The yaml string should be formatted in the same way that it is for
+    upsert_topic_data_file - this function exists just to allow you to specify
+    the data as a string so you can roll it into a migration file in the
+    data_fixtures app.
+
+    Args:
+    - yaml_data (str): the yaml to process
+    Returns:
+    - None
+    """
+
+    topics = yaml.safe_load(yaml_data)
+
+    _walk_topic_map(topics["topics"])

learning_resources/etl/prolearn_test.py

@@ -7,6 +7,7 @@
 
 import pytest
 
+from data_fixtures.utils import upsert_topic_data_file
 from learning_resources.constants import (
     Availability,
     CertificationType,
@@ -40,7 +41,6 @@
     LearningResourcePlatformFactory,
 )
 from learning_resources.models import LearningResourceOfferor, LearningResourcePlatform
-from learning_resources.utils import upsert_topic_data_file
 from main.test_utils import assert_json_equal
 from main.utils import clean_data
 

learning_resources/management/commands/update_departments_schools.py

@@ -2,7 +2,7 @@
 
 from django.core.management import BaseCommand
 
-from learning_resources.utils import (
+from data_fixtures.utils import (
     upsert_department_data,
     upsert_school_data,
 )

learning_resources/management/commands/update_offered_by.py

@@ -2,7 +2,7 @@
 
 from django.core.management import BaseCommand
 
-from learning_resources.utils import upsert_offered_by_data
+from data_fixtures.utils import upsert_offered_by_data
 from main.utils import clear_search_cache, now_in_utc
 
 

learning_resources/management/commands/update_platforms.py

@@ -2,7 +2,7 @@
 
 from django.core.management import BaseCommand
 
-from learning_resources.utils import upsert_platform_data
+from data_fixtures.utils import upsert_platform_data
 from main.utils import clear_search_cache, now_in_utc
 
 

learning_resources/test_utils.py

@@ -1,7 +1,7 @@
 """Test utility functions for learning resources."""
 
+from data_fixtures.utils import upsert_topic_data_file
 from learning_resources.factories import LearningResourceOfferorFactory
-from learning_resources.utils import upsert_topic_data_file
 
 
 def set_up_topics(**kwargs):