Check recipe flags at schedule creation (including fix for youzim.it)

dispatcher/backend/src/common/constants.py

@@ -97,3 +97,6 @@
 REQ_TIMEOUT_NOTIFICATIONS = int(os.getenv("REQ_TIMEOUT_NOTIFICATIONS", 5))
 REQ_TIMEOUT_CMS = int(os.getenv("REQ_TIMEOUT_CMS", 10))
 REQ_TIMEOUT_GHCR = int(os.getenv("REQ_TIMEOUT_GHCR", 10))
+
+# OFFLINERS
+ZIMIT_DISABLE_ZIM_FILENAME_CHECK = os.getenv("ZIMIT_DISABLE_ZIM_FILENAME_CHECK", "")

dispatcher/backend/src/common/schemas/offliners/zimit.py

@@ -1,5 +1,6 @@
 from marshmallow import fields, validate
 
+from common.constants import ZIMIT_DISABLE_ZIM_FILENAME_CHECK
 from common.schemas import SerializableSchema, String, StringEnum
 from common.schemas.fields import (
     validate_output,
@@ -190,7 +191,9 @@ class Meta:
             "Make sure to end with _{period}.zim",
         },
         data_key="zim-file",
-        validate=validate_zim_filename,
+        validate=validate_zim_filename
+        if not ZIMIT_DISABLE_ZIM_FILENAME_CHECK
+        else None,
     )
 
     tags = String(

dispatcher/backend/src/routes/schedules/schedule.py

@@ -125,6 +125,10 @@ def post(self, session: so.Session, token: AccessToken.Payload):
 
         try:
             document = ScheduleSchema().load(request.get_json())
+            flags_schema = ScheduleConfigSchema.get_offliner_schema(
+                document["config"]["task_name"]
+            )
+            flags_schema().load(document["config"]["flags"])
         except ValidationError as e:
             raise InvalidRequestJSON(e.messages)
 

dispatcher/backend/src/tests/integration/routes/schedules/test_freecodecamp.py

@@ -1,9 +1,9 @@
 class TestFreeCodeCamp:
-    def test_create_freecodecamp_schedule(
+    def test_create_freecodecamp_schedule_ok(
         self, client, access_token, garbage_collector
     ):
         schedule = {
-            "name": "fcc_javascript_test",
+            "name": "fcc_javascript_test_ok",
             "category": "freecodecamp",
             "enabled": False,
             "tags": [],
@@ -14,7 +14,13 @@ def test_create_freecodecamp_schedule(
                 "image": {"name": "openzim/freecodecamp", "tag": "1.0.0"},
                 "monitor": False,
                 "platform": None,
-                "flags": {},
+                "flags": {
+                    "course": ("somecourse"),
+                    "language": "eng",
+                    "name": "acourse",
+                    "title": "a title",
+                    "description": "a description",
+                },
                 "resources": {"cpu": 3, "memory": 1024, "disk": 0},
             },
             "periodicity": "quarterly",
@@ -24,9 +30,10 @@ def test_create_freecodecamp_schedule(
         response = client.post(
             url, json=schedule, headers={"Authorization": access_token}
         )
-        assert response.status_code == 201
         response_data = response.get_json()
-        garbage_collector.add_schedule_id(response_data["_id"])
+        if "_id" in response_data:
+            garbage_collector.add_schedule_id(response_data["_id"])
+        assert response.status_code == 201
 
         patch_data = {
             "enabled": True,
@@ -50,3 +57,48 @@ def test_create_freecodecamp_schedule(
             url, json=patch_data, headers={"Authorization": access_token}
         )
         assert response.status_code == 204
+
+    def test_create_freecodecamp_schedule_ko(
+        self, client, access_token, garbage_collector
+    ):
+        schedule = {
+            "name": "fcc_javascript_test_ko",
+            "category": "freecodecamp",
+            "enabled": False,
+            "tags": [],
+            "language": {"code": "fr", "name_en": "French", "name_native": "Français"},
+            "config": {
+                "task_name": "freecodecamp",
+                "warehouse_path": "/freecodecamp",
+                "image": {"name": "openzim/freecodecamp", "tag": "1.0.0"},
+                "monitor": False,
+                "platform": None,
+                "flags": {
+                    "course": ("somecourse"),
+                    "language": "eng",
+                    "name": "acourse",
+                    "title": "a title",
+                    "description": (
+                        "a description which is way way way way way way way way way "
+                        "way way way way way way way way way too long"
+                    ),
+                },
+                "resources": {"cpu": 3, "memory": 1024, "disk": 0},
+            },
+            "periodicity": "quarterly",
+        }
+
+        url = "/schedules/"
+        response = client.post(
+            url, json=schedule, headers={"Authorization": access_token}
+        )
+        response_data = response.get_json()
+        if "_id" in response_data:
+            garbage_collector.add_schedule_id(response_data["_id"])
+        assert response.status_code == 400
+        assert "error_description" in response_data
+        assert "description" in response_data["error_description"]
+        assert (
+            "Longer than maximum length 80."
+            in response_data["error_description"]["description"]
+        )

dispatcher/backend/src/tests/integration/routes/schedules/test_schedule.py

@@ -265,15 +265,25 @@ def test_create_schedule_missing_keys(self, client, access_token, key):
                 "flags": {},
                 "image": {"name": "openzim/phet", "tag": "latest"},
                 "monitor": False,
+                "platform": None,
+                "resources": {"cpu": 3, "memory": 1024, "disk": 0},
             },
+            "periodicity": "quarterly",
         }
 
         del schedule[key]
         url = "/schedules/"
         response = client.post(
             url, json=schedule, headers={"Authorization": access_token}
         )
+        response_data = response.get_json()
         assert response.status_code == 400
+        assert "error_description" in response_data
+        assert key in response_data["error_description"]
+        assert (
+            "Missing data for required field."
+            in response_data["error_description"][key]
+        )
 
     @pytest.mark.parametrize("key", ["warehouse_path", "flags", "image"])
     def test_create_schedule_missing_config_keys(self, client, access_token, key):
@@ -293,15 +303,62 @@ def test_create_schedule_missing_config_keys(self, client, access_token, key):
                 "flags": {},
                 "image": {"name": "openzim/phet", "tag": "latest"},
                 "monitor": False,
+                "platform": None,
+                "resources": {"cpu": 3, "memory": 1024, "disk": 0},
             },
+            "periodicity": "quarterly",
         }
 
         del schedule["config"][key]
         url = "/schedules/"
         response = client.post(
             url, json=schedule, headers={"Authorization": access_token}
         )
+        response_data = response.get_json()
         assert response.status_code == 400
+        assert "error_description" in response_data
+        assert "config" in response_data["error_description"]
+        assert key in response_data["error_description"]["config"]
+        assert (
+            "Missing data for required field."
+            in response_data["error_description"]["config"][key]
+        )
+
+    def test_create_schedule_flags_ko(self, client, access_token):
+        schedule = {
+            "name": "ifixit flags ko",
+            "category": "ifixit",
+            "enabled": False,
+            "tags": [],
+            "language": {
+                "code": "en",
+                "name_en": "English",
+                "name_native": "English",
+            },
+            "config": {
+                "task_name": "ifixit",
+                "warehouse_path": "/ifixit",
+                "flags": {},
+                "image": {"name": "openzim/ifixit", "tag": "latest"},
+                "monitor": False,
+                "platform": "ifixit",
+                "resources": {"cpu": 3, "memory": 1024, "disk": 0},
+            },
+            "periodicity": "quarterly",
+        }
+
+        url = "/schedules/"
+        response = client.post(
+            url, json=schedule, headers={"Authorization": access_token}
+        )
+        response_data = response.get_json()
+        assert response.status_code == 400
+        assert "error_description" in response_data
+        assert "language" in response_data["error_description"]
+        assert (
+            "Missing data for required field."
+            in response_data["error_description"]["language"]
+        )
 
     def test_image_names(self, client, schedule, access_token):
         url = "/schedules/{}/image-names".format(schedule["name"])

dispatcher/backend/src/tests/integration/routes/schedules/test_zimit.py

@@ -0,0 +1,111 @@
+class TestZimit:
+    def test_create_zimit_schedule_ok(self, client, access_token, garbage_collector):
+        schedule = {
+            "name": "zimit_test_ok",
+            "category": "other",
+            "enabled": False,
+            "tags": [],
+            "language": {"code": "fr", "name_en": "French", "name_native": "Français"},
+            "config": {
+                "task_name": "zimit",
+                "warehouse_path": "/other",
+                "image": {"name": "openzim/zimit", "tag": "1.0.0"},
+                "monitor": False,
+                "platform": None,
+                "flags": {
+                    "name": "acme",
+                    "url": "https://www.acme.com",
+                    "zim-file": "acme_en_all_{period}.zim",
+                },
+                "resources": {"cpu": 3, "memory": 1024, "disk": 0},
+            },
+            "periodicity": "quarterly",
+        }
+
+        url = "/schedules/"
+        response = client.post(
+            url, json=schedule, headers={"Authorization": access_token}
+        )
+        response_data = response.get_json()
+        print(response_data)
+        if "_id" in response_data:
+            garbage_collector.add_schedule_id(response_data["_id"])
+        assert response.status_code == 201
+
+    # below test is green only if ZIMIT_DISABLE_ZIM_FILENAME_CHECK is set
+    # def test_create_zimit_schedule_bad_name_ok(
+    #     self, client, access_token, garbage_collector
+    # ):
+    #     schedule = {
+    #         "name": "zimit_test_bad_name_ok",
+    #         "category": "other",
+    #         "enabled": False,
+    #         "tags": [],
+    #         "language": {
+    #             "code": "fr",
+    #             "name_en": "French",
+    #             "name_native": "Français"
+    #         },
+    #         "config": {
+    #             "task_name": "zimit",
+    #             "warehouse_path": "/other",
+    #             "image": {"name": "openzim/zimit", "tag": "1.0.0"},
+    #             "monitor": False,
+    #             "platform": None,
+    #             "flags": {
+    #                 "name": "acme",
+    #                 "url": "https://www.acme.com",
+    #                 "zim-file": "bad_name",
+    #             },
+    #             "resources": {"cpu": 3, "memory": 1024, "disk": 0},
+    #         },
+    #         "periodicity": "quarterly",
+    #     }
+
+    #     url = "/schedules/"
+    #     response = client.post(
+    #         url, json=schedule, headers={"Authorization": access_token}
+    #     )
+    #     response_data = response.get_json()
+    #     print(response_data)
+    #     if "_id" in response_data:
+    #         garbage_collector.add_schedule_id(response_data["_id"])
+    #     assert response.status_code == 201
+
+    # below test becomes red if ZIMIT_DISABLE_ZIM_FILENAME_CHECK is set
+    def test_create_zimit_schedule_bad_name_nok(
+        self, client, access_token, garbage_collector
+    ):
+        schedule = {
+            "name": "zimit_test_bad_name_nok",
+            "category": "other",
+            "enabled": False,
+            "tags": [],
+            "language": {"code": "fr", "name_en": "French", "name_native": "Français"},
+            "config": {
+                "task_name": "zimit",
+                "warehouse_path": "/other",
+                "image": {"name": "openzim/zimit", "tag": "1.0.0"},
+                "monitor": False,
+                "platform": None,
+                "flags": {
+                    "name": "acme",
+                    "url": "https://www.acme.com",
+                    "zim-file": "bad_name",
+                },
+                "resources": {"cpu": 3, "memory": 1024, "disk": 0},
+            },
+            "periodicity": "quarterly",
+        }
+
+        url = "/schedules/"
+        response = client.post(
+            url, json=schedule, headers={"Authorization": access_token}
+        )
+        response_data = response.get_json()
+        print(response_data)
+        if "_id" in response_data:
+            garbage_collector.add_schedule_id(response_data["_id"])
+        assert response.status_code == 400
+        assert "error_description" in response_data
+        assert "zim-file" in response_data["error_description"]