Merge pull request #834 from openzim/enhance_offliners_defs

Enhance offliners definitions

dispatcher/backend/src/common/schemas/__init__.py

@@ -1,24 +1,52 @@
 from marshmallow import Schema, fields
+from marshmallow.validate import And, ValidationError, Validator
+
+
+class NoNullChar(Validator):
+    """Validate that string value does not contains Unicode null character"""
+
+    def __call__(self, value: str) -> str:
+        if "\u0000" in value:
+            raise ValidationError("Null character is not allowed")
+
+        return value
+
+
+class String(fields.String):
+    """A custom String field for our needs
+
+    In addition to base type checks, it also ensures that value does not contains
+    Unicode null character
+    """
+
+    @property
+    def _validate_all(self):
+        return And(
+            NoNullChar(),
+            *self.validators,
+            error=self.error_messages["validator_failed"]
+        )
 
 
 class ListOfStringEnum(fields.List):
     pass
 
 
-class StringEnum(fields.String):
+class StringEnum(String):
     pass
 
 
-class HexColor(fields.String):
+class HexColor(String):
     pass
 
 
-class LongString(fields.String):
+class LongString(String):
     pass
 
 
 class SerializableSchema(Schema):
     MAPPING = {
+        String: "text",
         fields.String: "text",
         LongString: "long-text",
         StringEnum: "string-enum",

dispatcher/backend/src/common/schemas/fields.py

@@ -9,6 +9,7 @@
     WarehousePath,
 )
 from common.roles import ROLES
+from common.schemas import String
 
 # validators
 validate_priority = validate.Range(min=0, max=10)
@@ -65,11 +66,11 @@ def validate_multiple_of_100(value):
     required=False, load_default=20, validate=validate.Range(min=0, max=200)
 )
 priority_field = fields.Integer(required=False, validate=validate_priority)
-worker_field = fields.String(required=False, validate=validate_worker_name)
-schedule_name_field = fields.String(validate=validate_schedule_name)
-category_field = fields.String(required=False, validate=validate_category)
-periodicity_field = fields.String(required=False, validate=validate_periodicity)
-tag_field = fields.List(fields.String(validate=validate_not_empty), required=False)
-offliner_field = fields.String(required=False, validate=validate_offliner)
+worker_field = String(required=False, validate=validate_worker_name)
+schedule_name_field = String(validate=validate_schedule_name)
+category_field = String(required=False, validate=validate_category)
+periodicity_field = String(required=False, validate=validate_periodicity)
+tag_field = fields.List(String(validate=validate_not_empty), required=False)
+offliner_field = String(required=False, validate=validate_offliner)
 email_field = fields.Email(required=False, validate=validate_not_empty)
-username_field = fields.String(required=True, validate=validate_not_empty)
+username_field = String(required=True, validate=validate_not_empty)

dispatcher/backend/src/common/schemas/models.py

@@ -3,7 +3,7 @@
 from marshmallow import Schema, fields, pre_load, validate, validates_schema
 
 from common.enum import DockerImageName, Offliner, Platform
-from common.schemas import SerializableSchema
+from common.schemas import SerializableSchema, String
 from common.schemas.fields import (
     validate_category,
     validate_cpu,
@@ -37,23 +37,23 @@
 
 
 class LanguageSchema(Schema):
-    code = fields.String(required=True, validate=validate_lang_code)
-    name_en = fields.String(required=True, validate=validate_not_empty)
-    name_native = fields.String(required=True, validate=validate_not_empty)
+    code = String(required=True, validate=validate_lang_code)
+    name_en = String(required=True, validate=validate_not_empty)
+    name_native = String(required=True, validate=validate_not_empty)
 
 
 class ResourcesSchema(Schema):
     cpu = fields.Integer(required=True, validate=validate_cpu)
     memory = fields.Integer(required=True, validate=validate_memory)
     disk = fields.Integer(required=True, validate=validate_disk)
     shm = fields.Integer(required=False, validate=validate_memory)
-    cap_add = fields.List(fields.String(), required=False)
-    cap_drop = fields.List(fields.String(), required=False)
+    cap_add = fields.List(String(), required=False)
+    cap_drop = fields.List(String(), required=False)
 
 
 class DockerImageSchema(Schema):
-    name = fields.String(required=True, validate=validate.OneOf(DockerImageName.all()))
-    tag = fields.String(required=True)
+    name = String(required=True, validate=validate.OneOf(DockerImageName.all()))
+    tag = String(required=True)
 
     @pre_load
     def strip_prefix(self, in_data, **kwargs):
@@ -65,12 +65,12 @@ def strip_prefix(self, in_data, **kwargs):
 
 
 class ScheduleConfigSchema(SerializableSchema):
-    task_name = fields.String(required=True, validate=validate_offliner)
-    warehouse_path = fields.String(required=True, validate=validate_warehouse_path)
+    task_name = String(required=True, validate=validate_offliner)
+    warehouse_path = String(required=True, validate=validate_warehouse_path)
     image = fields.Nested(DockerImageSchema(), required=True)
     resources = fields.Nested(ResourcesSchema(), required=True)
     flags = fields.Dict(required=True)
-    platform = fields.String(required=True, allow_none=True, validate=validate_platform)
+    platform = String(required=True, allow_none=True, validate=validate_platform)
     monitor = fields.Boolean(required=True, truthy=[True], falsy=[False])
 
     @staticmethod
@@ -101,7 +101,7 @@ def validate(self, data, **kwargs):
 class EventNotificationSchema(SerializableSchema):
     mailgun = fields.List(fields.Email(), required=False)
     webhook = fields.List(fields.Url(), required=False)
-    slack = fields.List(fields.String(validate=validate_slack_target), required=False)
+    slack = fields.List(String(validate=validate_slack_target), required=False)
 
 
 class ScheduleNotificationSchema(SerializableSchema):
@@ -111,12 +111,12 @@ class ScheduleNotificationSchema(SerializableSchema):
 
 
 class ScheduleSchema(Schema):
-    name = fields.String(required=True, validate=validate_schedule_name)
+    name = String(required=True, validate=validate_schedule_name)
     language = fields.Nested(LanguageSchema(), required=True)
-    category = fields.String(required=True, validate=validate_category)
-    periodicity = fields.String(required=True, validate=validate_periodicity)
+    category = String(required=True, validate=validate_category)
+    periodicity = String(required=True, validate=validate_periodicity)
     tags = fields.List(
-        fields.String(validate=validate_not_empty), required=True, dump_default=[]
+        String(validate=validate_not_empty), required=True, dump_default=[]
     )
     enabled = fields.Boolean(required=True, truthy=[True], falsy=[False])
     config = fields.Nested(ScheduleConfigSchema(), required=True)

dispatcher/backend/src/common/schemas/offliners/freecodecamp.py

@@ -1,54 +1,67 @@
-from marshmallow import fields
+from marshmallow import fields, validate
 
-from common.schemas import SerializableSchema
+from common.schemas import LongString, SerializableSchema, String, StringEnum
 from common.schemas.fields import (
     validate_output,
     validate_zim_description,
     validate_zim_filename,
     validate_zim_longdescription,
 )
 
+FCC_LANG_MAP = {
+    "ara": "arabic",
+    "cmn": "chinese",
+    "lzh": "chinese-traditional",
+    "eng": "english",
+    "spa": "espanol",
+    "deu": "german",
+    "ita": "italian",
+    "jpn": "japanese",
+    "por": "portuguese",
+    "ukr": "ukranian",
+}
+
 
 class FreeCodeCampFlagsSchema(SerializableSchema):
     class Meta:
         ordered = True
 
-    course = fields.String(
+    course = String(
         metadata={
             "label": "Course(s)",
             "description": "Course or course list (separated by commas)",
         },
         required=True,
     )
 
-    language = fields.String(
+    language = StringEnum(
         metadata={
             "label": "Language",
-            "description": "Language of zim file and curriculum. Either (without "
-            "quotes) 'ara' (arabic), 'cmn' (chinese), 'lzh' (chinese-traditional), "
-            "'eng' (english), 'spa' (espanol), 'deu' (german), 'ita' (italian), "
-            "'jpn' (japanese), 'por' (portuguese), 'ukr' (ukranian).",
+            "description": "Language of zim file and curriculum. One of "
+            + ", ".join([f"'{key}' ({desc})" for key, desc in FCC_LANG_MAP.items()])
+            + ".",
         },
         required=True,
+        validate=validate.OneOf(list(FCC_LANG_MAP.keys())),
     )
 
-    name = fields.String(
+    name = String(
         metadata={
             "label": "Name",
             "description": "ZIM name",
         },
         required=True,
     )
 
-    title = fields.String(
+    title = String(
         metadata={
             "label": "Title",
             "description": "ZIM title",
         },
         required=True,
     )
 
-    description = fields.String(
+    description = String(
         metadata={
             "label": "Description",
             "description": "Description for your ZIM",
@@ -57,7 +70,7 @@ class Meta:
         validate=validate_zim_description,
     )
 
-    long_description = fields.String(
+    long_description = LongString(
         metadata={
             "label": "Long description",
             "description": "Optional long description for your ZIM",
@@ -66,14 +79,14 @@ class Meta:
         data_key="long-description",
     )
 
-    creator = fields.String(
+    creator = String(
         metadata={
             "label": "Content Creator",
             "description": "Name of content creator. “freeCodeCamp” otherwise",
         }
     )
 
-    publisher = fields.String(
+    publisher = String(
         metadata={
             "label": "Publisher",
             "description": "Custom publisher name (ZIM metadata). “OpenZIM” otherwise",
@@ -86,7 +99,7 @@ class Meta:
         metadata={"label": "Debug", "description": "Enable verbose output"},
     )
 
-    output_dir = fields.String(
+    output_dir = String(
         metadata={
             "label": "Output folder",
             "placeholder": "/output",
@@ -98,7 +111,7 @@ class Meta:
         validate=validate_output,
     )
 
-    zim_file = fields.String(
+    zim_file = String(
         metadata={
             "label": "ZIM filename",
             "description": "ZIM file name (based on --name if not provided). "

dispatcher/backend/src/common/schemas/offliners/gutenberg.py

@@ -1,13 +1,14 @@
 from marshmallow import fields
 
-from common.schemas import SerializableSchema
+from common.schemas import SerializableSchema, String
+from common.schemas.fields import validate_zim_description
 
 
 class GutenbergFlagsSchema(SerializableSchema):
     class Meta:
         ordered = True
 
-    languages = fields.String(
+    languages = String(
         metadata={
             "label": "Languages",
             "description": (
@@ -17,7 +18,7 @@ class Meta:
         },
     )
 
-    formats = fields.String(
+    formats = String(
         metadata={
             "label": "Formats",
             "description": (
@@ -27,20 +28,21 @@ class Meta:
         },
     )
 
-    zim_title = fields.String(
+    zim_title = String(
         metadata={
             "label": "Title",
             "description": "Custom title for your project and ZIM.",
         },
         data_key="zim-title",
     )
 
-    zim_desc = fields.String(
+    zim_desc = String(
         metadata={"label": "Description", "description": "Description for ZIM"},
         data_key="zim-desc",
+        validate=validate_zim_description,
     )
 
-    books = fields.String(
+    books = String(
         metadata={
             "label": "Books",
             "description": (