diff --git a/dispatcher/backend/src/common/schemas/__init__.py b/dispatcher/backend/src/common/schemas/__init__.py index ea7466db0..e394a8d62 100644 --- a/dispatcher/backend/src/common/schemas/__init__.py +++ b/dispatcher/backend/src/common/schemas/__init__.py @@ -1,24 +1,52 @@ from marshmallow import Schema, fields +from marshmallow.validate import And, ValidationError, Validator + + +class NoNullChar(Validator): + """Validate that string value does not contains Unicode null character""" + + def __call__(self, value: str) -> str: + if "\u0000" in value: + raise ValidationError("Null character is not allowed") + + return value + + +class String(fields.String): + """A custom String field for our needs + + In addition to base type checks, it also ensures that value does not contains + Unicode null character + """ + + @property + def _validate_all(self): + return And( + NoNullChar(), + *self.validators, + error=self.error_messages["validator_failed"] + ) class ListOfStringEnum(fields.List): pass -class StringEnum(fields.String): +class StringEnum(String): pass -class HexColor(fields.String): +class HexColor(String): pass -class LongString(fields.String): +class LongString(String): pass class SerializableSchema(Schema): MAPPING = { + String: "text", fields.String: "text", LongString: "long-text", StringEnum: "string-enum", diff --git a/dispatcher/backend/src/common/schemas/fields.py b/dispatcher/backend/src/common/schemas/fields.py index e124d57bf..876ada4c5 100644 --- a/dispatcher/backend/src/common/schemas/fields.py +++ b/dispatcher/backend/src/common/schemas/fields.py @@ -9,6 +9,7 @@ WarehousePath, ) from common.roles import ROLES +from common.schemas import String # validators validate_priority = validate.Range(min=0, max=10) @@ -65,11 +66,11 @@ def validate_multiple_of_100(value): required=False, load_default=20, validate=validate.Range(min=0, max=200) ) priority_field = fields.Integer(required=False, validate=validate_priority) -worker_field = fields.String(required=False, validate=validate_worker_name) -schedule_name_field = fields.String(validate=validate_schedule_name) -category_field = fields.String(required=False, validate=validate_category) -periodicity_field = fields.String(required=False, validate=validate_periodicity) -tag_field = fields.List(fields.String(validate=validate_not_empty), required=False) -offliner_field = fields.String(required=False, validate=validate_offliner) +worker_field = String(required=False, validate=validate_worker_name) +schedule_name_field = String(validate=validate_schedule_name) +category_field = String(required=False, validate=validate_category) +periodicity_field = String(required=False, validate=validate_periodicity) +tag_field = fields.List(String(validate=validate_not_empty), required=False) +offliner_field = String(required=False, validate=validate_offliner) email_field = fields.Email(required=False, validate=validate_not_empty) -username_field = fields.String(required=True, validate=validate_not_empty) +username_field = String(required=True, validate=validate_not_empty) diff --git a/dispatcher/backend/src/common/schemas/models.py b/dispatcher/backend/src/common/schemas/models.py index 113d9f38a..bec68c172 100644 --- a/dispatcher/backend/src/common/schemas/models.py +++ b/dispatcher/backend/src/common/schemas/models.py @@ -3,7 +3,7 @@ from marshmallow import Schema, fields, pre_load, validate, validates_schema from common.enum import DockerImageName, Offliner, Platform -from common.schemas import SerializableSchema +from common.schemas import SerializableSchema, String from common.schemas.fields import ( validate_category, validate_cpu, @@ -37,9 +37,9 @@ class LanguageSchema(Schema): - code = fields.String(required=True, validate=validate_lang_code) - name_en = fields.String(required=True, validate=validate_not_empty) - name_native = fields.String(required=True, validate=validate_not_empty) + code = String(required=True, validate=validate_lang_code) + name_en = String(required=True, validate=validate_not_empty) + name_native = String(required=True, validate=validate_not_empty) class ResourcesSchema(Schema): @@ -47,13 +47,13 @@ class ResourcesSchema(Schema): memory = fields.Integer(required=True, validate=validate_memory) disk = fields.Integer(required=True, validate=validate_disk) shm = fields.Integer(required=False, validate=validate_memory) - cap_add = fields.List(fields.String(), required=False) - cap_drop = fields.List(fields.String(), required=False) + cap_add = fields.List(String(), required=False) + cap_drop = fields.List(String(), required=False) class DockerImageSchema(Schema): - name = fields.String(required=True, validate=validate.OneOf(DockerImageName.all())) - tag = fields.String(required=True) + name = String(required=True, validate=validate.OneOf(DockerImageName.all())) + tag = String(required=True) @pre_load def strip_prefix(self, in_data, **kwargs): @@ -65,12 +65,12 @@ def strip_prefix(self, in_data, **kwargs): class ScheduleConfigSchema(SerializableSchema): - task_name = fields.String(required=True, validate=validate_offliner) - warehouse_path = fields.String(required=True, validate=validate_warehouse_path) + task_name = String(required=True, validate=validate_offliner) + warehouse_path = String(required=True, validate=validate_warehouse_path) image = fields.Nested(DockerImageSchema(), required=True) resources = fields.Nested(ResourcesSchema(), required=True) flags = fields.Dict(required=True) - platform = fields.String(required=True, allow_none=True, validate=validate_platform) + platform = String(required=True, allow_none=True, validate=validate_platform) monitor = fields.Boolean(required=True, truthy=[True], falsy=[False]) @staticmethod @@ -101,7 +101,7 @@ def validate(self, data, **kwargs): class EventNotificationSchema(SerializableSchema): mailgun = fields.List(fields.Email(), required=False) webhook = fields.List(fields.Url(), required=False) - slack = fields.List(fields.String(validate=validate_slack_target), required=False) + slack = fields.List(String(validate=validate_slack_target), required=False) class ScheduleNotificationSchema(SerializableSchema): @@ -111,12 +111,12 @@ class ScheduleNotificationSchema(SerializableSchema): class ScheduleSchema(Schema): - name = fields.String(required=True, validate=validate_schedule_name) + name = String(required=True, validate=validate_schedule_name) language = fields.Nested(LanguageSchema(), required=True) - category = fields.String(required=True, validate=validate_category) - periodicity = fields.String(required=True, validate=validate_periodicity) + category = String(required=True, validate=validate_category) + periodicity = String(required=True, validate=validate_periodicity) tags = fields.List( - fields.String(validate=validate_not_empty), required=True, dump_default=[] + String(validate=validate_not_empty), required=True, dump_default=[] ) enabled = fields.Boolean(required=True, truthy=[True], falsy=[False]) config = fields.Nested(ScheduleConfigSchema(), required=True) diff --git a/dispatcher/backend/src/common/schemas/offliners/freecodecamp.py b/dispatcher/backend/src/common/schemas/offliners/freecodecamp.py index b89bfb851..f332859ab 100644 --- a/dispatcher/backend/src/common/schemas/offliners/freecodecamp.py +++ b/dispatcher/backend/src/common/schemas/offliners/freecodecamp.py @@ -1,6 +1,6 @@ -from marshmallow import fields +from marshmallow import fields, validate -from common.schemas import SerializableSchema +from common.schemas import LongString, SerializableSchema, String, StringEnum from common.schemas.fields import ( validate_output, validate_zim_description, @@ -8,12 +8,25 @@ validate_zim_longdescription, ) +FCC_LANG_MAP = { + "ara": "arabic", + "cmn": "chinese", + "lzh": "chinese-traditional", + "eng": "english", + "spa": "espanol", + "deu": "german", + "ita": "italian", + "jpn": "japanese", + "por": "portuguese", + "ukr": "ukranian", +} + class FreeCodeCampFlagsSchema(SerializableSchema): class Meta: ordered = True - course = fields.String( + course = String( metadata={ "label": "Course(s)", "description": "Course or course list (separated by commas)", @@ -21,18 +34,18 @@ class Meta: required=True, ) - language = fields.String( + language = StringEnum( metadata={ "label": "Language", - "description": "Language of zim file and curriculum. Either (without " - "quotes) 'ara' (arabic), 'cmn' (chinese), 'lzh' (chinese-traditional), " - "'eng' (english), 'spa' (espanol), 'deu' (german), 'ita' (italian), " - "'jpn' (japanese), 'por' (portuguese), 'ukr' (ukranian).", + "description": "Language of zim file and curriculum. One of " + + ", ".join([f"'{key}' ({desc})" for key, desc in FCC_LANG_MAP.items()]) + + ".", }, required=True, + validate=validate.OneOf(list(FCC_LANG_MAP.keys())), ) - name = fields.String( + name = String( metadata={ "label": "Name", "description": "ZIM name", @@ -40,7 +53,7 @@ class Meta: required=True, ) - title = fields.String( + title = String( metadata={ "label": "Title", "description": "ZIM title", @@ -48,7 +61,7 @@ class Meta: required=True, ) - description = fields.String( + description = String( metadata={ "label": "Description", "description": "Description for your ZIM", @@ -57,7 +70,7 @@ class Meta: validate=validate_zim_description, ) - long_description = fields.String( + long_description = LongString( metadata={ "label": "Long description", "description": "Optional long description for your ZIM", @@ -66,14 +79,14 @@ class Meta: data_key="long-description", ) - creator = fields.String( + creator = String( metadata={ "label": "Content Creator", "description": "Name of content creator. “freeCodeCamp” otherwise", } ) - publisher = fields.String( + publisher = String( metadata={ "label": "Publisher", "description": "Custom publisher name (ZIM metadata). “OpenZIM” otherwise", @@ -86,7 +99,7 @@ class Meta: metadata={"label": "Debug", "description": "Enable verbose output"}, ) - output_dir = fields.String( + output_dir = String( metadata={ "label": "Output folder", "placeholder": "/output", @@ -98,7 +111,7 @@ class Meta: validate=validate_output, ) - zim_file = fields.String( + zim_file = String( metadata={ "label": "ZIM filename", "description": "ZIM file name (based on --name if not provided). " diff --git a/dispatcher/backend/src/common/schemas/offliners/gutenberg.py b/dispatcher/backend/src/common/schemas/offliners/gutenberg.py index 84572a490..055984de9 100644 --- a/dispatcher/backend/src/common/schemas/offliners/gutenberg.py +++ b/dispatcher/backend/src/common/schemas/offliners/gutenberg.py @@ -1,13 +1,14 @@ from marshmallow import fields -from common.schemas import SerializableSchema +from common.schemas import SerializableSchema, String +from common.schemas.fields import validate_zim_description class GutenbergFlagsSchema(SerializableSchema): class Meta: ordered = True - languages = fields.String( + languages = String( metadata={ "label": "Languages", "description": ( @@ -17,7 +18,7 @@ class Meta: }, ) - formats = fields.String( + formats = String( metadata={ "label": "Formats", "description": ( @@ -27,7 +28,7 @@ class Meta: }, ) - zim_title = fields.String( + zim_title = String( metadata={ "label": "Title", "description": "Custom title for your project and ZIM.", @@ -35,12 +36,13 @@ class Meta: data_key="zim-title", ) - zim_desc = fields.String( + zim_desc = String( metadata={"label": "Description", "description": "Description for ZIM"}, data_key="zim-desc", + validate=validate_zim_description, ) - books = fields.String( + books = String( metadata={ "label": "Books", "description": ( diff --git a/dispatcher/backend/src/common/schemas/offliners/ifixit.py b/dispatcher/backend/src/common/schemas/offliners/ifixit.py index f893fc45d..8719ec191 100644 --- a/dispatcher/backend/src/common/schemas/offliners/ifixit.py +++ b/dispatcher/backend/src/common/schemas/offliners/ifixit.py @@ -1,7 +1,11 @@ from marshmallow import fields, validate -from common.schemas import SerializableSchema -from common.schemas.fields import validate_output, validate_zim_filename +from common.schemas import SerializableSchema, String +from common.schemas.fields import ( + validate_output, + validate_zim_description, + validate_zim_filename, +) def validate_percent(value): @@ -12,7 +16,7 @@ class IFixitFlagsSchema(SerializableSchema): class Meta: ordered = True - language = fields.String( + language = String( metadata={ "label": "Language", "description": "iFixIt website to build from", @@ -20,7 +24,7 @@ class Meta: required=True, ) - name = fields.String( + name = String( metadata={ "label": "Name", "description": "ZIM name. Used as identifier and filename " @@ -28,7 +32,7 @@ class Meta: }, ) - title = fields.String( + title = String( metadata={ "label": "Title", "description": "Custom title for your ZIM. " @@ -36,12 +40,13 @@ class Meta: }, ) - description = fields.String( + description = String( metadata={ "label": "Description", "description": "Custom description for your ZIM. " "iFixIt homepage description (meta) otherwise", }, + validate=validate_zim_description, ) icon = fields.Url( @@ -52,21 +57,21 @@ class Meta: } ) - creator = fields.String( + creator = String( metadata={ "label": "Creator", "description": "Name of content creator. “iFixit” otherwise", }, ) - publisher = fields.String( + publisher = String( metadata={ "label": "Publisher", "description": "Custom publisher name (ZIM metadata). “openZIM” otherwise", }, ) - tag = fields.String( + tag = String( metadata={ "label": "ZIM Tags", "description": "List of semi-colon-separated Tags for the ZIM file. " @@ -74,7 +79,7 @@ class Meta: } ) - output = fields.String( + output = String( metadata={ "label": "Output folder", "placeholder": "/output", @@ -85,7 +90,7 @@ class Meta: validate=validate_output, ) - tmp_dir = fields.String( + tmp_dir = String( metadata={ "label": "Temp folder", "placeholder": "/output", @@ -98,7 +103,7 @@ class Meta: data_key="tmp-dir", ) - zim_file = fields.String( + zim_file = String( metadata={ "label": "ZIM filename", "description": "ZIM file name (based on --name if not provided). " @@ -117,7 +122,7 @@ class Meta: data_key="optimization-cache", ) - stats_filename = fields.String( + stats_filename = String( metadata={ "label": "Stats filename", "placeholder": "/output/task_progress.json", @@ -187,7 +192,7 @@ class Meta: validate=validate_percent, ) - category = fields.String( + category = String( metadata={ "label": "Categories", "description": "Only scrape those categories (comma-separated). " @@ -202,7 +207,7 @@ class Meta: data_key="no-category", ) - guide = fields.String( + guide = String( metadata={ "label": "Guides", "description": "Only scrape this guide (comma-separated)). " @@ -217,7 +222,7 @@ class Meta: data_key="no-guide", ) - info = fields.String( + info = String( metadata={ "label": "Info", "description": "Only scrape this info (comma-separated)). " diff --git a/dispatcher/backend/src/common/schemas/offliners/kolibri.py b/dispatcher/backend/src/common/schemas/offliners/kolibri.py index a9d32bae7..c73b4ce92 100644 --- a/dispatcher/backend/src/common/schemas/offliners/kolibri.py +++ b/dispatcher/backend/src/common/schemas/offliners/kolibri.py @@ -1,14 +1,19 @@ from marshmallow import fields -from common.schemas import SerializableSchema -from common.schemas.fields import validate_output, validate_zim_filename +from common.schemas import LongString, SerializableSchema, String +from common.schemas.fields import ( + validate_output, + validate_zim_description, + validate_zim_filename, + validate_zim_longdescription, +) class KolibriFlagsSchema(SerializableSchema): class Meta: ordered = True - channel_id = fields.String( + channel_id = String( metadata={ "label": "Channel ID", "description": "The Kolibri channel ID that you want to scrape", @@ -17,7 +22,7 @@ class Meta: required=True, ) - root_id = fields.String( + root_id = String( metadata={ "label": "Root ID", "description": "The node ID (usually Topic) from where to start " @@ -26,7 +31,7 @@ class Meta: data_key="root-id", ) - name = fields.String( + name = String( metadata={ "label": "Name", "description": "ZIM name. Used as identifier " @@ -35,22 +40,23 @@ class Meta: required=True, ) - title = fields.String( + title = String( metadata={ "label": "Title", "description": "Custom title for your ZIM. Kolibri channel name otherwise", } ) - description = fields.String( + description = String( metadata={ "label": "Description", "description": "Custom description for your ZIM. " "Kolibri channel description otherwise", - } + }, + validate=validate_zim_description, ) - long_description = fields.String( + long_description = LongString( metadata={ "label": "Long description", "description": "Custom long description for your ZIM. " @@ -58,6 +64,7 @@ class Meta: "too long to fit entirely in ZIM description", }, data_key="long-description", + validate=validate_zim_longdescription, ) favicon = fields.Url( @@ -91,7 +98,7 @@ class Meta: required=False, ) - creator = fields.String( + creator = String( metadata={ "label": "Content Creator", "description": "Name of content creator. Kolibri " @@ -99,14 +106,14 @@ class Meta: } ) - publisher = fields.String( + publisher = String( metadata={ "label": "Publisher", "description": "Custom publisher name (ZIM metadata). “OpenZIM” otherwise", } ) - tags = fields.String( + tags = String( metadata={ "label": "ZIM Tags", "description": "List of comma-separated Tags for the ZIM file. " @@ -148,7 +155,7 @@ class Meta: falsy=[False], ) - output = fields.String( + output = String( metadata={ "label": "Output folder", "placeholder": "/output", @@ -159,7 +166,7 @@ class Meta: validate=validate_output, ) - tmp_dir = fields.String( + tmp_dir = String( metadata={ "label": "Temp folder", "placeholder": "/output", @@ -172,7 +179,7 @@ class Meta: data_key="tmp-dir", ) - zim_file = fields.String( + zim_file = String( metadata={ "label": "ZIM filename", "description": "ZIM file name (based on --name if not provided). " diff --git a/dispatcher/backend/src/common/schemas/offliners/mwoffliner.py b/dispatcher/backend/src/common/schemas/offliners/mwoffliner.py index 1df7949ff..c0ffe873a 100644 --- a/dispatcher/backend/src/common/schemas/offliners/mwoffliner.py +++ b/dispatcher/backend/src/common/schemas/offliners/mwoffliner.py @@ -1,6 +1,12 @@ from marshmallow import fields, validate -from common.schemas import ListOfStringEnum, LongString, SerializableSchema, StringEnum +from common.schemas import ( + ListOfStringEnum, + LongString, + SerializableSchema, + String, + StringEnum, +) from common.schemas.fields import ( validate_output, validate_zim_description, @@ -42,20 +48,20 @@ class Meta: "to ignore (one per line)", } ) - customMainPage = fields.String( + customMainPage = String( metadata={ "label": "Main Page", "description": "Article Name to use as home page. " "Automatically built or guessed otherwise.", } ) - customZimTitle = fields.String( + customZimTitle = String( metadata={ "label": "ZIM Title", "description": "Custom ZIM title. Wiki name otherwise.", } ) - customZimDescription = fields.String( + customZimDescription = String( metadata={"label": "ZIM Description", "description": "Max length is 80 chars"}, validate=validate_zim_description, ) @@ -73,32 +79,32 @@ class Meta: "Will be resized to 48x48px.", } ) - customZimTags = fields.String( + customZimTags = String( metadata={ "label": "ZIM Tags", "description": "Semi-colon separated list of ZIM tags", } ) - customZimLanguage = fields.String( + customZimLanguage = String( metadata={ "label": "ZIM Language metadata", "description": "Custom ISO-639-3 language code for the ZIM", } ) - publisher = fields.String( + publisher = String( metadata={ "label": "Publisher", "description": "ZIM publisher metadata. `Kiwix` otherwise.", } ) - filenamePrefix = fields.String( + filenamePrefix = String( metadata={ "label": "Filename prefix", "description": "Custome filename up to the formats and date parts.", } ) formats = ListOfStringEnum( - fields.String( + String( validate=validate.OneOf( [ "nodet,nopic:mini", @@ -139,7 +145,7 @@ class Meta: } ) - addNamespaces = fields.String( + addNamespaces = String( metadata={ "label": "Add Namespaces", "description": "Include addional namespaces (comma separated numbers)", @@ -170,51 +176,51 @@ class Meta: }, ) - mwWikiPath = fields.String( + mwWikiPath = String( metadata={ "label": "Wiki Path", "description": "Mediawiki wiki base path. Otherwise `/wiki/`.", } ) - mwApiPath = fields.String( + mwApiPath = String( metadata={ "label": "API Path", "description": "Mediawiki API path. Otherwise `/w/api.php`.", } ) - mwModulePath = fields.String( + mwModulePath = String( metadata={ "label": "Module Path", "description": "Mediawiki module load path. Otherwise `/w/load.php`.", } ) - mwDomain = fields.String( + mwDomain = String( metadata={ "label": "User Domain", "description": "Mediawiki user domain (for private wikis)", } ) - mwUsername = fields.String( + mwUsername = String( metadata={ "label": "Username", "description": "Mediawiki username (for private wikis)", } ) - mwPassword = fields.String( + mwPassword = String( metadata={ "label": "Password", "description": "Mediawiki user password (for private wikis)", } ) - osTmpDir = fields.String( + osTmpDir = String( metadata={ "label": "OS Temp Dir", "description": "Override default operating system temporary " "directory path environnement variable", } ) - outputDirectory = fields.String( + outputDirectory = String( metadata={ "label": "Output folder", "placeholder": "/output", diff --git a/dispatcher/backend/src/common/schemas/offliners/nautilus.py b/dispatcher/backend/src/common/schemas/offliners/nautilus.py index d969b9c0b..c24611b37 100644 --- a/dispatcher/backend/src/common/schemas/offliners/nautilus.py +++ b/dispatcher/backend/src/common/schemas/offliners/nautilus.py @@ -1,7 +1,11 @@ from marshmallow import fields -from common.schemas import HexColor, SerializableSchema -from common.schemas.fields import validate_output, validate_zim_filename +from common.schemas import HexColor, SerializableSchema, String +from common.schemas.fields import ( + validate_output, + validate_zim_description, + validate_zim_filename, +) class NautilusFlagsSchema(SerializableSchema): @@ -26,7 +30,7 @@ class Meta: required=False, ) - name = fields.String( + name = String( metadata={ "label": "ZIM Name", "description": "Used as identifier and filename (date will be appended)", @@ -55,7 +59,7 @@ class Meta: data_key="show-description", ) - output = fields.String( + output = String( metadata={ "label": "Output folder", "placeholder": "/output", @@ -67,7 +71,7 @@ class Meta: dump_default="/output", validate=validate_output, ) - zim_file = fields.String( + zim_file = String( metadata={ "label": "ZIM filename", "description": "ZIM file name (based on --name if not provided)", @@ -75,13 +79,13 @@ class Meta: data_key="zim-file", validate=validate_zim_filename, ) - language = fields.String( + language = String( metadata={ "label": "Language", "description": "ISO-639-3 (3 chars) language code of content", } ) - locale = fields.String( + locale = String( metadata={ "label": "Locale", "description": ( @@ -90,25 +94,26 @@ class Meta: ), } ) - title = fields.String( + title = String( metadata={ "label": "Title", "description": "Title for your project and ZIM. Otherwise --name.", } ) - description = fields.String( + description = String( metadata={ "label": "Description", "description": "Description for your project and ZIM.", - } + }, + validate=validate_zim_description, ) - creator = fields.String( + creator = String( metadata={ "label": "Content Creator", "description": "Name of content creator.", } ) - tags = fields.String( + tags = String( metadata={ "label": "ZIM Tags", "description": "List of comma-separated Tags for the ZIM file.", diff --git a/dispatcher/backend/src/common/schemas/offliners/openedx.py b/dispatcher/backend/src/common/schemas/offliners/openedx.py index eab814ef3..0be71e566 100644 --- a/dispatcher/backend/src/common/schemas/offliners/openedx.py +++ b/dispatcher/backend/src/common/schemas/offliners/openedx.py @@ -1,7 +1,11 @@ from marshmallow import fields, validate -from common.schemas import SerializableSchema, StringEnum -from common.schemas.fields import validate_output, validate_zim_filename +from common.schemas import SerializableSchema, String, StringEnum +from common.schemas.fields import ( + validate_output, + validate_zim_description, + validate_zim_filename, +) class OpenedxFlagsSchema(SerializableSchema): @@ -17,7 +21,7 @@ class Meta: required=True, ) - email = fields.String( + email = String( metadata={ "label": "Registered e-mail", "description": "The registered e-mail ID on the openedx instance", @@ -26,7 +30,7 @@ class Meta: required=True, ) - password = fields.String( + password = String( metadata={ "label": "Password", "description": "Password to the account registered on the openedx instance", @@ -36,7 +40,7 @@ class Meta: required=True, ) - instance_login_page = fields.String( + instance_login_page = String( metadata={ "label": "Login page path", "description": "The login path in the instance. Must start with /", @@ -45,7 +49,7 @@ class Meta: data_key="instance-login-page", ) - instance_course_page = fields.String( + instance_course_page = String( metadata={ "label": "Course page path", "description": ( @@ -56,7 +60,7 @@ class Meta: data_key="instance-course-page", ) - instance_course_prefix = fields.String( + instance_course_prefix = String( metadata={ "label": "Course prefix path", "description": ( @@ -152,7 +156,7 @@ class Meta: data_key="autoplay", ) - name = fields.String( + name = String( metadata={ "label": "Name", "description": ( @@ -164,7 +168,7 @@ class Meta: required=True, ) - title = fields.String( + title = String( metadata={ "label": "Title", "description": "Custom title for your ZIM. Based on MOOC otherwise", @@ -172,15 +176,16 @@ class Meta: data_key="title", ) - description = fields.String( + description = String( metadata={ "label": "Description", "description": "Custom description for your ZIM. Based on MOOC otherwise", }, data_key="description", + validate=validate_zim_description, ) - creator = fields.String( + creator = String( metadata={ "label": "Content Creator", "description": "Name of content creator. Defaults to edX", @@ -188,7 +193,7 @@ class Meta: data_key="creator", ) - tags = fields.String( + tags = String( metadata={ "label": "ZIM Tags", "description": ( @@ -220,7 +225,7 @@ class Meta: data_key="use-any-optimized-version", ) - output = fields.String( + output = String( metadata={ "label": "Output folder", "placeholder": "/output", @@ -232,7 +237,7 @@ class Meta: data_key="output", ) - tmp_dir = fields.String( + tmp_dir = String( metadata={ "label": "Temp folder", "description": ( @@ -245,7 +250,7 @@ class Meta: data_key="tmp-dir", ) - zim_file = fields.String( + zim_file = String( metadata={ "label": "ZIM filename", "description": "ZIM file name (based on ZIM name if not provided)", @@ -268,7 +273,7 @@ class Meta: validate=validate.Range(min=1), ) - locale = fields.String( + locale = String( metadata={ "label": "Locale", "description": "The locale to use for the translations in ZIM", diff --git a/dispatcher/backend/src/common/schemas/offliners/sotoki.py b/dispatcher/backend/src/common/schemas/offliners/sotoki.py index d39b6e752..8c6af97cf 100644 --- a/dispatcher/backend/src/common/schemas/offliners/sotoki.py +++ b/dispatcher/backend/src/common/schemas/offliners/sotoki.py @@ -1,14 +1,18 @@ from marshmallow import fields, validate -from common.schemas import SerializableSchema -from common.schemas.fields import validate_output, validate_zim_filename +from common.schemas import SerializableSchema, String +from common.schemas.fields import ( + validate_output, + validate_zim_description, + validate_zim_filename, +) class SotokiFlagsSchema(SerializableSchema): class Meta: ordered = True - domain = fields.String( + domain = String( metadata={ "label": "Domain", "description": "Domain name from StackExchange to scrape.", @@ -16,7 +20,7 @@ class Meta: required=True, ) - name = fields.String( + name = String( metadata={ "label": "Name", "description": "ZIM name. Used as identifier and filename " @@ -24,18 +28,19 @@ class Meta: }, ) - title = fields.String( + title = String( metadata={ "label": "Title", "description": "Custom title for your ZIM. Site name otherwise", }, ) - description = fields.String( + description = String( metadata={ "label": "Description", "description": "Custom description for your ZIM. Site tagline otherwise", }, + validate=validate_zim_description, ) favicon = fields.Url( @@ -45,21 +50,21 @@ class Meta: } ) - creator = fields.String( + creator = String( metadata={ "label": "Creator", "description": "Name of content creator. “Stack Exchange” otherwise", }, ) - publisher = fields.String( + publisher = String( metadata={ "label": "Publisher", "description": "Custom publisher name (ZIM metadata). “OpenZIM” otherwise", }, ) - tags = fields.String( + tags = String( metadata={ "label": "ZIM Tags", "description": "List of comma-separated Tags for the ZIM file. " @@ -136,7 +141,7 @@ class Meta: data_key="censor-words-list", ) - output = fields.String( + output = String( metadata={ "label": "Output folder", "placeholder": "/output", @@ -155,7 +160,7 @@ class Meta: } ) - tmp_dir = fields.String( + tmp_dir = String( metadata={ "label": "Temp folder", "placeholder": "/output", @@ -168,7 +173,7 @@ class Meta: data_key="tmp-dir", ) - zim_file = fields.String( + zim_file = String( metadata={ "label": "ZIM filename", "description": "ZIM file name (based on --name if not provided). " @@ -194,7 +199,7 @@ class Meta: }, ) - stats_filename = fields.String( + stats_filename = String( metadata={ "label": "Stats filename", "placeholder": "/output/task_progress.json", @@ -207,7 +212,7 @@ class Meta: validate=validate.Equal("/output/task_progress.json"), ) - redis_url = fields.String( + redis_url = String( metadata={ "label": "Redis URL", "description": "Redis URL to use as database. " @@ -219,7 +224,7 @@ class Meta: data_key="redis-url", ) - defrag_redis = fields.String( + defrag_redis = String( metadata={ "label": "Defrag redis", "description": "Keep it as ENV:REDIS_PID", diff --git a/dispatcher/backend/src/common/schemas/offliners/ted.py b/dispatcher/backend/src/common/schemas/offliners/ted.py index 21ec21829..13c1d2476 100644 --- a/dispatcher/backend/src/common/schemas/offliners/ted.py +++ b/dispatcher/backend/src/common/schemas/offliners/ted.py @@ -1,7 +1,11 @@ from marshmallow import ValidationError, fields, validate, validates_schema -from common.schemas import SerializableSchema, StringEnum -from common.schemas.fields import validate_output, validate_zim_filename +from common.schemas import SerializableSchema, String, StringEnum +from common.schemas.fields import ( + validate_output, + validate_zim_description, + validate_zim_filename, +) class TedFlagsSchema(SerializableSchema): @@ -18,7 +22,7 @@ class Meta: data_key="indiv-zims", ) - topics = fields.String( + topics = String( metadata={ "label": "Topics", "description": ( @@ -28,7 +32,7 @@ class Meta: }, ) - playlists = fields.String( + playlists = String( metadata={ "label": "TED Playlists", "description": ( @@ -38,7 +42,7 @@ class Meta: }, ) - languages = fields.String( + languages = String( metadata={ "label": "Languages", "description": "Comma-seperated list of languages to filter videos", @@ -57,7 +61,7 @@ class Meta: }, ) - subtitles = fields.String( + subtitles = String( metadata={ "label": "Subtitles Setting", "description": ( @@ -99,7 +103,7 @@ class Meta: }, ) - name = fields.String( + name = String( metadata={ "label": "Name", "description": ( @@ -109,7 +113,7 @@ class Meta: }, ) - name_format = fields.String( + name_format = String( metadata={ "label": "Name Format", "description": ( @@ -121,14 +125,14 @@ class Meta: data_key="name-format", ) - title = fields.String( + title = String( metadata={ "label": "Title", "description": "Custom title for your ZIM. Based on selection otherwise", } ) - title_format = fields.String( + title_format = String( metadata={ "label": "Title Format", "description": "Custom title format for individual ZIMs", @@ -136,16 +140,17 @@ class Meta: data_key="title-format", ) - description = fields.String( + description = String( metadata={ "label": "Description", "description": ( "Custom description for your ZIM. Based on selection otherwise" ), - } + }, + validate=validate_zim_description, ) - description_format = fields.String( + description_format = String( metadata={ "label": "Description Format", "description": "Custom description format for individual ZIMs", @@ -153,14 +158,14 @@ class Meta: data_key="description-format", ) - creator = fields.String( + creator = String( metadata={ "label": "Content Creator", "description": "Name of content creator. Defaults to TED", } ) - tags = fields.String( + tags = String( metadata={ "label": "ZIM Tags", "description": ( @@ -191,7 +196,7 @@ class Meta: data_key="use-any-optimized-version", ) - output = fields.String( + output = String( metadata={ "label": "Output folder", "placeholder": "/output", @@ -202,7 +207,7 @@ class Meta: validate=validate_output, ) - tmp_dir = fields.String( + tmp_dir = String( metadata={ "label": "Temp folder", "description": ( @@ -215,7 +220,7 @@ class Meta: data_key="tmp-dir", ) - metadata_from = fields.String( + metadata_from = String( metadata={ "label": "Metadata JSON", "description": ( @@ -226,7 +231,7 @@ class Meta: data_key="metadata-from", ) - zim_file = fields.String( + zim_file = String( metadata={ "label": "ZIM filename", "description": "ZIM file name (based on ZIM name if not provided)", @@ -235,7 +240,7 @@ class Meta: validate=validate_zim_filename, ) - zim_file_format = fields.String( + zim_file_format = String( metadata={ "label": "ZIM filename format", "description": ( @@ -260,7 +265,7 @@ class Meta: validate=validate.Range(min=1), ) - locale = fields.String( + locale = String( metadata={ "label": "Locale", "description": "The locale to use for the translations in ZIM", diff --git a/dispatcher/backend/src/common/schemas/offliners/wikihow.py b/dispatcher/backend/src/common/schemas/offliners/wikihow.py index b719547dc..0e128436d 100644 --- a/dispatcher/backend/src/common/schemas/offliners/wikihow.py +++ b/dispatcher/backend/src/common/schemas/offliners/wikihow.py @@ -1,14 +1,18 @@ from marshmallow import fields, validate -from common.schemas import SerializableSchema -from common.schemas.fields import validate_output, validate_zim_filename +from common.schemas import SerializableSchema, String +from common.schemas.fields import ( + validate_output, + validate_zim_description, + validate_zim_filename, +) class WikihowFlagsSchema(SerializableSchema): class Meta: ordered = True - language = fields.String( + language = String( metadata={ "label": "Language", "description": "wikiHow website to build from. 2-letters language code.", @@ -16,7 +20,7 @@ class Meta: required=True, ) - name = fields.String( + name = String( metadata={ "label": "Name", "description": "ZIM name. Used as identifier and filename " @@ -24,7 +28,7 @@ class Meta: }, ) - title = fields.String( + title = String( metadata={ "label": "Title", "description": "Custom title for your ZIM. " @@ -32,12 +36,13 @@ class Meta: }, ) - description = fields.String( + description = String( metadata={ "label": "Description", "description": "Custom description for your ZIM. " "Wikihow homepage description (meta) otherwise", }, + validate=validate_zim_description, ) icon = fields.Url( @@ -48,21 +53,21 @@ class Meta: } ) - creator = fields.String( + creator = String( metadata={ "label": "Creator", "description": "Name of content creator. “wikiHow” otherwise", }, ) - publisher = fields.String( + publisher = String( metadata={ "label": "Publisher", "description": "Custom publisher name (ZIM metadata). “openZIM” otherwise", }, ) - tag = fields.String( + tag = String( metadata={ "label": "ZIM Tags", "description": "List of semi-colon-separated Tags for the ZIM file. " @@ -116,7 +121,7 @@ class Meta: data_key="low-quality", ) - output = fields.String( + output = String( metadata={ "label": "Output folder", "placeholder": "/output", @@ -127,7 +132,7 @@ class Meta: validate=validate_output, ) - tmp_dir = fields.String( + tmp_dir = String( metadata={ "label": "Temp folder", "placeholder": "/output", @@ -140,7 +145,7 @@ class Meta: data_key="tmp-dir", ) - zim_file = fields.String( + zim_file = String( metadata={ "label": "ZIM filename", "description": "ZIM file name (based on --name if not provided). " @@ -159,7 +164,7 @@ class Meta: data_key="optimization-cache", ) - category = fields.String( + category = String( metadata={ "label": "Categories", "description": "Only scrape those categories (comma-separated). " @@ -169,7 +174,7 @@ class Meta: }, ) - stats_filename = fields.String( + stats_filename = String( metadata={ "label": "Stats filename", "placeholder": "/output/task_progress.json", diff --git a/dispatcher/backend/src/common/schemas/offliners/youtube.py b/dispatcher/backend/src/common/schemas/offliners/youtube.py index 84eb21add..46b65cc44 100644 --- a/dispatcher/backend/src/common/schemas/offliners/youtube.py +++ b/dispatcher/backend/src/common/schemas/offliners/youtube.py @@ -1,7 +1,11 @@ from marshmallow import ValidationError, fields, validate, validates_schema -from common.schemas import HexColor, SerializableSchema, StringEnum -from common.schemas.fields import validate_output, validate_zim_filename +from common.schemas import HexColor, SerializableSchema, String, StringEnum +from common.schemas.fields import ( + validate_output, + validate_zim_description, + validate_zim_filename, +) class YoutubeFlagsSchema(SerializableSchema): @@ -27,7 +31,7 @@ class Meta: data_key="type", required=True, ) - ident = fields.String( + ident = String( metadata={ "label": "Youtube ID", "description": "Youtube ID of the collection. " @@ -36,20 +40,20 @@ class Meta: data_key="id", required=True, ) - api_key = fields.String( + api_key = String( metadata={"label": "API Key", "description": "Youtube API Token"}, data_key="api-key", required=True, ) - name = fields.String( + name = String( metadata={ "label": "ZIM Name", "description": "Used as identifier and filename (date will be appended)", "placeholder": "mychannel_eng_all", }, ) - playlists_name = fields.String( + playlists_name = String( metadata={ "label": "Playlists name", "description": "Format for building individual --name argument. " @@ -83,7 +87,7 @@ class Meta: }, ) - dateafter = fields.String( + dateafter = String( metadata={ "label": "Only after date", "description": "Custom filter to download videos uploaded on " @@ -135,7 +139,7 @@ class Meta: "(home never have autoplay).", }, ) - output = fields.String( + output = String( metadata={ "label": "Output folder", "placeholder": "/output", @@ -145,7 +149,7 @@ class Meta: dump_default="/output", validate=validate_output, ) - tmp_dir = fields.String( + tmp_dir = String( metadata={ "label": "Temp folder", "placeholder": "/output", @@ -158,7 +162,7 @@ class Meta: data_key="tmp-dir", ) - zim_file = fields.String( + zim_file = String( metadata={ "label": "ZIM filename", "description": "ZIM file name (based on --name if not provided). " @@ -167,7 +171,7 @@ class Meta: data_key="zim-file", validate=validate_zim_filename, ) - playlists_zim_file = fields.String( + playlists_zim_file = String( metadata={ "label": "Playlists ZIM filename", "description": "Format for building individual --zim-file argument. " @@ -176,13 +180,13 @@ class Meta: data_key="playlists-zim-file", ) - language = fields.String( + language = String( metadata={ "label": "Language", "description": "ISO-639-3 (3 chars) language code of content", } ) - locale = fields.String( + locale = String( metadata={ "label": "Locale", "description": "Locale name to use for translations (if avail) " @@ -190,14 +194,14 @@ class Meta: } ) - title = fields.String( + title = String( metadata={ "label": "Title", "description": "Custom title for your project and ZIM. Default to " "Channel name (of first video if playlists)", } ) - playlists_title = fields.String( + playlists_title = String( metadata={ "label": "Playlists title", "description": "Custom title format for individual playlist ZIM", @@ -205,10 +209,11 @@ class Meta: data_key="playlists-title", ) - description = fields.String( - metadata={"label": "Description", "description": "Description for ZIM"} + description = String( + metadata={"label": "Description", "description": "Description for ZIM"}, + validate=validate_zim_description, ) - playlists_description = fields.String( + playlists_description = String( metadata={ "label": "Playlists description", "description": "Custom description format for individual playlist ZIM", @@ -216,14 +221,14 @@ class Meta: data_key="playlists-description", ) - creator = fields.String( + creator = String( metadata={ "label": "Content Creator", "description": "Name of content creator. Defaults to Channel name " "or “Youtue Channels”", } ) - tags = fields.String( + tags = String( metadata={ "label": "ZIM Tags", "description": "List of Tags for the ZIM file. " @@ -231,7 +236,7 @@ class Meta: } ) - metadata_from = fields.String( + metadata_from = String( metadata={ "label": "Metadata JSON", "description": "File path or URL to a JSON file holding custom metadata " diff --git a/dispatcher/backend/src/common/schemas/offliners/zimit.py b/dispatcher/backend/src/common/schemas/offliners/zimit.py index d71dd4d7b..545562504 100644 --- a/dispatcher/backend/src/common/schemas/offliners/zimit.py +++ b/dispatcher/backend/src/common/schemas/offliners/zimit.py @@ -1,7 +1,11 @@ from marshmallow import fields, validate -from common.schemas import SerializableSchema, StringEnum -from common.schemas.fields import validate_output, validate_zim_filename +from common.schemas import SerializableSchema, String, StringEnum +from common.schemas.fields import ( + validate_output, + validate_zim_description, + validate_zim_filename, +) # https://github.com/puppeteer/puppeteer/blob/main/src/common/DeviceDescriptors.ts # https://github.com/puppeteer/puppeteer/blob/ @@ -141,7 +145,7 @@ class Meta: required=True, ) - name = fields.String( + name = String( metadata={ "label": "Name", "description": "Name of the ZIM. " @@ -150,7 +154,7 @@ class Meta: required=True, ) - lang = fields.String( + lang = String( metadata={ "label": "Language", "description": "ISO-639-3 (3 chars) language code of content. " @@ -158,14 +162,16 @@ class Meta: } ) - title = fields.String( + title = String( metadata={ "label": "Title", "description": "Custom title for ZIM. Defaults to title of main page", } ) - description = fields.String( - metadata={"label": "Description", "description": "Description for ZIM"} + + description = String( + metadata={"label": "Description", "description": "Description for ZIM"}, + validate=validate_zim_description, ) favicon = fields.Url( @@ -177,7 +183,7 @@ class Meta: required=False, ) - zim_file = fields.String( + zim_file = String( metadata={ "label": "ZIM filename", "description": "ZIM file name (based on --name if not provided). " @@ -187,21 +193,21 @@ class Meta: validate=validate_zim_filename, ) - tags = fields.String( + tags = String( metadata={ "label": "ZIM Tags", "description": "List of Tags for the ZIM file.", } ) - creator = fields.String( + creator = String( metadata={ "label": "Content Creator", "description": "Name of content creator.", } ) - source = fields.String( + source = String( metadata={ "label": "Content Source", "description": "Source name/URL of content", @@ -216,7 +222,7 @@ class Meta: required=False, ) - wait_until = fields.String( + wait_until = String( metadata={ "label": "WaitUntil", "description": "Puppeteer page.goto() condition to wait for " @@ -273,7 +279,7 @@ class Meta: ), ) - include = fields.String( + include = String( metadata={ "label": "Include", "description": "Regex of page URLs that should be " @@ -282,7 +288,7 @@ class Meta: required=False, ) - exclude = fields.String( + exclude = String( metadata={ "label": "Exclude", "description": "Regex of page URLs that should be excluded from the crawl", @@ -312,7 +318,7 @@ class Meta: validate=validate_devicelist, ) - user_agent = fields.String( + user_agent = String( metadata={ "label": "User Agent", "description": "Override user-agent with specified", @@ -321,7 +327,7 @@ class Meta: required=False, ) - user_agent_suffix = fields.String( + user_agent_suffix = String( metadata={ "label": "User Agent Suffix", "description": "Append suffix to existing browser user-agent. " @@ -341,7 +347,7 @@ class Meta: required=False, ) - behaviors = fields.String( + behaviors = String( metadata={ "label": "Behaviors", "description": "Which background behaviors to enable on each page. " @@ -419,7 +425,7 @@ class Meta: required=False, ) - output = fields.String( + output = String( metadata={ "label": "Output folder", "placeholder": "/output", @@ -430,7 +436,7 @@ class Meta: validate=validate_output, ) - stats_filename = fields.String( + stats_filename = String( metadata={ "label": "Stats filename", "placeholder": "/output/task_progress.json", @@ -453,7 +459,7 @@ class Meta: required=False, ) - admin_email = fields.String( + admin_email = String( metadata={ "label": "Admin Email", "description": "Admin Email for crawler: used in UserAgent " diff --git a/dispatcher/backend/src/common/schemas/parameters.py b/dispatcher/backend/src/common/schemas/parameters.py index cffea75e3..123dff943 100644 --- a/dispatcher/backend/src/common/schemas/parameters.py +++ b/dispatcher/backend/src/common/schemas/parameters.py @@ -1,5 +1,6 @@ from marshmallow import Schema, fields +from common.schemas import String from common.schemas.fields import ( category_field, email_field, @@ -63,7 +64,7 @@ class RequestedTaskSchema(Schema): # requested-tasks for worker class WorkerRequestedTaskSchema(Schema): - worker = fields.String(required=True, validate=validate_worker_name) + worker = String(required=True, validate=validate_worker_name) avail_cpu = fields.Integer(required=True, validate=validate_cpu) avail_memory = fields.Integer(required=True, validate=validate_memory) avail_disk = fields.Integer(required=True, validate=validate_disk) @@ -87,7 +88,7 @@ class SchedulesSchema(Schema): limit = limit_field_20_200 category = fields.List(category_field, required=False) tag = tag_field - lang = fields.List(fields.String(validate=validate_not_empty), required=False) + lang = fields.List(String(validate=validate_not_empty), required=False) name = schedule_name_field @@ -100,11 +101,9 @@ class UpdateSchema(Schema): tags = tag_field enabled = fields.Boolean(required=False, truthy={True}, falsy={False}) task_name = offliner_field - warehouse_path = fields.String(required=False, validate=validate_warehouse_path) + warehouse_path = String(required=False, validate=validate_warehouse_path) image = fields.Nested(DockerImageSchema, required=False) - platform = fields.String( - required=False, validate=validate_platform, allow_none=True - ) + platform = String(required=False, validate=validate_platform, allow_none=True) resources = fields.Nested(ResourcesSchema, required=False) monitor = fields.Boolean(required=False, truthy={True}, falsy={False}) flags = fields.Dict(required=False) @@ -112,46 +111,46 @@ class UpdateSchema(Schema): # schedule clone class CloneSchema(Schema): - name = fields.String(required=True, validate=validate_schedule_name) + name = String(required=True, validate=validate_schedule_name) # tasks GET class TasksSchema(Schema): skip = skip_field limit = limit_field_20_200 - status = fields.List(fields.String(validate=validate_status), required=False) + status = fields.List(String(validate=validate_status), required=False) schedule_name = schedule_name_field # tasks POST class TaskCreateSchema(Schema): - worker_name = fields.String(required=True, validate=validate_worker_name) + worker_name = String(required=True, validate=validate_worker_name) # tasks PATCH class TasKUpdateSchema(Schema): - event = fields.String(required=True, validate=validate_event) + event = String(required=True, validate=validate_event) payload = fields.Dict(required=True) # users keys POST class KeySchema(Schema): - name = fields.String(required=True, validate=validate_not_empty) - key = fields.String(required=True, validate=validate_not_empty) + name = String(required=True, validate=validate_not_empty) + key = String(required=True, validate=validate_not_empty) # users POST class UserCreateSchema(Schema): username = username_field - password = fields.String(required=True, validate=validate_not_empty) + password = String(required=True, validate=validate_not_empty) email = email_field - role = fields.String(required=True, validate=validate_role) + role = String(required=True, validate=validate_role) # users PATCH class UserUpdateSchema(Schema): email = email_field - role = fields.String(required=False, validate=validate_role) + role = String(required=False, validate=validate_role) # workers checkin diff --git a/dispatcher/backend/src/routes/auth/validate.py b/dispatcher/backend/src/routes/auth/validate.py index 13c116e79..a6f9b74dd 100644 --- a/dispatcher/backend/src/routes/auth/validate.py +++ b/dispatcher/backend/src/routes/auth/validate.py @@ -6,11 +6,12 @@ import sqlalchemy as sa import sqlalchemy.orm as so from flask import Response, request -from marshmallow import Schema, ValidationError, fields, validate +from marshmallow import Schema, ValidationError, validate import db.models as dbm import errors.http as http_errors from common import getnow +from common.schemas import String from routes import errors from utils.check import raise_if_none @@ -22,8 +23,8 @@ def ssh_key(session: so.Session): # validate request json class KeySchema(Schema): - username = fields.String(required=True, validate=validate.Length(min=1)) - key = fields.String(required=True, validate=validate.Length(min=1)) + username = String(required=True, validate=validate.Length(min=1)) + key = String(required=True, validate=validate.Length(min=1)) try: request_json = KeySchema().load(request.get_json()) diff --git a/dispatcher/backend/src/tests/integration/routes/schedules/test_schedule.py b/dispatcher/backend/src/tests/integration/routes/schedules/test_schedule.py index 963e520d7..463055ef4 100644 --- a/dispatcher/backend/src/tests/integration/routes/schedules/test_schedule.py +++ b/dispatcher/backend/src/tests/integration/routes/schedules/test_schedule.py @@ -87,6 +87,19 @@ def schedule(make_schedule): "image": {"name": "openzim/youtube", "tag": "latest"}, "resources": {"cpu": -1, "memory": MIN_RAM, "disk": ONE_GiB}, }, + {"name": "new\u0000name"}, + { + "flags": { + "mwUrl": "https://fr.wiki\u0000pedia.org", + "adminEmail": "hello@test.de", + } + }, + { + "flags": { + "mwUrl": "https://fr.wikipedia.org", + "adminEmail": "he\u0000llo@test.de", + } + }, ]