From 3783f1b57be4f4eebbc15b86d87442756400595b Mon Sep 17 00:00:00 2001 From: benoit74 Date: Thu, 9 Nov 2023 10:41:02 +0100 Subject: [PATCH] Add an environment variable to disable ZIM filename check in zimit offliner --- dispatcher/backend/src/common/constants.py | 3 + .../src/common/schemas/offliners/zimit.py | 6 +- .../routes/schedules/test_zimit.py | 111 ++++++++++++++++++ 3 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 dispatcher/backend/src/tests/integration/routes/schedules/test_zimit.py diff --git a/dispatcher/backend/src/common/constants.py b/dispatcher/backend/src/common/constants.py index 43d6c5e9..66c95881 100644 --- a/dispatcher/backend/src/common/constants.py +++ b/dispatcher/backend/src/common/constants.py @@ -97,3 +97,6 @@ REQ_TIMEOUT_NOTIFICATIONS = int(os.getenv("REQ_TIMEOUT_NOTIFICATIONS", 5)) REQ_TIMEOUT_CMS = int(os.getenv("REQ_TIMEOUT_CMS", 10)) REQ_TIMEOUT_GHCR = int(os.getenv("REQ_TIMEOUT_GHCR", 10)) + +# OFFLINERS +ZIMIT_DISABLE_ZIM_FILENAME_CHECK = os.getenv("ZIMIT_DISABLE_ZIM_FILENAME_CHECK", "") diff --git a/dispatcher/backend/src/common/schemas/offliners/zimit.py b/dispatcher/backend/src/common/schemas/offliners/zimit.py index 54556250..5b53420a 100644 --- a/dispatcher/backend/src/common/schemas/offliners/zimit.py +++ b/dispatcher/backend/src/common/schemas/offliners/zimit.py @@ -7,6 +7,8 @@ validate_zim_filename, ) +from common.constants import ZIMIT_DISABLE_ZIM_FILENAME_CHECK + # https://github.com/puppeteer/puppeteer/blob/main/src/common/DeviceDescriptors.ts # https://github.com/puppeteer/puppeteer/blob/ # main/packages/puppeteer-core/src/common/Device.ts @@ -190,7 +192,9 @@ class Meta: "Make sure to end with _{period}.zim", }, data_key="zim-file", - validate=validate_zim_filename, + validate=validate_zim_filename + if not ZIMIT_DISABLE_ZIM_FILENAME_CHECK + else None, ) tags = String( diff --git a/dispatcher/backend/src/tests/integration/routes/schedules/test_zimit.py b/dispatcher/backend/src/tests/integration/routes/schedules/test_zimit.py new file mode 100644 index 00000000..74de9fac --- /dev/null +++ b/dispatcher/backend/src/tests/integration/routes/schedules/test_zimit.py @@ -0,0 +1,111 @@ +class TestZimit: + def test_create_zimit_schedule_ok(self, client, access_token, garbage_collector): + schedule = { + "name": "zimit_test_ok", + "category": "other", + "enabled": False, + "tags": [], + "language": {"code": "fr", "name_en": "French", "name_native": "Français"}, + "config": { + "task_name": "zimit", + "warehouse_path": "/other", + "image": {"name": "openzim/zimit", "tag": "1.0.0"}, + "monitor": False, + "platform": None, + "flags": { + "name": "acme", + "url": "https://www.acme.com", + "zim-file": "acme_en_all_{period}.zim", + }, + "resources": {"cpu": 3, "memory": 1024, "disk": 0}, + }, + "periodicity": "quarterly", + } + + url = "/schedules/" + response = client.post( + url, json=schedule, headers={"Authorization": access_token} + ) + response_data = response.get_json() + print(response_data) + if "_id" in response_data: + garbage_collector.add_schedule_id(response_data["_id"]) + assert response.status_code == 201 + + # below test is green only if ZIMIT_DISABLE_ZIM_FILENAME_CHECK is set + # def test_create_zimit_schedule_bad_name_ok( + # self, client, access_token, garbage_collector + # ): + # schedule = { + # "name": "zimit_test_bad_name_ok", + # "category": "other", + # "enabled": False, + # "tags": [], + # "language": { + # "code": "fr", + # "name_en": "French", + # "name_native": "Français" + # }, + # "config": { + # "task_name": "zimit", + # "warehouse_path": "/other", + # "image": {"name": "openzim/zimit", "tag": "1.0.0"}, + # "monitor": False, + # "platform": None, + # "flags": { + # "name": "acme", + # "url": "https://www.acme.com", + # "zim-file": "bad_name", + # }, + # "resources": {"cpu": 3, "memory": 1024, "disk": 0}, + # }, + # "periodicity": "quarterly", + # } + + # url = "/schedules/" + # response = client.post( + # url, json=schedule, headers={"Authorization": access_token} + # ) + # response_data = response.get_json() + # print(response_data) + # if "_id" in response_data: + # garbage_collector.add_schedule_id(response_data["_id"]) + # assert response.status_code == 201 + + # below test becomes red if ZIMIT_DISABLE_ZIM_FILENAME_CHECK is set + def test_create_zimit_schedule_bad_name_nok( + self, client, access_token, garbage_collector + ): + schedule = { + "name": "zimit_test_bad_name_nok", + "category": "other", + "enabled": False, + "tags": [], + "language": {"code": "fr", "name_en": "French", "name_native": "Français"}, + "config": { + "task_name": "zimit", + "warehouse_path": "/other", + "image": {"name": "openzim/zimit", "tag": "1.0.0"}, + "monitor": False, + "platform": None, + "flags": { + "name": "acme", + "url": "https://www.acme.com", + "zim-file": "bad_name", + }, + "resources": {"cpu": 3, "memory": 1024, "disk": 0}, + }, + "periodicity": "quarterly", + } + + url = "/schedules/" + response = client.post( + url, json=schedule, headers={"Authorization": access_token} + ) + response_data = response.get_json() + print(response_data) + if "_id" in response_data: + garbage_collector.add_schedule_id(response_data["_id"]) + assert response.status_code == 400 + assert "error_description" in response_data + assert "zim-file" in response_data["error_description"]