Merge pull request #917 from openzim/artifacts_upload

Artifacts upload

.github/workflows/QA.yml

@@ -19,7 +19,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install black==23.3.0 flake8==6.0.0 isort==5.12.0
+          pip install black==24.2.0 flake8==7.0.0 isort==5.13.2
           black --version
           flake8 --version
           isort --version

dispatcher/backend/maint-scripts/report_youtube_api_keys.py

@@ -55,9 +55,11 @@ def report_youtube_api_keys(session: so.Session, *, display_unknown_secrets=Fals
         if hashed_api_key not in schedules_by_api_key.keys():
             schedules_by_api_key[hashed_api_key] = {
                 "api_key": api_key,
-                "key_name": known_api_keys[hashed_api_key]
-                if hashed_api_key in known_api_keys
-                else "unknown",
+                "key_name": (
+                    known_api_keys[hashed_api_key]
+                    if hashed_api_key in known_api_keys
+                    else "unknown"
+                ),
                 "schedules": [],
             }
         schedules_by_api_key[hashed_api_key]["schedules"].append(schedule.name)
@@ -69,9 +71,11 @@ def report_youtube_api_keys(session: so.Session, *, display_unknown_secrets=Fals
     for hashed_api_key, data in schedules_by_api_key.items():
         report_data["keys"].append(
             {
-                "name": known_api_keys[hashed_api_key]
-                if hashed_api_key in known_api_keys.keys()
-                else "unknown",
+                "name": (
+                    known_api_keys[hashed_api_key]
+                    if hashed_api_key in known_api_keys.keys()
+                    else "unknown"
+                ),
                 "schedules": sorted(data["schedules"]),
             }
         )

dispatcher/backend/src/common/constants.py

@@ -23,16 +23,24 @@
     "ZIM_UPLOAD_URI", "sftp://[email protected]:1522/zim"
 )
 try:
+    # ZIM files expiration, 0 to disable expiration
     ZIM_EXPIRATION = int(os.getenv("ZIM_EXPIRATION", "0"))
 except Exception:
     ZIM_EXPIRATION = 0
 LOGS_UPLOAD_URI = os.getenv(
     "LOGS_UPLOAD_URI", "sftp://[email protected]:1522/logs"
 )
 try:
+    # log files expiration, 0 to disable expiration
     LOGS_EXPIRATION = int(os.getenv("LOGS_EXPIRATION", "30"))
 except Exception:
     LOGS_EXPIRATION = 30
+ARTIFACTS_UPLOAD_URI = os.getenv("ARTIFACTS_UPLOAD_URI", None)
+try:
+    # artifact files expiration, 0 to disable expiration
+    ARTIFACTS_EXPIRATION = int(os.getenv("ARTIFACTS_EXPIRATION", "30"))
+except Exception:
+    ARTIFACTS_EXPIRATION = 30
 
 # empty ZIMCHECK_OPTION means no zimcheck
 ZIMCHECK_OPTION = os.getenv("ZIMCHECK_OPTION", "")

dispatcher/backend/src/common/emailing.py

@@ -51,12 +51,14 @@ def send_email_via_mailgun(
             url=f"{MAILGUN_API_URL}/messages",
             auth=("api", MAILGUN_API_KEY),
             data=data,
-            files=[
-                ("attachment", (fpath.name, open(fpath, "rb").read()))
-                for fpath in attachments
-            ]
-            if attachments
-            else [],
+            files=(
+                [
+                    ("attachment", (fpath.name, open(fpath, "rb").read()))
+                    for fpath in attachments
+                ]
+                if attachments
+                else []
+            ),
             timeout=REQ_TIMEOUT_NOTIFICATIONS,
         )
         resp.raise_for_status()

dispatcher/backend/src/common/schemas/models.py

@@ -73,6 +73,7 @@ class ScheduleConfigSchema(SerializableSchema):
     resources = fields.Nested(ResourcesSchema(), required=True)
     flags = fields.Dict(required=True)
     platform = String(required=True, allow_none=True, validate=validate_platform)
+    artifacts_globs = fields.List(String(validate=validate_not_empty), required=False)
     monitor = fields.Boolean(required=True, truthy=[True], falsy=[False])
 
     @staticmethod
@@ -86,9 +87,11 @@ def get_offliner_schema(offliner):
             Offliner.nautilus: NautilusFlagsSchema,
             Offliner.ted: TedFlagsSchema,
             Offliner.openedx: OpenedxFlagsSchema,
-            Offliner.zimit: ZimitFlagsSchemaRelaxed
-            if constants.ZIMIT_USE_RELAXED_SCHEMA
-            else ZimitFlagsSchema,
+            Offliner.zimit: (
+                ZimitFlagsSchemaRelaxed
+                if constants.ZIMIT_USE_RELAXED_SCHEMA
+                else ZimitFlagsSchema
+            ),
             Offliner.kolibri: KolibriFlagsSchema,
             Offliner.wikihow: WikihowFlagsSchema,
             Offliner.ifixit: IFixitFlagsSchema,

dispatcher/backend/src/common/schemas/orms.py

@@ -180,9 +180,9 @@ def get_duration(schedule: dbm.Schedule):
                 duration_res["default"] = ScheduleFullSchema.get_one_duration(duration)
             if duration.worker:
                 duration_res["available"] = True
-                duration_res["workers"][
-                    duration.worker.name
-                ] = ScheduleFullSchema.get_one_duration(duration)
+                duration_res["workers"][duration.worker.name] = (
+                    ScheduleFullSchema.get_one_duration(duration)
+                )
         return duration_res
 
     def get_is_requested(schedule: dbm.Schedule):

dispatcher/backend/src/common/schemas/parameters.py

@@ -107,6 +107,9 @@ class UpdateSchema(Schema):
     resources = fields.Nested(ResourcesSchema, required=False)
     monitor = fields.Boolean(required=False, truthy={True}, falsy={False})
     flags = fields.Dict(required=False)
+    artifacts_globs = fields.List(
+        String(validate=validate_not_empty), required=False, allow_none=True
+    )
 
 
 # schedule clone

dispatcher/backend/src/common/utils.py

@@ -117,6 +117,9 @@ def add_to_debug_if_present(
         task=task, kwargs_key="timeout", container_key="timeout"
     )
     add_to_container_if_present(task=task, kwargs_key="log", container_key="log")
+    add_to_container_if_present(
+        task=task, kwargs_key="artifacts", container_key="artifacts"
+    )
     add_to_debug_if_present(task=task, kwargs_key="task_log", debug_key="log")
     add_to_debug_if_present(task=task, kwargs_key="task_name", debug_key="task_name")
     add_to_debug_if_present(task=task, kwargs_key="task_args", debug_key="task_args")
@@ -348,10 +351,17 @@ def task_checked_file_event_handler(session: so.Session, task_id: UUID, payload:
 
 def task_update_event_handler(session: so.Session, task_id: UUID, payload: dict):
     timestamp = get_timestamp_from_event(payload)
-    log = payload.get("log")  # filename / S3 key of text file at upload_uri[logs]
-    logger.info(f"Task update: {task_id}, log: {log}")
-
-    save_event(session, task_id, TaskStatus.update, timestamp, log=log)
+    if "log" in payload:
+        log = payload.get("log")  # filename / S3 key of text file at upload_uri[logs]
+        logger.info(f"Task update: {task_id}, log: {log}")
+        save_event(session, task_id, TaskStatus.update, timestamp, log=log)
+
+    if "artifacts" in payload:
+        artifacts = payload.get(
+            "artifacts"
+        )  # filename / S3 key of text file at upload_uri[logs]
+        logger.info(f"Task update: {task_id}, artifacts: {artifacts}")
+        save_event(session, task_id, TaskStatus.update, timestamp, artifacts=artifacts)
 
 
 def handle_others(task_id: UUID, event: str, payload: dict):

dispatcher/backend/src/migrations/versions/0055fc77f5d8_add_unique_constraint_on_schedule_.py

@@ -5,6 +5,7 @@
 Create Date: 2023-05-04 12:36:14.802432
 
 """
+
 from alembic import op
 
 # revision identifiers, used by Alembic.

dispatcher/backend/src/migrations/versions/0ad2bf164dae_alter_config_columns_types.py

@@ -5,6 +5,7 @@
 Create Date: 2023-04-26 12:21:57.205579
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import postgresql

dispatcher/backend/src/migrations/versions/3c4d546fde7c_nullable_schedule_id_in_requested_tasks.py

@@ -5,6 +5,7 @@
 Create Date: 2023-05-18 13:35:13.362937
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 

dispatcher/backend/src/migrations/versions/43f385b318d4_add_original_schedule_name.py

@@ -5,6 +5,7 @@
 Create Date: 2023-09-26 07:56:45.008277
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 

dispatcher/backend/src/migrations/versions/4de2adfc3d11_create_some_indexes.py

@@ -5,6 +5,7 @@
 Create Date: 2023-04-06 05:13:40.448095
 
 """
+
 from alembic import op
 
 # revision identifiers, used by Alembic.

dispatcher/backend/src/migrations/versions/62227054989d_remove_mongo_backup_columns.py

@@ -5,6 +5,7 @@
 Create Date: 2023-05-12 15:29:06.706797
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import postgresql

dispatcher/backend/src/migrations/versions/6970d8681400_nullable_shedule_id_on_task.py

@@ -5,6 +5,7 @@
 Create Date: 2023-05-18 10:53:05.286686
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 

dispatcher/backend/src/migrations/versions/76a074cc939b_drop_task_id_in_scheduleduration.py

@@ -5,6 +5,7 @@
 Create Date: 2023-05-04 11:27:30.248671
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 

dispatcher/backend/src/migrations/versions/8279abb2d6dc_create_schedules_tasks_and_requested_.py

@@ -5,6 +5,7 @@
 Create Date: 2023-04-13 10:10:52.820099
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import postgresql

dispatcher/backend/src/migrations/versions/96203f95bc36_add_worker_table.py

@@ -5,6 +5,7 @@
 Create Date: 2023-04-06 15:56:02.435956
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import postgresql

dispatcher/backend/src/migrations/versions/acf1446fa2aa_keep_deleted_users_and_workers_in_the_.py

@@ -5,6 +5,7 @@
 Create Date: 2023-04-28 11:29:30.878410
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 

dispatcher/backend/src/migrations/versions/cbedcefd6059_store_mongo_id_as_well.py

@@ -5,6 +5,7 @@
 Create Date: 2023-04-03 18:01:18.286681
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 

dispatcher/backend/src/migrations/versions/ceae21f592b7_remove_ssh_key_last_used.py

@@ -5,6 +5,7 @@
 Create Date: 2023-09-29 10:59:39.739351
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import postgresql

dispatcher/backend/src/migrations/versions/d13a178652d5_apply_a_single_naming_convention.py

@@ -5,6 +5,7 @@
 Create Date: 2023-04-06 04:57:11.449333
 
 """
+
 from alembic import op
 
 # revision identifiers, used by Alembic.

dispatcher/backend/src/migrations/versions/dc811d96975c_added_user_sshkey_tables.py

@@ -5,6 +5,7 @@
 Create Date: 2023-03-31 14:03:52.822993
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import postgresql

dispatcher/backend/src/migrations/versions/e1d3894be9ea_set_fields_as_non_nullable.py

@@ -5,6 +5,7 @@
 Create Date: 2023-05-04 14:08:19.014269
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import postgresql

dispatcher/backend/src/migrations/versions/f1a7eae2c347_user_username_is_mandatory_unique.py

@@ -5,6 +5,7 @@
 Create Date: 2023-03-31 18:05:44.896451
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 

dispatcher/backend/src/migrations/versions/fe65a1b0f953_store_refresh_tokens.py

@@ -5,6 +5,7 @@
 Create Date: 2023-04-03 21:12:33.745734
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import postgresql