Merge pull request #830 from openzim/missing_schedule_name

Fix missing schedule name
openzim · Sep 27, 2023 · 74341bc · 74341bc
2 parents 3c5a520 + 592c2b5
commit 74341bc
Show file tree

Hide file tree

Showing 21 changed files with 381 additions and 11 deletions.
diff --git a/dev/README.md b/dev/README.md
@@ -76,10 +76,11 @@ Other containers might be restarted the same way.
 
 ## Browse the web UI
 
-Open [the web UI](http://localhost:8001) in your favorite browser.
+To develop: open [the development web UI](http://localhost:8002). This version has hot reload of UI code changes.
 
-You can login with username `admin` and password  `admin`.
+To test build version: open [the web UI](http://localhost:8001) in your favorite browser.
 
+You can login with username `admin` and password `admin`.
 
 ## Run tests
 

diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml
@@ -41,6 +41,19 @@ services:
       ZIMFARM_WEBAPI: http://localhost:8000/v1
     depends_on:
       - backend
+  frontend-ui-dev:
+    build:
+      dockerfile: ../../dev/frontend-ui-dev/Dockerfile
+      context: ../dispatcher/frontend-ui
+    container_name: zf_frontend-ui-dev
+    volumes:
+      - ../dispatcher/frontend-ui/src:/app/src
+    ports:
+      - 8002:80
+    environment:
+      ZIMFARM_WEBAPI: http://localhost:8000/v1
+    depends_on:
+      - backend
   backend-tools:
     build:
       dockerfile: ../../dev/backend-tools-tests/Dockerfile

diff --git a/dev/frontend-ui-dev/Dockerfile b/dev/frontend-ui-dev/Dockerfile
@@ -0,0 +1,13 @@
+FROM node:14-alpine as builder
+
+RUN apk --no-cache add yarn python3
+WORKDIR /app
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+COPY package.json yarn.lock /app/
+RUN yarn install && yarn cache clean
+COPY *.js /app/
+COPY public /app/public
+COPY src /app/src
+ENV ENVIRON_PATH /app/public/environ.json
+ENTRYPOINT [ "entrypoint.sh" ]
+CMD ["yarn", "serve", "--host", "0.0.0.0", "--port", "80"]
diff --git a/...atcher/backend/src/import_schedules.ipynb → dev/import_schedules.ipynb b/...atcher/backend/src/import_schedules.ipynb → dev/import_schedules.ipynb
@@ -16,6 +16,18 @@
     "```"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from pathlib import Path\n",
+    "if os.getcwd().endswith(\"/dev\"):\n",
+    "    os.chdir(Path(os.getcwd()) / Path(\"../dispatcher/backend/src\"))"
+   ]
+  },
   {
    "attachments": {},
    "cell_type": "markdown",
@@ -26,9 +38,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "env: POSTGRES_URI=postgresql+psycopg://zimfarm:zimpass@localhost:5432/zimfarm\n"
+     ]
+    }
+   ],
    "source": [
     "%env POSTGRES_URI=postgresql+psycopg://zimfarm:zimpass@localhost:5432/zimfarm\n",
     "\n",
@@ -141,7 +161,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.2"
+   "version": "3.8.17"
   },
   "orig_nbformat": 4
  },

diff --git a/dev/tests.ipynb b/dev/tests.ipynb
@@ -0,0 +1,96 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Just some test instructions, useful for instance to dry run SQLAlchemy code"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from pathlib import Path\n",
+    "if os.getcwd().endswith(\"/dev\"):\n",
+    "    os.chdir(Path(os.getcwd()) / Path(\"../dispatcher/backend/src\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "env: POSTGRES_URI=postgresql+psycopg://zimfarm:zimpass@localhost:5432/zimfarm\n"
+     ]
+    }
+   ],
+   "source": [
+    "%env POSTGRES_URI=postgresql+psycopg://zimfarm:zimpass@localhost:5432/zimfarm\n",
+    "\n",
+    "import json\n",
+    "import pathlib\n",
+    "import sqlalchemy as sa\n",
+    "import sqlalchemy.orm as so\n",
+    "\n",
+    "from db import Session\n",
+    "from db.models import Schedule, RequestedTask"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "UPDATE requested_task SET original_schedule_name=(SELECT schedule.name \n",
+      "FROM schedule \n",
+      "WHERE schedule.id = requested_task.schedule_id) WHERE requested_task.schedule_id IS NOT NULL\n"
+     ]
+    }
+   ],
+   "source": [
+    "stmt = (\n",
+    "    sa.update(RequestedTask)\n",
+    "    .where(\n",
+    "        RequestedTask.schedule_id != None\n",
+    "    )\n",
+    "    .values(original_schedule_name=sa.select(Schedule.name).where(Schedule.id == RequestedTask.schedule_id).scalar_subquery())\n",
+    ")\n",
+    "print(stmt)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.17"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/dispatcher/backend/src/common/external.py b/dispatcher/backend/src/common/external.py
@@ -202,6 +202,11 @@ def advertise_book_to_cms(task: dbm.Task, file_name):
             except Exception as exc:
                 logger.error(f"Unable to parse CMS response: {exc}")
                 logger.exception(exc)
+        else:
+            logger.error(
+                f"CMS returned an error {resp.status_code} for book"
+                f"{file_data['info']['id']}"
+            )
 
     # record request result
     task.files[file_name] = file_data

diff --git a/dispatcher/backend/src/common/schemas/orms.py b/dispatcher/backend/src/common/schemas/orms.py
@@ -71,6 +71,7 @@ class TaskLightSchema(m.Schema):
     worker_name = mf.String(data_key="worker")
     updated_at = MadeAwareDateTime()
     config = mf.Nested(ConfigWithOnlyTaskNameAndResourcesSchema, only=["resources"])
+    original_schedule_name = mf.String()
 
 
 class TaskFullSchema(TaskLightSchema):
@@ -101,6 +102,7 @@ class RequestedTaskLightSchema(m.Schema):
     requested_by = mf.String()
     priority = mf.Integer()
     schedule_name = mf.String()
+    original_schedule_name = mf.String()
     worker = mf.String()
 
 

diff --git a/dispatcher/backend/src/db/models.py b/dispatcher/backend/src/db/models.py
@@ -225,6 +225,7 @@ class Task(Base):
     notification: Mapped[Dict[str, Any]]
     files: Mapped[Dict[str, Any]]
     upload: Mapped[Dict[str, Any]]
+    original_schedule_name: Mapped[str]
 
     schedule_id: Mapped[Optional[UUID]] = mapped_column(
         ForeignKey("schedule.id"), init=False
@@ -358,6 +359,7 @@ class RequestedTask(Base):
     config: Mapped[Dict[str, Any]] = mapped_column(MutableDict.as_mutable(JSON))
     upload: Mapped[Dict[str, Any]]
     notification: Mapped[Dict[str, Any]]
+    original_schedule_name: Mapped[str]
 
     schedule_id: Mapped[Optional[UUID]] = mapped_column(
         ForeignKey("schedule.id"), init=False

diff --git a/dispatcher/backend/src/migrations/versions/43f385b318d4_add_original_schedule_name.py b/dispatcher/backend/src/migrations/versions/43f385b318d4_add_original_schedule_name.py
@@ -0,0 +1,84 @@
+"""add_original_schedule_name
+
+Revision ID: 43f385b318d4
+Revises: 15354d56545a
+Create Date: 2023-09-26 07:56:45.008277
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+from db.models import RequestedTask, Schedule, Task
+
+# revision identifiers, used by Alembic.
+revision = "43f385b318d4"
+down_revision = "15354d56545a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    session = sa.orm.Session(bind=bind)
+
+    # add original_schedule_name as nullable
+    op.add_column(
+        "requested_task",
+        sa.Column("original_schedule_name", sa.String(), nullable=True),
+    )
+
+    # set original_schedule_name for requested tasks with existing schedule
+    session.execute(
+        sa.update(RequestedTask)
+        .where(RequestedTask.schedule_id is not None)
+        .values(
+            original_schedule_name=sa.select(Schedule.name)
+            .where(Schedule.id == RequestedTask.schedule_id)
+            .scalar_subquery()
+        )
+    )
+
+    # set original_schedule_name for requested tasks without existing schedule
+    session.execute(
+        sa.update(RequestedTask)
+        .where(RequestedTask.schedule_id is None)
+        .values(original_schedule_name="<unknown>")
+    )
+
+    # set original_schedule_name as not nullable
+    op.alter_column("requested_task", "original_schedule_name", nullable=False)
+
+    # add original_schedule_name as nullable
+    op.add_column(
+        "task", sa.Column("original_schedule_name", sa.String(), nullable=True)
+    )
+
+    # set original_schedule_name for requested tasks with existing schedule
+    session.execute(
+        sa.update(Task)
+        .where(Task.schedule_id is not None)
+        .values(
+            original_schedule_name=sa.select(Schedule.name)
+            .where(Schedule.id == Task.schedule_id)
+            .scalar_subquery()
+        )
+    )
+
+    # set original_schedule_name for requested tasks without existing schedule
+    session.execute(
+        sa.update(Task)
+        .where(Task.schedule_id is None)
+        .values(original_schedule_name="<unknown>")
+    )
+
+    # set original_schedule_name as not nullable
+    op.alter_column("task", "original_schedule_name", nullable=False)
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("task", "original_schedule_name")
+    op.drop_column("requested_task", "original_schedule_name")
+    # ### end Alembic commands ###
diff --git a/dispatcher/backend/src/routes/requested_tasks/requested_task.py b/dispatcher/backend/src/routes/requested_tasks/requested_task.py
@@ -85,6 +85,7 @@ def list_of_requested_tasks(session: so.Session, token: AccessToken.Payload = No
             dbm.RequestedTask.timestamp,
             dbm.RequestedTask.requested_by,
             dbm.RequestedTask.priority,
+            dbm.RequestedTask.original_schedule_name,
             dbm.Schedule.name.label("schedule_name"),
             dbm.Worker.name.label("worker"),
         )

diff --git a/dispatcher/backend/src/routes/tasks/task.py b/dispatcher/backend/src/routes/tasks/task.py
@@ -53,6 +53,7 @@ def get(self, session: so.Session):
                 dbm.Task.id,
                 dbm.Task.status,
                 dbm.Task.timestamp,
+                dbm.Task.original_schedule_name,
                 # dbm.Task.config,
                 so.Bundle(
                     "config",
@@ -117,6 +118,7 @@ def get(
                 dbm.Task.files,
                 dbm.Task.upload,
                 dbm.Task.updated_at,
+                dbm.Task.original_schedule_name,
                 dbm.Schedule.name.label("schedule_name"),
                 dbm.Worker.name.label("worker_name"),
             )
@@ -169,6 +171,7 @@ def post(self, session: so.Session, task_id: UUID, token: AccessToken.Payload):
             notification=requested_task.notification,
             files={},
             upload=requested_task.upload,
+            original_schedule_name=requested_task.original_schedule_name,
         )
         task.id = requested_task.id
         task.schedule_id = requested_task.schedule_id