diff --git a/dev/README.md b/dev/README.md index 43dbd87b0..2f580fd6f 100644 --- a/dev/README.md +++ b/dev/README.md @@ -76,10 +76,11 @@ Other containers might be restarted the same way. ## Browse the web UI -Open [the web UI](http://localhost:8001) in your favorite browser. +To develop: open [the development web UI](http://localhost:8002). This version has hot reload of UI code changes. -You can login with username `admin` and password `admin`. +To test build version: open [the web UI](http://localhost:8001) in your favorite browser. +You can login with username `admin` and password `admin`. ## Run tests diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index 9c317e30b..9fd6ad8c5 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -41,6 +41,19 @@ services: ZIMFARM_WEBAPI: http://localhost:8000/v1 depends_on: - backend + frontend-ui-dev: + build: + dockerfile: ../../dev/frontend-ui-dev/Dockerfile + context: ../dispatcher/frontend-ui + container_name: zf_frontend-ui-dev + volumes: + - ../dispatcher/frontend-ui/src:/app/src + ports: + - 8002:80 + environment: + ZIMFARM_WEBAPI: http://localhost:8000/v1 + depends_on: + - backend backend-tools: build: dockerfile: ../../dev/backend-tools-tests/Dockerfile diff --git a/dev/frontend-ui-dev/Dockerfile b/dev/frontend-ui-dev/Dockerfile new file mode 100644 index 000000000..2ff42aec2 --- /dev/null +++ b/dev/frontend-ui-dev/Dockerfile @@ -0,0 +1,13 @@ +FROM node:14-alpine as builder + +RUN apk --no-cache add yarn python3 +WORKDIR /app +COPY entrypoint.sh /usr/local/bin/entrypoint.sh +COPY package.json yarn.lock /app/ +RUN yarn install && yarn cache clean +COPY *.js /app/ +COPY public /app/public +COPY src /app/src +ENV ENVIRON_PATH /app/public/environ.json +ENTRYPOINT [ "entrypoint.sh" ] +CMD ["yarn", "serve", "--host", "0.0.0.0", "--port", "80"] \ No newline at end of file diff --git a/dispatcher/backend/src/import_schedules.ipynb b/dev/import_schedules.ipynb similarity index 89% rename from dispatcher/backend/src/import_schedules.ipynb rename to dev/import_schedules.ipynb index d9657737e..dd8e66b09 100644 --- a/dispatcher/backend/src/import_schedules.ipynb +++ b/dev/import_schedules.ipynb @@ -16,6 +16,18 @@ "```" ] }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from pathlib import Path\n", + "if os.getcwd().endswith(\"/dev\"):\n", + " os.chdir(Path(os.getcwd()) / Path(\"../dispatcher/backend/src\"))" + ] + }, { "attachments": {}, "cell_type": "markdown", @@ -26,9 +38,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "env: POSTGRES_URI=postgresql+psycopg://zimfarm:zimpass@localhost:5432/zimfarm\n" + ] + } + ], "source": [ "%env POSTGRES_URI=postgresql+psycopg://zimfarm:zimpass@localhost:5432/zimfarm\n", "\n", @@ -141,7 +161,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.2" + "version": "3.8.17" }, "orig_nbformat": 4 }, diff --git a/dev/tests.ipynb b/dev/tests.ipynb new file mode 100644 index 000000000..e89c8981c --- /dev/null +++ b/dev/tests.ipynb @@ -0,0 +1,96 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Just some test instructions, useful for instance to dry run SQLAlchemy code" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from pathlib import Path\n", + "if os.getcwd().endswith(\"/dev\"):\n", + " os.chdir(Path(os.getcwd()) / Path(\"../dispatcher/backend/src\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "env: POSTGRES_URI=postgresql+psycopg://zimfarm:zimpass@localhost:5432/zimfarm\n" + ] + } + ], + "source": [ + "%env POSTGRES_URI=postgresql+psycopg://zimfarm:zimpass@localhost:5432/zimfarm\n", + "\n", + "import json\n", + "import pathlib\n", + "import sqlalchemy as sa\n", + "import sqlalchemy.orm as so\n", + "\n", + "from db import Session\n", + "from db.models import Schedule, RequestedTask" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "UPDATE requested_task SET original_schedule_name=(SELECT schedule.name \n", + "FROM schedule \n", + "WHERE schedule.id = requested_task.schedule_id) WHERE requested_task.schedule_id IS NOT NULL\n" + ] + } + ], + "source": [ + "stmt = (\n", + " sa.update(RequestedTask)\n", + " .where(\n", + " RequestedTask.schedule_id != None\n", + " )\n", + " .values(original_schedule_name=sa.select(Schedule.name).where(Schedule.id == RequestedTask.schedule_id).scalar_subquery())\n", + ")\n", + "print(stmt)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.17" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/dispatcher/backend/src/common/external.py b/dispatcher/backend/src/common/external.py index 6fc1aea7c..8a4d76fd3 100644 --- a/dispatcher/backend/src/common/external.py +++ b/dispatcher/backend/src/common/external.py @@ -202,6 +202,11 @@ def advertise_book_to_cms(task: dbm.Task, file_name): except Exception as exc: logger.error(f"Unable to parse CMS response: {exc}") logger.exception(exc) + else: + logger.error( + f"CMS returned an error {resp.status_code} for book" + f"{file_data['info']['id']}" + ) # record request result task.files[file_name] = file_data diff --git a/dispatcher/backend/src/common/schemas/orms.py b/dispatcher/backend/src/common/schemas/orms.py index ebf49c85b..d24ae79d2 100644 --- a/dispatcher/backend/src/common/schemas/orms.py +++ b/dispatcher/backend/src/common/schemas/orms.py @@ -71,6 +71,7 @@ class TaskLightSchema(m.Schema): worker_name = mf.String(data_key="worker") updated_at = MadeAwareDateTime() config = mf.Nested(ConfigWithOnlyTaskNameAndResourcesSchema, only=["resources"]) + original_schedule_name = mf.String() class TaskFullSchema(TaskLightSchema): @@ -101,6 +102,7 @@ class RequestedTaskLightSchema(m.Schema): requested_by = mf.String() priority = mf.Integer() schedule_name = mf.String() + original_schedule_name = mf.String() worker = mf.String() diff --git a/dispatcher/backend/src/db/models.py b/dispatcher/backend/src/db/models.py index 36e2429eb..1c77f30a9 100644 --- a/dispatcher/backend/src/db/models.py +++ b/dispatcher/backend/src/db/models.py @@ -225,6 +225,7 @@ class Task(Base): notification: Mapped[Dict[str, Any]] files: Mapped[Dict[str, Any]] upload: Mapped[Dict[str, Any]] + original_schedule_name: Mapped[str] schedule_id: Mapped[Optional[UUID]] = mapped_column( ForeignKey("schedule.id"), init=False @@ -358,6 +359,7 @@ class RequestedTask(Base): config: Mapped[Dict[str, Any]] = mapped_column(MutableDict.as_mutable(JSON)) upload: Mapped[Dict[str, Any]] notification: Mapped[Dict[str, Any]] + original_schedule_name: Mapped[str] schedule_id: Mapped[Optional[UUID]] = mapped_column( ForeignKey("schedule.id"), init=False diff --git a/dispatcher/backend/src/migrations/versions/43f385b318d4_add_original_schedule_name.py b/dispatcher/backend/src/migrations/versions/43f385b318d4_add_original_schedule_name.py new file mode 100644 index 000000000..f4721d513 --- /dev/null +++ b/dispatcher/backend/src/migrations/versions/43f385b318d4_add_original_schedule_name.py @@ -0,0 +1,84 @@ +"""add_original_schedule_name + +Revision ID: 43f385b318d4 +Revises: 15354d56545a +Create Date: 2023-09-26 07:56:45.008277 + +""" +import sqlalchemy as sa +from alembic import op + +from db.models import RequestedTask, Schedule, Task + +# revision identifiers, used by Alembic. +revision = "43f385b318d4" +down_revision = "15354d56545a" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + bind = op.get_bind() + session = sa.orm.Session(bind=bind) + + # add original_schedule_name as nullable + op.add_column( + "requested_task", + sa.Column("original_schedule_name", sa.String(), nullable=True), + ) + + # set original_schedule_name for requested tasks with existing schedule + session.execute( + sa.update(RequestedTask) + .where(RequestedTask.schedule_id is not None) + .values( + original_schedule_name=sa.select(Schedule.name) + .where(Schedule.id == RequestedTask.schedule_id) + .scalar_subquery() + ) + ) + + # set original_schedule_name for requested tasks without existing schedule + session.execute( + sa.update(RequestedTask) + .where(RequestedTask.schedule_id is None) + .values(original_schedule_name="") + ) + + # set original_schedule_name as not nullable + op.alter_column("requested_task", "original_schedule_name", nullable=False) + + # add original_schedule_name as nullable + op.add_column( + "task", sa.Column("original_schedule_name", sa.String(), nullable=True) + ) + + # set original_schedule_name for requested tasks with existing schedule + session.execute( + sa.update(Task) + .where(Task.schedule_id is not None) + .values( + original_schedule_name=sa.select(Schedule.name) + .where(Schedule.id == Task.schedule_id) + .scalar_subquery() + ) + ) + + # set original_schedule_name for requested tasks without existing schedule + session.execute( + sa.update(Task) + .where(Task.schedule_id is None) + .values(original_schedule_name="") + ) + + # set original_schedule_name as not nullable + op.alter_column("task", "original_schedule_name", nullable=False) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("task", "original_schedule_name") + op.drop_column("requested_task", "original_schedule_name") + # ### end Alembic commands ### diff --git a/dispatcher/backend/src/routes/requested_tasks/requested_task.py b/dispatcher/backend/src/routes/requested_tasks/requested_task.py index ac950a633..347192bc3 100644 --- a/dispatcher/backend/src/routes/requested_tasks/requested_task.py +++ b/dispatcher/backend/src/routes/requested_tasks/requested_task.py @@ -85,6 +85,7 @@ def list_of_requested_tasks(session: so.Session, token: AccessToken.Payload = No dbm.RequestedTask.timestamp, dbm.RequestedTask.requested_by, dbm.RequestedTask.priority, + dbm.RequestedTask.original_schedule_name, dbm.Schedule.name.label("schedule_name"), dbm.Worker.name.label("worker"), ) diff --git a/dispatcher/backend/src/routes/tasks/task.py b/dispatcher/backend/src/routes/tasks/task.py index 896056cec..55d7f8f56 100644 --- a/dispatcher/backend/src/routes/tasks/task.py +++ b/dispatcher/backend/src/routes/tasks/task.py @@ -53,6 +53,7 @@ def get(self, session: so.Session): dbm.Task.id, dbm.Task.status, dbm.Task.timestamp, + dbm.Task.original_schedule_name, # dbm.Task.config, so.Bundle( "config", @@ -117,6 +118,7 @@ def get( dbm.Task.files, dbm.Task.upload, dbm.Task.updated_at, + dbm.Task.original_schedule_name, dbm.Schedule.name.label("schedule_name"), dbm.Worker.name.label("worker_name"), ) @@ -169,6 +171,7 @@ def post(self, session: so.Session, task_id: UUID, token: AccessToken.Payload): notification=requested_task.notification, files={}, upload=requested_task.upload, + original_schedule_name=requested_task.original_schedule_name, ) task.id = requested_task.id task.schedule_id = requested_task.schedule_id diff --git a/dispatcher/backend/src/tests/integration/routes/business_logic/test_requested_task_bl.py b/dispatcher/backend/src/tests/integration/routes/business_logic/test_requested_task_bl.py new file mode 100644 index 000000000..52c1bae90 --- /dev/null +++ b/dispatcher/backend/src/tests/integration/routes/business_logic/test_requested_task_bl.py @@ -0,0 +1,62 @@ +import json + +import pytest + + +class TestRequestedTaskBusiness: + @pytest.fixture(scope="module") + def headers(self, access_token): + return {"Authorization": access_token, "Content-Type": "application/json"} + + @pytest.fixture(scope="module") + def requested_task(self, client, headers, schedule): + response = client.post( + "/requested-tasks/", + headers=headers, + data=json.dumps({"schedule_names": [schedule["name"]]}), + ) + assert response.status_code == 201 + assert "requested" in response.json + assert len(response.json["requested"]) == 1 + requested_task_id = response.json["requested"][0] + yield requested_task_id + + response = client.delete( + f"/requested-tasks/{requested_task_id}", + headers=headers, + ) + assert response.status_code == 200 + + def test_requested_task_with_schedule( + self, client, headers, schedule, requested_task + ): + url = f"/requested-tasks/{requested_task}" + response = client.get( + url, + headers=headers, + ) + assert response.status_code == 200 + assert "schedule_name" in response.json + assert response.json["schedule_name"] == schedule["name"] + assert "original_schedule_name" in response.json + assert response.json["original_schedule_name"] == schedule["name"] + + def test_requested_task_without_schedule( + self, client, headers, schedule, requested_task + ): + url = f"/schedules/{schedule['name']}" + response = client.delete( + url, + headers=headers, + ) + assert response.status_code == 204 + url = f"/requested-tasks/{requested_task}" + response = client.get( + url, + headers=headers, + ) + assert response.status_code == 200 + assert "schedule_name" in response.json + assert response.json["schedule_name"] == "none" + assert "original_schedule_name" in response.json + assert response.json["original_schedule_name"] == schedule["name"] diff --git a/dispatcher/backend/src/tests/integration/routes/business_logic/test_task_bl.py b/dispatcher/backend/src/tests/integration/routes/business_logic/test_task_bl.py new file mode 100644 index 000000000..9c8859b8e --- /dev/null +++ b/dispatcher/backend/src/tests/integration/routes/business_logic/test_task_bl.py @@ -0,0 +1,52 @@ +import pytest + +SCHEDULE_NAME = "a_schedule_for_tasks" + + +class TestTaskBusiness: + @pytest.fixture(scope="module") + def headers(self, access_token): + return {"Authorization": access_token, "Content-Type": "application/json"} + + @pytest.fixture(scope="module") + def task(self, client, headers, worker, make_requested_task, garbage_collector): + requested_task = make_requested_task(SCHEDULE_NAME) + url = "/tasks/{}".format(str(requested_task["_id"])) + response = client.post( + url, headers=headers, query_string={"worker_name": "worker_name"} + ) + assert response.status_code == 201 + assert "_id" in response.json + task_id = response.json["_id"] + garbage_collector.add_task_id(task_id) + yield task_id + + def test_task_with_schedule(self, client, headers, task): + url = f"/tasks/{task}" + response = client.get( + url, + headers=headers, + ) + assert response.status_code == 200 + assert "schedule_name" in response.json + assert response.json["schedule_name"] == SCHEDULE_NAME + assert "original_schedule_name" in response.json + assert response.json["original_schedule_name"] == SCHEDULE_NAME + + def test_task_without_schedule(self, client, headers, task): + url = f"/schedules/{SCHEDULE_NAME}" + response = client.delete( + url, + headers=headers, + ) + assert response.status_code == 204 + url = f"/tasks/{task}" + response = client.get( + url, + headers=headers, + ) + assert response.status_code == 200 + assert "schedule_name" in response.json + assert response.json["schedule_name"] is None + assert "original_schedule_name" in response.json + assert response.json["original_schedule_name"] == SCHEDULE_NAME diff --git a/dispatcher/backend/src/tests/integration/routes/conftest.py b/dispatcher/backend/src/tests/integration/routes/conftest.py index 8f326a160..f0a0eb939 100644 --- a/dispatcher/backend/src/tests/integration/routes/conftest.py +++ b/dispatcher/backend/src/tests/integration/routes/conftest.py @@ -275,6 +275,7 @@ def _make_requested_task( config=config, upload={}, notification={}, + original_schedule_name=schedule_name, ) requested_task.schedule = schedule session.add(requested_task) diff --git a/dispatcher/backend/src/tests/integration/routes/requested_tasks/test_requested_task.py b/dispatcher/backend/src/tests/integration/routes/requested_tasks/test_requested_task.py index 5730323cf..e4015630d 100644 --- a/dispatcher/backend/src/tests/integration/routes/requested_tasks/test_requested_task.py +++ b/dispatcher/backend/src/tests/integration/routes/requested_tasks/test_requested_task.py @@ -12,6 +12,7 @@ def _assert_requested_task(self, task, item): "_id", "status", "schedule_name", + "original_schedule_name", "timestamp", "config", "requested_by", @@ -21,6 +22,7 @@ def _assert_requested_task(self, task, item): assert item["_id"] == str(task["_id"]) assert item["status"] == task["status"] assert item["schedule_name"] == task["schedule_name"] + assert item["original_schedule_name"] == task["schedule_name"] @pytest.mark.parametrize( "query_param", [{"matching_cpu": "-2"}, {"matching_memory": -1}] @@ -160,6 +162,7 @@ def test_get(self, client, requested_task): assert data["status"] == requested_task["status"] assert "schedule_name" in data assert data["schedule_name"] == requested_task["schedule_name"] + assert data["original_schedule_name"] == requested_task["schedule_name"] assert "timestamp" in data assert "events" in data @@ -170,7 +173,9 @@ def requested_task(self, make_requested_task): requested_task = make_requested_task() return requested_task - def test_create_from_schedule(self, client, access_token, schedule): + def test_create_from_schedule( + self, client, access_token, schedule, garbage_collector + ): url = "/requested-tasks/" headers = {"Authorization": access_token, "Content-Type": "application/json"} response = client.post( @@ -179,6 +184,10 @@ def test_create_from_schedule(self, client, access_token, schedule): data=json.dumps({"schedule_names": [schedule["name"]]}), ) assert response.status_code == 201 + assert "requested" in response.json + assert len(response.json["requested"]) == 1 + requested_task_id = response.json["requested"][0] + garbage_collector.add_requested_task_id(requested_task_id) def test_create_with_wrong_schedule(self, client, access_token, schedule): url = "/requested-tasks/" diff --git a/dispatcher/backend/src/tests/integration/routes/tasks/conftest.py b/dispatcher/backend/src/tests/integration/routes/tasks/conftest.py index 5a669b446..9abc1e678 100644 --- a/dispatcher/backend/src/tests/integration/routes/tasks/conftest.py +++ b/dispatcher/backend/src/tests/integration/routes/tasks/conftest.py @@ -73,6 +73,7 @@ def _make_task( notification={}, files=files, upload={}, + original_schedule_name=schedule_name, ) task.schedule_id = schedule.id task.worker_id = worker_obj.id diff --git a/dispatcher/backend/src/tests/integration/routes/tasks/test_task.py b/dispatcher/backend/src/tests/integration/routes/tasks/test_task.py index 8fdfc8fc1..d5406e93f 100644 --- a/dispatcher/backend/src/tests/integration/routes/tasks/test_task.py +++ b/dispatcher/backend/src/tests/integration/routes/tasks/test_task.py @@ -43,9 +43,11 @@ def _assert_task(self, task, item): "schedule_name", "worker", "updated_at", + "original_schedule_name", } assert item["_id"] == str(task["_id"]) assert item["status"] == task["status"] + assert item["original_schedule_name"] == task["schedule_name"] @pytest.mark.parametrize("query_param", [{"schedule_id": "a"}, {"status": 123}]) def test_bad_request(self, client, query_param): @@ -146,6 +148,7 @@ def test_get(self, client, task): assert data["_id"] == str(task["_id"]) assert data["status"] == task["status"] assert data["schedule_name"] == task["schedule_name"] + assert data["original_schedule_name"] == task["schedule_name"] assert "timestamp" in data assert "events" in data diff --git a/dispatcher/backend/src/utils/scheduling.py b/dispatcher/backend/src/utils/scheduling.py index 6957dec48..d22a43cb8 100644 --- a/dispatcher/backend/src/utils/scheduling.py +++ b/dispatcher/backend/src/utils/scheduling.py @@ -168,6 +168,7 @@ def request_a_schedule( }, notification=schedule.notification if schedule.notification else {}, updated_at=now, + original_schedule_name=schedule.name, ) requested_task.schedule = schedule diff --git a/dispatcher/frontend-ui/Dockerfile b/dispatcher/frontend-ui/Dockerfile index 26f6692d8..19e1d6c31 100644 --- a/dispatcher/frontend-ui/Dockerfile +++ b/dispatcher/frontend-ui/Dockerfile @@ -20,6 +20,7 @@ COPY nginx-default.conf /etc/nginx/conf.d/default.conf COPY entrypoint.sh /usr/local/bin/entrypoint.sh ENV ZIMFARM_WEBAPI https://api.farm.openzim.org/v1 +ENV ENVIRON_PATH /usr/share/nginx/html/environ.json EXPOSE 80 ENTRYPOINT ["entrypoint.sh"] diff --git a/dispatcher/frontend-ui/entrypoint.sh b/dispatcher/frontend-ui/entrypoint.sh index 9dde45353..281a85447 100755 --- a/dispatcher/frontend-ui/entrypoint.sh +++ b/dispatcher/frontend-ui/entrypoint.sh @@ -1,11 +1,10 @@ #!/bin/sh -JS_PATH=/usr/share/nginx/html/environ.json -echo "dump ZIMFARM_* environ variables to $JS_PATH" +echo "dump ZIMFARM_* environ variables to $ENVIRON_PATH" -python3 -c 'import os; import json; print(json.dumps({k: v for k, v in os.environ.items() if k.startswith("ZIMFARM_")}, indent=4))' > $JS_PATH +python3 -c 'import os; import json; print(json.dumps({k: v for k, v in os.environ.items() if k.startswith("ZIMFARM_")}, indent=4))' > $ENVIRON_PATH -cat $JS_PATH +cat $ENVIRON_PATH echo "-----" exec "$@" diff --git a/dispatcher/frontend-ui/src/components/PipelineTable.vue b/dispatcher/frontend-ui/src/components/PipelineTable.vue index 163f9da76..caecddd3e 100644 --- a/dispatcher/frontend-ui/src/components/PipelineTable.vue +++ b/dispatcher/frontend-ui/src/components/PipelineTable.vue @@ -32,7 +32,8 @@ - {{ task.schedule_name }} + {{ task.original_schedule_name }} + {{ task.schedule_name }} {{ task.timestamp.requested | from_now }} {{ task.requested_by }}