From d406164252a1b09c6a598408fc8f60232639930a Mon Sep 17 00:00:00 2001 From: Tom Koscielniak Date: Mon, 8 Apr 2024 10:18:09 +0200 Subject: [PATCH 01/46] Change from Flask to FastAPI --- src/api.py | 26 +++++++++++----------- src/requirements.txt | 2 ++ tests/test_api.py | 52 ++++++++++++++++++-------------------------- 3 files changed, 36 insertions(+), 44 deletions(-) diff --git a/src/api.py b/src/api.py index 842dd1e..150994a 100644 --- a/src/api.py +++ b/src/api.py @@ -1,26 +1,26 @@ +from fastapi.params import Query from flask import Flask, request from src import service +from fastapi import FastAPI -app = Flask(__name__) +app = FastAPI() # Sample url: https://tmt.org/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke # or for plans: https://tmt.org/?plan-url=https://github.com/teemtee/tmt&plan-name=/plans/features/basic -@app.route("/", methods=["GET"]) -def find_test(): - test_url = request.args.get("test-url", default=None) - test_name = request.args.get("test-name", default=None) - test_ref = request.args.get("test-ref", default="main") +@app.get("/") +def find_test( + test_url: str = Query(None, alias="test-url"), + test_name: str = Query(None, alias="test-name"), + test_ref: str = Query("main", alias="test-ref"), + plan_url: str = Query(None, alias="plan-url"), + plan_name: str = Query(None, alias="plan-name"), + plan_ref: str = Query("main", alias="plan-ref"), + out_format: str = Query("json", alias="format") +): if (test_url is None and test_name is not None) or (test_url is not None and test_name is None): return "Invalid arguments!" - plan_url = request.args.get("plan-url", default=None) - plan_name = request.args.get("plan-name", default=None) - plan_ref = request.args.get("plan-ref", default="main") if (plan_url is None and plan_name is not None) or (plan_url is not None and plan_name is None): return "Invalid arguments!" html_page = service.main(test_url, test_name, test_ref, plan_url, plan_name, plan_ref) return html_page - - -if __name__ == "__main__": - app.run(debug=False) diff --git a/src/requirements.txt b/src/requirements.txt index 7dd0a2b..bb9b458 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -3,3 +3,5 @@ fmf==1.3.0 pytest==8.0.2 requests==2.31.0 tmt==1.31.0 +FastAPI==0.70.0 +httpx==0.20.0 diff --git a/tests/test_api.py b/tests/test_api.py index 2ae382f..37c5cb6 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,27 +1,26 @@ import pytest from src.api import app +from fastapi.testclient import TestClient @pytest.fixture() def client(): - return app.test_client() + return TestClient(app) class TestApi: def test_basic_test_request(self, client): # ?test_url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke - response = client.get("/", query_string={"test-url": "https://github.com/teemtee/tmt", - "test-name": "/tests/core/smoke"}) - data = response.data.decode("utf-8") + response = client.get("/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke") + data = response.content.decode("utf-8") print(data) assert "500" not in data assert "https://github.com/teemtee/tmt/tree/main/tests/core/smoke/main.fmf" in data def test_basic_plan_request(self, client): # ?plan-url=https://github.com/teemtee/tmt&plan-name=/plans/features/basic&type=plan - response = client.get("/", query_string={"plan-url": "https://github.com/teemtee/tmt", - "plan-name": "/plans/features/basic"}) - data = response.data.decode("utf-8") + response = client.get("/?plan-url=https://github.com/teemtee/tmt&plan-name=/plans/features/basic&type=plan") + data = response.content.decode("utf-8") print(data) assert "500" not in data assert "https://github.com/teemtee/tmt/tree/main/plans/features/basic.fmf" in data @@ -29,45 +28,36 @@ def test_basic_plan_request(self, client): def test_basic_testplan_request(self, client): # ?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke& # ?plan-url=https://github.com/teemtee/tmt&plan-name=/plans/features/basic&type=plan - response = client.get("/", query_string={"test-url": "https://github.com/teemtee/tmt", - "test-name": "/tests/core/smoke", - "plan-url": "https://github.com/teemtee/tmt", - "plan-name": "/plans/features/basic"}) - data = response.data.decode("utf-8") + response = client.get("/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke&" + "?plan-url=https://github.com/teemtee/tmt&plan-name=/plans/features/basic&type=plan") + data = response.content.decode("utf-8") print(data) assert "500" not in data assert "https://github.com/teemtee/tmt/tree/main/tests/core/smoke/main.fmf" in data assert "https://github.com/teemtee/tmt/tree/main/plans/features/basic.fmf" in data def test_invalid_test_arguments(self, client): - response = client.get("/", query_string={"test-url": "https://github.com/teemtee/tmt", - "test-name": None}) - data = response.data.decode("utf-8") + response = client.get("/?test-url=https://github.com/teemtee/tmt") + data = response.content.decode("utf-8") assert "Invalid arguments!" in data - response = client.get("/", query_string={"test-url": None, - "test-name": "/tests/core/smoke"}) - data = response.data.decode("utf-8") + response = client.get("/?test-name=/tests/core/smoke") + data = response.content.decode("utf-8") assert "Invalid arguments!" in data def test_invalid_plan_arguments(self, client): - response = client.get("/", query_string={"plan-url": "https://github.com/teemtee/tmt", - "plan-name": None}) - data = response.data.decode("utf-8") + response = client.get("/?plan-url=https://github.com/teemtee/tmt") + data = response.content.decode("utf-8") assert "Invalid arguments!" in data - response = client.get("/", query_string={"plan-url": None, - "plan-name": "/plans/features/basic"}) - data = response.data.decode("utf-8") + response = client.get("/?plan-name=/plans/features/basic") + data = response.content.decode("utf-8") assert "Invalid arguments!" in data def test_invalid_testplan_arguments(self, client): - response = client.get("/", query_string={"test-url": "https://github.com/teemtee/tmt", - "test-name": None, - "plan-url": "https://github.com/teemtee/tmt", - "plan-name": "/plans/features/basic"}) - data = response.data.decode("utf-8") + response = client.get("/?test-url=https://github.com/teemtee/tmt&plan-url=https://github.com/teemtee/tmt&" + "plan-name=/plans/features/basic") + data = response.content.decode("utf-8") assert "Invalid arguments!" in data def test_invalid_argument_names(self, client): - response = client.get("/", query_string={"test_urlurl": "https://github.com/teemtee/tmt", - "test_nn": "/tests/core/smoke"}) + response = client.get("/?test_urlur=https://github.com/teemtee/tmt&test_nn=/tests/core/smoke") assert response.status_code == 500 From 5ba9815bcd251d2099ab195889eddb79c58765b8 Mon Sep 17 00:00:00 2001 From: Tom Koscielniak Date: Thu, 18 Apr 2024 09:33:50 +0200 Subject: [PATCH 02/46] Switch to FastApi+Celery and add containerization --- Dockerfile | 10 ++++++++ compose.yaml | 22 ++++++++++++++++++ src/Dockerfile | 7 ++++++ src/api.py | 42 ++++++++++++++++++++++++++-------- src/requirements.txt | 3 +++ src/service.py | 20 +++++++++++----- src/start_api.sh | 3 --- src/utils/git_handler.py | 26 ++++++++++++++++++--- start_api.sh | 5 ++++ tests/test_api.py | 25 +++++++++++++++++++- tests/unit/test_git_handler.py | 6 ++--- 11 files changed, 143 insertions(+), 26 deletions(-) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 src/Dockerfile delete mode 100644 src/start_api.sh create mode 100644 start_api.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..65d294c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.10 + +RUN mkdir /app +WORKDIR /app + +RUN pip install celery + +COPY ./src /app/src +RUN pip install -r /app/src/requirements.txt +CMD celery --app=src.api.service worker --concurrency=1 --loglevel=INFO \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..e083ed4 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,22 @@ +services: + web: + container_name: fastapi + build: ./src + environment: + - REDIS_URL=redis://redis:6379 + ports: + - 8001:5001 + redis: + container_name: redis + ports: + - 6379:6379 + image: redis:latest + + celery: + container_name: celery + build: . + environment: + - REDIS_URL=redis://redis:6379 + depends_on: + - redis + diff --git a/src/Dockerfile b/src/Dockerfile new file mode 100644 index 0000000..7067435 --- /dev/null +++ b/src/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.10 + +RUN mkdir /app +WORKDIR /app +COPY ./ /app/src +RUN pip install -r /app/src/requirements.txt +CMD uvicorn src.api:app --reload \ No newline at end of file diff --git a/src/api.py b/src/api.py index 150994a..dc9f883 100644 --- a/src/api.py +++ b/src/api.py @@ -1,26 +1,48 @@ from fastapi.params import Query -from flask import Flask, request from src import service from fastapi import FastAPI +from pydantic import BaseModel +from celery.result import AsyncResult app = FastAPI() +class TaskOut(BaseModel): + id: str + status: str + result: str | None = None + + # Sample url: https://tmt.org/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke # or for plans: https://tmt.org/?plan-url=https://github.com/teemtee/tmt&plan-name=/plans/features/basic @app.get("/") def find_test( - test_url: str = Query(None, alias="test-url"), - test_name: str = Query(None, alias="test-name"), - test_ref: str = Query("main", alias="test-ref"), - plan_url: str = Query(None, alias="plan-url"), - plan_name: str = Query(None, alias="plan-name"), - plan_ref: str = Query("main", alias="plan-ref"), - out_format: str = Query("json", alias="format") + test_url: str = Query(None, alias="test-url"), + test_name: str = Query(None, alias="test-name"), + test_ref: str = Query("default", alias="test-ref"), + plan_url: str = Query(None, alias="plan-url"), + plan_name: str = Query(None, alias="plan-name"), + plan_ref: str = Query("default", alias="plan-ref"), + out_format: str = Query("json", alias="format") ): if (test_url is None and test_name is not None) or (test_url is not None and test_name is None): return "Invalid arguments!" if (plan_url is None and plan_name is not None) or (plan_url is not None and plan_name is None): return "Invalid arguments!" - html_page = service.main(test_url, test_name, test_ref, plan_url, plan_name, plan_ref) - return html_page + # html_page = service.main(test_url, test_name, test_ref, plan_url, plan_name, plan_ref, out_format) + r = service.main.delay(test_url, test_name, test_ref, plan_url, plan_name, plan_ref, out_format) + return _to_task_out(r) + + +@app.get("/status") +def status(task_id: str) -> TaskOut: + r = service.main.app.AsyncResult(task_id) + return _to_task_out(r) + + +def _to_task_out(r: AsyncResult) -> TaskOut: + return TaskOut( + id=r.task_id, + status=r.status, + result=r.traceback if r.failed() else r.result, + ) diff --git a/src/requirements.txt b/src/requirements.txt index bb9b458..057076e 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -5,3 +5,6 @@ requests==2.31.0 tmt==1.31.0 FastAPI==0.70.0 httpx==0.20.0 +Redis +uvicorn +celery diff --git a/src/service.py b/src/service.py index e5368b4..e0bb9f8 100644 --- a/src/service.py +++ b/src/service.py @@ -1,12 +1,17 @@ -import sys +import os import tmt import logging from pathlib import Path from src import html_generator as html from src.utils import git_handler as utils +from celery.app import Celery logger = tmt.Logger(logging.Logger("tmt-logger")) +redis_url = os.getenv("REDIS_URL", "redis://localhost:6379") + +app = Celery(__name__, broker=redis_url, backend=redis_url) + def process_test_request(test_url: str, test_name: str, test_ref: str, return_html: bool) -> str | None | tmt.Test: """ @@ -22,17 +27,18 @@ def process_test_request(test_url: str, test_name: str, test_ref: str, return_ht logger.print("URL: " + test_url) logger.print("Name: " + test_name) - utils.get_git_repository(test_url, logger) + utils.get_git_repository(test_url, logger, test_ref) repo_name = test_url.rsplit('/', 1)[-1] logger.print("Looking for tree...") - tree = tmt.base.Tree(path=Path("../.tmp/" + repo_name), logger=logger) + tree = tmt.base.Tree(path=Path("./.tmp/" + repo_name), logger=logger) logger.print("Tree found!", color="green") - logger.print("Looking for the wanted test...") + logger.print("Initializing the tree...") test_list = tree.tests() wanted_test = None # Find the desired Test object + logger.print("Looking for the wanted test...") for test in test_list: if test.name == test_name: wanted_test = test @@ -60,7 +66,7 @@ def process_plan_request(plan_url: str, plan_name: str, plan_ref: str, return_ht logger.print("URL: " + plan_url) logger.print("Name: " + plan_name) - utils.get_git_repository(plan_url, logger) + utils.get_git_repository(plan_url, logger, plan_ref) repo_name = plan_url.rsplit('/', 1)[-1] logger.print("Looking for tree...") @@ -100,12 +106,14 @@ def process_testplan_request(test_url, test_name, test_ref, plan_url, plan_name, return html.generate_testplan_html_page(test, plan, logger=logger) +@app.task def main(test_url: str | None, test_name: str | None, test_ref: str | None, plan_url: str | None, plan_name: str | None, - plan_ref: str | None) -> str | None: + plan_ref: str | None, + out_format: str | None) -> str | None: logger.print("Starting...", color="blue") if test_name is not None and plan_name is None: return process_test_request(test_url, test_name, test_ref, True) diff --git a/src/start_api.sh b/src/start_api.sh deleted file mode 100644 index 6a0506e..0000000 --- a/src/start_api.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -source ../venv/bin/activate -flask --app api run diff --git a/src/utils/git_handler.py b/src/utils/git_handler.py index ba20c39..d5b47c5 100644 --- a/src/utils/git_handler.py +++ b/src/utils/git_handler.py @@ -6,10 +6,28 @@ from pathlib import Path -def clone_repository(url: str, logger: Logger) -> None: +def checkout_branch(path: Path, logger: Logger, ref: str) -> None: + """ + Checks out the given branch in the repository. + :param ref: Name of the ref to check out + :param path: Path to the repository + :param logger: Instance of Logger + :return: + """ + try: + common_instance = tmt.utils.Common(logger=logger) + common_instance.run( + command=tmt.utils.Command('git', 'checkout', ref), cwd=path) + except tmt.utils.RunError: + logger.print("Failed to clone the repository!", color="red") + raise Exception + + +def clone_repository(url: str, logger: Logger, ref: str) -> None: """ Clones the repository from the given URL. Raises FileExistsError if the repository is already cloned and raises Exception if the cloning fails. + :param ref: Name of the ref to check out :param url: URL to the repository :param logger: Instance of Logger :return: @@ -21,6 +39,8 @@ def clone_repository(url: str, logger: Logger) -> None: raise FileExistsError try: tmt.utils.git_clone(url=url, shallow=True, destination=path, logger=logger) + if ref != "default": + checkout_branch(ref, path, logger) except tmt.utils.GeneralError as e: logger.print("Failed to clone the repository!", color="red") raise Exception @@ -68,7 +88,7 @@ def clear_tmp_dir(logger: Logger) -> None: logger.print(".tmp directory cleared successfully!", color="green") -def get_git_repository(url: str, logger: Logger) -> Path: +def get_git_repository(url: str, logger: Logger, ref: str) -> Path: """ Clones the repository from the given URL and returns the path to the cloned repository. :param url: URL to the repository @@ -76,7 +96,7 @@ def get_git_repository(url: str, logger: Logger) -> Path: :return: Path to the cloned repository """ try: - clone_repository(url, logger) + clone_repository(url, logger, ref) except FileExistsError: pass return get_path_to_repository(url) diff --git a/start_api.sh b/start_api.sh new file mode 100644 index 0000000..acbf5f8 --- /dev/null +++ b/start_api.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# For local dev purposes +docker run --rm --name some-redis -p 6379:6379 redis:latest & +celery --app=src.api.service worker --concurrency=1 --loglevel=INFO & +uvicorn src.api:app --reload && fg diff --git a/tests/test_api.py b/tests/test_api.py index 37c5cb6..7d9c853 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,3 +1,5 @@ +import time + import pytest from src.api import app from fastapi.testclient import TestClient @@ -54,10 +56,31 @@ def test_invalid_plan_arguments(self, client): def test_invalid_testplan_arguments(self, client): response = client.get("/?test-url=https://github.com/teemtee/tmt&plan-url=https://github.com/teemtee/tmt&" - "plan-name=/plans/features/basic") + "plan-name=/plans/features/basic") data = response.content.decode("utf-8") assert "Invalid arguments!" in data def test_invalid_argument_names(self, client): response = client.get("/?test_urlur=https://github.com/teemtee/tmt&test_nn=/tests/core/smoke") assert response.status_code == 500 + + +class TestCelery: + def test_basic_test_request(self, client): + response = client.get("/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke") + data = response.content.decode("utf-8") + json_data = response.json() + while True: + if json_data["status"] == "PENDING": + response = client.get("/status?task_id=" + json_data["id"]) + json_data = response.json() + time.sleep(0.1) + elif json_data["status"] == "SUCCESS": + result = json_data["result"] + assert "500" not in result + assert "https://github.com/teemtee/tmt/tree/main/tests/core/smoke/main.fmf" in result + break + elif json_data["status"] == "FAILURE": + assert False + else: + assert False \ No newline at end of file diff --git a/tests/unit/test_git_handler.py b/tests/unit/test_git_handler.py index ec96a8e..35baa19 100644 --- a/tests/unit/test_git_handler.py +++ b/tests/unit/test_git_handler.py @@ -7,7 +7,7 @@ from src.utils import git_handler -class TestUtils: +class TestGitHandler: logger = tmt.Logger(logging.Logger("tmt-logger")) def test_clear_tmp_dir(self): @@ -24,7 +24,7 @@ def test_clone_repository(self): while git_handler.check_if_repository_exists("https://github.com/teemtee/tmt") is True: git_handler.clear_tmp_dir(self.logger) time.sleep(1) - git_handler.clone_repository("https://github.com/teemtee/tmt", logger=self.logger) + git_handler.clone_repository(url="https://github.com/teemtee/tmt", logger=self.logger, ref="default") with pytest.raises(FileExistsError): - git_handler.clone_repository("https://github.com/teemtee/tmt", logger=self.logger) + git_handler.clone_repository(url="https://github.com/teemtee/tmt", logger=self.logger, ref="default") From 8d324f9b201f37646aee0d89543ddea3ab2b9025 Mon Sep 17 00:00:00 2001 From: Tom Koscielniak Date: Mon, 22 Apr 2024 11:08:43 +0200 Subject: [PATCH 03/46] Add service dockerfile and compose --- Dockerfile | 4 ---- compose.yaml | 14 ++++++++++---- src/Dockerfile | 7 ------- 3 files changed, 10 insertions(+), 15 deletions(-) delete mode 100644 src/Dockerfile diff --git a/Dockerfile b/Dockerfile index 65d294c..8ac51c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,9 +2,5 @@ FROM python:3.10 RUN mkdir /app WORKDIR /app - -RUN pip install celery - COPY ./src /app/src RUN pip install -r /app/src/requirements.txt -CMD celery --app=src.api.service worker --concurrency=1 --loglevel=INFO \ No newline at end of file diff --git a/compose.yaml b/compose.yaml index e083ed4..b35eb87 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,22 +1,28 @@ services: web: - container_name: fastapi - build: ./src + container_name: uvicorn + build: . + command: uvicorn src.api:app --reload --host 0.0.0.0 --port 8000 environment: - REDIS_URL=redis://redis:6379 + - TMP_DIR_PATH=./.tmp/ + - HOSTNAME=http://localhost:8000 ports: - - 8001:5001 + - 8000:8000 redis: container_name: redis + image: redis:latest ports: - 6379:6379 - image: redis:latest celery: container_name: celery build: . + command: celery --app=src.api.service worker --concurrency=1 --loglevel=INFO environment: - REDIS_URL=redis://redis:6379 + - TMP_DIR_PATH=./.tmp/ + - HOSTNAME=http://localhost:8000 depends_on: - redis diff --git a/src/Dockerfile b/src/Dockerfile deleted file mode 100644 index 7067435..0000000 --- a/src/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM python:3.10 - -RUN mkdir /app -WORKDIR /app -COPY ./ /app/src -RUN pip install -r /app/src/requirements.txt -CMD uvicorn src.api:app --reload \ No newline at end of file From dd031394eead14dbba85c256cd0007d4dfac1f9f Mon Sep 17 00:00:00 2001 From: Tom Koscielniak Date: Mon, 22 Apr 2024 11:11:26 +0200 Subject: [PATCH 04/46] Refactor api tests and add os variables --- src/api.py | 21 ++++++++++++++++++-- src/html_generator.py | 29 ++++++++++++++++++++++++++++ src/service.py | 45 ++++++++++++++++++++++++------------------- tests/test_api.py | 13 +++++++++++-- 4 files changed, 84 insertions(+), 24 deletions(-) diff --git a/src/api.py b/src/api.py index dc9f883..6f98acf 100644 --- a/src/api.py +++ b/src/api.py @@ -1,16 +1,21 @@ +import os + from fastapi.params import Query from src import service +from src import html_generator from fastapi import FastAPI from pydantic import BaseModel from celery.result import AsyncResult app = FastAPI() +format_html = False class TaskOut(BaseModel): id: str status: str result: str | None = None + status_callback_url: str | None = None # Sample url: https://tmt.org/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke @@ -29,7 +34,15 @@ def find_test( return "Invalid arguments!" if (plan_url is None and plan_name is not None) or (plan_url is not None and plan_name is None): return "Invalid arguments!" - # html_page = service.main(test_url, test_name, test_ref, plan_url, plan_name, plan_ref, out_format) + if os.environ.get("USE_CELERY") == "false": + html_page = service.main(test_url, test_name, test_ref, plan_url, plan_name, plan_ref, out_format) + return html_page + if out_format == "html": + global format_html + format_html = True + else: + # To set it back to False after a html format request + format_html = False r = service.main.delay(test_url, test_name, test_ref, plan_url, plan_name, plan_ref, out_format) return _to_task_out(r) @@ -40,9 +53,13 @@ def status(task_id: str) -> TaskOut: return _to_task_out(r) -def _to_task_out(r: AsyncResult) -> TaskOut: +def _to_task_out(r: AsyncResult) -> TaskOut | str: + if format_html: + status_callback_url = f'{os.getenv("HOSTNAME")}/status?task_id={r.task_id}' + return html_generator.generate_status_callback(r, status_callback_url) return TaskOut( id=r.task_id, status=r.status, result=r.traceback if r.failed() else r.result, + status_callback_url=f'{os.getenv("HOSTNAME")}/status?task_id={r.task_id}' ) diff --git a/src/html_generator.py b/src/html_generator.py index 3989ce3..d49cdc6 100644 --- a/src/html_generator.py +++ b/src/html_generator.py @@ -1,7 +1,36 @@ import tmt +from celery.result import AsyncResult from tmt import Test, Logger, Plan +def generate_status_callback(r: AsyncResult, status_callback_url: str) -> str: + """ + This function generates the status callback for the HTML file + :param r: AsyncResult object + :param status_callback_url: URL for the status callback + :return: + """ + if r.status == "PENDING": + return (f''' + + HTML File + + + + Processing... Try this url again in a few seconds: {status_callback_url} + ''') + else: + return (f''' + + HTML File + + + + Status: {r.status}
+ The result is:
{r.result} + ''') + + def generate_test_html_page(test: Test, logger: Logger) -> str: """ This function generates an HTML file with the input data for a test diff --git a/src/service.py b/src/service.py index e0bb9f8..97a5237 100644 --- a/src/service.py +++ b/src/service.py @@ -13,6 +13,28 @@ app = Celery(__name__, broker=redis_url, backend=redis_url) +def get_tree(url: str, name: str, ref: str) -> tmt.base.Tree: + """ + This function clones the repository and returns the Tree object + :param ref: Object ref + :param name: Object name + :param url: Object url + :return: + """ + logger.print("Cloning the repository for url: " + url) + logger.print("Parsing the url and name...") + logger.print("URL: " + url) + logger.print("Name: " + name) + + utils.get_git_repository(url, logger, ref) + + repo_name = url.rsplit('/', 1)[-1] + logger.print("Looking for tree...") + tree = tmt.base.Tree(path=Path(os.getenv("TMP_DIR_PATH") + repo_name), logger=logger) + logger.print("Tree found!", color="green") + return tree + + def process_test_request(test_url: str, test_name: str, test_ref: str, return_html: bool) -> str | None | tmt.Test: """ This function processes the request for a test and returns the HTML file or the Test object @@ -22,23 +44,14 @@ def process_test_request(test_url: str, test_name: str, test_ref: str, return_ht :param return_html: Specify if the function should return the HTML file or the Test object :return: """ - logger.print("Cloning the repository for url: " + test_url) - logger.print("Parsing the url and name...") - logger.print("URL: " + test_url) - logger.print("Name: " + test_name) - utils.get_git_repository(test_url, logger, test_ref) + tree = get_tree(test_url, test_name, test_ref) - repo_name = test_url.rsplit('/', 1)[-1] - logger.print("Looking for tree...") - tree = tmt.base.Tree(path=Path("./.tmp/" + repo_name), logger=logger) - logger.print("Tree found!", color="green") - logger.print("Initializing the tree...") + logger.print("Looking for the wanted test...") test_list = tree.tests() wanted_test = None # Find the desired Test object - logger.print("Looking for the wanted test...") for test in test_list: if test.name == test_name: wanted_test = test @@ -61,17 +74,9 @@ def process_plan_request(plan_url: str, plan_name: str, plan_ref: str, return_ht :param return_html: Specify if the function should return the HTML file or the Plan object :return: """ - logger.print("Cloning the repository for url: " + plan_url) - logger.print("Parsing the url and name...") - logger.print("URL: " + plan_url) - logger.print("Name: " + plan_name) - utils.get_git_repository(plan_url, logger, plan_ref) + tree = get_tree(plan_url, plan_name, plan_ref) - repo_name = plan_url.rsplit('/', 1)[-1] - logger.print("Looking for tree...") - tree = tmt.base.Tree(path=Path("../.tmp/" + repo_name), logger=logger) - logger.print("Tree found!", color="green") logger.print("Looking for the wanted plan...") plan_list = tree.plans() diff --git a/tests/test_api.py b/tests/test_api.py index 7d9c853..07696e1 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,3 +1,4 @@ +import os import time import pytest @@ -11,6 +12,11 @@ def client(): class TestApi: + @pytest.fixture(autouse=True) + def setup(self): + os.environ["USE_CELERY"] = "false" + os.environ["TMP_DIR_PATH"] = "../.tmp/" + def test_basic_test_request(self, client): # ?test_url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke response = client.get("/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke") @@ -66,9 +72,12 @@ def test_invalid_argument_names(self, client): class TestCelery: + @pytest.fixture(autouse=True) + def setup(self): + os.environ["USE_CELERY"] = "true" + def test_basic_test_request(self, client): response = client.get("/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke") - data = response.content.decode("utf-8") json_data = response.json() while True: if json_data["status"] == "PENDING": @@ -83,4 +92,4 @@ def test_basic_test_request(self, client): elif json_data["status"] == "FAILURE": assert False else: - assert False \ No newline at end of file + assert False From 41af0f300509f2528b78faf4421272cbb166c835 Mon Sep 17 00:00:00 2001 From: Tom Koscielniak Date: Mon, 22 Apr 2024 12:04:05 +0200 Subject: [PATCH 05/46] Add entrypoint and fix path in git pull location --- Dockerfile | 8 ++++++++ entrypoint.sh | 25 +++++++++++++++++++++++++ src/service.py | 4 ++-- 3 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 8ac51c6..a7c7b70 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,3 +4,11 @@ RUN mkdir /app WORKDIR /app COPY ./src /app/src RUN pip install -r /app/src/requirements.txt + +COPY /entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + + + +ENTRYPOINT ["/entrypoint.sh"] + diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..0078d54 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +# Name of container to start +APP=$1 + +[ -z "$APP" ] && { error "No api to run passed to entrypoint script"; exit 1; } + +case $APP in + uvicorn) + COMMAND="uvicorn src.api:app --reload --host 0.0.0.0 --port 8000" + ;; + celery) + COMMAND="celery --app=src.api.service worker --concurrency=1 --loglevel=INFO" + ;; + *) + echo "Unknown app '$APP'" + exit 1 + ;; +esac + +$COMMAND & +PID=$! + +wait $PID + diff --git a/src/service.py b/src/service.py index 97a5237..2804d19 100644 --- a/src/service.py +++ b/src/service.py @@ -30,7 +30,7 @@ def get_tree(url: str, name: str, ref: str) -> tmt.base.Tree: repo_name = url.rsplit('/', 1)[-1] logger.print("Looking for tree...") - tree = tmt.base.Tree(path=Path(os.getenv("TMP_DIR_PATH") + repo_name), logger=logger) + tree = tmt.base.Tree(path=Path(os.getenv("TMP_DIR_PATH") + "/.tmp/" + repo_name), logger=logger) logger.print("Tree found!", color="green") return tree @@ -81,7 +81,7 @@ def process_plan_request(plan_url: str, plan_name: str, plan_ref: str, return_ht plan_list = tree.plans() wanted_plan = None - # Find the desired Test object + # Find the desired Plan object for plan in plan_list: if plan.name == plan_name: wanted_plan = plan From b52618f28eccae7e1a75db2a811dd1ec8edaa059 Mon Sep 17 00:00:00 2001 From: Tom Koscielniak Date: Mon, 22 Apr 2024 12:43:37 +0200 Subject: [PATCH 06/46] Add envvar check in entrypoint and add html formatted response --- Dockerfile | 2 -- compose.yaml | 4 ++-- entrypoint.sh | 5 +++++ src/api.py | 16 ++++++++++------ src/html_generator.py | 2 +- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index a7c7b70..97d4e5b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,5 @@ RUN pip install -r /app/src/requirements.txt COPY /entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh - - ENTRYPOINT ["/entrypoint.sh"] diff --git a/compose.yaml b/compose.yaml index b35eb87..038cb3a 100644 --- a/compose.yaml +++ b/compose.yaml @@ -5,7 +5,7 @@ services: command: uvicorn src.api:app --reload --host 0.0.0.0 --port 8000 environment: - REDIS_URL=redis://redis:6379 - - TMP_DIR_PATH=./.tmp/ + - TMP_DIR_PATH=./ - HOSTNAME=http://localhost:8000 ports: - 8000:8000 @@ -21,7 +21,7 @@ services: command: celery --app=src.api.service worker --concurrency=1 --loglevel=INFO environment: - REDIS_URL=redis://redis:6379 - - TMP_DIR_PATH=./.tmp/ + - TMP_DIR_PATH=./ - HOSTNAME=http://localhost:8000 depends_on: - redis diff --git a/entrypoint.sh b/entrypoint.sh index 0078d54..7a47acb 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,6 +5,11 @@ APP=$1 [ -z "$APP" ] && { error "No api to run passed to entrypoint script"; exit 1; } +if [ -z "$REDIS_URL" ] || [ -z "$TMP_DIR_PATH" ] || [ -z "$HOSTNAME" ]; then + error "Missing one or more required environment variables" + exit 1 +fi + case $APP in uvicorn) COMMAND="uvicorn src.api:app --reload --host 0.0.0.0 --port 8000" diff --git a/src/api.py b/src/api.py index 6f98acf..bc9c58b 100644 --- a/src/api.py +++ b/src/api.py @@ -1,6 +1,8 @@ import os from fastapi.params import Query +from starlette.responses import HTMLResponse + from src import service from src import html_generator from fastapi import FastAPI @@ -37,26 +39,28 @@ def find_test( if os.environ.get("USE_CELERY") == "false": html_page = service.main(test_url, test_name, test_ref, plan_url, plan_name, plan_ref, out_format) return html_page + r = service.main.delay(test_url, test_name, test_ref, plan_url, plan_name, plan_ref, out_format) if out_format == "html": global format_html format_html = True + status_callback_url = f'{os.getenv("HOSTNAME")}/status?task_id={r.task_id}' + return HTMLResponse(content=html_generator.generate_status_callback(r, status_callback_url)) else: # To set it back to False after a html format request format_html = False - r = service.main.delay(test_url, test_name, test_ref, plan_url, plan_name, plan_ref, out_format) - return _to_task_out(r) + return _to_task_out(r) @app.get("/status") -def status(task_id: str) -> TaskOut: +def status(task_id: str) -> TaskOut | HTMLResponse: r = service.main.app.AsyncResult(task_id) + if format_html: + status_callback_url = f'{os.getenv("HOSTNAME")}/status?task_id={r.task_id}' + return HTMLResponse(content=html_generator.generate_status_callback(r, status_callback_url)) return _to_task_out(r) def _to_task_out(r: AsyncResult) -> TaskOut | str: - if format_html: - status_callback_url = f'{os.getenv("HOSTNAME")}/status?task_id={r.task_id}' - return html_generator.generate_status_callback(r, status_callback_url) return TaskOut( id=r.task_id, status=r.status, diff --git a/src/html_generator.py b/src/html_generator.py index d49cdc6..8ec54a4 100644 --- a/src/html_generator.py +++ b/src/html_generator.py @@ -17,7 +17,7 @@ def generate_status_callback(r: AsyncResult, status_callback_url: str) -> str: - Processing... Try this url again in a few seconds: {status_callback_url} + Processing... Try this url again in a few seconds: {status_callback_url} ''') else: return (f''' From 0ff329d8840c0cd74ac73cc396098aed9f5d892f Mon Sep 17 00:00:00 2001 From: Tom Koscielniak Date: Mon, 22 Apr 2024 22:04:17 +0200 Subject: [PATCH 07/46] Fix git clone paths and add html response --- entrypoint.sh | 5 ----- src/api.py | 4 ++-- src/html_generator.py | 2 +- src/service.py | 5 ++--- src/utils/git_handler.py | 7 +++++-- 5 files changed, 10 insertions(+), 13 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 7a47acb..0078d54 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,11 +5,6 @@ APP=$1 [ -z "$APP" ] && { error "No api to run passed to entrypoint script"; exit 1; } -if [ -z "$REDIS_URL" ] || [ -z "$TMP_DIR_PATH" ] || [ -z "$HOSTNAME" ]; then - error "Missing one or more required environment variables" - exit 1 -fi - case $APP in uvicorn) COMMAND="uvicorn src.api:app --reload --host 0.0.0.0 --port 8000" diff --git a/src/api.py b/src/api.py index bc9c58b..f656d4c 100644 --- a/src/api.py +++ b/src/api.py @@ -43,7 +43,7 @@ def find_test( if out_format == "html": global format_html format_html = True - status_callback_url = f'{os.getenv("HOSTNAME")}/status?task_id={r.task_id}' + status_callback_url = f'/status?task_id={r.task_id}' return HTMLResponse(content=html_generator.generate_status_callback(r, status_callback_url)) else: # To set it back to False after a html format request @@ -55,7 +55,7 @@ def find_test( def status(task_id: str) -> TaskOut | HTMLResponse: r = service.main.app.AsyncResult(task_id) if format_html: - status_callback_url = f'{os.getenv("HOSTNAME")}/status?task_id={r.task_id}' + status_callback_url = f'/status?task_id={r.task_id}' return HTMLResponse(content=html_generator.generate_status_callback(r, status_callback_url)) return _to_task_out(r) diff --git a/src/html_generator.py b/src/html_generator.py index 8ec54a4..8d3be36 100644 --- a/src/html_generator.py +++ b/src/html_generator.py @@ -17,7 +17,7 @@ def generate_status_callback(r: AsyncResult, status_callback_url: str) -> str: - Processing... Try this url again in a few seconds: {status_callback_url} + Processing... Try this clicking this url in a few seconds: {r.task_id} ''') else: return (f''' diff --git a/src/service.py b/src/service.py index 2804d19..4eea76e 100644 --- a/src/service.py +++ b/src/service.py @@ -26,11 +26,10 @@ def get_tree(url: str, name: str, ref: str) -> tmt.base.Tree: logger.print("URL: " + url) logger.print("Name: " + name) - utils.get_git_repository(url, logger, ref) + path = utils.get_git_repository(url, logger, ref) - repo_name = url.rsplit('/', 1)[-1] logger.print("Looking for tree...") - tree = tmt.base.Tree(path=Path(os.getenv("TMP_DIR_PATH") + "/.tmp/" + repo_name), logger=logger) + tree = tmt.base.Tree(path=path, logger=logger) logger.print("Tree found!", color="green") return tree diff --git a/src/utils/git_handler.py b/src/utils/git_handler.py index d5b47c5..56369a5 100644 --- a/src/utils/git_handler.py +++ b/src/utils/git_handler.py @@ -35,12 +35,14 @@ def clone_repository(url: str, logger: Logger, ref: str) -> None: logger.print("Cloning the repository...") path = get_path_to_repository(url) if check_if_repository_exists(url): + if ref != "default": + checkout_branch(ref=ref, path=path, logger=logger) logger.print("Repository already cloned!", color="yellow") raise FileExistsError try: tmt.utils.git_clone(url=url, shallow=True, destination=path, logger=logger) if ref != "default": - checkout_branch(ref, path, logger) + checkout_branch(ref=ref, path=path, logger=logger) except tmt.utils.GeneralError as e: logger.print("Failed to clone the repository!", color="red") raise Exception @@ -56,7 +58,7 @@ def get_path_to_repository(url: str) -> Path: repo_name = url.rsplit('/', 1)[-1] path = os.path.realpath(__file__) path = path.replace("src/utils/git_handler.py", "") - path = Path(path + "/.tmp/" + repo_name) + path = Path(path + os.getenv("CLONE_DIR_PATH", "./.repos/") + repo_name) return path @@ -80,6 +82,7 @@ def clear_tmp_dir(logger: Logger) -> None: path = os.path.realpath(__file__) path = path.replace("src/utils/git_handler.py", "") path = Path(path + "/.tmp") + # repo_name = .rsplit('/', 1)[-1] try: Popen(["rm", "-rf", path]) except Exception as e: From e249b27218c7afc6e60a44172e1ea22196dc49b0 Mon Sep 17 00:00:00 2001 From: Tom Koscielniak Date: Tue, 23 Apr 2024 00:15:52 +0200 Subject: [PATCH 08/46] Add json and yaml output and fix checkout and callback --- src/api.py | 17 +++--- src/generators/__init__.py | 0 src/{ => generators}/html_generator.py | 0 src/generators/json_generator.py | 80 ++++++++++++++++++++++++++ src/generators/yaml_generator.py | 79 +++++++++++++++++++++++++ src/service.py | 55 ++++++++++++------ src/utils/git_handler.py | 22 ++++--- tests/test_api.py | 23 ++++++-- tests/unit/test_git_handler.py | 40 ++++++++++++- 9 files changed, 278 insertions(+), 38 deletions(-) create mode 100644 src/generators/__init__.py rename src/{ => generators}/html_generator.py (100%) create mode 100644 src/generators/json_generator.py create mode 100644 src/generators/yaml_generator.py diff --git a/src/api.py b/src/api.py index f656d4c..78cd44f 100644 --- a/src/api.py +++ b/src/api.py @@ -4,7 +4,7 @@ from starlette.responses import HTMLResponse from src import service -from src import html_generator +from src.generators import html_generator from fastapi import FastAPI from pydantic import BaseModel from celery.result import AsyncResult @@ -40,22 +40,23 @@ def find_test( html_page = service.main(test_url, test_name, test_ref, plan_url, plan_name, plan_ref, out_format) return html_page r = service.main.delay(test_url, test_name, test_ref, plan_url, plan_name, plan_ref, out_format) + # Special handling of response if the format is html if out_format == "html": global format_html format_html = True - status_callback_url = f'/status?task_id={r.task_id}' + status_callback_url = f'/status?task-id={r.task_id}&html=true' return HTMLResponse(content=html_generator.generate_status_callback(r, status_callback_url)) else: - # To set it back to False after a html format request - format_html = False + format_html = False # To set it back to False after a html format request return _to_task_out(r) @app.get("/status") -def status(task_id: str) -> TaskOut | HTMLResponse: +def status(task_id: str = Query(None, alias="task-id"), + html: str = Query("false")) -> TaskOut | HTMLResponse: r = service.main.app.AsyncResult(task_id) - if format_html: - status_callback_url = f'/status?task_id={r.task_id}' + if html == "true": + status_callback_url = f'/status?task-id={r.task_id}&html=true' return HTMLResponse(content=html_generator.generate_status_callback(r, status_callback_url)) return _to_task_out(r) @@ -65,5 +66,5 @@ def _to_task_out(r: AsyncResult) -> TaskOut | str: id=r.task_id, status=r.status, result=r.traceback if r.failed() else r.result, - status_callback_url=f'{os.getenv("HOSTNAME")}/status?task_id={r.task_id}' + status_callback_url=f'{os.getenv("HOSTNAME")}/status?task-id={r.task_id}' ) diff --git a/src/generators/__init__.py b/src/generators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/html_generator.py b/src/generators/html_generator.py similarity index 100% rename from src/html_generator.py rename to src/generators/html_generator.py diff --git a/src/generators/json_generator.py b/src/generators/json_generator.py new file mode 100644 index 0000000..7ad034d --- /dev/null +++ b/src/generators/json_generator.py @@ -0,0 +1,80 @@ +import json + +import tmt +from tmt import Test, Logger, Plan + + +def generate_test_json(test: Test, logger: Logger) -> str: + """ + This function generates an JSON file with the input data for a test + :param test: Test object + :param logger: tmt.Logger instance + :return: + """ + logger.print("Generating the JSON file...") + full_url = test.web_link() + data = { + "name": test.name, + "summary": test.summary, + "description": test.description, + "url": full_url, + "ref": test.fmf_id.ref, + "contact": test.contact + } + data = json.dumps(data) + logger.print("JSON file generated successfully!", color="green") + return data + + +def generate_plan_json(plan: Plan, logger: Logger) -> str: + """ + This function generates an JSON file with the input data for a plan + :param plan: Plan object + :param logger: tmt.Logger instance + :return: + """ + logger.print("Generating the JSON file...") + full_url = plan.web_link() + data = { + "name": plan.name, + "summary": plan.summary, + "description": plan.description, + "url": full_url, + "ref": plan.fmf_id.ref + } + data = json.dumps(data) + logger.print("JSON file generated successfully!", color="green") + return data + + +def generate_testplan_json(test: tmt.Test, plan: tmt.Plan, logger: Logger) -> str: + """ + This function generates an JSON file with the input data for a test and a plan + :param test: Test object + :param plan: Plan object + :param logger: tmt.Logger instance + :return: + """ + logger.print("Generating the JSON file...") + full_url_test = test.web_link() + full_url_plan = plan.web_link() + data = { + "test": { + "name": test.name, + "summary": test.summary, + "description": test.description, + "url": full_url_test, + "ref": test.fmf_id.ref, + "contact": test.contact + }, + "plan": { + "name": plan.name, + "summary": plan.summary, + "description": plan.description, + "url": full_url_plan, + "ref": plan.fmf_id.ref + } + } + data = json.dumps(data) + logger.print("JSON file generated successfully!", color="green") + return data diff --git a/src/generators/yaml_generator.py b/src/generators/yaml_generator.py new file mode 100644 index 0000000..af45ae3 --- /dev/null +++ b/src/generators/yaml_generator.py @@ -0,0 +1,79 @@ +import tmt +from tmt import Logger +from tmt import utils + + +def generate_test_yaml(test: tmt.Test, logger: Logger) -> str: + """ + This function generates an YAML file with the input data for a test + :param test: Test object + :param logger: tmt.Logger instance + :return: + """ + logger.print("Generating the YAML file...") + full_url = test.web_link() + data = { + "name": test.name, + "summary": test.summary, + "description": test.description, + "url": full_url, + "ref": test.fmf_id.ref, + "contact": test.contact + } + data = tmt.utils.dict_to_yaml(data) + logger.print("YAML file generated successfully!", color="green") + return data + + +def generate_plan_yaml(plan: tmt.Plan, logger: Logger) -> str: + """ + This function generates an YAML file with the input data for a plan + :param plan: Plan object + :param logger: tmt.Logger instance + :return: + """ + logger.print("Generating the YAML file...") + full_url = plan.web_link() + data = { + "name": plan.name, + "summary": plan.summary, + "description": plan.description, + "url": full_url, + "ref": plan.fmf_id.ref + } + data = tmt.utils.dict_to_yaml(data) + logger.print("YAML file generated successfully!", color="green") + return data + + +def generate_testplan_yaml(test: tmt.Test, plan: tmt.Plan, logger: Logger) -> str: + """ + This function generates an YAML file with the input data for a test and a plan + :param test: Test object + :param plan: Plan object + :param logger: tmt.Logger instance + :return: + """ + logger.print("Generating the YAML file...") + full_url_test = test.web_link() + full_url_plan = plan.web_link() + data = { + "test": { + "name": test.name, + "summary": test.summary, + "description": test.description, + "url": full_url_test, + "ref": test.fmf_id.ref, + "contact": test.contact + }, + "plan": { + "name": plan.name, + "summary": plan.summary, + "description": plan.description, + "url": full_url_plan, + "ref": plan.fmf_id.ref + } + } + data = tmt.utils.dict_to_yaml(data) + logger.print("YAML file generated successfully!", color="green") + return data diff --git a/src/service.py b/src/service.py index 4eea76e..8496eb8 100644 --- a/src/service.py +++ b/src/service.py @@ -1,9 +1,9 @@ import os import tmt import logging -from pathlib import Path -from src import html_generator as html from src.utils import git_handler as utils +from src.generators import json_generator, html_generator as html +from src.generators import yaml_generator from celery.app import Celery logger = tmt.Logger(logging.Logger("tmt-logger")) @@ -34,13 +34,14 @@ def get_tree(url: str, name: str, ref: str) -> tmt.base.Tree: return tree -def process_test_request(test_url: str, test_name: str, test_ref: str, return_html: bool) -> str | None | tmt.Test: +def process_test_request(test_url: str, test_name: str, test_ref: str, return_object: bool, out_format: str) -> str | None | tmt.Test: """ This function processes the request for a test and returns the HTML file or the Test object :param test_url: Test url :param test_name: Test name :param test_ref: Test repo ref - :param return_html: Specify if the function should return the HTML file or the Test object + :param return_object: Specify if the function should return the HTML file or the Test object + :param out_format: Specifies output format :return: """ @@ -59,18 +60,25 @@ def process_test_request(test_url: str, test_name: str, test_ref: str, return_ht logger.print("Test not found!", color="red") return None logger.print("Test found!", color="green") - if not return_html: + if not return_object: return wanted_test - return html.generate_test_html_page(wanted_test, logger=logger) + match out_format: + case "html": + return html.generate_test_html_page(wanted_test, logger=logger) + case "json": + return json_generator.generate_test_json(wanted_test, logger=logger) + case "yaml": + return yaml_generator.generate_test_yaml(wanted_test, logger=logger) -def process_plan_request(plan_url: str, plan_name: str, plan_ref: str, return_html: bool) -> str | None | tmt.Plan: +def process_plan_request(plan_url: str, plan_name: str, plan_ref: str, return_object: bool, out_format: str) -> str | None | tmt.Plan: """ This function processes the request for a plan and returns the HTML file or the Plan object :param plan_url: Plan URL :param plan_name: Plan name :param plan_ref: Plan repo ref - :param return_html: Specify if the function should return the HTML file or the Plan object + :param return_object: Specify if the function should return the HTML file or the Plan object + :param out_format: Specifies output format :return: """ @@ -89,12 +97,18 @@ def process_plan_request(plan_url: str, plan_name: str, plan_ref: str, return_ht logger.print("Plan not found!", color="red") return None logger.print("Plan found!", color="green") - if not return_html: + if not return_object: return wanted_plan - return html.generate_plan_html_page(wanted_plan, logger=logger) + match out_format: + case "html": + return html.generate_plan_html_page(wanted_plan, logger=logger) + case "json": + return json_generator.generate_plan_json(wanted_plan, logger=logger) + case "yaml": + return yaml_generator.generate_plan_yaml(wanted_plan, logger=logger) -def process_testplan_request(test_url, test_name, test_ref, plan_url, plan_name, plan_ref) -> str | None: +def process_testplan_request(test_url, test_name, test_ref, plan_url, plan_name, plan_ref, out_format) -> str | None: """ This function processes the request for a test and a plan and returns the HTML file :param test_url: Test URL @@ -103,11 +117,18 @@ def process_testplan_request(test_url, test_name, test_ref, plan_url, plan_name, :param plan_url: Plan URL :param plan_name: Plan name :param plan_ref: Plan repo ref + :param out_format: Specifies output format :return: """ - test = process_test_request(test_url, test_name, test_ref, False) - plan = process_plan_request(plan_url, plan_name, plan_ref, False) - return html.generate_testplan_html_page(test, plan, logger=logger) + test = process_test_request(test_url, test_name, test_ref, False, out_format) + plan = process_plan_request(plan_url, plan_name, plan_ref, False, out_format) + match out_format: + case "html": + return html.generate_testplan_html_page(test, plan, logger=logger) + case "json": + return json_generator.generate_testplan_json(test, plan, logger=logger) + case "yaml": + return yaml_generator.generate_testplan_yaml(test, plan, logger=logger) @app.task @@ -120,11 +141,11 @@ def main(test_url: str | None, out_format: str | None) -> str | None: logger.print("Starting...", color="blue") if test_name is not None and plan_name is None: - return process_test_request(test_url, test_name, test_ref, True) + return process_test_request(test_url, test_name, test_ref, True, out_format) elif plan_name is not None and test_name is None: - return process_plan_request(plan_url, plan_name, plan_ref, True) + return process_plan_request(plan_url, plan_name, plan_ref, True, out_format) elif plan_name is not None and test_name is not None: - return process_testplan_request(test_url, test_name, test_ref, plan_url, plan_name, plan_ref) + return process_testplan_request(test_url, test_name, test_ref, plan_url, plan_name, plan_ref, out_format) if __name__ == "__main__": diff --git a/src/utils/git_handler.py b/src/utils/git_handler.py index 56369a5..2c9bda7 100644 --- a/src/utils/git_handler.py +++ b/src/utils/git_handler.py @@ -19,8 +19,8 @@ def checkout_branch(path: Path, logger: Logger, ref: str) -> None: common_instance.run( command=tmt.utils.Command('git', 'checkout', ref), cwd=path) except tmt.utils.RunError: - logger.print("Failed to clone the repository!", color="red") - raise Exception + logger.print("Failed to do checkout in the repository!", color="red") + raise AttributeError def clone_repository(url: str, logger: Logger, ref: str) -> None: @@ -36,12 +36,19 @@ def clone_repository(url: str, logger: Logger, ref: str) -> None: path = get_path_to_repository(url) if check_if_repository_exists(url): if ref != "default": - checkout_branch(ref=ref, path=path, logger=logger) + try: + checkout_branch(ref=ref, path=path, logger=logger) + except AttributeError: + raise AttributeError logger.print("Repository already cloned!", color="yellow") raise FileExistsError try: - tmt.utils.git_clone(url=url, shallow=True, destination=path, logger=logger) + tmt.utils.git_clone(url=url, destination=path, logger=logger) if ref != "default": + try: + checkout_branch(ref=ref, path=path, logger=logger) + except AttributeError: + raise AttributeError checkout_branch(ref=ref, path=path, logger=logger) except tmt.utils.GeneralError as e: logger.print("Failed to clone the repository!", color="red") @@ -81,14 +88,13 @@ def clear_tmp_dir(logger: Logger) -> None: logger.print("Clearing the .tmp directory...") path = os.path.realpath(__file__) path = path.replace("src/utils/git_handler.py", "") - path = Path(path + "/.tmp") - # repo_name = .rsplit('/', 1)[-1] + path = Path(path + os.getenv("CLONE_DIR_PATH", "./.repos/")) try: Popen(["rm", "-rf", path]) except Exception as e: - logger.print("Failed to clear the .tmp directory!", color="red") + logger.print("Failed to clear the repository clone directory!", color="red") raise e - logger.print(".tmp directory cleared successfully!", color="green") + logger.print("Repository clone directory cleared successfully!", color="green") def get_git_repository(url: str, logger: Logger, ref: str) -> Path: diff --git a/tests/test_api.py b/tests/test_api.py index 07696e1..72f7f91 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -15,9 +15,8 @@ class TestApi: @pytest.fixture(autouse=True) def setup(self): os.environ["USE_CELERY"] = "false" - os.environ["TMP_DIR_PATH"] = "../.tmp/" - def test_basic_test_request(self, client): + def test_basic_test_request_json(self, client): # ?test_url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke response = client.get("/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke") data = response.content.decode("utf-8") @@ -25,6 +24,22 @@ def test_basic_test_request(self, client): assert "500" not in data assert "https://github.com/teemtee/tmt/tree/main/tests/core/smoke/main.fmf" in data + def test_basic_test_request_html(self, client): + # ?test_url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke + response = client.get("/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke&format=html") + data = response.content.decode("utf-8") + print(data) + assert "500" not in data + assert f'' in data + + def test_basic_test_request_yaml(self, client): + # ?test_url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke + response = client.get("/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke&format=yaml") + data = response.content.decode("utf-8") + print(data) + assert "500" not in data + assert "url: https://github.com/teemtee/tmt/tree/main/tests/core/smoke/main.fmf" in data + def test_basic_plan_request(self, client): # ?plan-url=https://github.com/teemtee/tmt&plan-name=/plans/features/basic&type=plan response = client.get("/?plan-url=https://github.com/teemtee/tmt&plan-name=/plans/features/basic&type=plan") @@ -37,7 +52,7 @@ def test_basic_testplan_request(self, client): # ?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke& # ?plan-url=https://github.com/teemtee/tmt&plan-name=/plans/features/basic&type=plan response = client.get("/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke&" - "?plan-url=https://github.com/teemtee/tmt&plan-name=/plans/features/basic&type=plan") + "plan-url=https://github.com/teemtee/tmt&plan-name=/plans/features/basic&type=plan") data = response.content.decode("utf-8") print(data) assert "500" not in data @@ -81,7 +96,7 @@ def test_basic_test_request(self, client): json_data = response.json() while True: if json_data["status"] == "PENDING": - response = client.get("/status?task_id=" + json_data["id"]) + response = client.get("/status?task-id=" + json_data["id"]) json_data = response.json() time.sleep(0.1) elif json_data["status"] == "SUCCESS": diff --git a/tests/unit/test_git_handler.py b/tests/unit/test_git_handler.py index 35baa19..2cac4c7 100644 --- a/tests/unit/test_git_handler.py +++ b/tests/unit/test_git_handler.py @@ -1,4 +1,6 @@ +import os import time +from pathlib import Path import pytest import tmt @@ -11,12 +13,22 @@ class TestGitHandler: logger = tmt.Logger(logging.Logger("tmt-logger")) def test_clear_tmp_dir(self): + # Create test directory if it doesn't exist + try: + path = os.path.realpath(__file__) + path = path.replace("tests/unit/test_git_handler.py", "") + path = Path(path + os.getenv("CLONE_DIR_PATH", "./.repos/")) + os.mkdir(path) + except FileExistsError: + pass git_handler.clear_tmp_dir(self.logger) def test_get_path_to_repository(self): + self.test_clone_repository() assert git_handler.get_path_to_repository("https://github.com/teemtee/tmt").exists() def test_check_if_repository_exists(self): + self.test_clone_repository() assert git_handler.check_if_repository_exists("https://github.com/teemtee/tmt") def test_clone_repository(self): @@ -25,6 +37,32 @@ def test_clone_repository(self): git_handler.clear_tmp_dir(self.logger) time.sleep(1) git_handler.clone_repository(url="https://github.com/teemtee/tmt", logger=self.logger, ref="default") - with pytest.raises(FileExistsError): + + def test_clone_repository_even_if_exists(self): + try: git_handler.clone_repository(url="https://github.com/teemtee/tmt", logger=self.logger, ref="default") + except FileExistsError: + pass + + def test_clone_checkout_branch(self): + try: + git_handler.clone_repository(url="https://github.com/teemtee/tmt", logger=self.logger, ref="quay") + except FileExistsError: + pass + + def test_clone_checkout_branch_exception(self): + with pytest.raises(AttributeError): + git_handler.clone_repository(url="https://github.com/teemtee/tmt", logger=self.logger, ref="quadd") + + def test_checkout_branch(self): + self.test_clone_repository_even_if_exists() + git_handler.checkout_branch(ref="quay", path=git_handler.get_path_to_repository( + url="https://github.com/teemtee/tmt"), logger=self.logger) + + def test_checkout_branch_exception(self): + self.test_clone_repository_even_if_exists() + with pytest.raises(AttributeError): + git_handler.checkout_branch(ref="quaddy", path=git_handler.get_path_to_repository( + url="https://github.com/teemtee/tmt"), logger=self.logger) + From a9877028bf9658b85b429afd9b491084f4af8a39 Mon Sep 17 00:00:00 2001 From: Tom Koscielniak Date: Tue, 23 Apr 2024 10:46:10 +0200 Subject: [PATCH 09/46] Add envvars to README --- .gitignore | 2 +- README.md | 8 ++++++++ compose.yaml | 4 ---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 99452ee..dc4d3ac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ .pytest_cache -.tmp +.repos venv .idea .vscode diff --git a/README.md b/README.md index fd016ac..fbef137 100644 --- a/README.md +++ b/README.md @@ -39,3 +39,11 @@ If we want to display metadata for both tests and plans, we can combine the `tes and `plan-*` options together, they are not mutually exclusive. `test-url` and `test-name`, or `plan-url` and `plan-name` are required. + +## Environment variables +`REDIS_URL` - optional, passed to Celery on initialization as a `broker` and `backend` argument, +default value is `redis://localhost:6379` + +`CLONE_DIR_PATH` - optional, specifies the path where the repositories will be cloned, default value is `./.repos/` + +`USE_CELERY` - optional, specifies if the app should use Celery, set to `false` for running without Celery diff --git a/compose.yaml b/compose.yaml index 038cb3a..d092a53 100644 --- a/compose.yaml +++ b/compose.yaml @@ -5,8 +5,6 @@ services: command: uvicorn src.api:app --reload --host 0.0.0.0 --port 8000 environment: - REDIS_URL=redis://redis:6379 - - TMP_DIR_PATH=./ - - HOSTNAME=http://localhost:8000 ports: - 8000:8000 redis: @@ -21,8 +19,6 @@ services: command: celery --app=src.api.service worker --concurrency=1 --loglevel=INFO environment: - REDIS_URL=redis://redis:6379 - - TMP_DIR_PATH=./ - - HOSTNAME=http://localhost:8000 depends_on: - redis From 159c33580faa13f16485ddb01ff94a9a0686ddd6 Mon Sep 17 00:00:00 2001 From: Tom Koscielniak Date: Tue, 23 Apr 2024 13:11:48 +0200 Subject: [PATCH 10/46] Add hostname env for status callback --- compose.yaml | 4 +++- src/api.py | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/compose.yaml b/compose.yaml index d092a53..2c051f5 100644 --- a/compose.yaml +++ b/compose.yaml @@ -5,6 +5,7 @@ services: command: uvicorn src.api:app --reload --host 0.0.0.0 --port 8000 environment: - REDIS_URL=redis://redis:6379 + - API_HOSTNAME=http://localhost:8000 ports: - 8000:8000 redis: @@ -16,9 +17,10 @@ services: celery: container_name: celery build: . - command: celery --app=src.api.service worker --concurrency=1 --loglevel=INFO + command: celery --app=src.api.service worker --loglevel=INFO environment: - REDIS_URL=redis://redis:6379 + - API_HOSTNAME=http://localhost:8000 depends_on: - redis diff --git a/src/api.py b/src/api.py index 78cd44f..97580a9 100644 --- a/src/api.py +++ b/src/api.py @@ -32,10 +32,14 @@ def find_test( plan_ref: str = Query("default", alias="plan-ref"), out_format: str = Query("json", alias="format") ): + # Parameter validations if (test_url is None and test_name is not None) or (test_url is not None and test_name is None): return "Invalid arguments!" if (plan_url is None and plan_name is not None) or (plan_url is not None and plan_name is None): return "Invalid arguments!" + if plan_url is None and plan_name is None and test_url is None and test_name is None: + return "Missing arguments!" + # Disable Celery if not needed if os.environ.get("USE_CELERY") == "false": html_page = service.main(test_url, test_name, test_ref, plan_url, plan_name, plan_ref, out_format) return html_page @@ -44,7 +48,7 @@ def find_test( if out_format == "html": global format_html format_html = True - status_callback_url = f'/status?task-id={r.task_id}&html=true' + status_callback_url = f'{os.getenv("API_HOSTNAME")}/status?task-id={r.task_id}&html=true' return HTMLResponse(content=html_generator.generate_status_callback(r, status_callback_url)) else: format_html = False # To set it back to False after a html format request @@ -56,7 +60,7 @@ def status(task_id: str = Query(None, alias="task-id"), html: str = Query("false")) -> TaskOut | HTMLResponse: r = service.main.app.AsyncResult(task_id) if html == "true": - status_callback_url = f'/status?task-id={r.task_id}&html=true' + status_callback_url = f'{os.getenv("API_HOSTNAME")}/status?task-id={r.task_id}&html=true' return HTMLResponse(content=html_generator.generate_status_callback(r, status_callback_url)) return _to_task_out(r) @@ -66,5 +70,5 @@ def _to_task_out(r: AsyncResult) -> TaskOut | str: id=r.task_id, status=r.status, result=r.traceback if r.failed() else r.result, - status_callback_url=f'{os.getenv("HOSTNAME")}/status?task-id={r.task_id}' + status_callback_url=f'{os.getenv("API_HOSTNAME")}/status?task-id={r.task_id}' ) From ca081575f6cedd5074a43c373b2d8bec658833bf Mon Sep 17 00:00:00 2001 From: Tom Koscielniak Date: Tue, 23 Apr 2024 14:13:30 +0200 Subject: [PATCH 11/46] Add kube pod config --- src/api.py | 1 + src/generators/html_generator.py | 2 +- tmt-web.yaml | 39 ++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 tmt-web.yaml diff --git a/src/api.py b/src/api.py index 97580a9..cbeb09e 100644 --- a/src/api.py +++ b/src/api.py @@ -39,6 +39,7 @@ def find_test( return "Invalid arguments!" if plan_url is None and plan_name is None and test_url is None and test_name is None: return "Missing arguments!" + # TODO: forward to docs # Disable Celery if not needed if os.environ.get("USE_CELERY") == "false": html_page = service.main(test_url, test_name, test_ref, plan_url, plan_name, plan_ref, out_format) diff --git a/src/generators/html_generator.py b/src/generators/html_generator.py index 8d3be36..f5e5851 100644 --- a/src/generators/html_generator.py +++ b/src/generators/html_generator.py @@ -17,7 +17,7 @@ def generate_status_callback(r: AsyncResult, status_callback_url: str) -> str: - Processing... Try this clicking this url in a few seconds: {r.task_id} + Processing... Try this clicking this url in a few seconds: {status_callback_url} ''') else: return (f''' diff --git a/tmt-web.yaml b/tmt-web.yaml new file mode 100644 index 0000000..7546a0d --- /dev/null +++ b/tmt-web.yaml @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: "2024-04-23T11:20:25Z" + labels: + app: tmt-web + name: tmt-web +spec: + containers: + # Uvicorn + - image: quay.io/testing-farm/tmt-web:latest + args: + - uvicorn + name: uvicorn + ports: + - containerPort: 8000 + hostPort: 8000 + env: + - name: REDIS_URL + value: redis://redis:6379 + - name: API_HOSTNAME + value: http://localhost:8000 + # Celery + - image: quay.io/testing-farm/tmt-web:latest + args: + - celery + name: celery + env: + - name: REDIS_URL + value: redis://redis:6379 + - name: API_HOSTNAME + value: http://localhost:8000 + # Redis + - image: redis:latest + name: redis + ports: + - containerPort: 6379 + hostPort: 6379 + From 4d26aecfa0547162b885b95ba6b11e14fe0ae6f4 Mon Sep 17 00:00:00 2001 From: Tom Koscielniak Date: Tue, 14 May 2024 19:52:07 +0200 Subject: [PATCH 12/46] Add unit tests to complete coverage, add more metadata, refactor --- README.md | 2 ++ src/generators/html_generator.py | 44 ++++++++++++++++++++++++++++ src/generators/json_generator.py | 46 +++++++++++++++++++++++++++--- src/generators/yaml_generator.py | 46 +++++++++++++++++++++++++++--- src/service.py | 18 +++--------- src/utils/git_handler.py | 1 - tests/objects/sample_plan.fmf | 9 ++++++ tests/objects/sample_test/main.fmf | 2 ++ tests/objects/sample_test/test.sh | 6 ++++ tests/test_api.py | 22 +++++++------- tests/unit/test_git_handler.py | 2 ++ tests/unit/test_html_generator.py | 13 +++++++++ tests/unit/test_json_generator.py | 27 ++++++++++++++++++ tests/unit/test_yaml_generator.py | 27 ++++++++++++++++++ 14 files changed, 232 insertions(+), 33 deletions(-) create mode 100644 tests/objects/sample_plan.fmf create mode 100644 tests/objects/sample_test/main.fmf create mode 100755 tests/objects/sample_test/test.sh create mode 100644 tests/unit/test_html_generator.py create mode 100644 tests/unit/test_json_generator.py create mode 100644 tests/unit/test_yaml_generator.py diff --git a/README.md b/README.md index fbef137..18767b1 100644 --- a/README.md +++ b/README.md @@ -47,3 +47,5 @@ default value is `redis://localhost:6379` `CLONE_DIR_PATH` - optional, specifies the path where the repositories will be cloned, default value is `./.repos/` `USE_CELERY` - optional, specifies if the app should use Celery, set to `false` for running without Celery + +`API_HOSTNAME` - required, specifies the hostname of the API, used for creating the callback URL to the service diff --git a/src/generators/html_generator.py b/src/generators/html_generator.py index f5e5851..abb2b25 100644 --- a/src/generators/html_generator.py +++ b/src/generators/html_generator.py @@ -53,6 +53,16 @@ def generate_test_html_page(test: Test, logger: Logger) -> str: url: {full_url}
ref: {test.fmf_id.ref}
contact: {test.contact}
+ tag: {test.tag}
+ tier: {test.tier}
+ id: {test.id}
+ fmf-id:
+
    +
  • url: {test.fmf_id.url}
    • +
  • path: {test.fmf_id.path}
    • +
  • name: {test.fmf_id.name}
    • +
  • ref: {test.fmf_id.ref}
    • +
''') logger.print("HTML file generated successfully!", color="green") @@ -80,6 +90,17 @@ def generate_plan_html_page(plan: Plan, logger: Logger) -> str: description: {plan.description}
url: {full_url}
ref: {plan.fmf_id.ref}
+ contact: {plan.contact}
+ tag: {plan.tag}
+ tier: {plan.tier}
+ id: {plan.id}
+ fmf-id:
+
    +
  • url: {plan.fmf_id.url}
    • +
  • path: {plan.fmf_id.path}
    • +
  • name: {plan.fmf_id.name}
    • +
  • ref: {plan.fmf_id.ref}
    • +
''') logger.print("HTML file generated successfully!", color="green") @@ -104,18 +125,41 @@ def generate_testplan_html_page(test: tmt.Test, plan: tmt.Plan, logger: Logger) + Test metadata
name: {test.name}
summary: {test.summary}
description: {test.description}
url: {full_url_test}
ref: {test.fmf_id.ref}
contact: {test.contact}
+ tag: {test.tag}
+ tier: {test.tier}
+ id: {test.id}
+ fmf-id:
+
    +
  • url: {test.fmf_id.url}
    • +
  • path: {test.fmf_id.path}
    • +
  • name: {test.fmf_id.name}
    • +
  • ref: {test.fmf_id.ref}
    • +

+ Plan metadata
name: {plan.name}
summary: {plan.summary}
description: {plan.description}
url: {full_url_plan}
ref: {plan.fmf_id.ref}
+ contact: {plan.contact}
+ tag: {plan.tag}
+ tier: {plan.tier}
+ id: {plan.id}
+ fmf-id:
+
    +
  • url: {plan.fmf_id.url}
    • +
  • path: {plan.fmf_id.path}
    • +
  • name: {plan.fmf_id.name}
    • +
  • ref: {plan.fmf_id.ref}
    • +
''') logger.print("HTML file generated successfully!", color="green") diff --git a/src/generators/json_generator.py b/src/generators/json_generator.py index 7ad034d..b18e28d 100644 --- a/src/generators/json_generator.py +++ b/src/generators/json_generator.py @@ -19,7 +19,16 @@ def generate_test_json(test: Test, logger: Logger) -> str: "description": test.description, "url": full_url, "ref": test.fmf_id.ref, - "contact": test.contact + "contact": test.contact, + "tag": test.tag, + "tier": test.tier, + "id": test.id, + "fmf-id": { + "url": test.fmf_id.url, + "path": test.fmf_id.path, + "name": test.fmf_id.name, + "ref": test.fmf_id.ref, + } } data = json.dumps(data) logger.print("JSON file generated successfully!", color="green") @@ -40,7 +49,17 @@ def generate_plan_json(plan: Plan, logger: Logger) -> str: "summary": plan.summary, "description": plan.description, "url": full_url, - "ref": plan.fmf_id.ref + "ref": plan.fmf_id.ref, + "contact": plan.contact, + "tag": plan.tag, + "tier": plan.tier, + "id": plan.id, + "fmf-id": { + "url": plan.fmf_id.url, + "path": plan.fmf_id.path, + "name": plan.fmf_id.name, + "ref": plan.fmf_id.ref, + } } data = json.dumps(data) logger.print("JSON file generated successfully!", color="green") @@ -65,14 +84,33 @@ def generate_testplan_json(test: tmt.Test, plan: tmt.Plan, logger: Logger) -> st "description": test.description, "url": full_url_test, "ref": test.fmf_id.ref, - "contact": test.contact + "contact": test.contact, + "tag": test.tag, + "tier": test.tier, + "id": test.id, + "fmf-id": { + "url": test.fmf_id.url, + "path": test.fmf_id.path, + "name": test.fmf_id.name, + "ref": test.fmf_id.ref, + } }, "plan": { "name": plan.name, "summary": plan.summary, "description": plan.description, "url": full_url_plan, - "ref": plan.fmf_id.ref + "ref": plan.fmf_id.ref, + "contact": plan.contact, + "tag": plan.tag, + "tier": plan.tier, + "id": plan.id, + "fmf-id": { + "url": plan.fmf_id.url, + "path": plan.fmf_id.path, + "name": plan.fmf_id.name, + "ref": plan.fmf_id.ref, + } } } data = json.dumps(data) diff --git a/src/generators/yaml_generator.py b/src/generators/yaml_generator.py index af45ae3..7784178 100644 --- a/src/generators/yaml_generator.py +++ b/src/generators/yaml_generator.py @@ -18,7 +18,16 @@ def generate_test_yaml(test: tmt.Test, logger: Logger) -> str: "description": test.description, "url": full_url, "ref": test.fmf_id.ref, - "contact": test.contact + "contact": test.contact, + "tag": test.tag, + "tier": test.tier, + "id": test.id, + "fmf-id": { + "url": test.fmf_id.url, + "path": test.fmf_id.path, + "name": test.fmf_id.name, + "ref": test.fmf_id.ref, + } } data = tmt.utils.dict_to_yaml(data) logger.print("YAML file generated successfully!", color="green") @@ -39,7 +48,17 @@ def generate_plan_yaml(plan: tmt.Plan, logger: Logger) -> str: "summary": plan.summary, "description": plan.description, "url": full_url, - "ref": plan.fmf_id.ref + "ref": plan.fmf_id.ref, + "contact": plan.contact, + "tag": plan.tag, + "tier": plan.tier, + "id": plan.id, + "fmf-id": { + "url": plan.fmf_id.url, + "path": plan.fmf_id.path, + "name": plan.fmf_id.name, + "ref": plan.fmf_id.ref, + } } data = tmt.utils.dict_to_yaml(data) logger.print("YAML file generated successfully!", color="green") @@ -64,14 +83,33 @@ def generate_testplan_yaml(test: tmt.Test, plan: tmt.Plan, logger: Logger) -> st "description": test.description, "url": full_url_test, "ref": test.fmf_id.ref, - "contact": test.contact + "contact": test.contact, + "tag": test.tag, + "tier": test.tier, + "id": test.id, + "fmf-id": { + "url": test.fmf_id.url, + "path": test.fmf_id.path, + "name": test.fmf_id.name, + "ref": test.fmf_id.ref, + } }, "plan": { "name": plan.name, "summary": plan.summary, "description": plan.description, "url": full_url_plan, - "ref": plan.fmf_id.ref + "ref": plan.fmf_id.ref, + "contact": plan.contact, + "tag": plan.tag, + "tier": plan.tier, + "id": plan.id, + "fmf-id": { + "url": plan.fmf_id.url, + "path": plan.fmf_id.path, + "name": plan.fmf_id.name, + "ref": plan.fmf_id.ref, + } } } data = tmt.utils.dict_to_yaml(data) diff --git a/src/service.py b/src/service.py index 8496eb8..df234f1 100644 --- a/src/service.py +++ b/src/service.py @@ -49,14 +49,9 @@ def process_test_request(test_url: str, test_name: str, test_ref: str, return_ob logger.print("Looking for the wanted test...") - test_list = tree.tests() - wanted_test = None # Find the desired Test object - for test in test_list: - if test.name == test_name: - wanted_test = test - break - if wanted_test is None: + wanted_test = tree.tests(names=[test_name])[0] + if wanted_test is []: logger.print("Test not found!", color="red") return None logger.print("Test found!", color="green") @@ -86,14 +81,9 @@ def process_plan_request(plan_url: str, plan_name: str, plan_ref: str, return_ob logger.print("Looking for the wanted plan...") - plan_list = tree.plans() - wanted_plan = None # Find the desired Plan object - for plan in plan_list: - if plan.name == plan_name: - wanted_plan = plan - break - if wanted_plan is None: + wanted_plan = tree.plans(names=[plan_name])[0] + if wanted_plan is []: logger.print("Plan not found!", color="red") return None logger.print("Plan found!", color="green") diff --git a/src/utils/git_handler.py b/src/utils/git_handler.py index 2c9bda7..88e450c 100644 --- a/src/utils/git_handler.py +++ b/src/utils/git_handler.py @@ -1,4 +1,3 @@ -import sys from subprocess import Popen from tmt import Logger import os diff --git a/tests/objects/sample_plan.fmf b/tests/objects/sample_plan.fmf new file mode 100644 index 0000000..359a862 --- /dev/null +++ b/tests/objects/sample_plan.fmf @@ -0,0 +1,9 @@ +summary: Essential command line features +discover: + how: fmf + url: https://github.com/teemtee/tmt +prepare: + how: ansible + playbook: ansible/packages.yml +execute: + how: tmt diff --git a/tests/objects/sample_test/main.fmf b/tests/objects/sample_test/main.fmf new file mode 100644 index 0000000..aa0b710 --- /dev/null +++ b/tests/objects/sample_test/main.fmf @@ -0,0 +1,2 @@ +summary: Concise summary describing what the test does +test: ./test.sh diff --git a/tests/objects/sample_test/test.sh b/tests/objects/sample_test/test.sh new file mode 100755 index 0000000..b8ae7a2 --- /dev/null +++ b/tests/objects/sample_test/test.sh @@ -0,0 +1,6 @@ +#!/bin/sh -eux + +tmp=$(mktemp) +tmt --help > "$tmp" +grep -C3 'Test Management Tool' "$tmp" +rm "$tmp" diff --git a/tests/test_api.py b/tests/test_api.py index 72f7f91..5922e6c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -12,28 +12,28 @@ def client(): class TestApi: + """ + This class tests the behaviour of the API directly + """ @pytest.fixture(autouse=True) def setup(self): os.environ["USE_CELERY"] = "false" def test_basic_test_request_json(self, client): - # ?test_url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke - response = client.get("/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke") + response = client.get("/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke&test-ref=main") data = response.content.decode("utf-8") - print(data) assert "500" not in data assert "https://github.com/teemtee/tmt/tree/main/tests/core/smoke/main.fmf" in data def test_basic_test_request_html(self, client): - # ?test_url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke - response = client.get("/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke&format=html") + response = client.get( + "/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke&test-ref=main&format=html") data = response.content.decode("utf-8") print(data) assert "500" not in data assert f'' in data def test_basic_test_request_yaml(self, client): - # ?test_url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke response = client.get("/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke&format=yaml") data = response.content.decode("utf-8") print(data) @@ -41,7 +41,6 @@ def test_basic_test_request_yaml(self, client): assert "url: https://github.com/teemtee/tmt/tree/main/tests/core/smoke/main.fmf" in data def test_basic_plan_request(self, client): - # ?plan-url=https://github.com/teemtee/tmt&plan-name=/plans/features/basic&type=plan response = client.get("/?plan-url=https://github.com/teemtee/tmt&plan-name=/plans/features/basic&type=plan") data = response.content.decode("utf-8") print(data) @@ -49,8 +48,6 @@ def test_basic_plan_request(self, client): assert "https://github.com/teemtee/tmt/tree/main/plans/features/basic.fmf" in data def test_basic_testplan_request(self, client): - # ?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke& - # ?plan-url=https://github.com/teemtee/tmt&plan-name=/plans/features/basic&type=plan response = client.get("/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke&" "plan-url=https://github.com/teemtee/tmt&plan-name=/plans/features/basic&type=plan") data = response.content.decode("utf-8") @@ -83,10 +80,15 @@ def test_invalid_testplan_arguments(self, client): def test_invalid_argument_names(self, client): response = client.get("/?test_urlur=https://github.com/teemtee/tmt&test_nn=/tests/core/smoke") - assert response.status_code == 500 + data = response.content.decode("utf-8") + assert response.status_code == 200 + assert data == '"Missing arguments!"' class TestCelery: + """ + This class tests the API with the Celery instance + """ @pytest.fixture(autouse=True) def setup(self): os.environ["USE_CELERY"] = "true" diff --git a/tests/unit/test_git_handler.py b/tests/unit/test_git_handler.py index 2cac4c7..dc1a028 100644 --- a/tests/unit/test_git_handler.py +++ b/tests/unit/test_git_handler.py @@ -58,6 +58,8 @@ def test_checkout_branch(self): self.test_clone_repository_even_if_exists() git_handler.checkout_branch(ref="quay", path=git_handler.get_path_to_repository( url="https://github.com/teemtee/tmt"), logger=self.logger) + git_handler.checkout_branch(ref="main", path=git_handler.get_path_to_repository( + url="https://github.com/teemtee/tmt"), logger=self.logger) def test_checkout_branch_exception(self): self.test_clone_repository_even_if_exists() diff --git a/tests/unit/test_html_generator.py b/tests/unit/test_html_generator.py new file mode 100644 index 0000000..824dfd3 --- /dev/null +++ b/tests/unit/test_html_generator.py @@ -0,0 +1,13 @@ +import logging + +import tmt + +from src.generators import html_generator + + +class TestHtmlGenerator: + def test_generate_test_html(self): + logger = tmt.Logger(logging.Logger("tmt-logger")) + test = tmt.Tree(logger=logger).tests(names=["/tests/objects/sample_test"])[0] + data = html_generator.generate_test_html_page(test, logger) + assert 'name: /tests/objects/sample_test
' in data diff --git a/tests/unit/test_json_generator.py b/tests/unit/test_json_generator.py new file mode 100644 index 0000000..ffdde83 --- /dev/null +++ b/tests/unit/test_json_generator.py @@ -0,0 +1,27 @@ +import logging + +import tmt + +from src.generators import json_generator + + +class TestJsonGenerator: + def test_generate_test_json(self): + logger = tmt.Logger(logging.Logger("tmt-logger")) + test = tmt.Tree(logger=logger).tests(names=["/tests/objects/sample_test"])[0] + data = json_generator.generate_test_json(test, logger) + assert '"name": "/tests/objects/sample_test"' in data + + def test_generate_plan_json(self): + logger = tmt.Logger(logging.Logger("tmt-logger")) + plan = tmt.Tree(logger=logger).plans(names=["/tests/objects/sample_plan"])[0] + data = json_generator.generate_plan_json(plan, logger) + assert '"name": "/tests/objects/sample_plan"' in data + + def test_generate_testplan_json(self): + logger = tmt.Logger(logging.Logger("tmt-logger")) + test = tmt.Tree(logger=logger).tests(names=["/tests/objects/sample_test"])[0] + plan = tmt.Tree(logger=logger).plans(names=["/tests/objects/sample_plan"])[0] + data = json_generator.generate_testplan_json(test, plan, logger) + assert '"name": "/tests/objects/sample_test"' in data + assert '"name": "/tests/objects/sample_plan"' in data diff --git a/tests/unit/test_yaml_generator.py b/tests/unit/test_yaml_generator.py new file mode 100644 index 0000000..357d3d7 --- /dev/null +++ b/tests/unit/test_yaml_generator.py @@ -0,0 +1,27 @@ +import logging + +import tmt + +from src.generators import yaml_generator + + +class TestYamlGenerator: + def test_generate_test_yaml(self): + logger = tmt.Logger(logging.Logger("tmt-logger")) + test = tmt.Tree(logger=logger).tests(names=["/tests/objects/sample_test"])[0] + data = yaml_generator.generate_test_yaml(test, logger) + assert 'name: /tests/objects/sample_test' in data + + def test_generate_plan_yaml(self): + logger = tmt.Logger(logging.Logger("tmt-logger")) + plan = tmt.Tree(logger=logger).plans(names=["/tests/objects/sample_plan"])[0] + data = yaml_generator.generate_plan_yaml(plan, logger) + assert 'name: /tests/objects/sample_plan' in data + + def test_generate_testplan_yaml(self): + logger = tmt.Logger(logging.Logger("tmt-logger")) + test = tmt.Tree(logger=logger).tests(names=["/tests/objects/sample_test"])[0] + plan = tmt.Tree(logger=logger).plans(names=["/tests/objects/sample_plan"])[0] + data = yaml_generator.generate_testplan_yaml(test, plan, logger) + assert 'name: /tests/objects/sample_test' in data + assert 'name: /tests/objects/sample_plan' in data From 56b5948ed5098dd7a696cef4ae1fad0b326d2025 Mon Sep 17 00:00:00 2001 From: Tom Koscielniak Date: Tue, 14 May 2024 20:22:14 +0200 Subject: [PATCH 13/46] Add pytest action, edit Readme run instructions --- .github/workflows/test.yml | 34 ++++++++++++++++++++++++++++++++++ README.md | 9 +++++---- entrypoint.sh | 2 +- 3 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..4d6ffa2 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,34 @@ +name: Run tests + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ "3.9", "3.10", "3.11", "3.12" ] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest + pip install -r src/requirements.txt + - name: Start services + run: | + docker-compose up + - name: Test + run: | + pytest diff --git a/README.md b/README.md index 18767b1..b44c339 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,12 @@ # web Web app for checking tmt tests and plans # Run instructions -1. Create a virtual environment -2. Install the requirements -3. Use the `start_api.sh` script to start the api +To run the service locally for development purposes, use the following command: +```bash +docker-compose up --build +``` ## Tests -To run the tests, use the pytest command +To run the tests, use the `pytest` command # API API for checking tmt tests and plans metadata ## Version diff --git a/entrypoint.sh b/entrypoint.sh index 0078d54..c3742fe 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -10,7 +10,7 @@ case $APP in COMMAND="uvicorn src.api:app --reload --host 0.0.0.0 --port 8000" ;; celery) - COMMAND="celery --app=src.api.service worker --concurrency=1 --loglevel=INFO" + COMMAND="celery --app=src.api.service worker --loglevel=INFO" ;; *) echo "Unknown app '$APP'" From 934d4bdf7b114bd40d086d6f2ae12db2d11db68a Mon Sep 17 00:00:00 2001 From: Tom Koscielniak Date: Tue, 14 May 2024 20:52:34 +0200 Subject: [PATCH 14/46] Fix hanging actions and remove start script --- .github/workflows/test.yml | 2 +- start_api.sh | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 start_api.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4d6ffa2..7e3ed80 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,7 @@ jobs: pip install -r src/requirements.txt - name: Start services run: | - docker-compose up + docker-compose up -d - name: Test run: | pytest diff --git a/start_api.sh b/start_api.sh deleted file mode 100644 index acbf5f8..0000000 --- a/start_api.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -# For local dev purposes -docker run --rm --name some-redis -p 6379:6379 redis:latest & -celery --app=src.api.service worker --concurrency=1 --loglevel=INFO & -uvicorn src.api:app --reload && fg From 7f30676daf70d637fef3166f82f50c2e956fb10b Mon Sep 17 00:00:00 2001 From: Tom Koscielniak Date: Tue, 14 May 2024 20:55:28 +0200 Subject: [PATCH 15/46] Remove Python 3.9 worker --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7e3ed80..53fbc38 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] + python-version: [ "3.10", "3.11", "3.12" ] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} From 1f97e5d305069177a85c76bd7094bffc39a22705 Mon Sep 17 00:00:00 2001 From: Tom Koscielniak Date: Fri, 17 May 2024 17:04:45 +0200 Subject: [PATCH 16/46] Add path query parameter support --- src/api.py | 24 ++++++++++++-- src/generators/html_generator.py | 8 ++--- src/generators/json_generator.py | 8 ++--- src/generators/yaml_generator.py | 8 ++--- src/service.py | 57 ++++++++++++++++++++++++++------ tests/test_api.py | 7 ++++ 6 files changed, 87 insertions(+), 25 deletions(-) diff --git a/src/api.py b/src/api.py index cbeb09e..e7d62b4 100644 --- a/src/api.py +++ b/src/api.py @@ -27,9 +27,11 @@ def find_test( test_url: str = Query(None, alias="test-url"), test_name: str = Query(None, alias="test-name"), test_ref: str = Query("default", alias="test-ref"), + test_path: str = Query(None, alias="test-path"), plan_url: str = Query(None, alias="plan-url"), plan_name: str = Query(None, alias="plan-name"), plan_ref: str = Query("default", alias="plan-ref"), + plan_path: str = Query(None, alias="plan-path"), out_format: str = Query("json", alias="format") ): # Parameter validations @@ -42,9 +44,27 @@ def find_test( # TODO: forward to docs # Disable Celery if not needed if os.environ.get("USE_CELERY") == "false": - html_page = service.main(test_url, test_name, test_ref, plan_url, plan_name, plan_ref, out_format) + html_page = service.main( + test_url=test_url, + test_name=test_name, + test_ref=test_ref, + plan_url=plan_url, + plan_name=plan_name, + plan_ref=plan_ref, + out_format=out_format, + test_path=test_path, + plan_path=plan_path) return html_page - r = service.main.delay(test_url, test_name, test_ref, plan_url, plan_name, plan_ref, out_format) + r = service.main.delay( + test_url=test_url, + test_name=test_name, + test_ref=test_ref, + plan_url=plan_url, + plan_name=plan_name, + plan_ref=plan_ref, + out_format=out_format, + test_path=test_path, + plan_path=plan_path) # Special handling of response if the format is html if out_format == "html": global format_html diff --git a/src/generators/html_generator.py b/src/generators/html_generator.py index abb2b25..c38c872 100644 --- a/src/generators/html_generator.py +++ b/src/generators/html_generator.py @@ -59,7 +59,7 @@ def generate_test_html_page(test: Test, logger: Logger) -> str: fmf-id:
  • url: {test.fmf_id.url}
    • -
  • path: {test.fmf_id.path}
    • +
  • path: {test.fmf_id.path.as_posix() if test.fmf_id.path is not None else None}
  • name: {test.fmf_id.name}
  • ref: {test.fmf_id.ref}
@@ -97,7 +97,7 @@ def generate_plan_html_page(plan: Plan, logger: Logger) -> str: fmf-id:
  • url: {plan.fmf_id.url}
    • -
  • path: {plan.fmf_id.path}
    • +
  • path: {plan.fmf_id.path.as_posix() if plan.fmf_id.path is not None else None}
  • name: {plan.fmf_id.name}
  • ref: {plan.fmf_id.ref}
@@ -138,7 +138,7 @@ def generate_testplan_html_page(test: tmt.Test, plan: tmt.Plan, logger: Logger) fmf-id:
  • url: {test.fmf_id.url}
    • -
  • path: {test.fmf_id.path}
    • +
  • path: {test.fmf_id.path.as_posix() if test.fmf_id.path is not None else None}
  • name: {test.fmf_id.name}
  • ref: {test.fmf_id.ref}
@@ -156,7 +156,7 @@ def generate_testplan_html_page(test: tmt.Test, plan: tmt.Plan, logger: Logger) fmf-id:
  • url: {plan.fmf_id.url}
    • -
  • path: {plan.fmf_id.path}
    • +
  • path: {plan.fmf_id.path.as_posix() if plan.fmf_id.path is not None else None}
  • name: {plan.fmf_id.name}
  • ref: {plan.fmf_id.ref}
diff --git a/src/generators/json_generator.py b/src/generators/json_generator.py index b18e28d..4517711 100644 --- a/src/generators/json_generator.py +++ b/src/generators/json_generator.py @@ -25,7 +25,7 @@ def generate_test_json(test: Test, logger: Logger) -> str: "id": test.id, "fmf-id": { "url": test.fmf_id.url, - "path": test.fmf_id.path, + "path": test.fmf_id.path.as_posix() if test.fmf_id.path is not None else None, "name": test.fmf_id.name, "ref": test.fmf_id.ref, } @@ -56,7 +56,7 @@ def generate_plan_json(plan: Plan, logger: Logger) -> str: "id": plan.id, "fmf-id": { "url": plan.fmf_id.url, - "path": plan.fmf_id.path, + "path": plan.fmf_id.path.as_posix() if plan.fmf_id.path is not None else None, "name": plan.fmf_id.name, "ref": plan.fmf_id.ref, } @@ -90,7 +90,7 @@ def generate_testplan_json(test: tmt.Test, plan: tmt.Plan, logger: Logger) -> st "id": test.id, "fmf-id": { "url": test.fmf_id.url, - "path": test.fmf_id.path, + "path": test.fmf_id.path.as_posix() if test.fmf_id.path is not None else None, "name": test.fmf_id.name, "ref": test.fmf_id.ref, } @@ -107,7 +107,7 @@ def generate_testplan_json(test: tmt.Test, plan: tmt.Plan, logger: Logger) -> st "id": plan.id, "fmf-id": { "url": plan.fmf_id.url, - "path": plan.fmf_id.path, + "path": plan.fmf_id.path.as_posix() if plan.fmf_id.path is not None else None, "name": plan.fmf_id.name, "ref": plan.fmf_id.ref, } diff --git a/src/generators/yaml_generator.py b/src/generators/yaml_generator.py index 7784178..e392382 100644 --- a/src/generators/yaml_generator.py +++ b/src/generators/yaml_generator.py @@ -24,7 +24,7 @@ def generate_test_yaml(test: tmt.Test, logger: Logger) -> str: "id": test.id, "fmf-id": { "url": test.fmf_id.url, - "path": test.fmf_id.path, + "path": test.fmf_id.path.as_posix() if test.fmf_id.path is not None else None, "name": test.fmf_id.name, "ref": test.fmf_id.ref, } @@ -55,7 +55,7 @@ def generate_plan_yaml(plan: tmt.Plan, logger: Logger) -> str: "id": plan.id, "fmf-id": { "url": plan.fmf_id.url, - "path": plan.fmf_id.path, + "path": plan.fmf_id.path.as_posix() if plan.fmf_id.path is not None else None, "name": plan.fmf_id.name, "ref": plan.fmf_id.ref, } @@ -89,7 +89,7 @@ def generate_testplan_yaml(test: tmt.Test, plan: tmt.Plan, logger: Logger) -> st "id": test.id, "fmf-id": { "url": test.fmf_id.url, - "path": test.fmf_id.path, + "path": test.fmf_id.path.as_posix() if test.fmf_id.path is not None else None, "name": test.fmf_id.name, "ref": test.fmf_id.ref, } @@ -106,7 +106,7 @@ def generate_testplan_yaml(test: tmt.Test, plan: tmt.Plan, logger: Logger) -> st "id": plan.id, "fmf-id": { "url": plan.fmf_id.url, - "path": plan.fmf_id.path, + "path": plan.fmf_id.path.as_posix() if plan.fmf_id.path is not None else None, "name": plan.fmf_id.name, "ref": plan.fmf_id.ref, } diff --git a/src/service.py b/src/service.py index df234f1..6796fee 100644 --- a/src/service.py +++ b/src/service.py @@ -1,4 +1,6 @@ import os +from pathlib import Path + import tmt import logging from src.utils import git_handler as utils @@ -13,12 +15,13 @@ app = Celery(__name__, broker=redis_url, backend=redis_url) -def get_tree(url: str, name: str, ref: str) -> tmt.base.Tree: +def get_tree(url: str, name: str, ref: str, tree_path: str) -> tmt.base.Tree: """ This function clones the repository and returns the Tree object :param ref: Object ref :param name: Object name :param url: Object url + :param tree_path: Object path :return: """ logger.print("Cloning the repository for url: " + url) @@ -28,24 +31,37 @@ def get_tree(url: str, name: str, ref: str) -> tmt.base.Tree: path = utils.get_git_repository(url, logger, ref) + if tree_path is not None: + # If path is set, construct a path to the tmt Tree + if '.git' == path.suffix: + path = path.with_suffix('') + path = path.as_posix() + tree_path + path = Path(path) + logger.print("Looking for tree...") tree = tmt.base.Tree(path=path, logger=logger) logger.print("Tree found!", color="green") return tree -def process_test_request(test_url: str, test_name: str, test_ref: str, return_object: bool, out_format: str) -> str | None | tmt.Test: +def process_test_request(test_url: str, + test_name: str, + test_ref: str, + test_path: str, + return_object: bool, + out_format: str) -> str | None | tmt.Test: """ This function processes the request for a test and returns the HTML file or the Test object :param test_url: Test url :param test_name: Test name :param test_ref: Test repo ref + :param test_path: Test path :param return_object: Specify if the function should return the HTML file or the Test object :param out_format: Specifies output format :return: """ - tree = get_tree(test_url, test_name, test_ref) + tree = get_tree(test_url, test_name, test_ref, test_path) logger.print("Looking for the wanted test...") @@ -66,18 +82,24 @@ def process_test_request(test_url: str, test_name: str, test_ref: str, return_ob return yaml_generator.generate_test_yaml(wanted_test, logger=logger) -def process_plan_request(plan_url: str, plan_name: str, plan_ref: str, return_object: bool, out_format: str) -> str | None | tmt.Plan: +def process_plan_request(plan_url: str, + plan_name: str, + plan_ref: str, + plan_path:str, + return_object: bool, + out_format: str) -> str | None | tmt.Plan: """ This function processes the request for a plan and returns the HTML file or the Plan object :param plan_url: Plan URL :param plan_name: Plan name :param plan_ref: Plan repo ref + :param plan_path: Plan path :param return_object: Specify if the function should return the HTML file or the Plan object :param out_format: Specifies output format :return: """ - tree = get_tree(plan_url, plan_name, plan_ref) + tree = get_tree(plan_url, plan_name, plan_ref, plan_path) logger.print("Looking for the wanted plan...") @@ -98,20 +120,30 @@ def process_plan_request(plan_url: str, plan_name: str, plan_ref: str, return_ob return yaml_generator.generate_plan_yaml(wanted_plan, logger=logger) -def process_testplan_request(test_url, test_name, test_ref, plan_url, plan_name, plan_ref, out_format) -> str | None: +def process_testplan_request(test_url, + test_name, + test_ref, + test_path, + plan_url, + plan_name, + plan_ref, + plan_path, + out_format) -> str | None: """ This function processes the request for a test and a plan and returns the HTML file :param test_url: Test URL :param test_name: Test name :param test_ref: Test repo ref + :param test_path: Test path :param plan_url: Plan URL :param plan_name: Plan name :param plan_ref: Plan repo ref + :param plan_path: Plan path :param out_format: Specifies output format :return: """ - test = process_test_request(test_url, test_name, test_ref, False, out_format) - plan = process_plan_request(plan_url, plan_name, plan_ref, False, out_format) + test = process_test_request(test_url, test_name, test_ref, test_path, False, out_format) + plan = process_plan_request(plan_url, plan_name, plan_ref, plan_path, False, out_format) match out_format: case "html": return html.generate_testplan_html_page(test, plan, logger=logger) @@ -125,17 +157,20 @@ def process_testplan_request(test_url, test_name, test_ref, plan_url, plan_name, def main(test_url: str | None, test_name: str | None, test_ref: str | None, + test_path: str | None, plan_url: str | None, plan_name: str | None, plan_ref: str | None, + plan_path: str | None, out_format: str | None) -> str | None: logger.print("Starting...", color="blue") if test_name is not None and plan_name is None: - return process_test_request(test_url, test_name, test_ref, True, out_format) + return process_test_request(test_url, test_name, test_ref, test_path, True, out_format) elif plan_name is not None and test_name is None: - return process_plan_request(plan_url, plan_name, plan_ref, True, out_format) + return process_plan_request(plan_url, plan_name, plan_ref, plan_path, True, out_format) elif plan_name is not None and test_name is not None: - return process_testplan_request(test_url, test_name, test_ref, plan_url, plan_name, plan_ref, out_format) + return process_testplan_request(test_url, test_name, test_ref, test_path, + plan_url, plan_name, plan_ref, plan_path, out_format) if __name__ == "__main__": diff --git a/tests/test_api.py b/tests/test_api.py index 5922e6c..706b65f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -25,6 +25,13 @@ def test_basic_test_request_json(self, client): assert "500" not in data assert "https://github.com/teemtee/tmt/tree/main/tests/core/smoke/main.fmf" in data + def test_basic_test_request_json_with_path(self, client): + response = client.get("/?test-url=https://github.com/teemtee/tmt.git&test-name=/test/shell/weird&" + "test-path=/tests/execute/basic/data&test-ref=link-issues-to-jira") + data = response.content.decode("utf-8") + assert "500" not in data + assert "https://github.com/teemtee/tmt/tree/main/tests/execute/basic/data/test.fmf" in data + def test_basic_test_request_html(self, client): response = client.get( "/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke&test-ref=main&format=html") From 6aa0b1bcc5505f2108aa1282cf2584e2c3f0a6bd Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Tue, 13 Aug 2024 18:28:43 +0200 Subject: [PATCH 17/46] Add .venv to .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index dc4d3ac..6a48259 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .pytest_cache .repos venv +.venv .idea .vscode -__pycache__ \ No newline at end of file +__pycache__ From cb984bd25caf0e3d8dde62597e611a1f57a3288c Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Tue, 13 Aug 2024 18:29:05 +0200 Subject: [PATCH 18/46] Update requirements to latest, compatible version --- src/requirements.txt | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/requirements.txt b/src/requirements.txt index 057076e..212e33a 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,10 +1,8 @@ -Flask>=2.2.x -fmf==1.3.0 -pytest==8.0.2 -requests==2.31.0 -tmt==1.31.0 -FastAPI==0.70.0 -httpx==0.20.0 -Redis -uvicorn -celery +fmf~=1.4 +pytest~=8.3 +tmt~=1.35 +fastapi~=0.112 +httpx~=0.27 +redis~=5.0 +uvicorn~=0.30 +celery~=5.4 From 9e67ee28793fbd77e5138158a03817548da697d5 Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Tue, 13 Aug 2024 19:18:42 +0200 Subject: [PATCH 19/46] Split status to status and status/html --- src/api.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/api.py b/src/api.py index e7d62b4..9072b56 100644 --- a/src/api.py +++ b/src/api.py @@ -69,7 +69,7 @@ def find_test( if out_format == "html": global format_html format_html = True - status_callback_url = f'{os.getenv("API_HOSTNAME")}/status?task-id={r.task_id}&html=true' + status_callback_url = f'{os.getenv("API_HOSTNAME")}/status/html?task-id={r.task_id}' return HTMLResponse(content=html_generator.generate_status_callback(r, status_callback_url)) else: format_html = False # To set it back to False after a html format request @@ -77,14 +77,16 @@ def find_test( @app.get("/status") -def status(task_id: str = Query(None, alias="task-id"), - html: str = Query("false")) -> TaskOut | HTMLResponse: +def status(task_id: str = Query(None, alias="task-id")) -> TaskOut: r = service.main.app.AsyncResult(task_id) - if html == "true": - status_callback_url = f'{os.getenv("API_HOSTNAME")}/status?task-id={r.task_id}&html=true' - return HTMLResponse(content=html_generator.generate_status_callback(r, status_callback_url)) return _to_task_out(r) +@app.get("/status/html") +def status_html(task_id: str = Query(None, alias="task-id")) -> HTMLResponse: + r = service.main.app.AsyncResult(task_id) + status_callback_url = f'{os.getenv("API_HOSTNAME")}/status/html?task-id={r.task_id}' + return HTMLResponse(content=html_generator.generate_status_callback(r, status_callback_url)) + def _to_task_out(r: AsyncResult) -> TaskOut | str: return TaskOut( From 5c34317fff48cefe18d8cbdbe75013da9cb83afb Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Tue, 13 Aug 2024 19:23:11 +0200 Subject: [PATCH 20/46] Sort imports --- src/api.py | 6 +++--- src/generators/html_generator.py | 2 +- src/generators/json_generator.py | 2 +- src/generators/yaml_generator.py | 1 - src/service.py | 9 +++++---- src/utils/git_handler.py | 7 ++++--- 6 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/api.py b/src/api.py index 9072b56..669902d 100644 --- a/src/api.py +++ b/src/api.py @@ -1,13 +1,13 @@ import os +from celery.result import AsyncResult +from fastapi import FastAPI from fastapi.params import Query +from pydantic import BaseModel from starlette.responses import HTMLResponse from src import service from src.generators import html_generator -from fastapi import FastAPI -from pydantic import BaseModel -from celery.result import AsyncResult app = FastAPI() format_html = False diff --git a/src/generators/html_generator.py b/src/generators/html_generator.py index c38c872..6cfb844 100644 --- a/src/generators/html_generator.py +++ b/src/generators/html_generator.py @@ -1,6 +1,6 @@ import tmt from celery.result import AsyncResult -from tmt import Test, Logger, Plan +from tmt import Logger, Plan, Test def generate_status_callback(r: AsyncResult, status_callback_url: str) -> str: diff --git a/src/generators/json_generator.py b/src/generators/json_generator.py index 4517711..010a195 100644 --- a/src/generators/json_generator.py +++ b/src/generators/json_generator.py @@ -1,7 +1,7 @@ import json import tmt -from tmt import Test, Logger, Plan +from tmt import Logger, Plan, Test def generate_test_json(test: Test, logger: Logger) -> str: diff --git a/src/generators/yaml_generator.py b/src/generators/yaml_generator.py index e392382..f63cbf8 100644 --- a/src/generators/yaml_generator.py +++ b/src/generators/yaml_generator.py @@ -1,6 +1,5 @@ import tmt from tmt import Logger -from tmt import utils def generate_test_yaml(test: tmt.Test, logger: Logger) -> str: diff --git a/src/service.py b/src/service.py index 6796fee..177aefd 100644 --- a/src/service.py +++ b/src/service.py @@ -1,13 +1,14 @@ +import logging import os from pathlib import Path import tmt -import logging -from src.utils import git_handler as utils -from src.generators import json_generator, html_generator as html -from src.generators import yaml_generator from celery.app import Celery +from src.generators import html_generator as html +from src.generators import json_generator, yaml_generator +from src.utils import git_handler as utils + logger = tmt.Logger(logging.Logger("tmt-logger")) redis_url = os.getenv("REDIS_URL", "redis://localhost:6379") diff --git a/src/utils/git_handler.py b/src/utils/git_handler.py index 88e450c..64be8b2 100644 --- a/src/utils/git_handler.py +++ b/src/utils/git_handler.py @@ -1,8 +1,9 @@ -from subprocess import Popen -from tmt import Logger import os -import tmt.utils from pathlib import Path +from subprocess import Popen + +import tmt.utils +from tmt import Logger def checkout_branch(path: Path, logger: Logger, ref: str) -> None: From 09eef626f7046f93b031d8fadb81a918bd4c3374 Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Tue, 13 Aug 2024 19:59:41 +0200 Subject: [PATCH 21/46] Code style, typing, tmt.Path usage --- src/api.py | 5 ++-- src/generators/json_generator.py | 12 ++++---- src/generators/yaml_generator.py | 12 ++++---- src/service.py | 32 +++++++++++++------- src/utils/git_handler.py | 50 ++++++++++++++------------------ 5 files changed, 58 insertions(+), 53 deletions(-) diff --git a/src/api.py b/src/api.py index 669902d..d35c2ac 100644 --- a/src/api.py +++ b/src/api.py @@ -44,7 +44,7 @@ def find_test( # TODO: forward to docs # Disable Celery if not needed if os.environ.get("USE_CELERY") == "false": - html_page = service.main( + return service.main( test_url=test_url, test_name=test_name, test_ref=test_ref, @@ -54,7 +54,6 @@ def find_test( out_format=out_format, test_path=test_path, plan_path=plan_path) - return html_page r = service.main.delay( test_url=test_url, test_name=test_name, @@ -77,7 +76,7 @@ def find_test( @app.get("/status") -def status(task_id: str = Query(None, alias="task-id")) -> TaskOut: +def status(task_id: str = Query(None, alias="task-id")) -> TaskOut | str: r = service.main.app.AsyncResult(task_id) return _to_task_out(r) diff --git a/src/generators/json_generator.py b/src/generators/json_generator.py index 010a195..bf3a714 100644 --- a/src/generators/json_generator.py +++ b/src/generators/json_generator.py @@ -30,9 +30,9 @@ def generate_test_json(test: Test, logger: Logger) -> str: "ref": test.fmf_id.ref, } } - data = json.dumps(data) + json_data = json.dumps(data) logger.print("JSON file generated successfully!", color="green") - return data + return json_data def generate_plan_json(plan: Plan, logger: Logger) -> str: @@ -61,9 +61,9 @@ def generate_plan_json(plan: Plan, logger: Logger) -> str: "ref": plan.fmf_id.ref, } } - data = json.dumps(data) + json_data = json.dumps(data) logger.print("JSON file generated successfully!", color="green") - return data + return json_data def generate_testplan_json(test: tmt.Test, plan: tmt.Plan, logger: Logger) -> str: @@ -113,6 +113,6 @@ def generate_testplan_json(test: tmt.Test, plan: tmt.Plan, logger: Logger) -> st } } } - data = json.dumps(data) + json_data = json.dumps(data) logger.print("JSON file generated successfully!", color="green") - return data + return json_data diff --git a/src/generators/yaml_generator.py b/src/generators/yaml_generator.py index f63cbf8..0768520 100644 --- a/src/generators/yaml_generator.py +++ b/src/generators/yaml_generator.py @@ -28,9 +28,9 @@ def generate_test_yaml(test: tmt.Test, logger: Logger) -> str: "ref": test.fmf_id.ref, } } - data = tmt.utils.dict_to_yaml(data) + yaml_data = tmt.utils.dict_to_yaml(data) logger.print("YAML file generated successfully!", color="green") - return data + return yaml_data def generate_plan_yaml(plan: tmt.Plan, logger: Logger) -> str: @@ -59,9 +59,9 @@ def generate_plan_yaml(plan: tmt.Plan, logger: Logger) -> str: "ref": plan.fmf_id.ref, } } - data = tmt.utils.dict_to_yaml(data) + yaml_data = tmt.utils.dict_to_yaml(data) logger.print("YAML file generated successfully!", color="green") - return data + return yaml_data def generate_testplan_yaml(test: tmt.Test, plan: tmt.Plan, logger: Logger) -> str: @@ -111,6 +111,6 @@ def generate_testplan_yaml(test: tmt.Test, plan: tmt.Plan, logger: Logger) -> st } } } - data = tmt.utils.dict_to_yaml(data) + yaml_data = tmt.utils.dict_to_yaml(data) logger.print("YAML file generated successfully!", color="green") - return data + return yaml_data diff --git a/src/service.py b/src/service.py index 177aefd..aebbea7 100644 --- a/src/service.py +++ b/src/service.py @@ -1,15 +1,15 @@ import logging import os -from pathlib import Path import tmt from celery.app import Celery +from tmt.utils import Path from src.generators import html_generator as html from src.generators import json_generator, yaml_generator from src.utils import git_handler as utils -logger = tmt.Logger(logging.Logger("tmt-logger")) +logger = tmt.Logger(logging.getLogger("tmt-logger")) redis_url = os.getenv("REDIS_URL", "redis://localhost:6379") @@ -33,11 +33,12 @@ def get_tree(url: str, name: str, ref: str, tree_path: str) -> tmt.base.Tree: path = utils.get_git_repository(url, logger, ref) if tree_path is not None: + tree_path += '/' # If path is set, construct a path to the tmt Tree - if '.git' == path.suffix: + if path.suffix == '.git': path = path.with_suffix('') - path = path.as_posix() + tree_path - path = Path(path) + path = Path(path.as_posix() + tree_path) + logger.print("Looking for tree...") tree = tmt.base.Tree(path=path, logger=logger) @@ -50,7 +51,7 @@ def process_test_request(test_url: str, test_ref: str, test_path: str, return_object: bool, - out_format: str) -> str | None | tmt.Test: + out_format: str) -> str | tmt.Test | None: """ This function processes the request for a test and returns the HTML file or the Test object :param test_url: Test url @@ -68,7 +69,7 @@ def process_test_request(test_url: str, # Find the desired Test object wanted_test = tree.tests(names=[test_name])[0] - if wanted_test is []: + if wanted_test == []: logger.print("Test not found!", color="red") return None logger.print("Test found!", color="green") @@ -81,6 +82,7 @@ def process_test_request(test_url: str, return json_generator.generate_test_json(wanted_test, logger=logger) case "yaml": return yaml_generator.generate_test_yaml(wanted_test, logger=logger) + return None def process_plan_request(plan_url: str, @@ -106,7 +108,7 @@ def process_plan_request(plan_url: str, # Find the desired Plan object wanted_plan = tree.plans(names=[plan_name])[0] - if wanted_plan is []: + if wanted_plan == []: logger.print("Plan not found!", color="red") return None logger.print("Plan found!", color="green") @@ -116,9 +118,10 @@ def process_plan_request(plan_url: str, case "html": return html.generate_plan_html_page(wanted_plan, logger=logger) case "json": - return json_generator.generate_plan_json(wanted_plan, logger=logger) + return json_generator.generate_plan_json(wanted_plan, logger=logger) case "yaml": return yaml_generator.generate_plan_yaml(wanted_plan, logger=logger) + return None def process_testplan_request(test_url, @@ -144,7 +147,13 @@ def process_testplan_request(test_url, :return: """ test = process_test_request(test_url, test_name, test_ref, test_path, False, out_format) + if not isinstance(test, tmt.Test): + logger.print("Invalid test object", color="red") + return None plan = process_plan_request(plan_url, plan_name, plan_ref, plan_path, False, out_format) + if not isinstance(plan, tmt.Plan): + logger.print("Invalid plan object", color="red") + return None match out_format: case "html": return html.generate_testplan_html_page(test, plan, logger=logger) @@ -153,6 +162,8 @@ def process_testplan_request(test_url, case "yaml": return yaml_generator.generate_testplan_yaml(test, plan, logger=logger) + return None + @app.task def main(test_url: str | None, @@ -163,7 +174,7 @@ def main(test_url: str | None, plan_name: str | None, plan_ref: str | None, plan_path: str | None, - out_format: str | None) -> str | None: + out_format: str) -> str | tmt.Test | tmt.Plan | None: logger.print("Starting...", color="blue") if test_name is not None and plan_name is None: return process_test_request(test_url, test_name, test_ref, test_path, True, out_format) @@ -172,6 +183,7 @@ def main(test_url: str | None, elif plan_name is not None and test_name is not None: return process_testplan_request(test_url, test_name, test_ref, test_path, plan_url, plan_name, plan_ref, plan_path, out_format) + return None if __name__ == "__main__": diff --git a/src/utils/git_handler.py b/src/utils/git_handler.py index 64be8b2..e77e04d 100644 --- a/src/utils/git_handler.py +++ b/src/utils/git_handler.py @@ -1,9 +1,9 @@ +import contextlib import os -from pathlib import Path -from subprocess import Popen +from shutil import rmtree -import tmt.utils from tmt import Logger +from tmt.utils import Command, Common, GeneralError, Path, RunError, git_clone def checkout_branch(path: Path, logger: Logger, ref: str) -> None: @@ -15,12 +15,12 @@ def checkout_branch(path: Path, logger: Logger, ref: str) -> None: :return: """ try: - common_instance = tmt.utils.Common(logger=logger) + common_instance = Common(logger=logger) common_instance.run( - command=tmt.utils.Command('git', 'checkout', ref), cwd=path) - except tmt.utils.RunError: + command=Command('git', 'checkout', ref), cwd=path) + except RunError as err: logger.print("Failed to do checkout in the repository!", color="red") - raise AttributeError + raise AttributeError from err def clone_repository(url: str, logger: Logger, ref: str) -> None: @@ -38,21 +38,21 @@ def clone_repository(url: str, logger: Logger, ref: str) -> None: if ref != "default": try: checkout_branch(ref=ref, path=path, logger=logger) - except AttributeError: - raise AttributeError + except AttributeError as err: + raise AttributeError from err logger.print("Repository already cloned!", color="yellow") raise FileExistsError try: - tmt.utils.git_clone(url=url, destination=path, logger=logger) + git_clone(url=url, destination=path, logger=logger) if ref != "default": try: checkout_branch(ref=ref, path=path, logger=logger) - except AttributeError: - raise AttributeError + except AttributeError as err: + raise AttributeError from err checkout_branch(ref=ref, path=path, logger=logger) - except tmt.utils.GeneralError as e: + except GeneralError as e: logger.print("Failed to clone the repository!", color="red") - raise Exception + raise Exception from e logger.print("Repository cloned successfully!", color="green") @@ -62,11 +62,9 @@ def get_path_to_repository(url: str) -> Path: :param url: URL to the repository :return: Path to the cloned repository """ - repo_name = url.rsplit('/', 1)[-1] - path = os.path.realpath(__file__) - path = path.replace("src/utils/git_handler.py", "") - path = Path(path + os.getenv("CLONE_DIR_PATH", "./.repos/") + repo_name) - return path + repo_name = url.rstrip('/').rsplit('/', 1)[-1] + root_dir = Path(__file__).resolve().parents[2] # going up from src/utils/git_handler.py + return root_dir / os.getenv("CLONE_DIR_PATH", "./.repos/") / repo_name def check_if_repository_exists(url: str) -> bool: @@ -75,8 +73,7 @@ def check_if_repository_exists(url: str) -> bool: :param url: URL to the repository :return: True if the repository is already cloned, False otherwise """ - path = get_path_to_repository(url) - return path.exists() + return get_path_to_repository(url).exists() def clear_tmp_dir(logger: Logger) -> None: @@ -86,11 +83,10 @@ def clear_tmp_dir(logger: Logger) -> None: :return: """ logger.print("Clearing the .tmp directory...") - path = os.path.realpath(__file__) - path = path.replace("src/utils/git_handler.py", "") - path = Path(path + os.getenv("CLONE_DIR_PATH", "./.repos/")) + root_dir = Path(__file__).resolve().parents[2] # going up from src/utils/git_handler.py + path = root_dir / os.getenv("CLONE_DIR_PATH", "./.repos/") try: - Popen(["rm", "-rf", path]) + rmtree(path) except Exception as e: logger.print("Failed to clear the repository clone directory!", color="red") raise e @@ -104,10 +100,8 @@ def get_git_repository(url: str, logger: Logger, ref: str) -> Path: :param logger: Instance of Logger :return: Path to the cloned repository """ - try: + with contextlib.suppress(FileExistsError): clone_repository(url, logger, ref) - except FileExistsError: - pass return get_path_to_repository(url) From 99b517b9f25c2e0d3e464e4d2ad8102c5b173429 Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Wed, 14 Aug 2024 15:41:36 +0200 Subject: [PATCH 22/46] Use podman-compose in CI --- .github/workflows/test.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 53fbc38..8b06715 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,12 +23,13 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install pytest - pip install -r src/requirements.txt + sudo apt-get update -y + sudo apt-get install podman -y + python -m pip install -U pip podman-compose + python -m pip install -r src/requirements.txt - name: Start services run: | - docker-compose up -d + podman-compose up -d - name: Test run: | pytest From e1c4ad7479d77b7ec3cb1e2991507bf49f48efca Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Thu, 15 Aug 2024 15:03:56 +0200 Subject: [PATCH 23/46] Remove tmt-web yaml file --- compose.yaml | 9 +++++++-- tmt-web.yaml | 39 --------------------------------------- 2 files changed, 7 insertions(+), 41 deletions(-) delete mode 100644 tmt-web.yaml diff --git a/compose.yaml b/compose.yaml index 2c051f5..5f37a6c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,7 +1,9 @@ services: web: container_name: uvicorn - build: . + build: + context: . + dockerfile: ./Containerfile command: uvicorn src.api:app --reload --host 0.0.0.0 --port 8000 environment: - REDIS_URL=redis://redis:6379 @@ -16,7 +18,9 @@ services: celery: container_name: celery - build: . + build: + context: . + dockerfile: ./Containerfile command: celery --app=src.api.service worker --loglevel=INFO environment: - REDIS_URL=redis://redis:6379 @@ -24,3 +28,4 @@ services: depends_on: - redis + diff --git a/tmt-web.yaml b/tmt-web.yaml deleted file mode 100644 index 7546a0d..0000000 --- a/tmt-web.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - creationTimestamp: "2024-04-23T11:20:25Z" - labels: - app: tmt-web - name: tmt-web -spec: - containers: - # Uvicorn - - image: quay.io/testing-farm/tmt-web:latest - args: - - uvicorn - name: uvicorn - ports: - - containerPort: 8000 - hostPort: 8000 - env: - - name: REDIS_URL - value: redis://redis:6379 - - name: API_HOSTNAME - value: http://localhost:8000 - # Celery - - image: quay.io/testing-farm/tmt-web:latest - args: - - celery - name: celery - env: - - name: REDIS_URL - value: redis://redis:6379 - - name: API_HOSTNAME - value: http://localhost:8000 - # Redis - - image: redis:latest - name: redis - ports: - - containerPort: 6379 - hostPort: 6379 - From d7bdc9a1181d869a38965dcb1133ea5b52aa1276 Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Thu, 15 Aug 2024 15:17:57 +0200 Subject: [PATCH 24/46] Refactoring github workflow --- .github/workflows/test.yml | 80 +++++++++++++++++++++++++++++-------- Dockerfile => Containerfile | 3 +- src/requirements.txt | 5 +-- 3 files changed, 67 insertions(+), 21 deletions(-) rename Dockerfile => Containerfile (77%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8b06715..adedbe1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,30 +6,78 @@ on: pull_request: branches: [ "main" ] -permissions: - contents: read - jobs: - build: + test: runs-on: ubuntu-latest + strategy: matrix: - python-version: [ "3.10", "3.11", "3.12" ] + python-version: [ "3.10", "3.12" ] + steps: - - uses: actions/checkout@v4 + + - name: Check out code + uses: actions/checkout@v4 + + - name: Set up Podman and Buildah + run: | + sudo apt-get update + sudo apt-get install -y podman buildah + - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies + python-version: '${{ matrix.python-version }}' + + - name: Install pytest run: | - sudo apt-get update -y - sudo apt-get install podman -y - python -m pip install -U pip podman-compose + python -m pip install pytest python -m pip install -r src/requirements.txt - - name: Start services + + + - name: Build the web image + run: | + buildah bud -t tmt-web:latest --build-arg PYTHON_VERSION=${{ matrix.python-version }} . + + - name: Create Podman pod run: | - podman-compose up -d - - name: Test + podman pod create --name tmt-web-pod -p 8000:8000 -p 6379:6379 || true + # I don't even know, now it threw "docker://k8s.gcr.io/pause:3.5: Requesting bear token: invalid status code from registry 404" + # Exposing redis port as well for test_api.py::TestCelery::test_basic_test_request + + - name: Start Redis container + run: | + podman run -d --pod tmt-web-pod --name redis redis:latest + + - name: Start Celery container + run: | + podman run -d --pod tmt-web-pod --name celery \ + -e REDIS_URL=redis://localhost:6379 \ + -e API_HOSTNAME=http://localhost:8000 \ + tmt-web:latest celery --app=src.api.service worker --loglevel=INFO + + - name: Start Web container + run: | + podman run -d --pod tmt-web-pod --name web \ + -e REDIS_URL=redis://localhost:6379 \ + -e API_HOSTNAME=http://localhost:8000 \ + tmt-web:latest uvicorn src.api:app --reload --host 0.0.0.0 --port 8000 + + - name: Wait for services to be ready + run: | + for i in {1..30}; do + if curl -s http://localhost:8000/health; then + break + fi + sleep 4 + done + + - name: Run tests + run: | + python -m pytest + + - name: Cleanup + if: always() run: | - pytest + podman pod stop tmt-web-pod + podman pod rm tmt-web-pod diff --git a/Dockerfile b/Containerfile similarity index 77% rename from Dockerfile rename to Containerfile index 97d4e5b..dfc37c1 100644 --- a/Dockerfile +++ b/Containerfile @@ -1,4 +1,5 @@ -FROM python:3.10 +ARG PYTHON_VERSION=3.12 +FROM python:${PYTHON_VERSION} RUN mkdir /app WORKDIR /app diff --git a/src/requirements.txt b/src/requirements.txt index 212e33a..e3dc82a 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,8 +1,5 @@ -fmf~=1.4 -pytest~=8.3 tmt~=1.35 fastapi~=0.112 httpx~=0.27 -redis~=5.0 uvicorn~=0.30 -celery~=5.4 +celery[redis]~=5.4 From 62f2c1a1416d8afdcef24927216b493a0fe90386 Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Thu, 15 Aug 2024 19:01:02 +0200 Subject: [PATCH 25/46] Adding /health check --- src/api.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/api.py b/src/api.py index d35c2ac..68fd2a2 100644 --- a/src/api.py +++ b/src/api.py @@ -94,3 +94,7 @@ def _to_task_out(r: AsyncResult) -> TaskOut | str: result=r.traceback if r.failed() else r.result, status_callback_url=f'{os.getenv("API_HOSTNAME")}/status?task-id={r.task_id}' ) + +@app.get("/health") +def health_check(): + return {"status": "healthy"} From d40d34ec5887434715779bf3df6dbff1188393c6 Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Fri, 16 Aug 2024 13:04:08 +0200 Subject: [PATCH 26/46] Use Annotated Co-authored-by: Patrik Hagara --- src/api.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/api.py b/src/api.py index 68fd2a2..e527d96 100644 --- a/src/api.py +++ b/src/api.py @@ -1,4 +1,5 @@ import os +from typing import Annotated, Literal from celery.result import AsyncResult from fastapi import FastAPI @@ -24,7 +25,14 @@ class TaskOut(BaseModel): # or for plans: https://tmt.org/?plan-url=https://github.com/teemtee/tmt&plan-name=/plans/features/basic @app.get("/") def find_test( - test_url: str = Query(None, alias="test-url"), + test_url: Annotated[ + str | None, + Query( + alias="test-url", + title="Test URL", + description="URL of a Git repository containing test metadata", + ), + ] = None, test_name: str = Query(None, alias="test-name"), test_ref: str = Query("default", alias="test-ref"), test_path: str = Query(None, alias="test-path"), @@ -32,7 +40,7 @@ def find_test( plan_name: str = Query(None, alias="plan-name"), plan_ref: str = Query("default", alias="plan-ref"), plan_path: str = Query(None, alias="plan-path"), - out_format: str = Query("json", alias="format") + out_format: Annotated[Literal["html", "json", "yaml"], Query(alias="format")] = "json", ): # Parameter validations if (test_url is None and test_name is not None) or (test_url is not None and test_name is None): From ab640a03eb2da43210e807eba8d4b24b0f78089e Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Fri, 16 Aug 2024 13:43:36 +0200 Subject: [PATCH 27/46] Adding metadata, modifying file structure --- .github/workflows/test.yml | 12 +- .gitignore | 3 + Containerfile | 5 +- entrypoint.sh | 4 +- pyproject.toml | 105 ++++++++++++++++++ src/requirements.txt | 5 - src/{ => tmt_web}/__init__.py | 0 src/{ => tmt_web}/api.py | 0 src/{ => tmt_web}/generators/__init__.py | 0 .../generators/html_generator.py | 0 .../generators/json_generator.py | 0 .../generators/yaml_generator.py | 0 src/{ => tmt_web}/service.py | 0 src/{ => tmt_web}/utils/__init__.py | 0 src/{ => tmt_web}/utils/git_handler.py | 0 15 files changed, 119 insertions(+), 15 deletions(-) create mode 100644 pyproject.toml delete mode 100644 src/requirements.txt rename src/{ => tmt_web}/__init__.py (100%) rename src/{ => tmt_web}/api.py (100%) rename src/{ => tmt_web}/generators/__init__.py (100%) rename src/{ => tmt_web}/generators/html_generator.py (100%) rename src/{ => tmt_web}/generators/json_generator.py (100%) rename src/{ => tmt_web}/generators/yaml_generator.py (100%) rename src/{ => tmt_web}/service.py (100%) rename src/{ => tmt_web}/utils/__init__.py (100%) rename src/{ => tmt_web}/utils/git_handler.py (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index adedbe1..3417792 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,11 +28,12 @@ jobs: uses: actions/setup-python@v5 with: python-version: '${{ matrix.python-version }}' + cache: 'pip' - name: Install pytest run: | - python -m pip install pytest - python -m pip install -r src/requirements.txt + python -m pip install pytest hatch + python -m pip install . - name: Build the web image @@ -41,8 +42,7 @@ jobs: - name: Create Podman pod run: | - podman pod create --name tmt-web-pod -p 8000:8000 -p 6379:6379 || true - # I don't even know, now it threw "docker://k8s.gcr.io/pause:3.5: Requesting bear token: invalid status code from registry 404" + podman pod create --name tmt-web-pod --infra-image=registry.k8s.io/pause:3.9 -p 8000:8000 -p 6379:6379 # Exposing redis port as well for test_api.py::TestCelery::test_basic_test_request - name: Start Redis container @@ -54,14 +54,14 @@ jobs: podman run -d --pod tmt-web-pod --name celery \ -e REDIS_URL=redis://localhost:6379 \ -e API_HOSTNAME=http://localhost:8000 \ - tmt-web:latest celery --app=src.api.service worker --loglevel=INFO + tmt-web:latest celery --app=tmt_web.api.service worker --loglevel=INFO - name: Start Web container run: | podman run -d --pod tmt-web-pod --name web \ -e REDIS_URL=redis://localhost:6379 \ -e API_HOSTNAME=http://localhost:8000 \ - tmt-web:latest uvicorn src.api:app --reload --host 0.0.0.0 --port 8000 + tmt-web:latest uvicorn tmt_web.api:app --reload --host 0.0.0.0 --port 8000 - name: Wait for services to be ready run: | diff --git a/.gitignore b/.gitignore index 6a48259..9b1452a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,7 @@ venv .venv .idea .vscode +dist +.ruff_cache +.mypy_chache __pycache__ diff --git a/Containerfile b/Containerfile index dfc37c1..884de3d 100644 --- a/Containerfile +++ b/Containerfile @@ -3,8 +3,9 @@ FROM python:${PYTHON_VERSION} RUN mkdir /app WORKDIR /app -COPY ./src /app/src -RUN pip install -r /app/src/requirements.txt +COPY README.md LICENSE pyproject.toml src/ ./ + +RUN SETUPTOOLS_SCM_PRETEND_VERSION=0.1.0.dev0 pip install . COPY /entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh diff --git a/entrypoint.sh b/entrypoint.sh index c3742fe..ac40de3 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -7,10 +7,10 @@ APP=$1 case $APP in uvicorn) - COMMAND="uvicorn src.api:app --reload --host 0.0.0.0 --port 8000" + COMMAND="uvicorn tmt_web.api:app --reload --host 0.0.0.0 --port 8000" ;; celery) - COMMAND="celery --app=src.api.service worker --loglevel=INFO" + COMMAND="celery --app=tmt_web.api.service worker --loglevel=INFO" ;; *) echo "Unknown app '$APP'" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e47a339 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,105 @@ +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[project] +name = "tmt-web" +dynamic = ["version"] +description = 'Web app for checking tmt tests, plans and stories' +readme = "README.md" +requires-python = ">=3.8" +license = "MIT" +keywords = [] +authors = [ + { name = "Petr Splichal", email = "psplicha@redhat.com" }, + { name = "Tomas Koscielniak", email = "tkosciel@redhat.com" } +] +classifiers = [ + "Programming Language :: Python", + "Programming Language :: Python :: 3.12", +] +dependencies = [ + "tmt~=1.35", + "fastapi~=0.112", + "httpx~=0.27", + "uvicorn~=0.30", + "celery[redis]~=5.4", +] + +[project.urls] +Source = "https://github.com/teemtee/web teemtee/tmt-web" + +[tool.hatch.version] +source = "vcs" +raw-options.version_scheme = "release-branch-semver" + +[tool.hatch.envs.dev] +extra-dependencies = [ + "mypy~=1.11.1", + "ruff~=0.5.7", + "hatch", +] + +[tool.hatch.envs.dev.scripts] +check = "mypy --install-types --non-interactive src/tmt_web" + +[tool.ruff] +# Based on teemtee/tmt/pyproject.toml +line-length = 100 +target-version = "py312" +lint.select = [ + "F", # pyflakes + "E", # pycodestyle error + "W", # pycodestyle warning + "I", # isort + "N", # pep8-naming + #"D", # pydocstyle TODO + "UP", # pyupgrade + "YTT", # flake8-2020 + "ASYNC", # flake8-async + "S", # flake8-bandit + "B", # flake8-bugbear + "A", # flake8-builtins + "COM", # flake8-commas + "C4", # flake8-comprehensions + "DTZ", # flake8-datetimez + "T10", # flake8-debugger + "EXE", # flake8-executable + "ISC", # flake8-implicit-str-concat + "ICN", # flake8-import-conventions + "LOG", # flake8-logging + "G", # flake8-logging-format + "PIE", # flake8-pie + "PYI", # flake8-pyi + "PT", # flake8-pytest-style + "Q003", # avoidable-escaped-quote + "Q004", # unnecessary-escaped-quote + "RSE", # flake8-raise + "RET", # flake8-return + "SIM", # flake8-simplify + "TID", # flake8-tidy-imports + "INT", # flake8-gettext + "PGH", # pygrep-hooks + "PLC", # pylint-convention + "PLE", # pylint-error + "PLR", # pylint-refactor + "RUF", # ruff + ] +lint.ignore = [ + #"E501", # TODO line lenght + "PLR0913", # Too many arguments + "RET505", # Unnecessary 'else' after 'return' + "COM812", # Trailing comma missing +] + +[tool.ruff.lint.per-file-ignores] +# Less strict security checks in tests +"tests/*" = [ + "S101", # Assert usage + "PLR2004", # Magic value + "E501", # Line length + ] +"src/tmt_web/generators/html_generator.py" = [ + "W291", # Trailing whitespace + "E501", # Line length +] diff --git a/src/requirements.txt b/src/requirements.txt deleted file mode 100644 index e3dc82a..0000000 --- a/src/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -tmt~=1.35 -fastapi~=0.112 -httpx~=0.27 -uvicorn~=0.30 -celery[redis]~=5.4 diff --git a/src/__init__.py b/src/tmt_web/__init__.py similarity index 100% rename from src/__init__.py rename to src/tmt_web/__init__.py diff --git a/src/api.py b/src/tmt_web/api.py similarity index 100% rename from src/api.py rename to src/tmt_web/api.py diff --git a/src/generators/__init__.py b/src/tmt_web/generators/__init__.py similarity index 100% rename from src/generators/__init__.py rename to src/tmt_web/generators/__init__.py diff --git a/src/generators/html_generator.py b/src/tmt_web/generators/html_generator.py similarity index 100% rename from src/generators/html_generator.py rename to src/tmt_web/generators/html_generator.py diff --git a/src/generators/json_generator.py b/src/tmt_web/generators/json_generator.py similarity index 100% rename from src/generators/json_generator.py rename to src/tmt_web/generators/json_generator.py diff --git a/src/generators/yaml_generator.py b/src/tmt_web/generators/yaml_generator.py similarity index 100% rename from src/generators/yaml_generator.py rename to src/tmt_web/generators/yaml_generator.py diff --git a/src/service.py b/src/tmt_web/service.py similarity index 100% rename from src/service.py rename to src/tmt_web/service.py diff --git a/src/utils/__init__.py b/src/tmt_web/utils/__init__.py similarity index 100% rename from src/utils/__init__.py rename to src/tmt_web/utils/__init__.py diff --git a/src/utils/git_handler.py b/src/tmt_web/utils/git_handler.py similarity index 100% rename from src/utils/git_handler.py rename to src/tmt_web/utils/git_handler.py From 3206b80f2c671639b1ef3bab25923e4927abd062 Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Fri, 16 Aug 2024 13:55:33 +0200 Subject: [PATCH 28/46] Addressing issues found by Ruff --- src/tmt_web/api.py | 4 ++-- src/tmt_web/service.py | 10 +++++----- src/tmt_web/utils/git_handler.py | 6 +++--- tests/test_api.py | 15 ++++++++------- tests/unit/test_git_handler.py | 27 ++++++++++++++------------- tests/unit/test_html_generator.py | 4 ++-- tests/unit/test_json_generator.py | 8 ++++---- tests/unit/test_yaml_generator.py | 8 ++++---- 8 files changed, 42 insertions(+), 40 deletions(-) diff --git a/src/tmt_web/api.py b/src/tmt_web/api.py index e527d96..bca3f0b 100644 --- a/src/tmt_web/api.py +++ b/src/tmt_web/api.py @@ -7,8 +7,8 @@ from pydantic import BaseModel from starlette.responses import HTMLResponse -from src import service -from src.generators import html_generator +from tmt_web import service +from tmt_web.generators import html_generator app = FastAPI() format_html = False diff --git a/src/tmt_web/service.py b/src/tmt_web/service.py index aebbea7..eccbf2a 100644 --- a/src/tmt_web/service.py +++ b/src/tmt_web/service.py @@ -5,9 +5,9 @@ from celery.app import Celery from tmt.utils import Path -from src.generators import html_generator as html -from src.generators import json_generator, yaml_generator -from src.utils import git_handler as utils +from tmt_web.generators import html_generator as html +from tmt_web.generators import json_generator, yaml_generator +from tmt_web.utils import git_handler as utils logger = tmt.Logger(logging.getLogger("tmt-logger")) @@ -178,9 +178,9 @@ def main(test_url: str | None, logger.print("Starting...", color="blue") if test_name is not None and plan_name is None: return process_test_request(test_url, test_name, test_ref, test_path, True, out_format) - elif plan_name is not None and test_name is None: + if plan_name is not None and test_name is None: return process_plan_request(plan_url, plan_name, plan_ref, plan_path, True, out_format) - elif plan_name is not None and test_name is not None: + if plan_name is not None and test_name is not None: return process_testplan_request(test_url, test_name, test_ref, test_path, plan_url, plan_name, plan_ref, plan_path, out_format) return None diff --git a/src/tmt_web/utils/git_handler.py b/src/tmt_web/utils/git_handler.py index e77e04d..5574619 100644 --- a/src/tmt_web/utils/git_handler.py +++ b/src/tmt_web/utils/git_handler.py @@ -26,7 +26,7 @@ def checkout_branch(path: Path, logger: Logger, ref: str) -> None: def clone_repository(url: str, logger: Logger, ref: str) -> None: """ Clones the repository from the given URL. - Raises FileExistsError if the repository is already cloned and raises Exception if the cloning fails. + Raises FileExistsError if the repository is already cloned or Exception if the cloning fails. :param ref: Name of the ref to check out :param url: URL to the repository :param logger: Instance of Logger @@ -63,7 +63,7 @@ def get_path_to_repository(url: str) -> Path: :return: Path to the cloned repository """ repo_name = url.rstrip('/').rsplit('/', 1)[-1] - root_dir = Path(__file__).resolve().parents[2] # going up from src/utils/git_handler.py + root_dir = Path(__file__).resolve().parents[2] # going up from tmt_web/utils/git_handler.py return root_dir / os.getenv("CLONE_DIR_PATH", "./.repos/") / repo_name @@ -83,7 +83,7 @@ def clear_tmp_dir(logger: Logger) -> None: :return: """ logger.print("Clearing the .tmp directory...") - root_dir = Path(__file__).resolve().parents[2] # going up from src/utils/git_handler.py + root_dir = Path(__file__).resolve().parents[2] # going up from tmt_web/utils/git_handler.py path = root_dir / os.getenv("CLONE_DIR_PATH", "./.repos/") try: rmtree(path) diff --git a/tests/test_api.py b/tests/test_api.py index 706b65f..8ec5273 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -2,11 +2,12 @@ import time import pytest -from src.api import app from fastapi.testclient import TestClient +from tmt_web.api import app -@pytest.fixture() + +@pytest.fixture def client(): return TestClient(app) @@ -16,7 +17,7 @@ class TestApi: This class tests the behaviour of the API directly """ @pytest.fixture(autouse=True) - def setup(self): + def _setup(self): os.environ["USE_CELERY"] = "false" def test_basic_test_request_json(self, client): @@ -38,7 +39,7 @@ def test_basic_test_request_html(self, client): data = response.content.decode("utf-8") print(data) assert "500" not in data - assert f'' in data + assert '' in data def test_basic_test_request_yaml(self, client): response = client.get("/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke&format=yaml") @@ -97,7 +98,7 @@ class TestCelery: This class tests the API with the Celery instance """ @pytest.fixture(autouse=True) - def setup(self): + def _setup(self): os.environ["USE_CELERY"] = "true" def test_basic_test_request(self, client): @@ -114,6 +115,6 @@ def test_basic_test_request(self, client): assert "https://github.com/teemtee/tmt/tree/main/tests/core/smoke/main.fmf" in result break elif json_data["status"] == "FAILURE": - assert False + pytest.fail("status = FAILURE: " + json_data["result"]) else: - assert False + pytest.fail("Unknown status: " + json_data["status"]) diff --git a/tests/unit/test_git_handler.py b/tests/unit/test_git_handler.py index dc1a028..2dc6a53 100644 --- a/tests/unit/test_git_handler.py +++ b/tests/unit/test_git_handler.py @@ -1,16 +1,17 @@ +import contextlib +import logging import os import time from pathlib import Path import pytest import tmt -import logging -from src.utils import git_handler +from tmt_web.utils import git_handler class TestGitHandler: - logger = tmt.Logger(logging.Logger("tmt-logger")) + logger = tmt.Logger(logging.getLogger("tmt-logger")) def test_clear_tmp_dir(self): # Create test directory if it doesn't exist @@ -36,23 +37,23 @@ def test_clone_repository(self): while git_handler.check_if_repository_exists("https://github.com/teemtee/tmt") is True: git_handler.clear_tmp_dir(self.logger) time.sleep(1) - git_handler.clone_repository(url="https://github.com/teemtee/tmt", logger=self.logger, ref="default") + git_handler.clone_repository(url="https://github.com/teemtee/tmt", + logger=self.logger, ref="default") def test_clone_repository_even_if_exists(self): - try: - git_handler.clone_repository(url="https://github.com/teemtee/tmt", logger=self.logger, ref="default") - except FileExistsError: - pass + with contextlib.suppress(FileExistsError): + git_handler.clone_repository(url="https://github.com/teemtee/tmt", + logger=self.logger, ref="default") def test_clone_checkout_branch(self): - try: - git_handler.clone_repository(url="https://github.com/teemtee/tmt", logger=self.logger, ref="quay") - except FileExistsError: - pass + with contextlib.suppress(FileExistsError): + git_handler.clone_repository(url="https://github.com/teemtee/tmt", + logger=self.logger, ref="quay") def test_clone_checkout_branch_exception(self): with pytest.raises(AttributeError): - git_handler.clone_repository(url="https://github.com/teemtee/tmt", logger=self.logger, ref="quadd") + git_handler.clone_repository(url="https://github.com/teemtee/tmt", + logger=self.logger, ref="quadd") def test_checkout_branch(self): self.test_clone_repository_even_if_exists() diff --git a/tests/unit/test_html_generator.py b/tests/unit/test_html_generator.py index 824dfd3..757e672 100644 --- a/tests/unit/test_html_generator.py +++ b/tests/unit/test_html_generator.py @@ -2,12 +2,12 @@ import tmt -from src.generators import html_generator +from tmt_web.generators import html_generator class TestHtmlGenerator: def test_generate_test_html(self): - logger = tmt.Logger(logging.Logger("tmt-logger")) + logger = tmt.Logger(logging.getLogger("tmt-logger")) test = tmt.Tree(logger=logger).tests(names=["/tests/objects/sample_test"])[0] data = html_generator.generate_test_html_page(test, logger) assert 'name: /tests/objects/sample_test
' in data diff --git a/tests/unit/test_json_generator.py b/tests/unit/test_json_generator.py index ffdde83..3b98fab 100644 --- a/tests/unit/test_json_generator.py +++ b/tests/unit/test_json_generator.py @@ -2,24 +2,24 @@ import tmt -from src.generators import json_generator +from tmt_web.generators import json_generator class TestJsonGenerator: def test_generate_test_json(self): - logger = tmt.Logger(logging.Logger("tmt-logger")) + logger = tmt.Logger(logging.getLogger("tmt-logger")) test = tmt.Tree(logger=logger).tests(names=["/tests/objects/sample_test"])[0] data = json_generator.generate_test_json(test, logger) assert '"name": "/tests/objects/sample_test"' in data def test_generate_plan_json(self): - logger = tmt.Logger(logging.Logger("tmt-logger")) + logger = tmt.Logger(logging.getLogger("tmt-logger")) plan = tmt.Tree(logger=logger).plans(names=["/tests/objects/sample_plan"])[0] data = json_generator.generate_plan_json(plan, logger) assert '"name": "/tests/objects/sample_plan"' in data def test_generate_testplan_json(self): - logger = tmt.Logger(logging.Logger("tmt-logger")) + logger = tmt.Logger(logging.getLogger("tmt-logger")) test = tmt.Tree(logger=logger).tests(names=["/tests/objects/sample_test"])[0] plan = tmt.Tree(logger=logger).plans(names=["/tests/objects/sample_plan"])[0] data = json_generator.generate_testplan_json(test, plan, logger) diff --git a/tests/unit/test_yaml_generator.py b/tests/unit/test_yaml_generator.py index 357d3d7..cb62d04 100644 --- a/tests/unit/test_yaml_generator.py +++ b/tests/unit/test_yaml_generator.py @@ -2,24 +2,24 @@ import tmt -from src.generators import yaml_generator +from tmt_web.generators import yaml_generator class TestYamlGenerator: def test_generate_test_yaml(self): - logger = tmt.Logger(logging.Logger("tmt-logger")) + logger = tmt.Logger(logging.getLogger("tmt-logger")) test = tmt.Tree(logger=logger).tests(names=["/tests/objects/sample_test"])[0] data = yaml_generator.generate_test_yaml(test, logger) assert 'name: /tests/objects/sample_test' in data def test_generate_plan_yaml(self): - logger = tmt.Logger(logging.Logger("tmt-logger")) + logger = tmt.Logger(logging.getLogger("tmt-logger")) plan = tmt.Tree(logger=logger).plans(names=["/tests/objects/sample_plan"])[0] data = yaml_generator.generate_plan_yaml(plan, logger) assert 'name: /tests/objects/sample_plan' in data def test_generate_testplan_yaml(self): - logger = tmt.Logger(logging.Logger("tmt-logger")) + logger = tmt.Logger(logging.getLogger("tmt-logger")) test = tmt.Tree(logger=logger).tests(names=["/tests/objects/sample_test"])[0] plan = tmt.Tree(logger=logger).plans(names=["/tests/objects/sample_plan"])[0] data = yaml_generator.generate_testplan_yaml(test, plan, logger) From b41a0d3acb7e0e161bdc9a9c7b5dc3e2b99b97cb Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Mon, 19 Aug 2024 17:20:34 +0200 Subject: [PATCH 29/46] Resolving typing check errors --- src/tmt_web/api.py | 83 ++++++++++++++++++++---- src/tmt_web/generators/html_generator.py | 8 +-- src/tmt_web/service.py | 13 ++-- src/tmt_web/utils/git_handler.py | 9 ++- tests/unit/test_git_handler.py | 8 +-- 5 files changed, 92 insertions(+), 29 deletions(-) diff --git a/src/tmt_web/api.py b/src/tmt_web/api.py index bca3f0b..71ae698 100644 --- a/src/tmt_web/api.py +++ b/src/tmt_web/api.py @@ -1,5 +1,5 @@ import os -from typing import Annotated, Literal +from typing import Annotated, Any, Literal from celery.result import AsyncResult from fastapi import FastAPI @@ -33,15 +33,64 @@ def find_test( description="URL of a Git repository containing test metadata", ), ] = None, - test_name: str = Query(None, alias="test-name"), - test_ref: str = Query("default", alias="test-ref"), - test_path: str = Query(None, alias="test-path"), - plan_url: str = Query(None, alias="plan-url"), - plan_name: str = Query(None, alias="plan-name"), - plan_ref: str = Query("default", alias="plan-ref"), - plan_path: str = Query(None, alias="plan-path"), + test_name: Annotated[ + str | None, + Query( + alias="test-name", + title="Test name", + description="Name of the test", + ), + ] = None, + test_ref: Annotated[ + str | None, + Query( + alias="test-ref", + title="Test reference", + description="Reference of the test repository (Default: default)", + ), + ] = "default", + test_path: Annotated[ + str | None, + Query( + alias="test-path", + title="Test path", + description="Path to the test metadata directory", + ), + ] = None, + plan_url: Annotated[ + str | None, + Query( + alias="plan-url", + title="Plan URL", + description="URL of a Git repository containing plan metadata", + ), + ] = None, + plan_name: Annotated[ + str | None, + Query( + alias="plan-name", + title="Plan name", + description="Name of the plan", + ), + ] = None, + plan_ref: Annotated[ + str | None, + Query( + alias="plan-ref", + title="Plan reference", + description="Reference of the plan repository (Default: default)", + ), + ] = "default", + plan_path: Annotated[ + str | None, + Query( + alias="plan-path", + title="Plan path", + description="Path to the plan metadata directory", + ), + ] = None, out_format: Annotated[Literal["html", "json", "yaml"], Query(alias="format")] = "json", -): +) -> TaskOut | str | Any: # Parameter validations if (test_url is None and test_name is not None) or (test_url is not None and test_name is None): return "Invalid arguments!" @@ -84,18 +133,28 @@ def find_test( @app.get("/status") -def status(task_id: str = Query(None, alias="task-id")) -> TaskOut | str: +def status(task_id: Annotated[str | None, + Query( + alias="task-id", + title="Task ID", + ) + ]) -> TaskOut | str: r = service.main.app.AsyncResult(task_id) return _to_task_out(r) @app.get("/status/html") -def status_html(task_id: str = Query(None, alias="task-id")) -> HTMLResponse: +def status_html(task_id: Annotated[str | None, + Query( + alias="task-id", + title="Task ID", + ) + ]) -> HTMLResponse: r = service.main.app.AsyncResult(task_id) status_callback_url = f'{os.getenv("API_HOSTNAME")}/status/html?task-id={r.task_id}' return HTMLResponse(content=html_generator.generate_status_callback(r, status_callback_url)) -def _to_task_out(r: AsyncResult) -> TaskOut | str: +def _to_task_out(r: AsyncResult) -> TaskOut: # type: ignore [type-arg] return TaskOut( id=r.task_id, status=r.status, diff --git a/src/tmt_web/generators/html_generator.py b/src/tmt_web/generators/html_generator.py index 6cfb844..b476804 100644 --- a/src/tmt_web/generators/html_generator.py +++ b/src/tmt_web/generators/html_generator.py @@ -3,7 +3,7 @@ from tmt import Logger, Plan, Test -def generate_status_callback(r: AsyncResult, status_callback_url: str) -> str: +def generate_status_callback(r: AsyncResult, status_callback_url: str) -> str: # type: ignore [type-arg] """ This function generates the status callback for the HTML file :param r: AsyncResult object @@ -45,7 +45,7 @@ def generate_test_html_page(test: Test, logger: Logger) -> str: HTML File - + name: {test.name}
summary: {test.summary}
@@ -83,7 +83,7 @@ def generate_plan_html_page(plan: Plan, logger: Logger) -> str: HTML File - + name: {plan.name}
summary: {plan.summary}
@@ -123,7 +123,7 @@ def generate_testplan_html_page(test: tmt.Test, plan: tmt.Plan, logger: Logger) HTML File - + Test metadata
name: {test.name}
diff --git a/src/tmt_web/service.py b/src/tmt_web/service.py index eccbf2a..05ec207 100644 --- a/src/tmt_web/service.py +++ b/src/tmt_web/service.py @@ -2,8 +2,8 @@ import os import tmt -from celery.app import Celery -from tmt.utils import Path +from celery.app import Celery # type: ignore[attr-defined] +from tmt.utils import Path # type: ignore[attr-defined] from tmt_web.generators import html_generator as html from tmt_web.generators import json_generator, yaml_generator @@ -69,7 +69,7 @@ def process_test_request(test_url: str, # Find the desired Test object wanted_test = tree.tests(names=[test_name])[0] - if wanted_test == []: + if not wanted_test: logger.print("Test not found!", color="red") return None logger.print("Test found!", color="green") @@ -108,7 +108,7 @@ def process_plan_request(plan_url: str, # Find the desired Plan object wanted_plan = tree.plans(names=[plan_name])[0] - if wanted_plan == []: + if not wanted_plan: logger.print("Plan not found!", color="red") return None logger.print("Plan found!", color="green") @@ -176,10 +176,11 @@ def main(test_url: str | None, plan_path: str | None, out_format: str) -> str | tmt.Test | tmt.Plan | None: logger.print("Starting...", color="blue") + # TODO if test_name is not None and plan_name is None: - return process_test_request(test_url, test_name, test_ref, test_path, True, out_format) + return process_test_request(test_url, test_name, test_ref, test_path, True, out_format) # type: ignore [arg-type] if plan_name is not None and test_name is None: - return process_plan_request(plan_url, plan_name, plan_ref, plan_path, True, out_format) + return process_plan_request(plan_url, plan_name, plan_ref, plan_path, True, out_format) # type: ignore [arg-type] if plan_name is not None and test_name is not None: return process_testplan_request(test_url, test_name, test_ref, test_path, plan_url, plan_name, plan_ref, plan_path, out_format) diff --git a/src/tmt_web/utils/git_handler.py b/src/tmt_web/utils/git_handler.py index 5574619..d1c9dff 100644 --- a/src/tmt_web/utils/git_handler.py +++ b/src/tmt_web/utils/git_handler.py @@ -3,7 +3,14 @@ from shutil import rmtree from tmt import Logger -from tmt.utils import Command, Common, GeneralError, Path, RunError, git_clone +from tmt.utils import ( # type: ignore[attr-defined] + Command, + Common, + GeneralError, + Path, + RunError, + git_clone, +) def checkout_branch(path: Path, logger: Logger, ref: str) -> None: diff --git a/tests/unit/test_git_handler.py b/tests/unit/test_git_handler.py index 2dc6a53..73ae732 100644 --- a/tests/unit/test_git_handler.py +++ b/tests/unit/test_git_handler.py @@ -16,10 +16,8 @@ class TestGitHandler: def test_clear_tmp_dir(self): # Create test directory if it doesn't exist try: - path = os.path.realpath(__file__) - path = path.replace("tests/unit/test_git_handler.py", "") - path = Path(path + os.getenv("CLONE_DIR_PATH", "./.repos/")) - os.mkdir(path) + path = Path(__file__).resolve().parents[2].joinpath(os.getenv("CLONE_DIR_PATH", "./.repos/")) + path.mkdir(exist_ok=True) except FileExistsError: pass git_handler.clear_tmp_dir(self.logger) @@ -67,5 +65,3 @@ def test_checkout_branch_exception(self): with pytest.raises(AttributeError): git_handler.checkout_branch(ref="quaddy", path=git_handler.get_path_to_repository( url="https://github.com/teemtee/tmt"), logger=self.logger) - - From 350b2cbcea544c1ab0fd19b95ac84310ce0b61e9 Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Mon, 19 Aug 2024 17:22:57 +0200 Subject: [PATCH 30/46] Removing redundant whitespaces, lines --- Containerfile | 1 - README.md | 2 +- compose.yaml | 4 +--- entrypoint.sh | 1 - 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Containerfile b/Containerfile index 884de3d..9cd306c 100644 --- a/Containerfile +++ b/Containerfile @@ -11,4 +11,3 @@ COPY /entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] - diff --git a/README.md b/README.md index b44c339..7aa5638 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ and `plan-*` options together, they are not mutually exclusive. `test-url` and `test-name`, or `plan-url` and `plan-name` are required. ## Environment variables -`REDIS_URL` - optional, passed to Celery on initialization as a `broker` and `backend` argument, +`REDIS_URL` - optional, passed to Celery on initialization as a `broker` and `backend` argument, default value is `redis://localhost:6379` `CLONE_DIR_PATH` - optional, specifies the path where the repositories will be cloned, default value is `./.repos/` diff --git a/compose.yaml b/compose.yaml index 5f37a6c..d8ab219 100644 --- a/compose.yaml +++ b/compose.yaml @@ -18,7 +18,7 @@ services: celery: container_name: celery - build: + build: context: . dockerfile: ./Containerfile command: celery --app=src.api.service worker --loglevel=INFO @@ -27,5 +27,3 @@ services: - API_HOSTNAME=http://localhost:8000 depends_on: - redis - - diff --git a/entrypoint.sh b/entrypoint.sh index ac40de3..6e66c43 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -22,4 +22,3 @@ $COMMAND & PID=$! wait $PID - From 17f887824dc20ed58a81a2cf21021023272a0bdc Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Mon, 19 Aug 2024 17:23:59 +0200 Subject: [PATCH 31/46] Add pre-commit hooks and mypy configuration --- .pre-commit-config.yaml | 33 +++++++++++++++++++++++++++++++++ pyproject.toml | 19 +++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..f9fbdd4 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,33 @@ + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: "v4.6.0" + hooks: + - id: check-merge-conflict + - id: check-symlinks + - id: check-toml + - id: destroyed-symlinks + - id: detect-private-key + - id: end-of-file-fixer + - id: mixed-line-ending + - id: no-commit-to-branch + args: [--branch, main] + - id: trailing-whitespace + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: "v1.11.1" + hooks: + - id: mypy + language_version: "3.12" + additional_dependencies: + - 'tmt' + - 'pydantic' + - 'celery-types' + + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.6.1 + hooks: + - id: ruff + args: + - '--fix' + - '--show-fixes' diff --git a/pyproject.toml b/pyproject.toml index e47a339..56a28f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,10 +79,14 @@ lint.select = [ "SIM", # flake8-simplify "TID", # flake8-tidy-imports "INT", # flake8-gettext + "PTH", # flake8-use-pathlib "PGH", # pygrep-hooks "PLC", # pylint-convention "PLE", # pylint-error "PLR", # pylint-refactor + "FLY", # flynt + "FAST", # FastAPI (in preview) + "PERF", # Perflint "RUF", # ruff ] lint.ignore = [ @@ -103,3 +107,18 @@ lint.ignore = [ "W291", # Trailing whitespace "E501", # Line length ] + +[tool.mypy] +plugins = [ + "pydantic.mypy" +] + +follow_imports = "silent" +warn_redundant_casts = true +warn_unused_ignores = true +disallow_any_generics = true +check_untyped_defs = true +no_implicit_reexport = true + +python_version = "3.12" +files = ["src/", "tests/"] From 4c6db17bd86b76052d87b3874f5f5ea12c916937 Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Tue, 20 Aug 2024 12:16:26 +0200 Subject: [PATCH 32/46] Add a simple wrapper for local test to hatch env --- pyproject.toml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 56a28f5..04ec4fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,12 +35,25 @@ raw-options.version_scheme = "release-branch-semver" [tool.hatch.envs.dev] extra-dependencies = [ + "pre-commit", "mypy~=1.11.1", "ruff~=0.5.7", "hatch", + "podman-compose", + "pytest", +] +post-install-commands = [ + "pre-commit install", +] + +[tool.hatch.envs.test] +extra-dependencies = [ + "podman-compose", + "pytest", ] -[tool.hatch.envs.dev.scripts] +[tool.hatch.envs.test.scripts] +run = "podman-compose down && podman-compose up -d --build && pytest" check = "mypy --install-types --non-interactive src/tmt_web" [tool.ruff] From cf99aaff34a87bd37305ed1093dae7f4e0c97d52 Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Tue, 20 Aug 2024 13:44:35 +0200 Subject: [PATCH 33/46] Use Jinja in html_generator --- src/tmt_web/generators/html_generator.py | 180 ++---------------- src/tmt_web/generators/json_generator.py | 124 ++++-------- .../generators/templates/_base.html.j2 | 10 + .../templates/status_callback.html.j2 | 13 ++ .../generators/templates/testandplan.html.j2 | 27 +++ .../generators/templates/testorplan.html.j2 | 24 +++ src/tmt_web/service.py | 4 +- tests/unit/test_html_generator.py | 4 +- 8 files changed, 136 insertions(+), 250 deletions(-) create mode 100644 src/tmt_web/generators/templates/_base.html.j2 create mode 100644 src/tmt_web/generators/templates/status_callback.html.j2 create mode 100644 src/tmt_web/generators/templates/testandplan.html.j2 create mode 100644 src/tmt_web/generators/templates/testorplan.html.j2 diff --git a/src/tmt_web/generators/html_generator.py b/src/tmt_web/generators/html_generator.py index b476804..f0a0ffa 100644 --- a/src/tmt_web/generators/html_generator.py +++ b/src/tmt_web/generators/html_generator.py @@ -1,170 +1,32 @@ -import tmt +from pathlib import Path + from celery.result import AsyncResult +from jinja2 import Environment, FileSystemLoader from tmt import Logger, Plan, Test +templ_dir = Path(__file__).resolve().parent / 'templates' -def generate_status_callback(r: AsyncResult, status_callback_url: str) -> str: # type: ignore [type-arg] - """ - This function generates the status callback for the HTML file - :param r: AsyncResult object - :param status_callback_url: URL for the status callback - :return: - """ - if r.status == "PENDING": - return (f''' - - HTML File - - - - Processing... Try this clicking this url in a few seconds: {status_callback_url} - ''') - else: - return (f''' - - HTML File - - - - Status: {r.status}
- The result is:
{r.result} - ''') - - -def generate_test_html_page(test: Test, logger: Logger) -> str: - """ - This function generates an HTML file with the input data for a test - :param test: Test object - :param logger: tmt.Logger instance - :return: - """ - logger.print("Generating the HTML file...") - full_url = test.web_link() - # Adding the input data to the HTML file - file_html = (f''' - - HTML File - - - - name: {test.name}
- summary: {test.summary}
- description: {test.description}
- url: {full_url}
- ref: {test.fmf_id.ref}
- contact: {test.contact}
- tag: {test.tag}
- tier: {test.tier}
- id: {test.id}
- fmf-id:
-
    -
  • url: {test.fmf_id.url}
    • -
  • path: {test.fmf_id.path.as_posix() if test.fmf_id.path is not None else None}
    • -
  • name: {test.fmf_id.name}
    • -
  • ref: {test.fmf_id.ref}
    • -
- - ''') - logger.print("HTML file generated successfully!", color="green") - return file_html +env = Environment(loader=FileSystemLoader(str(templ_dir)), autoescape=True) +def render_template(template_name: str, **kwargs) -> str: + template = env.get_template(template_name) + return template.render(**kwargs) -def generate_plan_html_page(plan: Plan, logger: Logger) -> str: - """ - This function generates an HTML file with the input data for a plan - :param plan: Plan object - :param logger: tmt.Logger instance - :return: - """ +def generate_status_callback(r: AsyncResult, status_callback_url: str) -> str: # type: ignore [type-arg] + data = { + "status": r.status, + "status_callback_url": status_callback_url, + "result": r.result + } + return render_template("status_callback.html.j2", **data) + +def generate_html_page(obj: Test | Plan, logger: Logger) -> str: logger.print("Generating the HTML file...") - full_url = plan.web_link() - # Adding the input data to the HTML file - file_html = (f''' - - HTML File - - - - name: {plan.name}
- summary: {plan.summary}
- description: {plan.description}
- url: {full_url}
- ref: {plan.fmf_id.ref}
- contact: {plan.contact}
- tag: {plan.tag}
- tier: {plan.tier}
- id: {plan.id}
- fmf-id:
-
    -
  • url: {plan.fmf_id.url}
    • -
  • path: {plan.fmf_id.path.as_posix() if plan.fmf_id.path is not None else None}
    • -
  • name: {plan.fmf_id.name}
    • -
  • ref: {plan.fmf_id.ref}
    • -
- - ''') - logger.print("HTML file generated successfully!", color="green") - return file_html + return render_template('testorplan.html.j2', testorplan=obj) - -def generate_testplan_html_page(test: tmt.Test, plan: tmt.Plan, logger: Logger) -> str: - """ - This function generates an HTML file with the input data for a test and a plan - :param test: Test object - :param plan: Plan object - :param logger: tmt.Logger instance - :return: - """ +def generate_testplan_html_page(test: Test, plan: Plan, logger: Logger) -> str: logger.print("Generating the HTML file...") - full_url_test = test.web_link() - full_url_plan = plan.web_link() - # Adding the input data to the HTML file - file_html = (f''' - - HTML File - - - - Test metadata
- name: {test.name}
- summary: {test.summary}
- description: {test.description}
- url: {full_url_test}
- ref: {test.fmf_id.ref}
- contact: {test.contact}
- tag: {test.tag}
- tier: {test.tier}
- id: {test.id}
- fmf-id:
-
    -
  • url: {test.fmf_id.url}
    • -
  • path: {test.fmf_id.path.as_posix() if test.fmf_id.path is not None else None}
    • -
  • name: {test.fmf_id.name}
    • -
  • ref: {test.fmf_id.ref}
    • -
-
- Plan metadata
- name: {plan.name}
- summary: {plan.summary}
- description: {plan.description}
- url: {full_url_plan}
- ref: {plan.fmf_id.ref}
- contact: {plan.contact}
- tag: {plan.tag}
- tier: {plan.tier}
- id: {plan.id}
- fmf-id:
-
    -
  • url: {plan.fmf_id.url}
    • -
  • path: {plan.fmf_id.path.as_posix() if plan.fmf_id.path is not None else None}
    • -
  • name: {plan.fmf_id.name}
    • -
  • ref: {plan.fmf_id.ref}
    • -
- - ''') - logger.print("HTML file generated successfully!", color="green") - return file_html - + return render_template('testandplan.html.j2', test=test, plan=plan) if __name__ == "__main__": - print("This is not executable file!") + print("This is not an executable file!") diff --git a/src/tmt_web/generators/json_generator.py b/src/tmt_web/generators/json_generator.py index bf3a714..36b34eb 100644 --- a/src/tmt_web/generators/json_generator.py +++ b/src/tmt_web/generators/json_generator.py @@ -1,72 +1,58 @@ import json +from typing import Any -import tmt -from tmt import Logger, Plan, Test +import tmt.utils +from tmt import Plan, Test -def generate_test_json(test: Test, logger: Logger) -> str: +def _create_json_data(obj: Test | Plan, logger: tmt.Logger) -> dict[str, Any]: """ - This function generates an JSON file with the input data for a test - :param test: Test object - :param logger: tmt.Logger instance - :return: + Helper function to create the JSON data from a test or plan object """ logger.print("Generating the JSON file...") - full_url = test.web_link() - data = { - "name": test.name, - "summary": test.summary, - "description": test.description, + full_url = obj.web_link() + return { + "name": obj.name, + "summary": obj.summary, + "description": obj.description, "url": full_url, - "ref": test.fmf_id.ref, - "contact": test.contact, - "tag": test.tag, - "tier": test.tier, - "id": test.id, + "ref": obj.fmf_id.ref, + "contact": obj.contact, + "tag": obj.tag, + "tier": obj.tier, + "id": obj.id, "fmf-id": { - "url": test.fmf_id.url, - "path": test.fmf_id.path.as_posix() if test.fmf_id.path is not None else None, - "name": test.fmf_id.name, - "ref": test.fmf_id.ref, + "url": obj.fmf_id.url, + "path": obj.fmf_id.path.as_posix() if obj.fmf_id.path is not None else None, + "name": obj.fmf_id.name, + "ref": obj.fmf_id.ref, } } - json_data = json.dumps(data) - logger.print("JSON file generated successfully!", color="green") - return json_data -def generate_plan_json(plan: Plan, logger: Logger) -> str: +def generate_test_json(test: tmt.Test, logger: tmt.Logger) -> str: + """ + This function generates an JSON file with the input data for a test + :param test: Test object + :param logger: tmt.Logger instance + :return: + """ + data = _create_json_data(test, logger) + return json.dumps(data) + + +def generate_plan_json(plan: tmt.Plan, logger: tmt.Logger) -> str: """ This function generates an JSON file with the input data for a plan :param plan: Plan object :param logger: tmt.Logger instance :return: """ - logger.print("Generating the JSON file...") - full_url = plan.web_link() - data = { - "name": plan.name, - "summary": plan.summary, - "description": plan.description, - "url": full_url, - "ref": plan.fmf_id.ref, - "contact": plan.contact, - "tag": plan.tag, - "tier": plan.tier, - "id": plan.id, - "fmf-id": { - "url": plan.fmf_id.url, - "path": plan.fmf_id.path.as_posix() if plan.fmf_id.path is not None else None, - "name": plan.fmf_id.name, - "ref": plan.fmf_id.ref, - } - } - json_data = json.dumps(data) - logger.print("JSON file generated successfully!", color="green") - return json_data + data = _create_json_data(plan, logger) + return json.dumps(data) -def generate_testplan_json(test: tmt.Test, plan: tmt.Plan, logger: Logger) -> str: +def generate_testplan_json(test: tmt.Test, plan: tmt.Plan, logger: tmt.Logger) -> str: """ This function generates an JSON file with the input data for a test and a plan :param test: Test object @@ -75,44 +61,8 @@ def generate_testplan_json(test: tmt.Test, plan: tmt.Plan, logger: Logger) -> st :return: """ logger.print("Generating the JSON file...") - full_url_test = test.web_link() - full_url_plan = plan.web_link() data = { - "test": { - "name": test.name, - "summary": test.summary, - "description": test.description, - "url": full_url_test, - "ref": test.fmf_id.ref, - "contact": test.contact, - "tag": test.tag, - "tier": test.tier, - "id": test.id, - "fmf-id": { - "url": test.fmf_id.url, - "path": test.fmf_id.path.as_posix() if test.fmf_id.path is not None else None, - "name": test.fmf_id.name, - "ref": test.fmf_id.ref, - } - }, - "plan": { - "name": plan.name, - "summary": plan.summary, - "description": plan.description, - "url": full_url_plan, - "ref": plan.fmf_id.ref, - "contact": plan.contact, - "tag": plan.tag, - "tier": plan.tier, - "id": plan.id, - "fmf-id": { - "url": plan.fmf_id.url, - "path": plan.fmf_id.path.as_posix() if plan.fmf_id.path is not None else None, - "name": plan.fmf_id.name, - "ref": plan.fmf_id.ref, - } - } + "test": _create_json_data(test, logger), + "plan": _create_json_data(plan, logger), } - json_data = json.dumps(data) - logger.print("JSON file generated successfully!", color="green") - return json_data + return json.dumps(data) diff --git a/src/tmt_web/generators/templates/_base.html.j2 b/src/tmt_web/generators/templates/_base.html.j2 new file mode 100644 index 0000000..83ec3e5 --- /dev/null +++ b/src/tmt_web/generators/templates/_base.html.j2 @@ -0,0 +1,10 @@ + + + + {{ title | default('HTML File') }} + + + + {%- block content %}{% endblock %} + + diff --git a/src/tmt_web/generators/templates/status_callback.html.j2 b/src/tmt_web/generators/templates/status_callback.html.j2 new file mode 100644 index 0000000..5dab855 --- /dev/null +++ b/src/tmt_web/generators/templates/status_callback.html.j2 @@ -0,0 +1,13 @@ +{% extends '_base.html.j2' %} +{% set title = "Status" %} + +{% block content %} + {% if status == "PENDING" %} + Processing... Try this clicking this url in a few seconds: {{ status_callback_url }} + {%- elif status == "RETRYING" %} + Task is retrying... Please wait for a few seconds and try again: {{ status_callback_url }} + {%- else %} + Status: {{ status }}
+ The result is:
{{ result }} + {%- endif %} +{%- endblock %} diff --git a/src/tmt_web/generators/templates/testandplan.html.j2 b/src/tmt_web/generators/templates/testandplan.html.j2 new file mode 100644 index 0000000..bb4256c --- /dev/null +++ b/src/tmt_web/generators/templates/testandplan.html.j2 @@ -0,0 +1,27 @@ + +{% extends '_base.html.j2' %} +{% set title = "Test and Plan Information" %} + +{% block content %} +

Test Information

+

name: {{ test.name }}

+

summary: {{ test.summary }}

+

description: {{ test.description }}

+

url: {{ test.web_link() }}

+

ref: {{ test.fmf_id.ref }}

+

contact: {{ test.contact }}

+

tag: {{ test.tag }}

+

tier: {{ test.tier }}

+

id: {{ test.id }}

+ +

Plan Information

+

name: {{ plan.name }}

+

summary: {{ plan.summary }}

+

description: {{ plan.description }}

+

url: {{ plan.web_link() }}

+

ref: {{ plan.fmf_id.ref }}

+

contact: {{ plan.contact }}

+

tag: {{ plan.tag }}

+

tier: {{ plan.tier }}

+

id: {{ plan.id }}

+{%- endblock %} diff --git a/src/tmt_web/generators/templates/testorplan.html.j2 b/src/tmt_web/generators/templates/testorplan.html.j2 new file mode 100644 index 0000000..8d885f8 --- /dev/null +++ b/src/tmt_web/generators/templates/testorplan.html.j2 @@ -0,0 +1,24 @@ + +{% extends '_base.html.j2' %} +{% set title = testorplan.name %} + +{% block content %} +

name: {{ testorplan.name }}

+

summary: {{ testorplan.summary }}

+

description: {{ testorplan.description }}

+

url: {{ testorplan.web_link() }}

+

ref: {{ testorplan.fmf_id.ref }}

+

contact: {{ testorplan.contact }}

+

tag: {{ testorplan.tag }}

+

tier: {{ testorplan.tier }}

+

id: {{ testorplan.id }}

+ fmf-id:
+
    +
  • url: {{ testorplan.fmf_id.url }}
    • + {%- if testorplan.fmf_id.path %} +
  • path: {{ testorplan.fmf_id.path.as_posix() }}
    • + {%- else %} +
  • path: None
    • + {%- endif %} +
+{%- endblock %} diff --git a/src/tmt_web/service.py b/src/tmt_web/service.py index 05ec207..3652d65 100644 --- a/src/tmt_web/service.py +++ b/src/tmt_web/service.py @@ -77,7 +77,7 @@ def process_test_request(test_url: str, return wanted_test match out_format: case "html": - return html.generate_test_html_page(wanted_test, logger=logger) + return html.generate_html_page(wanted_test, logger=logger) case "json": return json_generator.generate_test_json(wanted_test, logger=logger) case "yaml": @@ -116,7 +116,7 @@ def process_plan_request(plan_url: str, return wanted_plan match out_format: case "html": - return html.generate_plan_html_page(wanted_plan, logger=logger) + return html.generate_html_page(wanted_plan, logger=logger) case "json": return json_generator.generate_plan_json(wanted_plan, logger=logger) case "yaml": diff --git a/tests/unit/test_html_generator.py b/tests/unit/test_html_generator.py index 757e672..cd1beb1 100644 --- a/tests/unit/test_html_generator.py +++ b/tests/unit/test_html_generator.py @@ -9,5 +9,5 @@ class TestHtmlGenerator: def test_generate_test_html(self): logger = tmt.Logger(logging.getLogger("tmt-logger")) test = tmt.Tree(logger=logger).tests(names=["/tests/objects/sample_test"])[0] - data = html_generator.generate_test_html_page(test, logger) - assert 'name: /tests/objects/sample_test
' in data + data = html_generator.generate_html_page(test, logger) + assert 'name: /tests/objects/sample_test

' in data From b317eb966d702f6bc7c54aae9d2985304d69cde1 Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Tue, 20 Aug 2024 13:53:49 +0200 Subject: [PATCH 34/46] Remove usage of Python <3.12, add classifiers --- .github/workflows/test.yml | 2 +- pyproject.toml | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3417792..e54b4e0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: - python-version: [ "3.10", "3.12" ] + python-version: ["3.12" ] # Can be extended with future Python versions steps: diff --git a/pyproject.toml b/pyproject.toml index 04ec4fe..83b8dee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "tmt-web" dynamic = ["version"] description = 'Web app for checking tmt tests, plans and stories' readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.12" license = "MIT" keywords = [] authors = [ @@ -15,8 +15,13 @@ authors = [ { name = "Tomas Koscielniak", email = "tkosciel@redhat.com" } ] classifiers = [ + "License :: OSI Approved :: MIT License", "Programming Language :: Python", "Programming Language :: Python :: 3.12", + "Framework :: FastAPI", + "Framework :: Celery", + "Topic :: Software Development :: Testing", + "Operating System :: POSIX :: Linux", ] dependencies = [ "tmt~=1.35", From c086e46e2e6be01f04803ffbcba9a21b8bd19fbf Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Tue, 20 Aug 2024 14:35:16 +0200 Subject: [PATCH 35/46] Add pre-commit to gh workflow --- .github/workflows/pre-commit.yml | 19 +++++++++++++++++++ .github/workflows/test.yml | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/pre-commit.yml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..db12211 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,19 @@ +name: pre-commit + +on: + pull_request: + push: + branches: [main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + env: + SKIP: no-commit-to-branch + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e54b4e0..3150fc4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: - python-version: ["3.12" ] # Can be extended with future Python versions + python-version: ["3.12"] # Can be extended with future Python versions steps: From 41dd0bad29291a36517b4b54a8540205f62afc87 Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Tue, 20 Aug 2024 14:40:52 +0200 Subject: [PATCH 36/46] Remove format_html variable --- src/tmt_web/api.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/tmt_web/api.py b/src/tmt_web/api.py index 71ae698..9dd07c9 100644 --- a/src/tmt_web/api.py +++ b/src/tmt_web/api.py @@ -11,7 +11,6 @@ from tmt_web.generators import html_generator app = FastAPI() -format_html = False class TaskOut(BaseModel): @@ -123,12 +122,9 @@ def find_test( plan_path=plan_path) # Special handling of response if the format is html if out_format == "html": - global format_html - format_html = True status_callback_url = f'{os.getenv("API_HOSTNAME")}/status/html?task-id={r.task_id}' return HTMLResponse(content=html_generator.generate_status_callback(r, status_callback_url)) else: - format_html = False # To set it back to False after a html format request return _to_task_out(r) From cf99bbc9c45c322dfbee0e264a4b48cf21974bba Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Tue, 3 Sep 2024 12:53:23 +0200 Subject: [PATCH 37/46] Re-use existing dict generation for yaml --- src/tmt_web/generators/json_generator.py | 4 +- src/tmt_web/generators/yaml_generator.py | 93 ++++-------------------- 2 files changed, 16 insertions(+), 81 deletions(-) diff --git a/src/tmt_web/generators/json_generator.py b/src/tmt_web/generators/json_generator.py index 36b34eb..c3d5ba1 100644 --- a/src/tmt_web/generators/json_generator.py +++ b/src/tmt_web/generators/json_generator.py @@ -9,7 +9,6 @@ def _create_json_data(obj: Test | Plan, logger: tmt.Logger) -> dict[str, Any]: """ Helper function to create the JSON data from a test or plan object """ - logger.print("Generating the JSON file...") full_url = obj.web_link() return { "name": obj.name, @@ -38,6 +37,7 @@ def generate_test_json(test: tmt.Test, logger: tmt.Logger) -> str: :return: """ data = _create_json_data(test, logger) + logger.print("Generating the JSON file...") return json.dumps(data) @@ -49,6 +49,7 @@ def generate_plan_json(plan: tmt.Plan, logger: tmt.Logger) -> str: :return: """ data = _create_json_data(plan, logger) + logger.print("Generating the JSON file...") return json.dumps(data) @@ -65,4 +66,5 @@ def generate_testplan_json(test: tmt.Test, plan: tmt.Plan, logger: tmt.Logger) - "test": _create_json_data(test, logger), "plan": _create_json_data(plan, logger), } + logger.print("Generating the JSON file...") return json.dumps(data) diff --git a/src/tmt_web/generators/yaml_generator.py b/src/tmt_web/generators/yaml_generator.py index 0768520..03050b6 100644 --- a/src/tmt_web/generators/yaml_generator.py +++ b/src/tmt_web/generators/yaml_generator.py @@ -1,6 +1,12 @@ import tmt from tmt import Logger +from tmt_web.generators.json_generator import _create_json_data + + +def print_success(logger) -> None: + logger.print("YAML file generated successfully!", color="green") + def generate_test_yaml(test: tmt.Test, logger: Logger) -> str: """ @@ -9,27 +15,8 @@ def generate_test_yaml(test: tmt.Test, logger: Logger) -> str: :param logger: tmt.Logger instance :return: """ - logger.print("Generating the YAML file...") - full_url = test.web_link() - data = { - "name": test.name, - "summary": test.summary, - "description": test.description, - "url": full_url, - "ref": test.fmf_id.ref, - "contact": test.contact, - "tag": test.tag, - "tier": test.tier, - "id": test.id, - "fmf-id": { - "url": test.fmf_id.url, - "path": test.fmf_id.path.as_posix() if test.fmf_id.path is not None else None, - "name": test.fmf_id.name, - "ref": test.fmf_id.ref, - } - } - yaml_data = tmt.utils.dict_to_yaml(data) - logger.print("YAML file generated successfully!", color="green") + yaml_data = tmt.utils.dict_to_yaml(_create_json_data(test, logger)) + print_success(logger) return yaml_data @@ -40,27 +27,8 @@ def generate_plan_yaml(plan: tmt.Plan, logger: Logger) -> str: :param logger: tmt.Logger instance :return: """ - logger.print("Generating the YAML file...") - full_url = plan.web_link() - data = { - "name": plan.name, - "summary": plan.summary, - "description": plan.description, - "url": full_url, - "ref": plan.fmf_id.ref, - "contact": plan.contact, - "tag": plan.tag, - "tier": plan.tier, - "id": plan.id, - "fmf-id": { - "url": plan.fmf_id.url, - "path": plan.fmf_id.path.as_posix() if plan.fmf_id.path is not None else None, - "name": plan.fmf_id.name, - "ref": plan.fmf_id.ref, - } - } - yaml_data = tmt.utils.dict_to_yaml(data) - logger.print("YAML file generated successfully!", color="green") + yaml_data = tmt.utils.dict_to_yaml(_create_json_data(plan, logger)) + print_success(logger) return yaml_data @@ -72,45 +40,10 @@ def generate_testplan_yaml(test: tmt.Test, plan: tmt.Plan, logger: Logger) -> st :param logger: tmt.Logger instance :return: """ - logger.print("Generating the YAML file...") - full_url_test = test.web_link() - full_url_plan = plan.web_link() data = { - "test": { - "name": test.name, - "summary": test.summary, - "description": test.description, - "url": full_url_test, - "ref": test.fmf_id.ref, - "contact": test.contact, - "tag": test.tag, - "tier": test.tier, - "id": test.id, - "fmf-id": { - "url": test.fmf_id.url, - "path": test.fmf_id.path.as_posix() if test.fmf_id.path is not None else None, - "name": test.fmf_id.name, - "ref": test.fmf_id.ref, - } - }, - "plan": { - "name": plan.name, - "summary": plan.summary, - "description": plan.description, - "url": full_url_plan, - "ref": plan.fmf_id.ref, - "contact": plan.contact, - "tag": plan.tag, - "tier": plan.tier, - "id": plan.id, - "fmf-id": { - "url": plan.fmf_id.url, - "path": plan.fmf_id.path.as_posix() if plan.fmf_id.path is not None else None, - "name": plan.fmf_id.name, - "ref": plan.fmf_id.ref, - } - } + "test": _create_json_data(test, logger), + "plan": _create_json_data(plan, logger), } yaml_data = tmt.utils.dict_to_yaml(data) - logger.print("YAML file generated successfully!", color="green") + print_success(logger) return yaml_data From 2b7a3826d9a41194acc84aa894aeaaffe5028794 Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Tue, 3 Sep 2024 13:05:42 +0200 Subject: [PATCH 38/46] Minor README changes --- README.md | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 7aa5638..c10cb63 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,33 @@ # web -Web app for checking tmt tests and plans -# Run instructions + +Web app for checking tmt tests, plans and stories + +## Run instructions + To run the service locally for development purposes, use the following command: + ```bash -docker-compose up --build +podman-compose up --build ``` + +add `-d` for the service to run in the background + ## Tests -To run the tests, use the `pytest` command + +To run the tests, use the `pytest` command (assuming the service is running). +Alternatively, if you have `hatch` installed, `hatch run test:run` command will rebuild, start the service and run the tests. + +## Environment variables + +`REDIS_URL` - optional, passed to Celery on initialization as a `broker` and `backend` argument, +default value is `redis://localhost:6379` + +`CLONE_DIR_PATH` - optional, specifies the path where the repositories will be cloned, default value is `./.repos/` + +`USE_CELERY` - optional, specifies if the app should use Celery, set to `false` for running without Celery + +`API_HOSTNAME` - required, specifies the hostname of the API, used for creating the callback URL to the service + # API API for checking tmt tests and plans metadata ## Version @@ -40,13 +61,3 @@ If we want to display metadata for both tests and plans, we can combine the `tes and `plan-*` options together, they are not mutually exclusive. `test-url` and `test-name`, or `plan-url` and `plan-name` are required. - -## Environment variables -`REDIS_URL` - optional, passed to Celery on initialization as a `broker` and `backend` argument, -default value is `redis://localhost:6379` - -`CLONE_DIR_PATH` - optional, specifies the path where the repositories will be cloned, default value is `./.repos/` - -`USE_CELERY` - optional, specifies if the app should use Celery, set to `false` for running without Celery - -`API_HOSTNAME` - required, specifies the hostname of the API, used for creating the callback URL to the service From 85f2daa837ae762a01e5c1a7e36f31df1134afac Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Tue, 3 Sep 2024 13:13:18 +0200 Subject: [PATCH 39/46] fixup! Add pre-commit hooks and mypy configuration --- .pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f9fbdd4..4cfe3b5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,6 +13,8 @@ repos: - id: no-commit-to-branch args: [--branch, main] - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + - repo: https://github.com/pre-commit/mirrors-mypy rev: "v1.11.1" From 3bcc90f2b4f6bfc6e073e4d6915c41dcfdf077fc Mon Sep 17 00:00:00 2001 From: Martin Hoyer Date: Tue, 3 Sep 2024 16:20:18 +0200 Subject: [PATCH 40/46] Make 'ref' optional --- src/tmt_web/api.py | 8 ++++---- src/tmt_web/generators/yaml_generator.py | 2 +- src/tmt_web/service.py | 2 +- src/tmt_web/utils/git_handler.py | 25 ++++++++++++++---------- tests/unit/test_git_handler.py | 4 ++-- 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/src/tmt_web/api.py b/src/tmt_web/api.py index 9dd07c9..807af8e 100644 --- a/src/tmt_web/api.py +++ b/src/tmt_web/api.py @@ -45,9 +45,9 @@ def find_test( Query( alias="test-ref", title="Test reference", - description="Reference of the test repository (Default: default)", + description="Reference of the test repository", ), - ] = "default", + ] = None, test_path: Annotated[ str | None, Query( @@ -77,9 +77,9 @@ def find_test( Query( alias="plan-ref", title="Plan reference", - description="Reference of the plan repository (Default: default)", + description="Reference of the plan repository", ), - ] = "default", + ] = None, plan_path: Annotated[ str | None, Query( diff --git a/src/tmt_web/generators/yaml_generator.py b/src/tmt_web/generators/yaml_generator.py index 03050b6..e2e60e7 100644 --- a/src/tmt_web/generators/yaml_generator.py +++ b/src/tmt_web/generators/yaml_generator.py @@ -4,7 +4,7 @@ from tmt_web.generators.json_generator import _create_json_data -def print_success(logger) -> None: +def print_success(logger: Logger) -> None: logger.print("YAML file generated successfully!", color="green") diff --git a/src/tmt_web/service.py b/src/tmt_web/service.py index 3652d65..f7799cf 100644 --- a/src/tmt_web/service.py +++ b/src/tmt_web/service.py @@ -16,7 +16,7 @@ app = Celery(__name__, broker=redis_url, backend=redis_url) -def get_tree(url: str, name: str, ref: str, tree_path: str) -> tmt.base.Tree: +def get_tree(url: str, name: str, ref: str | None, tree_path: str) -> tmt.base.Tree: """ This function clones the repository and returns the Tree object :param ref: Object ref diff --git a/src/tmt_web/utils/git_handler.py b/src/tmt_web/utils/git_handler.py index d1c9dff..b85cb89 100644 --- a/src/tmt_web/utils/git_handler.py +++ b/src/tmt_web/utils/git_handler.py @@ -23,40 +23,45 @@ def checkout_branch(path: Path, logger: Logger, ref: str) -> None: """ try: common_instance = Common(logger=logger) - common_instance.run( - command=Command('git', 'checkout', ref), cwd=path) + common_instance.run(command=Command("git", "checkout", ref), cwd=path) except RunError as err: logger.print("Failed to do checkout in the repository!", color="red") raise AttributeError from err -def clone_repository(url: str, logger: Logger, ref: str) -> None: +def clone_repository(url: str, logger: Logger, ref: str | None = None) -> None: """ - Clones the repository from the given URL. + Clones the repository from the given URL and optionally checks out a specific ref. Raises FileExistsError if the repository is already cloned or Exception if the cloning fails. - :param ref: Name of the ref to check out :param url: URL to the repository :param logger: Instance of Logger + :param ref: Optional name of the ref to check out :return: """ logger.print("Cloning the repository...") path = get_path_to_repository(url) + if check_if_repository_exists(url): - if ref != "default": + if ref: try: checkout_branch(ref=ref, path=path, logger=logger) + logger.print(f"Checked out ref: {ref}", color="green") except AttributeError as err: + logger.print(f"Failed to checkout ref: {ref}", color="red") raise AttributeError from err logger.print("Repository already cloned!", color="yellow") raise FileExistsError + try: git_clone(url=url, destination=path, logger=logger) - if ref != "default": + + if ref: try: checkout_branch(ref=ref, path=path, logger=logger) + logger.print(f"Checked out ref: {ref}", color="green") except AttributeError as err: + logger.print(f"Failed to checkout ref: {ref}", color="red") raise AttributeError from err - checkout_branch(ref=ref, path=path, logger=logger) except GeneralError as e: logger.print("Failed to clone the repository!", color="red") raise Exception from e @@ -69,7 +74,7 @@ def get_path_to_repository(url: str) -> Path: :param url: URL to the repository :return: Path to the cloned repository """ - repo_name = url.rstrip('/').rsplit('/', 1)[-1] + repo_name = url.rstrip("/").rsplit("/", 1)[-1] root_dir = Path(__file__).resolve().parents[2] # going up from tmt_web/utils/git_handler.py return root_dir / os.getenv("CLONE_DIR_PATH", "./.repos/") / repo_name @@ -100,7 +105,7 @@ def clear_tmp_dir(logger: Logger) -> None: logger.print("Repository clone directory cleared successfully!", color="green") -def get_git_repository(url: str, logger: Logger, ref: str) -> Path: +def get_git_repository(url: str, logger: Logger, ref: str | None) -> Path: """ Clones the repository from the given URL and returns the path to the cloned repository. :param url: URL to the repository diff --git a/tests/unit/test_git_handler.py b/tests/unit/test_git_handler.py index 73ae732..837550f 100644 --- a/tests/unit/test_git_handler.py +++ b/tests/unit/test_git_handler.py @@ -36,12 +36,12 @@ def test_clone_repository(self): git_handler.clear_tmp_dir(self.logger) time.sleep(1) git_handler.clone_repository(url="https://github.com/teemtee/tmt", - logger=self.logger, ref="default") + logger=self.logger) def test_clone_repository_even_if_exists(self): with contextlib.suppress(FileExistsError): git_handler.clone_repository(url="https://github.com/teemtee/tmt", - logger=self.logger, ref="default") + logger=self.logger) def test_clone_checkout_branch(self): with contextlib.suppress(FileExistsError): From dfd56bbe40766367ce8dff4a0cafd01f25a1c58a Mon Sep 17 00:00:00 2001 From: Sabart Otto Date: Tue, 8 Oct 2024 15:44:15 +0200 Subject: [PATCH 41/46] A bunch of minor code improvements (#6) * bump the dependencies to latest versions * fix the import of git_clone * no special reason for import alias * docstrings formatting and indentation problems * use f-strings instead * fix the order of imports * small improvements to readme * set execute permission to entrypoint.sh * small refactor of docstrings, unify quotation marks --- README.md | 22 ++++++++------- entrypoint.sh | 0 pyproject.toml | 8 +++--- src/tmt_web/api.py | 22 +++++++++------ src/tmt_web/generators/html_generator.py | 11 ++++++-- src/tmt_web/generators/json_generator.py | 11 +++++--- src/tmt_web/generators/yaml_generator.py | 9 ++++-- src/tmt_web/service.py | 35 +++++++++++++----------- src/tmt_web/utils/git_handler.py | 13 +++++++-- 9 files changed, 80 insertions(+), 51 deletions(-) mode change 100644 => 100755 entrypoint.sh diff --git a/README.md b/README.md index c10cb63..ad34a33 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # web -Web app for checking tmt tests, plans and stories +Web application for checking tmt tests, plans and stories. ## Run instructions @@ -15,18 +15,20 @@ add `-d` for the service to run in the background ## Tests To run the tests, use the `pytest` command (assuming the service is running). -Alternatively, if you have `hatch` installed, `hatch run test:run` command will rebuild, start the service and run the tests. -## Environment variables - -`REDIS_URL` - optional, passed to Celery on initialization as a `broker` and `backend` argument, -default value is `redis://localhost:6379` +Alternatively, if you have `hatch` installed, `hatch run test:run` command will +rebuild, start the service and run the tests. -`CLONE_DIR_PATH` - optional, specifies the path where the repositories will be cloned, default value is `./.repos/` - -`USE_CELERY` - optional, specifies if the app should use Celery, set to `false` for running without Celery +## Environment variables -`API_HOSTNAME` - required, specifies the hostname of the API, used for creating the callback URL to the service +- `REDIS_URL` - *optional*, passed to Celery on initialization as a `broker` and + `backend` argument, default value is: `redis://localhost:6379` +- `CLONE_DIR_PATH` - *optional*, specifies the path where the repositories will + be cloned, default value is: `./.repos/` +- `USE_CELERY` - *optional*, specifies if the app should use Celery, set to + `false` for running without Celery +- `API_HOSTNAME` - *required*, specifies the hostname of the API, used for + creating the callback URL to the service # API API for checking tmt tests and plans metadata diff --git a/entrypoint.sh b/entrypoint.sh old mode 100644 new mode 100755 diff --git a/pyproject.toml b/pyproject.toml index 83b8dee..f8c9d91 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,8 +24,8 @@ classifiers = [ "Operating System :: POSIX :: Linux", ] dependencies = [ - "tmt~=1.35", - "fastapi~=0.112", + "tmt~=1.36", + "fastapi~=0.115", "httpx~=0.27", "uvicorn~=0.30", "celery[redis]~=5.4", @@ -41,8 +41,8 @@ raw-options.version_scheme = "release-branch-semver" [tool.hatch.envs.dev] extra-dependencies = [ "pre-commit", - "mypy~=1.11.1", - "ruff~=0.5.7", + "mypy~=1.11.2", + "ruff~=0.6", "hatch", "podman-compose", "pytest", diff --git a/src/tmt_web/api.py b/src/tmt_web/api.py index 807af8e..d895b2d 100644 --- a/src/tmt_web/api.py +++ b/src/tmt_web/api.py @@ -110,16 +110,18 @@ def find_test( out_format=out_format, test_path=test_path, plan_path=plan_path) + r = service.main.delay( - test_url=test_url, - test_name=test_name, - test_ref=test_ref, - plan_url=plan_url, - plan_name=plan_name, - plan_ref=plan_ref, - out_format=out_format, - test_path=test_path, - plan_path=plan_path) + test_url=test_url, + test_name=test_name, + test_ref=test_ref, + plan_url=plan_url, + plan_name=plan_name, + plan_ref=plan_ref, + out_format=out_format, + test_path=test_path, + plan_path=plan_path) + # Special handling of response if the format is html if out_format == "html": status_callback_url = f'{os.getenv("API_HOSTNAME")}/status/html?task-id={r.task_id}' @@ -138,6 +140,7 @@ def status(task_id: Annotated[str | None, r = service.main.app.AsyncResult(task_id) return _to_task_out(r) + @app.get("/status/html") def status_html(task_id: Annotated[str | None, Query( @@ -158,6 +161,7 @@ def _to_task_out(r: AsyncResult) -> TaskOut: # type: ignore [type-arg] status_callback_url=f'{os.getenv("API_HOSTNAME")}/status?task-id={r.task_id}' ) + @app.get("/health") def health_check(): return {"status": "healthy"} diff --git a/src/tmt_web/generators/html_generator.py b/src/tmt_web/generators/html_generator.py index f0a0ffa..dde0efd 100644 --- a/src/tmt_web/generators/html_generator.py +++ b/src/tmt_web/generators/html_generator.py @@ -4,14 +4,16 @@ from jinja2 import Environment, FileSystemLoader from tmt import Logger, Plan, Test -templ_dir = Path(__file__).resolve().parent / 'templates' +templ_dir = Path(__file__).resolve().parent / "templates" env = Environment(loader=FileSystemLoader(str(templ_dir)), autoescape=True) + def render_template(template_name: str, **kwargs) -> str: template = env.get_template(template_name) return template.render(**kwargs) + def generate_status_callback(r: AsyncResult, status_callback_url: str) -> str: # type: ignore [type-arg] data = { "status": r.status, @@ -20,13 +22,16 @@ def generate_status_callback(r: AsyncResult, status_callback_url: str) -> str: } return render_template("status_callback.html.j2", **data) + def generate_html_page(obj: Test | Plan, logger: Logger) -> str: logger.print("Generating the HTML file...") - return render_template('testorplan.html.j2', testorplan=obj) + return render_template("testorplan.html.j2", testorplan=obj) + def generate_testplan_html_page(test: Test, plan: Plan, logger: Logger) -> str: logger.print("Generating the HTML file...") - return render_template('testandplan.html.j2', test=test, plan=plan) + return render_template("testandplan.html.j2", test=test, plan=plan) + if __name__ == "__main__": print("This is not an executable file!") diff --git a/src/tmt_web/generators/json_generator.py b/src/tmt_web/generators/json_generator.py index c3d5ba1..47ef5d2 100644 --- a/src/tmt_web/generators/json_generator.py +++ b/src/tmt_web/generators/json_generator.py @@ -7,7 +7,7 @@ def _create_json_data(obj: Test | Plan, logger: tmt.Logger) -> dict[str, Any]: """ - Helper function to create the JSON data from a test or plan object + Helper function to create the JSON data from a test or plan object. """ full_url = obj.web_link() return { @@ -31,7 +31,8 @@ def _create_json_data(obj: Test | Plan, logger: tmt.Logger) -> dict[str, Any]: def generate_test_json(test: tmt.Test, logger: tmt.Logger) -> str: """ - This function generates an JSON file with the input data for a test + This function generates an JSON file with the input data for a test. + :param test: Test object :param logger: tmt.Logger instance :return: @@ -43,7 +44,8 @@ def generate_test_json(test: tmt.Test, logger: tmt.Logger) -> str: def generate_plan_json(plan: tmt.Plan, logger: tmt.Logger) -> str: """ - This function generates an JSON file with the input data for a plan + This function generates an JSON file with the input data for a plan. + :param plan: Plan object :param logger: tmt.Logger instance :return: @@ -55,7 +57,8 @@ def generate_plan_json(plan: tmt.Plan, logger: tmt.Logger) -> str: def generate_testplan_json(test: tmt.Test, plan: tmt.Plan, logger: tmt.Logger) -> str: """ - This function generates an JSON file with the input data for a test and a plan + This function generates an JSON file with the input data for a test and a plan. + :param test: Test object :param plan: Plan object :param logger: tmt.Logger instance diff --git a/src/tmt_web/generators/yaml_generator.py b/src/tmt_web/generators/yaml_generator.py index e2e60e7..08d2e03 100644 --- a/src/tmt_web/generators/yaml_generator.py +++ b/src/tmt_web/generators/yaml_generator.py @@ -10,7 +10,8 @@ def print_success(logger: Logger) -> None: def generate_test_yaml(test: tmt.Test, logger: Logger) -> str: """ - This function generates an YAML file with the input data for a test + This function generates an YAML file with the input data for a test. + :param test: Test object :param logger: tmt.Logger instance :return: @@ -22,7 +23,8 @@ def generate_test_yaml(test: tmt.Test, logger: Logger) -> str: def generate_plan_yaml(plan: tmt.Plan, logger: Logger) -> str: """ - This function generates an YAML file with the input data for a plan + This function generates an YAML file with the input data for a plan. + :param plan: Plan object :param logger: tmt.Logger instance :return: @@ -34,7 +36,8 @@ def generate_plan_yaml(plan: tmt.Plan, logger: Logger) -> str: def generate_testplan_yaml(test: tmt.Test, plan: tmt.Plan, logger: Logger) -> str: """ - This function generates an YAML file with the input data for a test and a plan + This function generates an YAML file with the input data for a test and a plan. + :param test: Test object :param plan: Plan object :param logger: tmt.Logger instance diff --git a/src/tmt_web/service.py b/src/tmt_web/service.py index f7799cf..c91661d 100644 --- a/src/tmt_web/service.py +++ b/src/tmt_web/service.py @@ -5,9 +5,8 @@ from celery.app import Celery # type: ignore[attr-defined] from tmt.utils import Path # type: ignore[attr-defined] -from tmt_web.generators import html_generator as html -from tmt_web.generators import json_generator, yaml_generator -from tmt_web.utils import git_handler as utils +from tmt_web.generators import html_generator, json_generator, yaml_generator +from tmt_web.utils import git_handler logger = tmt.Logger(logging.getLogger("tmt-logger")) @@ -18,19 +17,20 @@ def get_tree(url: str, name: str, ref: str | None, tree_path: str) -> tmt.base.Tree: """ - This function clones the repository and returns the Tree object + This function clones the repository and returns the Tree object. + :param ref: Object ref :param name: Object name :param url: Object url :param tree_path: Object path :return: """ - logger.print("Cloning the repository for url: " + url) + logger.print(f"Cloning the repository for url: {url}") logger.print("Parsing the url and name...") - logger.print("URL: " + url) - logger.print("Name: " + name) + logger.print(f"URL: {url}") + logger.print(f"Name: {name}") - path = utils.get_git_repository(url, logger, ref) + path = git_handler.get_git_repository(url, logger, ref) if tree_path is not None: tree_path += '/' @@ -39,7 +39,6 @@ def get_tree(url: str, name: str, ref: str | None, tree_path: str) -> tmt.base.T path = path.with_suffix('') path = Path(path.as_posix() + tree_path) - logger.print("Looking for tree...") tree = tmt.base.Tree(path=path, logger=logger) logger.print("Tree found!", color="green") @@ -53,7 +52,8 @@ def process_test_request(test_url: str, return_object: bool, out_format: str) -> str | tmt.Test | None: """ - This function processes the request for a test and returns the HTML file or the Test object + This function processes the request for a test and returns the HTML file or the Test object. + :param test_url: Test url :param test_name: Test name :param test_ref: Test repo ref @@ -72,12 +72,13 @@ def process_test_request(test_url: str, if not wanted_test: logger.print("Test not found!", color="red") return None + logger.print("Test found!", color="green") if not return_object: return wanted_test match out_format: case "html": - return html.generate_html_page(wanted_test, logger=logger) + return html_generator.generate_html_page(wanted_test, logger=logger) case "json": return json_generator.generate_test_json(wanted_test, logger=logger) case "yaml": @@ -88,11 +89,12 @@ def process_test_request(test_url: str, def process_plan_request(plan_url: str, plan_name: str, plan_ref: str, - plan_path:str, + plan_path: str, return_object: bool, out_format: str) -> str | None | tmt.Plan: """ - This function processes the request for a plan and returns the HTML file or the Plan object + This function processes the request for a plan and returns the HTML file or the Plan object. + :param plan_url: Plan URL :param plan_name: Plan name :param plan_ref: Plan repo ref @@ -116,7 +118,7 @@ def process_plan_request(plan_url: str, return wanted_plan match out_format: case "html": - return html.generate_html_page(wanted_plan, logger=logger) + return html_generator.generate_html_page(wanted_plan, logger=logger) case "json": return json_generator.generate_plan_json(wanted_plan, logger=logger) case "yaml": @@ -134,7 +136,8 @@ def process_testplan_request(test_url, plan_path, out_format) -> str | None: """ - This function processes the request for a test and a plan and returns the HTML file + This function processes the request for a test and a plan and returns the HTML file. + :param test_url: Test URL :param test_name: Test name :param test_ref: Test repo ref @@ -156,7 +159,7 @@ def process_testplan_request(test_url, return None match out_format: case "html": - return html.generate_testplan_html_page(test, plan, logger=logger) + return html_generator.generate_testplan_html_page(test, plan, logger=logger) case "json": return json_generator.generate_testplan_json(test, plan, logger=logger) case "yaml": diff --git a/src/tmt_web/utils/git_handler.py b/src/tmt_web/utils/git_handler.py index b85cb89..d2d79a9 100644 --- a/src/tmt_web/utils/git_handler.py +++ b/src/tmt_web/utils/git_handler.py @@ -9,13 +9,14 @@ GeneralError, Path, RunError, - git_clone, + git, ) def checkout_branch(path: Path, logger: Logger, ref: str) -> None: """ Checks out the given branch in the repository. + :param ref: Name of the ref to check out :param path: Path to the repository :param logger: Instance of Logger @@ -32,7 +33,9 @@ def checkout_branch(path: Path, logger: Logger, ref: str) -> None: def clone_repository(url: str, logger: Logger, ref: str | None = None) -> None: """ Clones the repository from the given URL and optionally checks out a specific ref. + Raises FileExistsError if the repository is already cloned or Exception if the cloning fails. + :param url: URL to the repository :param logger: Instance of Logger :param ref: Optional name of the ref to check out @@ -53,7 +56,7 @@ def clone_repository(url: str, logger: Logger, ref: str | None = None) -> None: raise FileExistsError try: - git_clone(url=url, destination=path, logger=logger) + git.git_clone(url=url, destination=path, logger=logger) if ref: try: @@ -71,6 +74,7 @@ def clone_repository(url: str, logger: Logger, ref: str | None = None) -> None: def get_path_to_repository(url: str) -> Path: """ Returns the path to the cloned repository from the given URL. + :param url: URL to the repository :return: Path to the cloned repository """ @@ -82,6 +86,7 @@ def get_path_to_repository(url: str) -> Path: def check_if_repository_exists(url: str) -> bool: """ Checks if the repository from the given URL is already cloned. + :param url: URL to the repository :return: True if the repository is already cloned, False otherwise """ @@ -91,6 +96,7 @@ def check_if_repository_exists(url: str) -> bool: def clear_tmp_dir(logger: Logger) -> None: """ Clears the .tmp directory. + :param logger: Instance of Logger :return: """ @@ -102,18 +108,21 @@ def clear_tmp_dir(logger: Logger) -> None: except Exception as e: logger.print("Failed to clear the repository clone directory!", color="red") raise e + logger.print("Repository clone directory cleared successfully!", color="green") def get_git_repository(url: str, logger: Logger, ref: str | None) -> Path: """ Clones the repository from the given URL and returns the path to the cloned repository. + :param url: URL to the repository :param logger: Instance of Logger :return: Path to the cloned repository """ with contextlib.suppress(FileExistsError): clone_repository(url, logger, ref) + return get_path_to_repository(url) From ea3a988cf5869f4b06e5315d3873d30fe69ae227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20=C5=A0pl=C3=ADchal?= Date: Tue, 8 Oct 2024 16:20:34 +0200 Subject: [PATCH 42/46] Remove unnecesary module warnings --- src/tmt_web/generators/html_generator.py | 4 ---- src/tmt_web/service.py | 4 ---- src/tmt_web/utils/git_handler.py | 4 ---- 3 files changed, 12 deletions(-) diff --git a/src/tmt_web/generators/html_generator.py b/src/tmt_web/generators/html_generator.py index dde0efd..e9d3e3d 100644 --- a/src/tmt_web/generators/html_generator.py +++ b/src/tmt_web/generators/html_generator.py @@ -31,7 +31,3 @@ def generate_html_page(obj: Test | Plan, logger: Logger) -> str: def generate_testplan_html_page(test: Test, plan: Plan, logger: Logger) -> str: logger.print("Generating the HTML file...") return render_template("testandplan.html.j2", test=test, plan=plan) - - -if __name__ == "__main__": - print("This is not an executable file!") diff --git a/src/tmt_web/service.py b/src/tmt_web/service.py index c91661d..50d399f 100644 --- a/src/tmt_web/service.py +++ b/src/tmt_web/service.py @@ -188,7 +188,3 @@ def main(test_url: str | None, return process_testplan_request(test_url, test_name, test_ref, test_path, plan_url, plan_name, plan_ref, plan_path, out_format) return None - - -if __name__ == "__main__": - print("This is not executable file!") diff --git a/src/tmt_web/utils/git_handler.py b/src/tmt_web/utils/git_handler.py index d2d79a9..a78a852 100644 --- a/src/tmt_web/utils/git_handler.py +++ b/src/tmt_web/utils/git_handler.py @@ -124,7 +124,3 @@ def get_git_repository(url: str, logger: Logger, ref: str | None) -> Path: clone_repository(url, logger, ref) return get_path_to_repository(url) - - -if __name__ == "__main__": - print("This is not executable file!") From 7b567ab71c87950367a8d8f36779bc242b29aed0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20=C5=A0pl=C3=ADchal?= Date: Tue, 8 Oct 2024 17:52:08 +0200 Subject: [PATCH 43/46] Some minor README changes --- README.md | 64 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index ad34a33..b767632 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# web +# tmt web Web application for checking tmt tests, plans and stories. @@ -10,7 +10,7 @@ To run the service locally for development purposes, use the following command: podman-compose up --build ``` -add `-d` for the service to run in the background +Add `-d` for the service to run in the background. ## Tests @@ -30,36 +30,52 @@ rebuild, start the service and run the tests. - `API_HOSTNAME` - *required*, specifies the hostname of the API, used for creating the callback URL to the service -# API -API for checking tmt tests and plans metadata -## Version -The API version is defined by prefix in url, e.g. -`/v0.1/status` -## Endpoints -* `/` - returns ID of the created Celery task with additional metadata in JSON and callback url for `/status` endpoint, -returns the same in HTML format if `format` is set to `html` - * `test-url` - URL of the repo test is located in - accepts a `string` +## API + +The API version is defined by prefix in url, e.g. `/v0.1/status`. + +If we want to display metadata for both tests and plans, we can combine +the `test-*` and `plan-*` options together, they are not mutually +exclusive. + +`test-url` and `test-name`, or `plan-url` and `plan-name` are required. - * `test-ref` - Ref of the repository the test is located in - accepts a `string`, - defaults to default branch of the repo +### `/` + +Returns ID of the created Celery task with additional metadata in JSON +and callback url for `/status` endpoint, returns the same in HTML format +if `format` is set to `html`. + + * `test-url` - URL of the repo test is located in - accepts a `string` + * `test-ref` - Ref of the repository the test is located in - accepts + a `string`, defaults to default branch of the repo * `test-path` - Points to directory where `fmf` tree is stored * `test-name` - Name of the test - accepts a `string` + * `plan-url` - URL of the repo plan is located in - accepts a `string` - - * `plan-ref` - Ref of the repository the plan is located in - accepts a `string`, - defaults to default branch of the repo + * `plan-ref` - Ref of the repository the plan is located in - accepts + a `string`, defaults to default branch of the repo * `plan-path` - Points to directory where `fmf` tree is stored * `plan-name` - Name of the plan - accepts a `string` - * `format` - Format of the output - accepts a `string`, default is `json`, other options are `xml`, `html` - (serves as a basic human-readable output format) + + * `format` - Format of the output - accepts a `string`, default is + `json`, other options are `xml`, `html` (serves as a basic + human-readable output format) * `id` - Unique ID of the tmt object -* `/status` - returns a status of the tmt object being processed by the backend + +### `/status` + +Returns a status of the tmt object being processed by the backend. + * `task_id` - ID of the task - accepts a `string` -* `/status/html` - returns a status of the tmt object being processed by the backend in a simple HTML formatting + +### `/status/html` + +Returns a status of the tmt object being processed by the backend in a +simple HTML formatting. + * `task_id` - ID of the task - accepts a `string` -* `/health` - returns a health status of the service -If we want to display metadata for both tests and plans, we can combine the `test-*` -and `plan-*` options together, they are not mutually exclusive. +### `/health` -`test-url` and `test-name`, or `plan-url` and `plan-name` are required. +Returns a health status of the service. From dfe70211f520aa4604ca6dd2c93e9f35dc3e1a76 Mon Sep 17 00:00:00 2001 From: Sabart Otto Date: Thu, 24 Oct 2024 13:10:38 +0200 Subject: [PATCH 44/46] Second bunch of code improvements (#7) * Add missing docstrings for return values * Move the settings to one place * Fix the typo in gitignore * Fix the RET505 lint * Provide service arguments as dictionary * Add a doctype to HTML template * Change page default title * Handle various types of responses when celery is disabled --- .gitignore | 2 +- src/tmt_web/api.py | 59 ++++++++++--------- src/tmt_web/generators/json_generator.py | 6 +- .../generators/templates/_base.html.j2 | 6 +- src/tmt_web/generators/yaml_generator.py | 6 +- src/tmt_web/service.py | 20 +++---- src/tmt_web/settings.py | 5 ++ src/tmt_web/utils/git_handler.py | 10 ++-- 8 files changed, 60 insertions(+), 54 deletions(-) create mode 100644 src/tmt_web/settings.py diff --git a/.gitignore b/.gitignore index 9b1452a..1d6f257 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,5 @@ venv .vscode dist .ruff_cache -.mypy_chache +.mypy_cache __pycache__ diff --git a/src/tmt_web/api.py b/src/tmt_web/api.py index d895b2d..45f6639 100644 --- a/src/tmt_web/api.py +++ b/src/tmt_web/api.py @@ -5,9 +5,9 @@ from fastapi import FastAPI from fastapi.params import Query from pydantic import BaseModel -from starlette.responses import HTMLResponse +from starlette.responses import HTMLResponse, JSONResponse, PlainTextResponse -from tmt_web import service +from tmt_web import service, settings from tmt_web.generators import html_generator app = FastAPI() @@ -98,36 +98,39 @@ def find_test( if plan_url is None and plan_name is None and test_url is None and test_name is None: return "Missing arguments!" # TODO: forward to docs + + service_args = { + "test_url": test_url, + "test_name": test_name, + "test_ref": test_ref, + "plan_url": plan_url, + "plan_name": plan_name, + "plan_ref": plan_ref, + "out_format": out_format, + "test_path": test_path, + "plan_path": plan_path, + } + # Disable Celery if not needed if os.environ.get("USE_CELERY") == "false": - return service.main( - test_url=test_url, - test_name=test_name, - test_ref=test_ref, - plan_url=plan_url, - plan_name=plan_name, - plan_ref=plan_ref, - out_format=out_format, - test_path=test_path, - plan_path=plan_path) - - r = service.main.delay( - test_url=test_url, - test_name=test_name, - test_ref=test_ref, - plan_url=plan_url, - plan_name=plan_name, - plan_ref=plan_ref, - out_format=out_format, - test_path=test_path, - plan_path=plan_path) + response_by_output = { + "html": HTMLResponse, + "json": JSONResponse, + "yaml": PlainTextResponse, + } + + response = response_by_output.get(out_format, PlainTextResponse) + return response(service.main(**service_args)) + + r = service.main.delay(**service_args) # Special handling of response if the format is html + # TODO: Shouldn't be the "yaml" format also covered with a `PlainTextResponse`? if out_format == "html": - status_callback_url = f'{os.getenv("API_HOSTNAME")}/status/html?task-id={r.task_id}' + status_callback_url = f"{settings.API_HOSTNAME}/status/html?task-id={r.task_id}" return HTMLResponse(content=html_generator.generate_status_callback(r, status_callback_url)) - else: - return _to_task_out(r) + + return _to_task_out(r) @app.get("/status") @@ -149,7 +152,7 @@ def status_html(task_id: Annotated[str | None, ) ]) -> HTMLResponse: r = service.main.app.AsyncResult(task_id) - status_callback_url = f'{os.getenv("API_HOSTNAME")}/status/html?task-id={r.task_id}' + status_callback_url = f"{settings.API_HOSTNAME}/status/html?task-id={r.task_id}" return HTMLResponse(content=html_generator.generate_status_callback(r, status_callback_url)) @@ -158,7 +161,7 @@ def _to_task_out(r: AsyncResult) -> TaskOut: # type: ignore [type-arg] id=r.task_id, status=r.status, result=r.traceback if r.failed() else r.result, - status_callback_url=f'{os.getenv("API_HOSTNAME")}/status?task-id={r.task_id}' + status_callback_url="{settings.API_HOSTNAME}/status?task-id={r.task_id}", ) diff --git a/src/tmt_web/generators/json_generator.py b/src/tmt_web/generators/json_generator.py index 47ef5d2..3cf76e0 100644 --- a/src/tmt_web/generators/json_generator.py +++ b/src/tmt_web/generators/json_generator.py @@ -35,7 +35,7 @@ def generate_test_json(test: tmt.Test, logger: tmt.Logger) -> str: :param test: Test object :param logger: tmt.Logger instance - :return: + :return: JSON data for a given test """ data = _create_json_data(test, logger) logger.print("Generating the JSON file...") @@ -48,7 +48,7 @@ def generate_plan_json(plan: tmt.Plan, logger: tmt.Logger) -> str: :param plan: Plan object :param logger: tmt.Logger instance - :return: + :return: JSON data for a given plan """ data = _create_json_data(plan, logger) logger.print("Generating the JSON file...") @@ -62,7 +62,7 @@ def generate_testplan_json(test: tmt.Test, plan: tmt.Plan, logger: tmt.Logger) - :param test: Test object :param plan: Plan object :param logger: tmt.Logger instance - :return: + :return: JSON data for a given test and plan """ logger.print("Generating the JSON file...") data = { diff --git a/src/tmt_web/generators/templates/_base.html.j2 b/src/tmt_web/generators/templates/_base.html.j2 index 83ec3e5..9a52187 100644 --- a/src/tmt_web/generators/templates/_base.html.j2 +++ b/src/tmt_web/generators/templates/_base.html.j2 @@ -1,7 +1,7 @@ - - + + - {{ title | default('HTML File') }} + {{ title | default('Untitled Document') }} diff --git a/src/tmt_web/generators/yaml_generator.py b/src/tmt_web/generators/yaml_generator.py index 08d2e03..b3aed80 100644 --- a/src/tmt_web/generators/yaml_generator.py +++ b/src/tmt_web/generators/yaml_generator.py @@ -14,7 +14,7 @@ def generate_test_yaml(test: tmt.Test, logger: Logger) -> str: :param test: Test object :param logger: tmt.Logger instance - :return: + :return: YAML data for a given test """ yaml_data = tmt.utils.dict_to_yaml(_create_json_data(test, logger)) print_success(logger) @@ -27,7 +27,7 @@ def generate_plan_yaml(plan: tmt.Plan, logger: Logger) -> str: :param plan: Plan object :param logger: tmt.Logger instance - :return: + :return: YAML data for a given plan. """ yaml_data = tmt.utils.dict_to_yaml(_create_json_data(plan, logger)) print_success(logger) @@ -41,7 +41,7 @@ def generate_testplan_yaml(test: tmt.Test, plan: tmt.Plan, logger: Logger) -> st :param test: Test object :param plan: Plan object :param logger: tmt.Logger instance - :return: + :return: YAML data for a given test and plan """ data = { "test": _create_json_data(test, logger), diff --git a/src/tmt_web/service.py b/src/tmt_web/service.py index 50d399f..d6e7ab3 100644 --- a/src/tmt_web/service.py +++ b/src/tmt_web/service.py @@ -1,18 +1,16 @@ import logging -import os import tmt from celery.app import Celery # type: ignore[attr-defined] from tmt.utils import Path # type: ignore[attr-defined] +from tmt_web import settings from tmt_web.generators import html_generator, json_generator, yaml_generator from tmt_web.utils import git_handler logger = tmt.Logger(logging.getLogger("tmt-logger")) -redis_url = os.getenv("REDIS_URL", "redis://localhost:6379") - -app = Celery(__name__, broker=redis_url, backend=redis_url) +app = Celery(__name__, broker=settings.REDIS_URL, backend=settings.REDIS_URL) def get_tree(url: str, name: str, ref: str | None, tree_path: str) -> tmt.base.Tree: @@ -23,7 +21,7 @@ def get_tree(url: str, name: str, ref: str | None, tree_path: str) -> tmt.base.T :param name: Object name :param url: Object url :param tree_path: Object path - :return: + :return: returns a Tree object """ logger.print(f"Cloning the repository for url: {url}") logger.print("Parsing the url and name...") @@ -52,7 +50,8 @@ def process_test_request(test_url: str, return_object: bool, out_format: str) -> str | tmt.Test | None: """ - This function processes the request for a test and returns the HTML file or the Test object. + This function processes the request for a test and returns the data in specified output format + or the Test object. :param test_url: Test url :param test_name: Test name @@ -60,7 +59,7 @@ def process_test_request(test_url: str, :param test_path: Test path :param return_object: Specify if the function should return the HTML file or the Test object :param out_format: Specifies output format - :return: + :return: the data in specified output format or the Test object """ tree = get_tree(test_url, test_name, test_ref, test_path) @@ -93,7 +92,8 @@ def process_plan_request(plan_url: str, return_object: bool, out_format: str) -> str | None | tmt.Plan: """ - This function processes the request for a plan and returns the HTML file or the Plan object. + This function processes the request for a plan and returns the data in specified output format + or the Plan object. :param plan_url: Plan URL :param plan_name: Plan name @@ -101,7 +101,7 @@ def process_plan_request(plan_url: str, :param plan_path: Plan path :param return_object: Specify if the function should return the HTML file or the Plan object :param out_format: Specifies output format - :return: + :return: the data in specified output format or the Plan object """ tree = get_tree(plan_url, plan_name, plan_ref, plan_path) @@ -147,7 +147,7 @@ def process_testplan_request(test_url, :param plan_ref: Plan repo ref :param plan_path: Plan path :param out_format: Specifies output format - :return: + :return: page data in specified output format """ test = process_test_request(test_url, test_name, test_ref, test_path, False, out_format) if not isinstance(test, tmt.Test): diff --git a/src/tmt_web/settings.py b/src/tmt_web/settings.py new file mode 100644 index 0000000..be5faac --- /dev/null +++ b/src/tmt_web/settings.py @@ -0,0 +1,5 @@ +import os + +API_HOSTNAME = os.getenv("API_HOSTNAME", default="") +REDIS_URL = os.getenv("REDIS_URL", default="redis://localhost:6379") +CLONE_DIR_PATH = os.getenv("CLONE_DIR_PATH", default="./.repos/") diff --git a/src/tmt_web/utils/git_handler.py b/src/tmt_web/utils/git_handler.py index a78a852..1bf569c 100644 --- a/src/tmt_web/utils/git_handler.py +++ b/src/tmt_web/utils/git_handler.py @@ -1,5 +1,4 @@ import contextlib -import os from shutil import rmtree from tmt import Logger @@ -12,6 +11,8 @@ git, ) +from tmt_web import settings + def checkout_branch(path: Path, logger: Logger, ref: str) -> None: """ @@ -20,7 +21,6 @@ def checkout_branch(path: Path, logger: Logger, ref: str) -> None: :param ref: Name of the ref to check out :param path: Path to the repository :param logger: Instance of Logger - :return: """ try: common_instance = Common(logger=logger) @@ -39,7 +39,6 @@ def clone_repository(url: str, logger: Logger, ref: str | None = None) -> None: :param url: URL to the repository :param logger: Instance of Logger :param ref: Optional name of the ref to check out - :return: """ logger.print("Cloning the repository...") path = get_path_to_repository(url) @@ -80,7 +79,7 @@ def get_path_to_repository(url: str) -> Path: """ repo_name = url.rstrip("/").rsplit("/", 1)[-1] root_dir = Path(__file__).resolve().parents[2] # going up from tmt_web/utils/git_handler.py - return root_dir / os.getenv("CLONE_DIR_PATH", "./.repos/") / repo_name + return root_dir / settings.CLONE_DIR_PATH / repo_name def check_if_repository_exists(url: str) -> bool: @@ -98,11 +97,10 @@ def clear_tmp_dir(logger: Logger) -> None: Clears the .tmp directory. :param logger: Instance of Logger - :return: """ logger.print("Clearing the .tmp directory...") root_dir = Path(__file__).resolve().parents[2] # going up from tmt_web/utils/git_handler.py - path = root_dir / os.getenv("CLONE_DIR_PATH", "./.repos/") + path = root_dir / settings.CLONE_DIR_PATH try: rmtree(path) except Exception as e: From 8cd7674ffff3cad26c8217bc0d901ca4960eea6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20=C5=A0pl=C3=ADchal?= Date: Thu, 24 Oct 2024 15:04:33 +0200 Subject: [PATCH 45/46] Fix the `test-ref` value (branch was removed) --- tests/test_api.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index 8ec5273..1682362 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -27,8 +27,11 @@ def test_basic_test_request_json(self, client): assert "https://github.com/teemtee/tmt/tree/main/tests/core/smoke/main.fmf" in data def test_basic_test_request_json_with_path(self, client): - response = client.get("/?test-url=https://github.com/teemtee/tmt.git&test-name=/test/shell/weird&" - "test-path=/tests/execute/basic/data&test-ref=link-issues-to-jira") + response = client.get( + "/?test-url=https://github.com/teemtee/tmt.git" + "&test-name=/test/shell/weird" + "&test-path=/tests/execute/basic/data" + "&test-ref=main") data = response.content.decode("utf-8") assert "500" not in data assert "https://github.com/teemtee/tmt/tree/main/tests/execute/basic/data/test.fmf" in data From 15ca2d561171cedfbd230f4ea4f41506769330a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20=C5=A0pl=C3=ADchal?= Date: Thu, 24 Oct 2024 15:42:18 +0200 Subject: [PATCH 46/46] Fix expected test output --- tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_api.py b/tests/test_api.py index 1682362..30b3e3d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -42,7 +42,7 @@ def test_basic_test_request_html(self, client): data = response.content.decode("utf-8") print(data) assert "500" not in data - assert '' in data + assert '' in data def test_basic_test_request_yaml(self, client): response = client.get("/?test-url=https://github.com/teemtee/tmt&test-name=/tests/core/smoke&format=yaml")