From 3498333eecd18e73e3972a04341bb4c3cebd9552 Mon Sep 17 00:00:00 2001 From: Michael Herman Date: Wed, 24 Jan 2024 12:46:39 -0700 Subject: [PATCH] updates --- project/app/api/crud.py | 5 +- project/app/api/summaries.py | 9 +- project/app/summarizer.py | 21 +++ project/requirements.txt | 2 + project/tests/test_summaries.py | 127 ++++---------- project/tests/test_summaries_unit.py | 245 +++++++++++++++++++++++++++ 6 files changed, 308 insertions(+), 101 deletions(-) create mode 100644 project/app/summarizer.py create mode 100644 project/tests/test_summaries_unit.py diff --git a/project/app/api/crud.py b/project/app/api/crud.py index a746993..9b58efb 100644 --- a/project/app/api/crud.py +++ b/project/app/api/crud.py @@ -17,10 +17,7 @@ async def get_all() -> List: async def post(payload: SummaryPayloadSchema) -> int: - summary = TextSummary( - url=payload.url, - summary="dummy summary", - ) + summary = TextSummary(url=payload.url, summary="") await summary.save() return summary.id diff --git a/project/app/api/summaries.py b/project/app/api/summaries.py index 6fb3e7e..10ce945 100644 --- a/project/app/api/summaries.py +++ b/project/app/api/summaries.py @@ -1,9 +1,10 @@ from typing import List -from fastapi import APIRouter, HTTPException, Path +from fastapi import APIRouter, BackgroundTasks, HTTPException, Path from app.api import crud from app.models.tortoise import SummarySchema +from app.summarizer import generate_summary from app.models.pydantic import ( # isort:skip SummaryPayloadSchema, @@ -29,9 +30,13 @@ async def read_all_summaries() -> List[SummarySchema]: @router.post("/", response_model=SummaryResponseSchema, status_code=201) -async def create_summary(payload: SummaryPayloadSchema) -> SummaryResponseSchema: +async def create_summary( + payload: SummaryPayloadSchema, background_tasks: BackgroundTasks +) -> SummaryResponseSchema: summary_id = await crud.post(payload) + background_tasks.add_task(generate_summary, summary_id, str(payload.url)) + response_object = {"id": summary_id, "url": payload.url} return response_object diff --git a/project/app/summarizer.py b/project/app/summarizer.py new file mode 100644 index 0000000..85c794a --- /dev/null +++ b/project/app/summarizer.py @@ -0,0 +1,21 @@ +import nltk +from newspaper import Article + +from app.models.tortoise import TextSummary + + +async def generate_summary(summary_id: int, url: str) -> None: + article = Article(url) + article.download() + article.parse() + + try: + nltk.data.find("tokenizers/punkt") + except LookupError: + nltk.download("punkt") + finally: + article.nlp() + + summary = article.summary + + await TextSummary.filter(id=summary_id).update(summary=summary) diff --git a/project/requirements.txt b/project/requirements.txt index 2aeffa2..13223a3 100644 --- a/project/requirements.txt +++ b/project/requirements.txt @@ -6,8 +6,10 @@ flake8==7.0.0 gunicorn==21.0.1 httpx==0.26.0 isort==5.13.2 +newspaper3k==0.2.8 pydantic-settings==2.1.0 pytest==7.4.4 pytest-cov==4.1.0 +pytest-xdist==3.5.0 tortoise-orm==0.20.0 uvicorn==0.26.0 diff --git a/project/tests/test_summaries.py b/project/tests/test_summaries.py index d31b5d9..16284f3 100644 --- a/project/tests/test_summaries.py +++ b/project/tests/test_summaries.py @@ -2,8 +2,15 @@ import pytest +from app.api import summaries + + +def test_create_summary(test_app_with_db, monkeypatch): + def mock_generate_summary(summary_id, url): + return None + + monkeypatch.setattr(summaries, "generate_summary", mock_generate_summary) -def test_create_summary(test_app_with_db): response = test_app_with_db.post( "/summaries/", data=json.dumps({"url": "https://foo.bar"}) ) @@ -34,7 +41,12 @@ def test_create_summaries_invalid_json(test_app): ) -def test_read_summary(test_app_with_db): +def test_read_summary(test_app_with_db, monkeypatch): + def mock_generate_summary(summary_id, url): + return None + + monkeypatch.setattr(summaries, "generate_summary", mock_generate_summary) + response = test_app_with_db.post( "/summaries/", data=json.dumps({"url": "https://foo.bar"}) ) @@ -46,7 +58,6 @@ def test_read_summary(test_app_with_db): response_dict = response.json() assert response_dict["id"] == summary_id assert response_dict["url"] == "https://foo.bar/" - assert response_dict["summary"] assert response_dict["created_at"] @@ -71,7 +82,12 @@ def test_read_summary_incorrect_id(test_app_with_db): } -def test_read_all_summaries(test_app_with_db): +def test_read_all_summaries(test_app_with_db, monkeypatch): + def mock_generate_summary(summary_id, url): + return None + + monkeypatch.setattr(summaries, "generate_summary", mock_generate_summary) + response = test_app_with_db.post( "/summaries/", data=json.dumps({"url": "https://foo.bar"}) ) @@ -84,7 +100,12 @@ def test_read_all_summaries(test_app_with_db): assert len(list(filter(lambda d: d["id"] == summary_id, response_list))) == 1 -def test_remove_summary(test_app_with_db): +def test_remove_summary(test_app_with_db, monkeypatch): + def mock_generate_summary(summary_id, url): + return None + + monkeypatch.setattr(summaries, "generate_summary", mock_generate_summary) + response = test_app_with_db.post( "/summaries/", data=json.dumps({"url": "https://foo.bar"}) ) @@ -116,7 +137,12 @@ def test_remove_summary_incorrect_id(test_app_with_db): } -def test_update_summary(test_app_with_db): +def test_update_summary(test_app_with_db, monkeypatch): + def mock_generate_summary(summary_id, url): + return None + + monkeypatch.setattr(summaries, "generate_summary", mock_generate_summary) + response = test_app_with_db.post( "/summaries/", data=json.dumps({"url": "https://foo.bar"}) ) @@ -135,95 +161,6 @@ def test_update_summary(test_app_with_db): assert response_dict["created_at"] -# def test_update_summary_incorrect_id(test_app_with_db): -# response = test_app_with_db.put( -# "/summaries/999/", -# data=json.dumps({"url": "https://foo.bar", "summary": "updated!"}) -# ) -# assert response.status_code == 404 -# assert response.json()["detail"] == "Summary not found" - -# response = test_app_with_db.put( -# f"/summaries/0/", -# data=json.dumps({"url": "https://foo.bar", "summary": "updated!"}) -# ) -# assert response.status_code == 422 -# assert response.json() == { -# "detail": [ -# { -# "ctx": {"gt": 0}, -# "input": "0", -# "loc": ["path", "id"], -# "msg": "Input should be greater than 0", -# "type": "greater_than", -# "url": "https://errors.pydantic.dev/2.5/v/greater_than", -# } -# ] -# } - - -# def test_update_summary_invalid_json(test_app_with_db): -# response = test_app_with_db.post( -# "/summaries/", data=json.dumps({"url": "https://foo.bar"}) -# ) -# summary_id = response.json()["id"] - -# response = test_app_with_db.put( -# f"/summaries/{summary_id}/", -# data=json.dumps({}) -# ) -# assert response.status_code == 422 -# assert response.json() == { -# "detail": [ -# { -# "input": {}, -# "loc": ["body", "url"], -# "msg": "Field required", -# "type": "missing", -# "url": "https://errors.pydantic.dev/2.5/v/missing", -# }, -# { -# "input": {}, -# "loc": ["body", "summary"], -# "msg": "Field required", -# "type": "missing", -# "url": "https://errors.pydantic.dev/2.5/v/missing", -# } -# ] -# } - - -# def test_update_summary_invalid_keys(test_app_with_db): -# response = test_app_with_db.post( -# "/summaries/", data=json.dumps({"url": "https://foo.bar"}) -# ) -# summary_id = response.json()["id"] - -# response = test_app_with_db.put( -# f"/summaries/{summary_id}/", -# data=json.dumps({"url": "https://foo.bar"}) -# ) -# assert response.status_code == 422 -# assert response.json() == { -# "detail": [ -# { -# "input": {"url": "https://foo.bar"}, -# "loc": ["body", "summary"], -# "msg": "Field required", -# "type": "missing", -# "url": "https://errors.pydantic.dev/2.5/v/missing", -# } -# ] -# } - -# response = test_app_with_db.put( -# f"/summaries/{summary_id}/", -# data=json.dumps({"url": "invalid://url", "summary": "updated!"}) -# ) -# assert response.status_code == 422 -# assert response.json()["detail"][0]["msg"] == "URL scheme should be 'http' or 'https'" - - @pytest.mark.parametrize( "summary_id, payload, status_code, detail", [ diff --git a/project/tests/test_summaries_unit.py b/project/tests/test_summaries_unit.py new file mode 100644 index 0000000..9216b0c --- /dev/null +++ b/project/tests/test_summaries_unit.py @@ -0,0 +1,245 @@ +import json +from datetime import datetime + +import pytest + +from app.api import crud, summaries + + +def test_create_summary(test_app, monkeypatch): + def mock_generate_summary(summary_id, url): + return None + + monkeypatch.setattr(summaries, "generate_summary", mock_generate_summary) + + test_request_payload = {"url": "https://foo.bar"} + test_response_payload = {"id": 1, "url": "https://foo.bar/"} + + async def mock_post(payload): + return 1 + + monkeypatch.setattr(crud, "post", mock_post) + + response = test_app.post( + "/summaries/", + data=json.dumps(test_request_payload), + ) + + assert response.status_code == 201 + assert response.json() == test_response_payload + + +def test_create_summaries_invalid_json(test_app): + response = test_app.post("/summaries/", data=json.dumps({})) + assert response.status_code == 422 + assert response.json() == { + "detail": [ + { + "type": "missing", + "loc": ["body", "url"], + "msg": "Field required", + "input": {}, + "url": "https://errors.pydantic.dev/2.5/v/missing", + } + ] + } + + response = test_app.post("/summaries/", data=json.dumps({"url": "invalid://url"})) + assert response.status_code == 422 + assert ( + response.json()["detail"][0]["msg"] == "URL scheme should be 'http' or 'https'" + ) + + +def test_read_summary(test_app, monkeypatch): + test_data = { + "id": 1, + "url": "https://foo.bar", + "summary": "summary", + "created_at": datetime.utcnow().isoformat(), + } + + async def mock_get(id): + return test_data + + monkeypatch.setattr(crud, "get", mock_get) + + response = test_app.get("/summaries/1/") + assert response.status_code == 200 + assert response.json() == test_data + + +def test_read_summary_incorrect_id(test_app, monkeypatch): + async def mock_get(id): + return None + + monkeypatch.setattr(crud, "get", mock_get) + + response = test_app.get("/summaries/999/") + assert response.status_code == 404 + assert response.json()["detail"] == "Summary not found" + + +def test_read_all_summaries(test_app, monkeypatch): + test_data = [ + { + "id": 1, + "url": "https://foo.bar", + "summary": "summary", + "created_at": datetime.utcnow().isoformat(), + }, + { + "id": 2, + "url": "https://testdrivenn.io", + "summary": "summary", + "created_at": datetime.utcnow().isoformat(), + }, + ] + + async def mock_get_all(): + return test_data + + monkeypatch.setattr(crud, "get_all", mock_get_all) + + response = test_app.get("/summaries/") + assert response.status_code == 200 + assert response.json() == test_data + + +def test_remove_summary(test_app, monkeypatch): + async def mock_get(id): + return { + "id": 1, + "url": "https://foo.bar", + "summary": "summary", + "created_at": datetime.utcnow().isoformat(), + } + + monkeypatch.setattr(crud, "get", mock_get) + + async def mock_delete(id): + return id + + monkeypatch.setattr(crud, "delete", mock_delete) + + response = test_app.delete("/summaries/1/") + assert response.status_code == 200 + assert response.json() == {"id": 1, "url": "https://foo.bar/"} + + +def test_remove_summary_incorrect_id(test_app, monkeypatch): + async def mock_get(id): + return None + + monkeypatch.setattr(crud, "get", mock_get) + + response = test_app.delete("/summaries/999/") + assert response.status_code == 404 + assert response.json()["detail"] == "Summary not found" + + +def test_update_summary(test_app, monkeypatch): + test_request_payload = {"url": "https://foo.bar", "summary": "updated"} + test_response_payload = { + "id": 1, + "url": "https://foo.bar", + "summary": "summary", + "created_at": datetime.utcnow().isoformat(), + } + + async def mock_put(id, payload): + return test_response_payload + + monkeypatch.setattr(crud, "put", mock_put) + + response = test_app.put( + "/summaries/1/", + data=json.dumps(test_request_payload), + ) + assert response.status_code == 200 + assert response.json() == test_response_payload + + +@pytest.mark.parametrize( + "summary_id, payload, status_code, detail", + [ + [ + 999, + {"url": "https://foo.bar", "summary": "updated!"}, + 404, + "Summary not found", + ], + [ + 0, + {"url": "https://foo.bar", "summary": "updated!"}, + 422, + [ + { + "type": "greater_than", + "loc": ["path", "id"], + "msg": "Input should be greater than 0", + "input": "0", + "ctx": {"gt": 0}, + "url": "https://errors.pydantic.dev/2.5/v/greater_than", + } + ], + ], + [ + 1, + {}, + 422, + [ + { + "type": "missing", + "loc": ["body", "url"], + "msg": "Field required", + "input": {}, + "url": "https://errors.pydantic.dev/2.5/v/missing", + }, + { + "type": "missing", + "loc": ["body", "summary"], + "msg": "Field required", + "input": {}, + "url": "https://errors.pydantic.dev/2.5/v/missing", + }, + ], + ], + [ + 1, + {"url": "https://foo.bar"}, + 422, + [ + { + "type": "missing", + "loc": ["body", "summary"], + "msg": "Field required", + "input": {"url": "https://foo.bar"}, + "url": "https://errors.pydantic.dev/2.5/v/missing", + } + ], + ], + ], +) +def test_update_summary_invalid( + test_app, monkeypatch, summary_id, payload, status_code, detail +): + async def mock_put(id, payload): + return None + + monkeypatch.setattr(crud, "put", mock_put) + + response = test_app.put(f"/summaries/{summary_id}/", data=json.dumps(payload)) + assert response.status_code == status_code + assert response.json()["detail"] == detail + + +def test_update_summary_invalid_url(test_app): + response = test_app.put( + "/summaries/1/", + data=json.dumps({"url": "invalid://url", "summary": "updated!"}), + ) + assert response.status_code == 422 + assert ( + response.json()["detail"][0]["msg"] == "URL scheme should be 'http' or 'https'" + )