From 6b9e3eceecbb9000a5d96bbc5cb8e1626ce784a3 Mon Sep 17 00:00:00 2001 From: Sergii Smyrnov Date: Tue, 21 Nov 2023 15:03:42 +0100 Subject: [PATCH 01/11] Move dev requirements out of requirements.txt --- requirements-dev.txt | 7 +++++++ requirements.txt | 6 ------ 2 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 requirements-dev.txt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..cf1b592 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,7 @@ +-r requirements.txt +asynctest==0.13.0 +pytest==6.0.2 +pytest-cov==2.10.1 +pytest-aiohttp==0.3.0 +pre-commit==2.7.1 +coverage==5.3 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e2f8204..49e2ef9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,3 @@ fastapi==0.65.2 uvicorn==0.11.8 -asynctest==0.13.0 -pytest==6.0.2 -pytest-cov==2.10.1 -pytest-aiohttp==0.3.0 -coverage==5.3 elasticsearch==7.9.1 -pre-commit==2.7.1 \ No newline at end of file From c64d800f34d76b62cf9ab76323a476891a0bf117 Mon Sep 17 00:00:00 2001 From: Sergii Smyrnov Date: Tue, 21 Nov 2023 15:05:43 +0100 Subject: [PATCH 02/11] Add github action for tests and lining --- .github/workflows/main.yml | 28 ++++++++++++++++++++++++++++ requirements-dev.txt | 3 ++- 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..4393807 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,28 @@ +name: Unit tests + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + name: Set up Python + with: + python-version: "3.8" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + - name: Lint with ruff + run: | + ruff . + - name: Test with pytest + run: | + PYTHONPATH=`pwd` pytest --cache-clear --cov=. tests diff --git a/requirements-dev.txt b/requirements-dev.txt index cf1b592..b1a59c8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,4 +4,5 @@ pytest==6.0.2 pytest-cov==2.10.1 pytest-aiohttp==0.3.0 pre-commit==2.7.1 -coverage==5.3 \ No newline at end of file +coverage==5.3 +ruff==0.1.6 From ae7e5a623f770139e0d2e70cbf0b5f2ded8c8233 Mon Sep 17 00:00:00 2001 From: Sergii Smyrnov Date: Tue, 21 Nov 2023 15:38:59 +0100 Subject: [PATCH 03/11] Bump dev requirements --- requirements-dev.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index b1a59c8..f92be92 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,8 +1,8 @@ -r requirements.txt asynctest==0.13.0 -pytest==6.0.2 -pytest-cov==2.10.1 -pytest-aiohttp==0.3.0 -pre-commit==2.7.1 -coverage==5.3 +pytest==7.4.3 +pytest-cov==4.1.0 +pytest-aiohttp==1.0.5 +pre-commit==3.5.0 +coverage==7.3.2 ruff==0.1.6 From c0bd03e6022cbb6f9a55c5bf1245bcad740a0f97 Mon Sep 17 00:00:00 2001 From: Sergii Smyrnov Date: Tue, 21 Nov 2023 15:39:17 +0100 Subject: [PATCH 04/11] Bump fastapi and uvicorn --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 49e2ef9..07b59ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -fastapi==0.65.2 -uvicorn==0.11.8 +fastapi==0.104.1 +uvicorn==0.24.0.post1 elasticsearch==7.9.1 From af6dea85b407d25a31136be9b2b81e42f2624b7c Mon Sep 17 00:00:00 2001 From: Sergii Smyrnov Date: Tue, 21 Nov 2023 15:41:45 +0100 Subject: [PATCH 05/11] Add ruff config --- pyproject.toml | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 74b536d..38db817 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.black] include = '\.pyi?$' -target-version = ['py37'] +target-version = ['py311'] exclude = ''' /( \.git @@ -16,4 +16,31 @@ exclude = ''' | blib2to3 | tests/data )/ -''' \ No newline at end of file +''' + +[tool.ruff] +# Docs for Ruff are here https://github.com/charliermarsh/ruff +line-length = 120 + +# Enable Pyflakes `E` and `F` codes. `I` is for isort. +select = ["C", "E", "F", "W", "I001", "SIM", "TCH", "UP", "ERA", "PIE", "RET", "TID"] +ignore = ["C901"] + +exclude = [ + ".direnv", + ".git", + ".mypy_cache", + ".ruff_cache", + ".venv", + "__pypackages__", +] +per-file-ignores = {} + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +target-version = "py311" + +[tool.ruff.isort] +combine-as-imports = true +force-single-line = false \ No newline at end of file From bcf8cc3093fb83c21b7dc1210dc6259f7911c119 Mon Sep 17 00:00:00 2001 From: Sergii Smyrnov Date: Tue, 21 Nov 2023 16:11:02 +0100 Subject: [PATCH 06/11] Remove asynctest --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index f92be92..98ebe2f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,8 +1,8 @@ -r requirements.txt -asynctest==0.13.0 pytest==7.4.3 pytest-cov==4.1.0 pytest-aiohttp==1.0.5 +pytest-asyncio==0.21.1 pre-commit==3.5.0 coverage==7.3.2 ruff==0.1.6 From 9f2f52ab035a7628b0f148e93912e0fa913ff501 Mon Sep 17 00:00:00 2001 From: Sergii Smyrnov Date: Tue, 21 Nov 2023 16:11:53 +0100 Subject: [PATCH 07/11] Use 3.11 style typing --- quepid_es_proxy/elasticsearch/executor.py | 8 +++--- quepid_es_proxy/main.py | 25 ++++++------------ tests/units/test_main.py | 31 ++++++++--------------- 3 files changed, 22 insertions(+), 42 deletions(-) diff --git a/quepid_es_proxy/elasticsearch/executor.py b/quepid_es_proxy/elasticsearch/executor.py index 9431c0c..bfb0130 100644 --- a/quepid_es_proxy/elasticsearch/executor.py +++ b/quepid_es_proxy/elasticsearch/executor.py @@ -1,5 +1,3 @@ -from typing import List, Optional - from .connection import get_connection @@ -8,9 +6,9 @@ async def search( from_: int, size: int, explain: bool = False, - source: Optional[List[str]] = None, - query: Optional[str] = None, - q: Optional[str] = None, + source: list[str] | None = None, + query: str | None = None, + q: str | None = None, ): conn = await get_connection() payload = { diff --git a/quepid_es_proxy/main.py b/quepid_es_proxy/main.py index 62a523f..42ecdad 100644 --- a/quepid_es_proxy/main.py +++ b/quepid_es_proxy/main.py @@ -1,8 +1,6 @@ -from typing import List, Optional, Union - from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware -from pydantic import BaseModel +from pydantic import BaseModel, Field from .auth import basic_auth from .elasticsearch import executor @@ -10,7 +8,7 @@ app = FastAPI() # Replace "*" to the list of your origins, e.g. -# origins = ["quepid.yourcompany.com", "localhost:8080"] +# origins = ["quepid.yourcompany.com", "localhost:8080"] # noqa: ERA001 origins = "*" app.add_middleware( @@ -30,20 +28,15 @@ async def root(): class ProxyRequst(BaseModel): explain: bool - from_: int + from_: int = Field(..., alias="from") size: int - source: Union[str, List[str], None] - query: Optional[dict] - - class Config: - fields = {"from_": "from", "source": "_source"} + source: str | list[str] | None = Field(None, alias="_source") + query: dict | None @app.post("/{index_name}") -async def search_proxy( - index_name: str, body: ProxyRequst, username: str = Depends(basic_auth) -) -> dict: - result = await executor.search( +async def search_proxy(index_name: str, body: ProxyRequst, username: str = Depends(basic_auth)) -> dict: + return await executor.search( index_name, body.from_, body.size, @@ -52,7 +45,6 @@ async def search_proxy( {"query": body.query} if body.query else None, None, ) - return result @app.get("/{index_name}") @@ -63,7 +55,7 @@ async def explain_missing_documents( size: int, username: str = Depends(basic_auth), ) -> dict: - result = await executor.search( + return await executor.search( index_name, 0, size, @@ -72,7 +64,6 @@ async def explain_missing_documents( None, q, ) - return result @app.post("/{index_name}/_doc/{doc_id}/_explain") diff --git a/tests/units/test_main.py b/tests/units/test_main.py index d1fb693..bf48926 100644 --- a/tests/units/test_main.py +++ b/tests/units/test_main.py @@ -1,23 +1,20 @@ +from unittest.mock import patch + import pytest -from asynctest import patch +@pytest.mark.asyncio @pytest.mark.parametrize( "query,expected_query", [(None, None), ({"day": "Friday"}, {"query": {"day": "Friday"}})], ) async def test_search_proxy(environment, query, expected_query): - from quepid_es_proxy import main - with patch.object( - main.executor, "search", return_value={"test": "passed"} - ) as search_mock: + with patch.object(main.executor, "search", return_value={"test": "passed"}) as search_mock: result = await main.search_proxy( "big-index", - body=main.ProxyRequst( - **{"explain": True, "from": 3, "size": 7, "query": query} - ), + body=main.ProxyRequst(**{"explain": True, "from": 3, "size": 7, "query": query}), ) assert result == {"test": "passed"} @@ -32,29 +29,23 @@ async def test_search_proxy(environment, query, expected_query): ) +@pytest.mark.asyncio async def test_explain_missing_documents(): from quepid_es_proxy import main - with patch.object( - main.executor, "search", return_value={"test": "passed"} - ) as search_mock: + with patch.object(main.executor, "search", return_value={"test": "passed"}) as search_mock: result = await main.explain_missing_documents( index_name="index-123", _source="_id,title", q="title:Berlin", size=2 ) assert result == {"test": "passed"} - search_mock.assert_awaited_once_with( - "index-123", 0, 2, False, "_id,title", None, "title:Berlin" - ) + search_mock.assert_awaited_once_with("index-123", 0, 2, False, "_id,title", None, "title:Berlin") +@pytest.mark.asyncio async def test_explain(): from quepid_es_proxy import main - with patch.object( - main.executor, "explain", return_value={"test": "passed again!"} - ) as explain_mock: - result = await main.explain( - index_name="index-123", doc_id="123_321", query={"match": "all"} - ) + with patch.object(main.executor, "explain", return_value={"test": "passed again!"}) as explain_mock: + result = await main.explain(index_name="index-123", doc_id="123_321", query={"match": "all"}) assert result == {"test": "passed again!"} explain_mock.assert_awaited_once_with("index-123", "123_321", {"match": "all"}) From c8b1691b73355494f13c63ea5cecef6769095a09 Mon Sep 17 00:00:00 2001 From: Sergii Smyrnov Date: Tue, 21 Nov 2023 16:12:48 +0100 Subject: [PATCH 08/11] Remove the usage of pyfmt. Add lint target --- Makefile | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index c2c44b5..259ca02 100644 --- a/Makefile +++ b/Makefile @@ -1,28 +1,18 @@ -.PHONY: doit prepare-env run-tests +.PHONY: lint prepare-env tests # Removes the existing virtualenv, creates a new one, install dependencies. prepare-env: rm -rf .venv - python3.8 -m venv .venv + python3.11 -m venv .venv .venv/bin/pip install -U pip - .venv/bin/pip install -r requirements.txt + .venv/bin/pip install -r requirements-dev.txt -doit: - # Dependencies should be installed from requirements-dev.txt. - # Sorts imports in python files. - docker run -v `pwd`:`pwd` -w `pwd` quay.io/amboss-mededu/pyfmt:0.7 isort . - docker run -v `pwd`:`pwd` -w `pwd` quay.io/amboss-mededu/pyfmt:0.7 black --exclude .venv . - # Linting. - docker run -v `pwd`:`pwd` -w `pwd` quay.io/amboss-mededu/pyfmt:0.7 flake8 --exclude .venv --exclude=.venv --max-line-length=120 . - # Formats python files again after flake8. - docker run -v `pwd`:`pwd` -w `pwd` quay.io/amboss-mededu/pyfmt:0.7 black --exclude .venv . - # Static type checking. - docker run -v `pwd`:`pwd` -w `pwd` quay.io/amboss-mededu/pyfmt:0.7 mypy --ignore-missing-imports quepid_es_proxy +lint: + .venv/bin/ruff format . - -run-tests: - PYTHONPATH=`pwd` .venv/bin/pytest -W ignore tests/units --cov-report xml:cov.xml --cov . +tests: + PYTHONPATH=`pwd` .venv/bin/pytest -vv -W ignore tests/units --cov-report xml:cov.xml --cov . run-server: PROXY_USERNAME="lab_user" \ From 05d4b9efb590d54f4782d265b7342e5aeb917a7c Mon Sep 17 00:00:00 2001 From: Sergii Smyrnov Date: Tue, 21 Nov 2023 16:14:35 +0100 Subject: [PATCH 09/11] Use python 3.11.4 --- .github/workflows/main.yml | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4393807..c0b3ebc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/setup-python@v4 name: Set up Python with: - python-version: "3.8" + python-version: "3.11" - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/Dockerfile b/Dockerfile index a5f8367..01d5b5b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8.5-slim-buster +FROM python:3.11.4-slim-buster WORKDIR /app From bca97de3e499b82036234b9ce89b61ecdaf0a48f Mon Sep 17 00:00:00 2001 From: Sergii Smyrnov Date: Tue, 21 Nov 2023 16:18:46 +0100 Subject: [PATCH 10/11] Use ruff pre-commit --- .pre-commit-config.yaml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0d4dcb4..138a3a2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,11 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks repos: - - repo: https://github.com/psf/black - rev: 20.8b1 + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.1.6 hooks: - - id: black - language_version: python3.8 - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.3 - hooks: - - id: flake8 + # Run the linter. + - id: ruff + # Run the formatter. + - id: ruff-format From f92f4a9ab94e2d65e5acebe666206b376a404a15 Mon Sep 17 00:00:00 2001 From: Sergii Smyrnov Date: Tue, 21 Nov 2023 16:19:10 +0100 Subject: [PATCH 11/11] Update README. Public docker image is not available. --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6e444c2..1ba8992 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,9 @@ rarely available publicly or can be tunneled to the labeler's computer. ## Run with Docker -The image of the proxy is publicly available on [https://quay.io/amboss-mededu/quepid_es_proxy](https://quay.io/amboss-mededu/quepid_es_proxy). +~~The image of the proxy is publicly available on [https://quay.io/amboss-mededu/quepid_es_proxy](https://quay.io/amboss-mededu/quepid_es_proxy).~~ +Currently the image is not publicly available. Please build it locally. + To run the proxy docker execute ```bash docker run \ @@ -19,7 +21,7 @@ docker run \ -e "ES_USE_SSL=true" \ -e "WEB_CONCURRENCY=2" \ -p 5000:5000 \ -quay.io/amboss-mededu/quepid_es_proxy +quepid_es_proxy ``` The proxy is now available with basic auth now on `http://username_is_here:password_is_here@localhost:5000/`. Use this address in Quepid instead of Elasticsearch. @@ -36,12 +38,12 @@ docker run \ -e "WEB_CONCURRENCY=2" \ -p 5000:5000 \ --network="elasticsearch-docker-network" \ -quay.io/amboss-mededu/quepid_es_proxy +quepid_es_proxy ``` ## Run locally with Python virtual environment. -Proxy uses Python 3.8. +Proxy uses Python 3.11. First prepare a virtual environment `make prepare-env`. The proxy will be available with the default credentials on