From 1cae9ae187c18cd9af9462add78e31fd795d1987 Mon Sep 17 00:00:00 2001 From: Alexandre Busquets Date: Wed, 2 Aug 2023 05:59:44 +0200 Subject: [PATCH 1/4] Add github action --- .github/workflows/main.yml | 47 ++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 29 ++++++++++++++++------- src/pytest.github.ini | 25 ++++++++++++++++++++ 3 files changed, 93 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/main.yml create mode 100644 src/pytest.github.ini diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..246e51d --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,47 @@ +name: Check code and Pytest + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:latest + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: filmin + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + + steps: + - uses: actions/checkout@v3 + + - name: psycopg prerequisites + run: sudo apt-get install libpq-dev + + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install poetry ruff + poetry install + + - name: Lint with ruff + run: | + ruff --format=github src/ + + - name: Test with pytest + working-directory: ./src + run: | + poetry run pytest -c pytest.github.ini diff --git a/docker-compose.yml b/docker-compose.yml index 1528822..2961112 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.7' +version: "3.7" services: api: container_name: api.filmin @@ -13,8 +13,9 @@ services: - env-api-dev environment: DEBUGPY: ${DEBUGPY:-true} - DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@${DB_HOST:-postgres}:5432/${POSTGRES_DB:-filmin} - WAIT_HOSTS: postgres:5432 + DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@${DB_HOST:-postgres}-woop:5432/${POSTGRES_DB:-filmin} + TEST_DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@${DB_HOST:-postgres}-test:5432/${POSTGRES_DB:-filmin} + WAIT_HOSTS: postgres:5432,postgres-test:5432,redis:6379 WAIT_LOGGER_LEVEL: error WAIT_TIMEOUT: 60 WAIT_SLEEP_INTERVAL: 5 @@ -22,6 +23,10 @@ services: REDIS_URL: ${REDIS_URL-redis://redis:6379/0} REDIS_USER: ${REDIS_USER-default} REDIS_PASSWORD: ${REDIS_PASSWORD-redis} + depends_on: + - postgres + - postgres-test + - redis volumes: - ./src/:/app:cached @@ -55,7 +60,16 @@ services: PGPASSWORD: ${PGPASSWORD-postgres} PGUSER: ${PGUSER-postgres} PHOST: localhost - command: ["postgres", "-c", "max_connections=1000", "-c", "log_statement=all", "-c", "log_destination=stderr"] + command: + [ + "postgres", + "-c", + "max_connections=1000", + "-c", + "log_statement=all", + "-c", + "log_destination=stderr", + ] ports: - 5432:5432 volumes: @@ -81,13 +95,12 @@ services: environment: REDIS_PASSWORD: ${REDIS_PASSWORD-redis} command: - - /bin/sh - - -c - - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD + - /bin/sh + - -c + - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD volumes: - redis_data:/data:delegated - volumes: postgres_data: postgres_test_data: diff --git a/src/pytest.github.ini b/src/pytest.github.ini new file mode 100644 index 0000000..d512add --- /dev/null +++ b/src/pytest.github.ini @@ -0,0 +1,25 @@ +[pytest] +norecursedirs = versions +testpaths = tests +python_files = tests.py test_*.py + +env = + APP_ENV=test + DATABASE_URL=postgresql+asyncpg://postgres:postgres@127.0.0.1:5432/filmin + + +addopts = + -p no:warnings + --cov=. + --no-cov-on-fail + --cov-report term-missing + --cov-report term:skip-covered + --cov-report xml + --cov-branch + + +; http://doc.pytest.org/en/latest/example/markers.html +markers = + unit_test: Pure unit tests. + integration_test: Tests that access a database, API, etc. + functional_test: End to end tests that needs a browser. From 4747285b08b8328a072a3e6e329c16e0487136a3 Mon Sep 17 00:00:00 2001 From: abusquets Date: Thu, 3 Aug 2023 05:23:12 +0200 Subject: [PATCH 2/4] feat: add memory cache for testing --- src/infra/cache/memory_cache.py | 39 +++++++++++++++++++++++++++++++++ src/tests/conftest.py | 16 +++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 src/infra/cache/memory_cache.py diff --git a/src/infra/cache/memory_cache.py b/src/infra/cache/memory_cache.py new file mode 100644 index 0000000..bd54490 --- /dev/null +++ b/src/infra/cache/memory_cache.py @@ -0,0 +1,39 @@ +from threading import RLock +import time + +from typing import Any, Dict, Union + +from infra.cache.ports import AbstractCacheRepository, EncodableT + + +class MemoryCache(AbstractCacheRepository): + def __init__(self) -> None: + self._data: Dict[str, tuple[EncodableT, float]] = {} + self.lock = RLock() + + async def init(self) -> None: + pass + + async def close(self) -> None: + with self.lock: + self._data = {} + + async def get(self, key: str) -> Any: + with self.lock: + ret = self._data.get(key) + if ret is not None: + if ret[1] < time.time(): + return ret[0] + else: + self._data.pop(key, None) + + async def set(self, key: str, value: Union[EncodableT, None], expire: int) -> None: + with self.lock: + if value is None: + self._data.pop(key, None) + else: + self._data[key] = (value, time.time() + expire) + + async def delete(self, key: str) -> None: + with self.lock: + self._data.pop(key, None) diff --git a/src/tests/conftest.py b/src/tests/conftest.py index c0fe2a8..d493633 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -10,16 +10,19 @@ from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_scoped_session, create_async_engine from sqlalchemy.orm import sessionmaker -from app.app_container import AppContainer +from app.app_container import AppContainer, AppContainerMixin from app.asgi import app from auth.utils import create_access_token from config import settings from core.data.repositories.ports.user import AbstractUserRepository from core.domain.schemas.user import User from core.schemas.user.create_user import CreateUserInDTO +from infra.cache.memory_cache import MemoryCache +from infra.cache.ports import AbstractCacheRepository import infra.database.sqlalchemy.models # noqa from infra.database.sqlalchemy.sqlalchemy import metadata +from utils.di import di_singleton @pytest.fixture(scope='session') @@ -129,3 +132,14 @@ async def async_normal_client(normal_user_access_token: str) -> AsyncClient: async with AsyncClient(app=app, base_url='http://test/api/v1') as ac: ac.headers.update({'Authorization': f'Bearer {normal_user_access_token}'}) yield ac + + +@pytest.fixture(autouse=True) +def memory_cache_database(monkeypatch: pytest.MonkeyPatch) -> Generator[None, None, None]: + @di_singleton + def fake_cache_gateway() -> AbstractCacheRepository: + return MemoryCache() + + monkeypatch.setattr(AppContainerMixin, '_get_cache_repository', lambda _: fake_cache_gateway()) + + yield From 67fb071e80509d9900ebbd23809215eeefe6d4db Mon Sep 17 00:00:00 2001 From: abusquets Date: Thu, 3 Aug 2023 05:57:46 +0200 Subject: [PATCH 3/4] fix: upgrade pre-commit --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 768dcca..edcbc16 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: - repo: https://github.com/myint/autoflake - rev: v1.4 + rev: v2.2.0 hooks: - id: autoflake args: @@ -31,14 +31,14 @@ repos: - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.272 + rev: v0.0.280 hooks: - id: ruff args: - --fix # Enables autofix - repo: https://github.com/psf/black # Refer to this repository for futher documentation about black hook - rev: 23.3.0 + rev: 23.7.0 hooks: - id: black From 4ccd85efaaf6bcc2414db1c94346e501732b08e1 Mon Sep 17 00:00:00 2001 From: abusquets Date: Thu, 3 Aug 2023 05:58:19 +0200 Subject: [PATCH 4/4] fix: format not detected by pre-commit --- src/shared/api/schemas/page.py | 2 +- src/shared/repository/sqlalchemy.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/shared/api/schemas/page.py b/src/shared/api/schemas/page.py index 041bbd8..2903f51 100644 --- a/src/shared/api/schemas/page.py +++ b/src/shared/api/schemas/page.py @@ -1,5 +1,5 @@ from enum import IntEnum -from typing import Generic, List, Type, TypeVar +from typing import Generic, List, TypeVar from pydantic import BaseModel, Field diff --git a/src/shared/repository/sqlalchemy.py b/src/shared/repository/sqlalchemy.py index cc8d129..eb38b5a 100644 --- a/src/shared/repository/sqlalchemy.py +++ b/src/shared/repository/sqlalchemy.py @@ -1,10 +1,10 @@ -from typing import Any, AsyncContextManager, AsyncIterator, Callable, List, Optional, Self, Tuple, Type, TypeVar, cast +from typing import Any, AsyncContextManager, Callable, List, Optional, Self, Tuple, Type, TypeVar, cast from pydantic import BaseModel from sqlalchemy.exc import NoResultFound from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import noload -from sqlalchemy.sql import Select, func, select, update +from sqlalchemy.sql import Select, func, select from shared.exceptions import NotFound from shared.repository.ports.generic import AbstractRepository, FilterBy