Merge branch 'develop' into M2-3148

# Conflicts:
#	src/apps/answers/crud/answers.py
#	src/apps/applets/crud/applets.py

.env.default

@@ -25,8 +25,8 @@ CORS__ALLOW_HEADERS=*
 
 
 # Authentication
-AUTHENTICATION__ACCESS_TOKEN__SECRET_KEY="e51bcf5f4cb8550ff3f6a8bb4dfe112a3da2cf5142929e1b281cd974c88fa66c"
-AUTHENTICATION__REFRESH_TOKEN__SECRET_KEY="5da342d54ed5659f123cdd1cefe439c5aaf7e317a0aba1405375c07d32e097cc"
+AUTHENTICATION__ACCESS_TOKEN__SECRET_KEY="secret1"
+AUTHENTICATION__REFRESH_TOKEN__SECRET_KEY="secret2"
 AUTHENTICATION__ACCESS_TOKEN__EXPIRATION=30
 AUTHENTICATION__REFRESH_TOKEN__EXPIRATION=540
 AUTHENTICATION__ALGORITHM="HS256"
@@ -53,12 +53,14 @@ MAILING__VALIDATE_CERTS=False
 # FCM Notification api key
 NOTIFICATION__API_KEY=
 
-# CDN configs
-CDN__SECRET_KEY=
-CDN__ACCESS_KEY=
+# CDN configs (container by default)
+CDN__ENDPOINT_URL=http://localhost:9000
+CDN__SECRET_KEY=miniosecret
+CDN__ACCESS_KEY=minioaccess
 CDN__REGION=
-CDN__BUCKET_ANSWERS=
-CDN__BUCKET=
+CDN__BUCKET_ANSWER=media
+CDN__BUCKET=media
+CDN__STORAGE_ADDRESS=http://localhost:9000
 CDN__LEGACY_REGION=
 CDN__LEGACY_BUCKET=
 CDN__LEGACY_SECRET_KEY=

.github/workflows/code_quality.yaml

@@ -6,10 +6,22 @@ jobs:
  build:
  runs-on: ubuntu-latest
  steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v4
+ with:
+ repository: 'awslabs/git-secrets'
+ ref: 'master'
+ - name: Install git-secrets
+ run: sudo make install
+ - uses: actions/checkout@v4
  - uses: actions/setup-python@v3
  with:
  python-version: '3.10'
+ - name: Install git-secrets in the repository
+ run: git secrets --install
+ - name: Install git-secrets aws register in the repository 
+ run: git secrets --register-aws
+ - name: Scan aws secrets
+ run: git secrets --scan
  - name: Install pipenv
  run: python -m pip install --upgrade pip && pip install pipenv
  - name: Install deps

.pre-commit-config.yaml

@@ -29,8 +29,8 @@ repos:
  entry: mypy
  language: system
  types: [python]
- exclude: "apps/girderformindlogger"
- 
+ exclude: "apps/girderformindlogger|apps/migrate"
+
  - id: git-secrets
  name: git-secrets
  entry: git secrets --scan

Makefile

@@ -37,6 +37,10 @@ cq:
 migrate:
  python src/apps/migrate/run.py
 
+.PHONY: migrate_answer
+migrate_answer:
+ python src/apps/migrate/answers/run.py
+
 # ###############
 # Docker
 # ###############

Pipfile

@@ -7,7 +7,7 @@ name = "pypi"
 aioredis = "~=2.0.1"
 alembic = "~=1.8.1"
 asyncpg = "~=0.27.0"
-asynctest = "*"
+asynctest = "==0.13.0"
 boto3 = "==1.26.10"
 fastapi = "==0.87.0"
 # The latest version of the fastapi is not taken because of the issue
@@ -16,6 +16,7 @@ fastapi = "==0.87.0"
 fastapi-mail = "~=1.2.2"
 httpx = "~=0.23"
 jinja2 = "~=3.1.2"
+bcrypt = "==4.0.1"
 passlib = {version = "~=1.7.4", extras = ["bcrypt"]}
 pillow = "~=9.3.0"
 psutil = "~=5.9.4"
@@ -24,17 +25,20 @@ pydantic = {extras = ["email"], version = "~=1.10.2"}
 python-jose = {version = "~=3.3.0", extras = ["cryptography"]}
 python-multipart = "~=0.0.5"
 sentry-sdk = "~=1.6"
-sqlalchemy = {extras = ["asyncio"], version = "~=1.4.46"}
+sqlalchemy = {extras = ["asyncio"], version = "==1.4.49"}
 uvicorn = {extras = ["standard"], version = "==0.19"}
-aiohttp = "*"
-firebase-admin = "*"
-aio-pika = "*"
-pyld = "*"
-azure-storage-blob = "*"
-taskiq-fastapi = "*"
-taskiq-redis = "*"
-taskiq-aio-pika = "*"
+taskiq = {extras = ["reload"], version = "==0.9.1"}
+aiohttp = "==3.8.5"
+firebase-admin = "==6.2.0"
+aio-pika = "==9.3.0"
+pyld = "==2.0.3"
+azure-storage-blob = "==12.18.2"
+taskiq-fastapi = "==0.3.0"
+taskiq-redis = "==0.5.0"
+taskiq-aio-pika = "==0.4.0"
 sqlalchemy-utils = "==0.41.1"
+typer = {extras = ["all"], version = "==0.9.0"}
+aiofiles = "==23.2.1"
 
 [dev-packages]
 black = "~=22.6"
@@ -78,9 +82,10 @@ pycryptodomex = "==3.9.7"
 psycopg2-binary = "==2.9.6"
 cachetools = "==5.3.0"
 pyld = "==2.0.3"
-types-requests = "*"
-taskiq = {extras = ["reload"], version = "*"}
-types-pytz = "*"
-
+types-requests = "==2.31.0.10"
+types-pytz = "==2023.3.1.1"
+gevent = "~=23.9"
+types-aiofiles = "==23.2.0.0"
+types-cachetools = "==5.3.0.7"
 [requires]
 python_version = "3.10"

README.md

@@ -12,15 +12,13 @@
 - ✅ [Redis](https://redis.io)
 - ✅ [Docker](https://docs.docker.com/get-docker/)
 - ✅ [Pydantic](https://pydantic-docs.helpmanual.io)
-- ✅ [FastAPI](https://fastapi.tiangolo.com/)
 - ✅ [SQLAlchemy](https://www.sqlalchemy.org/)
 
 And
 
 - ✅ [The 12-Factor App](https://12factor.net)
-- ✅ [Domain driven design](https://www.amazon.com/Domain-Driven-Design-Tackling-Complexity-Software-ebook/dp/B00794TAUG)
 
-</br>
+<br/>
 
 🔌 **Code quality tools:**
 - ✅ [flake8](https://github.com/pycqa/flake8)
@@ -29,7 +27,7 @@ And
 - ✅ [mypy](https://github.com/python/mypy)
 - ✅ [pytest](https://github.com/pytest-dev/pytest)
 
-</br>
+<br/>
 
 ## ✋ <span style="color:#9DB7FF">Mandatory steps</span>
 
@@ -49,23 +47,22 @@ git clone [email protected]:ChildMindInstitute/mindlogger-backend-refactor.git
 
 
 #### 2.1 Description 📜
-| Key | Default value | Description |
-| --- |------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| PYTHONPATH | src/ | This variable makes all folders inside `src/` reachable in a runtime. </br> ***NOTE:*** You don't need to do this if you use Docker as far as it is hardcoded in `Dockerfile` |
-| DATABASE__HOST | postgres | Database Host |
-| DATABASE__USER | postgres | User name for Postgresql Database user |
-| DATABASE__PASSWORD | postgres | Password for Postgresql Database user |
-| DATABASE__DB | mindlogger_backend | Database name |
-| CORS__ALLOW_ORIGINS | `*` | Represents the list of allowed origins. Set the `Access-Control-Allow-Origin` header. Example: `https://dev.com,http://localohst:8000` |
-| CORS__ALLOW_CREDENTIALS | true | Set the `Access-Control-Allow-Credentials` header |
-| CORS__ALLOW_METHODS | `*` | Set the `Access-Control-Allow-Methods` header |
-| CORS__ALLOW_HEADERS | `*` | Set the `Access-Control-Allow-Headers` header |
-| AUTHENTICATION__SECRET_KEY | e51bcf5f4cb8550ff3f6a8bb4dfe112a | Access token's salt |
-| AUTHENTICATION__REFRESH_SECRET_KEY | 5da342d54ed5659f123cdd1cefe439c5aaf7e317a0aba1405375c07d32e097cc | Refresh token salt |
-| AUTHENTICATION__ALGORITHM | HS256 | The JWT's algorithm |
-| AUTHENTICATION__ACCESS_TOKEN_EXPIRATION_TIME | 30 | Time in minutes after which the access token will stop working |
-| AUTHENTICATION__REFRESH_TOKEN_EXPIRATION_TIME | 30 | Time in minutes after which the refresh token will stop working |
-| ADMIN_DOMAIN | - | Admin panel domain |
+| Key | Default value | Description |
+| --- |--------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| DATABASE__HOST | postgres | Database Host |
+| DATABASE__USER | postgres | User name for Postgresql Database user |
+| DATABASE__PASSWORD | postgres | Password for Postgresql Database user |
+| DATABASE__DB | mindlogger_backend | Database name |
+| CORS__ALLOW_ORIGINS | `*` | Represents the list of allowed origins. Set the `Access-Control-Allow-Origin` header. Example: `https://dev.com,http://localohst:8000` |
+| CORS__ALLOW_CREDENTIALS | true | Set the `Access-Control-Allow-Credentials` header |
+| CORS__ALLOW_METHODS | `*` | Set the `Access-Control-Allow-Methods` header |
+| CORS__ALLOW_HEADERS | `*` | Set the `Access-Control-Allow-Headers` header |
+| AUTHENTICATION__ACCESS_TOKEN__SECRET_KEY | secret1 | Access token's salt |
+| AUTHENTICATION__REFRESH_TOKEN__SECRET_KEY | secret2 | Refresh token salt |
+| AUTHENTICATION__ALGORITHM | HS256 | The JWT's algorithm |
+| AUTHENTICATION__ACCESS_TOKEN__EXPIRATION | 30 | Time in minutes after which the access token will stop working |
+| AUTHENTICATION__REFRESH_TOKEN__EXPIRATION | 30 | Time in minutes after which the refresh token will stop working |
+| ADMIN_DOMAIN | - | Admin panel domain |
 
 ##### ✋ Mandatory:
 
@@ -82,7 +79,7 @@ cp .env.default .env
 ```
 
 
-</br>
+<br/>
 
 
 ## 👨‍🦯 <span style="color:#9DB7FF">Local development</span>
@@ -115,7 +112,7 @@ pipenv shell
 pipenv sync --dev
 ```
 
-</br>
+<br/>
 
 > 🛑 **NOTE:** if you don't use `pipenv` for some reason remember that you will not have automatically exported variables from your `.env` file.
 >
@@ -136,7 +133,7 @@ set -o allexport; source .env; set +o allexport
 
 > 🛑 **NOTE:** Please do not forget about environment variables! Now all environment variables for the Postgres Database which runs in docker are already passed to docker-compose.yaml from the .env file.
 
-</br>
+<br/>
 
 
 ### 3. Provide code quality ✨
@@ -177,7 +174,7 @@ P.S. You don't need to do this additional step if you run application via Docker
 uvicorn src.main:app --proxy-headers --port {PORT} --reload
 ```
 
-</br>
+<br/>
 
 ### 5. Running Tests ▶️
 
@@ -216,8 +213,18 @@ psql# create user test;
 # Set password for the user
 psql# alter user test with password 'test';
 ```
-</br>
-</br>
+
+#### Test coverage
+
+To correctly calculate test coverage, you need to run the coverage with the `--concurrency=thread,gevent` parameter:
+
+```bash
+coverage run --concurrency=thread,gevent -m pytest
+coverage report -m
+```
+
+<br/>
+<br/>
 
 ## 🐳 <span style="color:#9DB7FF">Docker development</span>
 
@@ -631,4 +638,4 @@ postgresql+asyncpg://<username>:<password>@<hostname>:port/database
 For AWS S3 bucket next fields are required: 
 `storage_region`,`storage_bucket`, `storage_access_key`,`storage_secret_key`.
 ### 3. Azure Blob
-In case of Azure blob, specify your connection string into field `storage_secret_key`
+In case of Azure blob, specify your connection string into field `storage_secret_key`

compose/fastapi/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/x86_64 python:3.10.8-slim
+FROM --platform=linux/x86_64 python:3.10.8-slim as base
 
 ARG PIPENV_EXTRA_ARGS
 
@@ -7,11 +7,9 @@ ENV PYTHONPATH="src/"
 
 WORKDIR /app/
 
-RUN apt-get update \
+RUN apt-get -y update && apt-get -y upgrade \
  # dependencies for building Python packages
- && apt-get install -y build-essential curl \
- # cleaning up unused files
- && rm -rf /var/lib/apt/lists/*
+ && apt-get install -y build-essential curl
 
 # Add local non-root user to avoid issue with files
 # created inside a container being owned by root.
@@ -42,3 +40,11 @@ RUN sed -i 's/\r$//g' /fastapi-start && chmod +x /fastapi-start
 
 # Select internal user
 USER code
+
+# worker instructions
+FROM base as worker
+
+USER root
+RUN apt-get install -y --no-install-recommends ffmpeg && apt-get install -y imagemagick
+
+USER code

compose/minio/create_bucket.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+#/usr/bin/mc alias set myminio http://minio:9000 minioaccess miniosecret;
+#/usr/bin/mc mb myminio/media;
+#/usr/bin/mc policy set public myminio/media;
+#exit 0;
+
+/usr/bin/mc config host add local http://minio:9000 minioaccess miniosecret;
+#/usr/bin/mc rm -r --force local/${CDN__BUCKET};
+/usr/bin/mc mb -p local/${CDN__BUCKET};
+/usr/bin/mc policy set download local/${CDN__BUCKET};
+/usr/bin/mc policy set public local/${CDN__BUCKET};
+/usr/bin/mc anonymous set upload local/${CDN__BUCKET};
+/usr/bin/mc anonymous set download local/${CDN__BUCKET};
+/usr/bin/mc anonymous set public local/${CDN__BUCKET};
+
+exit 0;

compose/postgres/Dockerfile

@@ -0,0 +1,7 @@
+FROM postgres:15.4
+
+RUN apt-get update && apt-get install -y curl
+
+RUN apt-get -y install postgresql-15-cron
+
+COPY ./compose/postgres/init-db /docker-entrypoint-initdb.d

compose/postgres/init-db/002-pg-cron.sh

@@ -0,0 +1,6 @@
+dbname="$POSTGRES_DB"
+
+echo "shared_preload_libraries = 'pg_cron'" >> /var/lib/postgresql/data/postgresql.conf
+echo "cron.database_name = '$dbname'" >> /var/lib/postgresql/data/postgresql.conf
+
+pg_ctl restart

compose/postgres/init-db/003-main.sql

@@ -0,0 +1 @@
+CREATE EXTENSION pg_cron;

conftest.py

@@ -1,9 +1,9 @@
-import asyncio
 import os
 
 import pytest
 from alembic import command
 from alembic.config import Config
+from pytest_asyncio import is_async_test
 
 alembic_configs = [Config("alembic.ini"), Config("alembic_arbitrary.ini")]
 
@@ -20,12 +20,11 @@ def after():
  os.environ.pop("PYTEST_APP_TESTING")
 
 
-@pytest.fixture(scope="session")
-def event_loop():
- policy = asyncio.get_event_loop_policy()
- loop = policy.new_event_loop()
- yield loop
- loop.close()
+def pytest_collection_modifyitems(items):
+ pytest_asyncio_tests = (item for item in items if is_async_test(item))
+ session_scope_marker = pytest.mark.asyncio(scope="session")
+ for async_test in pytest_asyncio_tests:
+ async_test.add_marker(session_scope_marker)
 
 
 def pytest_sessionstart():