diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..01be899 --- /dev/null +++ b/.env.example @@ -0,0 +1,8 @@ +SUPERADMIN_GH_PAT=your-github-pat +GH_OAUTH_ID=your-github-oauth-app-id +GH_OAUTH_SECRET=your-github-oauth-app-secret +OPENAI_API_KEY=your-openai-api-key +SUPERADMIN_PWD='Dumm1PassW0rdz!' +POSTGRES_DB=postgres +POSTGRES_USER=postgres +POSTGRES_PASSWORD='An0th3rDumm1PassW0rdz!' diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 4245eff..f259ca9 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -24,19 +24,18 @@ jobs: run: poetry export -f requirements.txt --without-hashes --output requirements.txt - name: Build & run docker env: - POSTGRES_DB: ${{ secrets.POSTGRES_DB }} - POSTGRES_USER: ${{ secrets.POSTGRES_USER }} - POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} - SUPERUSER_LOGIN: ${{ secrets.SUPERUSER_LOGIN }} - SUPERUSER_ID: ${{ secrets.SUPERUSER_ID }} - SUPERUSER_PWD: ${{ secrets.SUPERUSER_PWD }} + SUPERADMIN_GH_PAT: ${{ secrets.SUPERADMIN_GH_PAT }} + SUPERADMIN_PWD: dummy_pwd GH_OAUTH_ID: ${{ secrets.GH_OAUTH_ID }} GH_OAUTH_SECRET: ${{ secrets.GH_OAUTH_SECRET }} + POSTGRES_DB: postgres + POSTGRES_USER: postgres + POSTGRES_PASSWORD: pg_pwd OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: docker-compose up -d --build - name: Docker sanity check - run: sleep 20 && nc -vz api.localhost 8050 + run: sleep 20 && nc -vz localhost 8050 - name: Debug run: docker-compose logs - name: Ping server - run: curl http://api.localhost:8050/docs + run: curl http://localhost:8050/docs diff --git a/.github/workflows/scripts.yml b/.github/workflows/scripts.yml index b262654..cf9acd4 100644 --- a/.github/workflows/scripts.yml +++ b/.github/workflows/scripts.yml @@ -24,12 +24,7 @@ jobs: run: poetry export -f requirements.txt --without-hashes --output requirements.txt - name: Build & run docker env: - POSTGRES_DB: ${{ secrets.POSTGRES_DB }} - POSTGRES_LOGIN: ${{ secrets.POSTGRES_LOGIN }} - POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} - SUPERUSER_LOGIN: ${{ secrets.SUPERUSER_LOGIN }} - SUPERUSER_ID: ${{ secrets.SUPERUSER_ID }} - SUPERUSER_PWD: ${{ secrets.SUPERUSER_PWD }} + SUPERADMIN_GH_PAT: ${{ secrets.SUPERADMIN_GH_PAT }} GH_OAUTH_ID: ${{ secrets.GH_OAUTH_ID }} GH_OAUTH_SECRET: ${{ secrets.GH_OAUTH_SECRET }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -43,5 +38,5 @@ jobs: - name: Run integration test env: SUPERUSER_LOGIN: ${{ secrets.SUPERUSER_LOGIN }} - SUPERUSER_PWD: ${{ secrets.SUPERUSER_PWD }} + SUPERUSER_PWD: superadmin_pwd run: python scripts/test_e2e.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0f6adb5..cf5dffc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,18 +21,14 @@ jobs: run: poetry export -f requirements.txt --without-hashes --with dev --output requirements.txt - name: Build & run docker env: - SUPERUSER_LOGIN: ${{ secrets.SUPERUSER_LOGIN }} - SUPERUSER_ID: ${{ secrets.SUPERUSER_ID }} - SUPERUSER_PWD: ${{ secrets.SUPERUSER_PWD }} + SUPERADMIN_GH_PAT: ${{ secrets.SUPERADMIN_GH_PAT }} GH_OAUTH_ID: ${{ secrets.GH_OAUTH_ID }} GH_OAUTH_SECRET: ${{ secrets.GH_OAUTH_SECRET }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: docker compose -f docker-compose.test.yml up -d --build - name: Run docker test env: - SUPERUSER_LOGIN: ${{ secrets.SUPERUSER_LOGIN }} - SUPERUSER_ID: ${{ secrets.SUPERUSER_ID }} - SUPERUSER_PWD: ${{ secrets.SUPERUSER_PWD }} + SUPERADMIN_GH_PAT: ${{ secrets.SUPERADMIN_GH_PAT }} GH_OAUTH_ID: ${{ secrets.GH_OAUTH_ID }} GH_OAUTH_SECRET: ${{ secrets.GH_OAUTH_SECRET }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} diff --git a/.gitignore b/.gitignore index 3279764..3e7913c 100644 --- a/.gitignore +++ b/.gitignore @@ -166,3 +166,7 @@ cython_debug/ src/app/requirements.txt src/requirements-dev.txt requirements.txt +# Ollama +.ollama +# SQL dump +*.sql diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 142edf9..43be40c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,7 +10,9 @@ Whatever the way you wish to contribute to the project, please respect the [code - [`src/app`](https://github.com/quack-ai/contribution-api/blob/main/src/app) - The actual API codebase - [`src/tests`](https://github.com/quack-ai/contribution-api/blob/main/src/tests) - The API unit tests -- [`./traefik`](https://github.com/quack-ai/contribution-api/blob/main/traefik) - Configuration files for the reverse proxy +- [`.github`](https://github.com/quack-ai/contribution-api/blob/main/.github) - Configuration for CI (GitHub Workflows) +- [`docs`](https://github.com/quack-ai/contribution-api/blob/main/docs) - Everything related to documentation +- [`scripts`](https://github.com/quack-ai/contribution-api/blob/main/scripts) - Custom scripts ## Continuous Integration @@ -21,8 +23,9 @@ This project uses the following integrations to ensure proper codebase maintenan - [Codacy](https://www.codacy.com/) - analyzes commits for code quality - [Codecov](https://codecov.io/) - reports back coverage results - [Sentry](https://docs.sentry.io/platforms/python/) - automatically reports errors back to us -- [LogTail](https://betterstack.com/logtail) - manage logs - [PostgreSQL](https://www.postgresql.org/) - storing and interacting with the metadata database +- [PostHog](https://posthog.com/) - product analytics +- [Slack](https://slack.com/) - event notifications - [Traefik](https://traefik.io/) - the reverse proxy and load balancer As a contributor, you will only have to ensure coverage of your code by adding appropriate unit testing of your code. diff --git a/README.md b/README.md index 8909b52..85cc936 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ You can run the API containers using this command: make run ``` -You can now navigate to [`http://api.localhost:8050/docs`](http://api.localhost:8050/docs) to interact with the API (or do it through HTTP requests) and explore the documentation. +You can now navigate to [`http://localhost:8050/docs`](http://localhost:8050/docs) to interact with the API (or do it through HTTP requests) and explore the documentation. In order to stop the service, run: ```shell @@ -77,42 +77,34 @@ The project was designed so that everything runs with Docker orchestration (stan In order to run the project, you will need to specific some information, which can be done using a `.env` file. This file will have to hold the following information: -- `POSTGRES_DB`: the name of the [PostgreSQL](https://www.postgresql.org/) database that will be created -- `POSTGRES_USER`: the login to the PostgreSQL database -- `POSTGRES_PASSWORD`: the password to the PostgreSQL database -- `SUPERUSER_LOGIN`: the login of the initial admin access -- `SUPERUSER_ID`: the GitHub ID of the initial admin access -- `SUPERUSER_PWD`: the password of the initial admin access -- `GH_OAUTH_ID`: the ID of the GitHub Oauth app -- `GH_OAUTH_SECRET`: the secret of the GitHub Oauth app -- `OPENAI_API_KEY`: your API key for Open AI +- `SUPERADMIN_GH_PAT`: the GitHub token of the initial admin access (Generate a new token on [GitHub](https://github.com/settings/tokens?type=beta), with no extra permissions = read-only) +- `SUPERADMIN_PWD`*: the password of the initial admin access +- `GH_OAUTH_ID`: the Client ID of the GitHub Oauth app (Create an OAuth app on [GitHub](https://github.com/settings/applications/new), pointing to your Quack dashboard w/ callback URL) +- `GH_OAUTH_SECRET`: the secret of the GitHub Oauth app (Generate a new client secret on the created OAuth app) +- `POSTGRES_DB`*: a name for the [PostgreSQL](https://www.postgresql.org/) database that will be created +- `POSTGRES_USER`*: a login for the PostgreSQL database +- `POSTGRES_PASSWORD`*: a password for the PostgreSQL database +- `OPENAI_API_KEY`: your API key for Open AI (Create new secret key on [OpenAI](https://platform.openai.com/api-keys)) -Optionally, the following information can be added: -- `SENTRY_DSN`: the URL of the [Sentry](https://sentry.io/) project, which monitors back-end errors and report them back. -- `SERVER_NAME`: the server tag to apply to events. +_* marks the values where you can pick what you want._ -So your `.env` file should look like something similar to: -``` -POSTGRES_DB=review_db -POSTGRES_USER=admin -POSTGRES_PASSWORD=my_password -SUPERUSER_LOGIN=superadmin -SUPERUSER_PWD=super_password -SUPERUSER_ID=1 -SENTRY_DSN='https://replace.with.you.sentry.dsn/' -SENTRY_SERVER_NAME=my_storage_bucket_name -GH_OAUTH_ID=your_github_oauth_app_id -GH_OAUTH_SECRET=your_github_oauth_app_secret -OPENAI_API_KEY='you-openai-key' -``` +Optionally, the following information can be added: +- `SECRET_KEY`*: if set, tokens can be reused between sessions. All instances sharing the same secret key can use the same token. +- `SENTRY_DSN`: the DSN for your [Sentry](https://sentry.io/) project, which monitors back-end errors and report them back. +- `SERVER_NAME`*: the server tag that will be used to report events to Sentry. +- `POSTHOG_KEY`: the project API key for PostHog [PostHog](https://eu.posthog.com/settings/project-details). +- `SLACK_API_TOKEN`: the App key for your Slack bot (Create New App on [Slack](https://api.slack.com/apps), go to OAuth & Permissions and generate a bot User OAuth Token). +- `SLACK_CHANNEL`: the Slack channel where your bot will post events (defaults to `#general`, you have to invite the App to your channel). +- `DEBUG`: if set to false, silence debug logs. -The file should be placed at the root folder of your local copy of the project. +So your `.env` file should look like something similar to [`.env.example`](.env.example) +The file should be placed in the folder of your `./docker-compose.yml`. ## More goodies ### Documentation -The full package documentation is available [here](https://quack-ai.github.io/contribution-api) for detailed specifications. +Your API documentation gets a swagger automatically available on [here](http://localhost:8050/docs) for detailed specifications. ## Contributing diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 3f96dd6..4111ff0 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -9,14 +9,12 @@ services: ports: - "8050:8050" environment: - - POSTGRES_URL=postgresql+asyncpg://dummy_login:dummy_pwd@test_db/dummy_db - - SUPERUSER_LOGIN=${SUPERUSER_LOGIN} - - SUPERUSER_ID=${SUPERUSER_ID} - - SUPERUSER_PWD=${SUPERUSER_PWD} + - SUPERADMIN_GH_PAT=${SUPERADMIN_GH_PAT} + - SUPERADMIN_PWD=superadmin_pwd - GH_OAUTH_ID=${GH_OAUTH_ID} - GH_OAUTH_SECRET=${GH_OAUTH_SECRET} + - POSTGRES_URL=postgresql+asyncpg://dummy_login:dummy_pwd@test_db/dummy_db - OPENAI_API_KEY=${OPENAI_API_KEY} - - SERVER_NAME=dummy_server - DEBUG=true depends_on: test_db: diff --git a/docker-compose.yml b/docker-compose.yml index 8262423..bc483d3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,27 +9,22 @@ services: ports: - "8050:8050" environment: - - POSTGRES_URL=postgresql+asyncpg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/${POSTGRES_DB} - - SUPERUSER_LOGIN=${SUPERUSER_LOGIN} - - SUPERUSER_ID=${SUPERUSER_ID} - - SUPERUSER_PWD=${SUPERUSER_PWD} + - SUPERADMIN_GH_PAT=${SUPERADMIN_GH_PAT} + - SUPERADMIN_PWD=${SUPERADMIN_PWD} - GH_OAUTH_ID=${GH_OAUTH_ID} - GH_OAUTH_SECRET=${GH_OAUTH_SECRET} + - POSTGRES_URL=postgresql+asyncpg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/${POSTGRES_DB} - OPENAI_API_KEY=${OPENAI_API_KEY} - - SERVER_NAME=${SERVER_NAME} - SECRET_KEY=${SECRET_KEY} + - SENTRY_DSN=${SENTRY_DSN} + - SERVER_NAME=${SERVER_NAME} + - POSTHOG_KEY=${POSTHOG_KEY} - SLACK_API_TOKEN=${SLACK_API_TOKEN} - SLACK_CHANNEL=${SLACK_CHANNEL} - DEBUG=true depends_on: db: condition: service_healthy - labels: - - "traefik.enable=true" - - "traefik.http.routers.api.rule=Host(`api.localhost`) && PathPrefix(`/`)" - - "traefik.http.routers.api.service=backend@docker" - - "traefik.http.routers.api.tls={}" - - "traefik.http.services.backend.loadbalancer.server.port=8050" db: image: postgres:15-alpine @@ -47,17 +42,5 @@ services: timeout: 3s retries: 3 - traefik: - image: traefik:v2.9.6 - volumes: - - "./traefik:/etc/traefik" - - "/var/run/docker.sock:/var/run/docker.sock:ro" - ports: - # http(s) traffic - - "80:80" - - "443:443" - # traefik dashboard - - "8080:8080" - volumes: postgres_data: diff --git a/src/app/api/api_v1/endpoints/login.py b/src/app/api/api_v1/endpoints/login.py index 8a053bb..e388b19 100644 --- a/src/app/api/api_v1/endpoints/login.py +++ b/src/app/api/api_v1/endpoints/login.py @@ -31,7 +31,7 @@ async def authorize_github( redirect_uri: HttpUrl, ) -> RedirectResponse: return RedirectResponse( - f"{settings.GH_AUTHORIZE_ENDPOINT}?scope={scope}&client_id={settings.GH_OAUTH_ID}&redirect_uri={redirect_uri}" + f"https://github.com/login/oauth/authorize?scope={scope}&client_id={settings.GH_OAUTH_ID}&redirect_uri={redirect_uri}" ) diff --git a/src/app/core/config.py b/src/app/core/config.py index c318e22..97706af 100644 --- a/src/app/core/config.py +++ b/src/app/core/config.py @@ -5,6 +5,7 @@ import os import secrets +import socket from typing import Union from pydantic import BaseSettings, validator @@ -21,24 +22,14 @@ class Settings(BaseSettings): VERSION: str = "0.1.0.dev0" API_V1_STR: str = "/api/v1" CORS_ORIGIN: str = "*" - # Ext API endpoints - GH_AUTHORIZE_ENDPOINT: str = "https://github.com/login/oauth/authorize" + # Authentication + SUPERADMIN_GH_PAT: str = os.environ["SUPERADMIN_GH_PAT"] + SUPERADMIN_PWD: str = os.environ["SUPERADMIN_PWD"] GH_OAUTH_ID: str = os.environ["GH_OAUTH_ID"] GH_OAUTH_SECRET: str = os.environ["GH_OAUTH_SECRET"] GH_TOKEN: Union[str, None] = os.environ.get("GH_TOKEN") - # Security - SECRET_KEY: str = os.environ.get("SECRET_KEY", secrets.token_urlsafe(32)) - ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 - ACCESS_TOKEN_UNLIMITED_MINUTES: int = 60 * 24 * 365 - JWT_ENCODING_ALGORITHM: str = "HS256" # DB POSTGRES_URL: str = os.environ["POSTGRES_URL"] - SUPERUSER_LOGIN: str = os.environ["SUPERUSER_LOGIN"] - SUPERUSER_ID: int = int(os.environ["SUPERUSER_ID"]) - SUPERUSER_PWD: str = os.environ["SUPERUSER_PWD"] - # Compute - OPENAI_API_KEY: str = os.environ["OPENAI_API_KEY"] - OPENAI_MODEL: OpenAIModel = OpenAIModel.GPT3_5 @validator("POSTGRES_URL", pre=True) @classmethod @@ -48,8 +39,18 @@ def sqlachmey_uri(cls, v: str) -> str: return v.replace("postgres://", "postgresql+asyncpg://", 1) return v + # Security + SECRET_KEY: str = os.environ.get("SECRET_KEY", secrets.token_urlsafe(32)) + ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 + ACCESS_TOKEN_UNLIMITED_MINUTES: int = 60 * 24 * 365 + JWT_ENCODING_ALGORITHM: str = "HS256" + # Compute + OPENAI_API_KEY: str = os.environ["OPENAI_API_KEY"] + OPENAI_MODEL: OpenAIModel = OpenAIModel.GPT4_TURBO + + # Error monitoring SENTRY_DSN: Union[str, None] = os.environ.get("SENTRY_DSN") - SERVER_NAME: str = os.environ["SERVER_NAME"] + SERVER_NAME: str = os.environ.get("SERVER_NAME", socket.gethostname()) @validator("SENTRY_DSN", pre=True) @classmethod @@ -58,6 +59,7 @@ def sentry_dsn_can_be_blank(cls, v: str) -> Union[str, None]: return None return v + # Product analytics POSTHOG_KEY: Union[str, None] = os.environ.get("POSTHOG_KEY") @validator("POSTHOG_KEY", pre=True) @@ -67,6 +69,7 @@ def posthog_key_can_be_blank(cls, v: str) -> Union[str, None]: return None return v + # Event notifications SLACK_API_TOKEN: Union[str, None] = os.environ.get("SLACK_API_TOKEN") SLACK_CHANNEL: str = os.environ.get("SLACK_CHANNEL", "#general") diff --git a/src/app/db.py b/src/app/db.py index aeb8754..a225707 100644 --- a/src/app/db.py +++ b/src/app/db.py @@ -11,6 +11,7 @@ from app.core.config import settings from app.core.security import hash_password from app.models import User, UserScope +from app.services.github import gh_client __all__ = ["get_session", "init_db"] @@ -29,14 +30,12 @@ async def init_db(): await conn.run_sync(SQLModel.metadata.create_all) async with AsyncSession(engine) as session: - statement = select(User).where(User.login == settings.SUPERUSER_LOGIN) + # Fetch authenticated GitHub User + gh_user = gh_client.get_my_user(settings.SUPERADMIN_GH_PAT) + statement = select(User).where(User.login == gh_user["login"]) results = await session.execute(statement=statement) current_user = results.scalar_one_or_none() if not current_user: - pwd = await hash_password(settings.SUPERUSER_PWD) - session.add( - User( - id=settings.SUPERUSER_ID, login=settings.SUPERUSER_LOGIN, hashed_password=pwd, scope=UserScope.ADMIN - ) - ) + pwd = await hash_password(settings.SUPERADMIN_PWD) + session.add(User(id=gh_user["id"], login=gh_user["login"], hashed_password=pwd, scope=UserScope.ADMIN)) await session.commit() diff --git a/src/app/schemas/guidelines.py b/src/app/schemas/guidelines.py index ae1fd28..8b54787 100644 --- a/src/app/schemas/guidelines.py +++ b/src/app/schemas/guidelines.py @@ -21,8 +21,12 @@ class ExampleRequest(TextContent): class GuidelineExample(BaseModel): - positive: str = Field(..., min_length=3) - negative: str = Field(..., min_length=3) + positive: str = Field( + ..., min_length=3, description="a minimal code snippet where the instruction was correctly followed." + ) + negative: str = Field( + ..., min_length=3, description="the same snippet with minimal modifications that invalidates the instruction." + ) class GuidelineContent(BaseModel): diff --git a/src/app/schemas/services.py b/src/app/schemas/services.py index 32b1e00..f5ffd22 100644 --- a/src/app/schemas/services.py +++ b/src/app/schemas/services.py @@ -13,10 +13,10 @@ class OpenAIModel(str, Enum): # https://platform.openai.com/docs/models/overview - GPT3_5: str = "gpt-3.5-turbo-1106" - GPT3_5_LEGACY: str = "gpt-3.5-turbo-0613" - GPT4: str = "gpt-4-1106-preview" - GPT4_LEGACY: str = "gpt-4-0613" + GPT3_5_TURBO: str = "gpt-3.5-turbo-1106" + GPT3_5_TURBO_LEGACY: str = "gpt-3.5-turbo-0613" + GPT4_TURBO: str = "gpt-4-1106-preview" + GPT4: str = "gpt-4-0613" class OpenAIChatRole(str, Enum): @@ -71,7 +71,7 @@ class _ResponseFormat(BaseModel): class ChatCompletion(BaseModel): - model: OpenAIModel = OpenAIModel.GPT3_5 + model: OpenAIModel = OpenAIModel.GPT3_5_TURBO messages: List[OpenAIMessage] functions: List[OpenAIFunction] function_call: Dict[str, str] diff --git a/traefik/traefik.yml b/traefik/traefik.yml deleted file mode 100644 index feda6c2..0000000 --- a/traefik/traefik.yml +++ /dev/null @@ -1,28 +0,0 @@ -entryPoints: - web: - address: :80 - http: - redirections: - entryPoint: - to: websecure - scheme: https - permanent: true - - websecure: - address: :443 - -providers: - docker: - endpoint: "unix:///var/run/docker.sock" - exposedByDefault: false - -log: - level: DEBUG - -api: - dashboard: true - insecure: true - -monitoring: - dashboard: true - insecure: true