diff --git a/.github/workflows/after-success.yml b/.github/workflows/after-success.yml new file mode 100644 index 00000000..c05bb6d1 --- /dev/null +++ b/.github/workflows/after-success.yml @@ -0,0 +1,14 @@ +name: After Success + +on: + workflow_run: + workflows: ["PR Tests"] + types: + - completed + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Confirm Success + run: echo "Previous workflow was successful!" diff --git a/.github/workflows/aws-preview.yml b/.github/workflows/aws-preview.yml index 0526abba..5654fa83 100644 --- a/.github/workflows/aws-preview.yml +++ b/.github/workflows/aws-preview.yml @@ -2,7 +2,7 @@ name: Deploy Backend to Preview ECS on: workflow_run: - workflows: ["Build And Test"] + workflows: ["PR Tests"] types: - completed @@ -14,11 +14,12 @@ env: permissions: id-token: write # This is required for requesting the JWT contents: read # This is required for actions/checkout + actions: write jobs: deploy: runs-on: ubuntu-latest - environment: production + environment: Preview strategy: fail-fast: true diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml new file mode 100644 index 00000000..88342daf --- /dev/null +++ b/.github/workflows/pr-tests.yml @@ -0,0 +1,16 @@ +name: PR Tests + +on: + pull_request: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Run a simple test + run: echo "Running tests..." diff --git a/.github/workflows/server-test.yml b/.github/workflows/server-test.yml deleted file mode 100644 index 34365c55..00000000 --- a/.github/workflows/server-test.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Build And Test - -on: - pull_request: - branches: [ "main" ] - paths: - - .github/workflows/aws-preview.yml - - server/** - - petercat_utils/** - - subscriber/** - -jobs: - build: - runs-on: ubuntu-latest - environment: production - strategy: - fail-fast: true - - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Setup Server - run: | - cd server - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - - name: Test with pytest - run: | - pip install pytest pytest-cov - pytest \ No newline at end of file diff --git a/.gitignore b/.gitignore index b30bf697..717b791f 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ venv .next/ out/ +.ruff_cache/ # production build diff --git a/Makefile b/Makefile deleted file mode 100644 index 9c30432a..00000000 --- a/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -test: - pytest backend/tests - -dev: - docker compose -f docker/docker-compose.yml build backend-core - docker compose -f docker/docker-compose.yml up --build -d - -dev-ps: - docker compose -f docker/docker-compose.yml ps - -dev-init: - rm -rf docker/volumes/db/data - docker compose -f docker/docker-compose.yml build backend-core - docker compose -f docker/docker-compose.yml up --build - -prod: - docker compose build backend-core - docker compose -f docker-compose.yml up --build - -test-type: - @if command -v python3 &>/dev/null; then \ - python3 -m pyright; \ - else \ - python -m pyright; \ - fi diff --git a/docs/guides/self_hosting_aws.md b/docs/guides/self_hosting_aws.md index e69de29b..e1517df3 100644 --- a/docs/guides/self_hosting_aws.md +++ b/docs/guides/self_hosting_aws.md @@ -0,0 +1,25 @@ +## Supabase Budget + +|Resources| Instances | Pricing | +|---------|------|------| +| Supabase | 1 | $0 + + +## Infrastructure Budget + +|Resources| Instances | Pricing | +|---------|------|------| +| EC2 Container Registry (ECR) | | https://aws.amazon.com/cn/ecr/pricing/ +| Route 53 | 1 | $0.53 +| Secrets Manager | 1 | $0.40 +| S3 | Very Few | https://aws.amazon.com/cn/s3/pricing/ +| CloudFront | 2 | $0 +| Lambda | 4 | https://aws.amazon.com/cn/lambda/pricing/ + + +## LLM Budget + +|Resources| Instances | Pricing | +|---------|------|------| +| OpenAI | - | https://openai.com/pricing/ +| Gemini flash | - | $0 diff --git a/petercat_utils/rag_helper/task.py b/petercat_utils/rag_helper/task.py index fd26995b..e73bee7f 100644 --- a/petercat_utils/rag_helper/task.py +++ b/petercat_utils/rag_helper/task.py @@ -1,5 +1,6 @@ import json from typing import Optional +from github import Github import boto3 @@ -7,15 +8,14 @@ from .git_issue_task import GitIssueTask from .git_task import GitTask -# Create SQS client -sqs = boto3.client("sqs") - -from github import Github - from ..utils.env import get_env_variable from ..data_class import TaskStatus, TaskType from ..db.client.supabase import get_client +# Create SQS client +sqs = boto3.client("sqs") + + g = Github() TABLE_NAME = "rag_tasks" diff --git a/pyproject.toml b/pyproject.toml index 808ee806..ee604801 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,11 +6,26 @@ authors = ["raoha.rh "] readme = "README.md" packages = [{include = "petercat_utils"}] +[tool.ruff] +builtins = ["_"] + +[pytest] +testpaths = ["tests"] +pythonpath = "." +consider_namespace_packages = "True" +python_files = "test_*.py" +cov="com" +cov-report=["xml","html"] +md_report = true +md_report_verbose = 0 +md_report_color = "auto" + [tool.poetry.dependencies] python = "^3.8" langchain_community = "^0.2.11" langchain_openai = "^0.1.20" langchain_core = "0.2.28" +langchain = "^0.2.12" supabase = "2.6.0" pydantic = "2.7.0" PyGithub = "2.3.0" diff --git a/server/agent/bot_builder.py b/server/agent/bot_builder.py index a7b94598..339873ba 100644 --- a/server/agent/bot_builder.py +++ b/server/agent/bot_builder.py @@ -1,9 +1,9 @@ from typing import AsyncIterator, Optional from petercat_utils.data_class import ChatData -from ..agent.base import AgentBuilder -from ..prompts.bot_builder import generate_prompt_by_user_id -from ..tools import bot_builder +from agent.base import AgentBuilder +from prompts.bot_builder import generate_prompt_by_user_id +from tools import bot_builder TOOL_MAPPING = { diff --git a/server/agent/qa_chat.py b/server/agent/qa_chat.py index 52717a72..1be7d8f9 100644 --- a/server/agent/qa_chat.py +++ b/server/agent/qa_chat.py @@ -2,9 +2,9 @@ from petercat_utils import get_client from petercat_utils.data_class import ChatData -from ..agent.base import AgentBuilder -from ..prompts.bot_template import generate_prompt_by_repo_name -from ..tools import issue, sourcecode, knowledge, git_info +from agent.base import AgentBuilder +from prompts.bot_template import generate_prompt_by_repo_name +from tools import issue, sourcecode, knowledge, git_info def get_tools(bot_id: str, token: Optional[str]): diff --git a/server/auth/get_user_info.py b/server/auth/get_user_info.py index 92aa3aad..7198ad98 100644 --- a/server/auth/get_user_info.py +++ b/server/auth/get_user_info.py @@ -1,5 +1,5 @@ from typing import Annotated -from fastapi import Cookie, HTTPException +from fastapi import Cookie import httpx import secrets import random @@ -8,7 +8,8 @@ from .get_oauth_token import get_oauth_token from petercat_utils import get_client, get_env_variable -random_str = lambda N: ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(N)) +def random_str(N): + return ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(N)) AUTH0_DOMAIN = get_env_variable("AUTH0_DOMAIN") @@ -70,7 +71,7 @@ async def get_user_id(petercat_user_token: Annotated[str | None, Cookie()] = Non user_info = await getUserInfoByToken(petercat_user_token) return user_info['id'] - except Exception as e: + except Exception: return None async def get_user_access_token(petercat_user_token: Annotated[str | None, Cookie()] = None): @@ -83,5 +84,5 @@ async def get_user_access_token(petercat_user_token: Annotated[str | None, Cooki access_token = await getUserAccessToken(user_id=user_info['id']) print(f"get_user_access_token: user_info={user_info}, access_token={access_token}") return access_token - except Exception as e: + except Exception: return None diff --git a/server/bot/builder.py b/server/bot/builder.py index 6c270835..35603ac8 100644 --- a/server/bot/builder.py +++ b/server/bot/builder.py @@ -5,7 +5,7 @@ from petercat_utils.data_class import RAGGitDocConfig from petercat_utils import git_doc_task -from ..prompts.bot_template import generate_prompt_by_repo_name +from prompts.bot_template import generate_prompt_by_repo_name g = Github() diff --git a/server/dao/authorizationDAO.py b/server/dao/authorizationDAO.py index 44f752ef..9fff5100 100644 --- a/server/dao/authorizationDAO.py +++ b/server/dao/authorizationDAO.py @@ -2,8 +2,8 @@ from petercat_utils.db.client.supabase import get_client -from ..dao.BaseDAO import BaseDAO -from ..models.authorization import Authorization +from dao.BaseDAO import BaseDAO +from models.authorization import Authorization class AuthorizationDAO(BaseDAO): client: Client diff --git a/server/dao/repositoryConfigDAO.py b/server/dao/repositoryConfigDAO.py index 89cc1372..dceb1d57 100644 --- a/server/dao/repositoryConfigDAO.py +++ b/server/dao/repositoryConfigDAO.py @@ -1,8 +1,8 @@ from supabase.client import Client from petercat_utils.db.client.supabase import get_client -from ..dao.BaseDAO import BaseDAO -from ..models.repository import RepositoryConfig +from dao.BaseDAO import BaseDAO +from models.repository import RepositoryConfig class RepositoryConfigDAO(BaseDAO): client: Client diff --git a/server/event_handler/discussion.py b/server/event_handler/discussion.py index c13f43fd..d0f8648d 100644 --- a/server/event_handler/discussion.py +++ b/server/event_handler/discussion.py @@ -5,7 +5,7 @@ from petercat_utils.data_class import ChatData, Message, TextContentBlock -from ..agent.qa_chat import agent_chat +from agent.qa_chat import agent_chat class DiscussionEventHandler: diff --git a/server/event_handler/issue.py b/server/event_handler/issue.py index 3db2b954..72eae3b5 100644 --- a/server/event_handler/issue.py +++ b/server/event_handler/issue.py @@ -4,7 +4,7 @@ from petercat_utils.data_class import ChatData, Message, TextContentBlock -from ..agent.qa_chat import agent_chat +from agent.qa_chat import agent_chat class IssueEventHandler: diff --git a/server/main.py b/server/main.py index 90beb75d..80748ea4 100644 --- a/server/main.py +++ b/server/main.py @@ -1,6 +1,4 @@ -import os -import uvicorn from fastapi import FastAPI from starlette.middleware.sessions import SessionMiddleware from fastapi.middleware.cors import CORSMiddleware @@ -9,7 +7,7 @@ # Import fastapi routers -from .routers import bot, health_checker, github, rag, auth, chat, task +from routers import bot, health_checker, github, rag, auth, chat, task AUTH0_DOMAIN = get_env_variable("AUTH0_DOMAIN") API_AUDIENCE = get_env_variable("API_IDENTIFIER") diff --git a/server/models/authorization.py b/server/models/authorization.py index c75423bd..f517c947 100644 --- a/server/models/authorization.py +++ b/server/models/authorization.py @@ -1,7 +1,7 @@ from datetime import datetime import json from pydantic import BaseModel, field_serializer -from typing import Any, Dict +from typing import Dict class Authorization(BaseModel): token: str diff --git a/server/pytest.ini b/server/pytest.ini index 7c1d8c1a..993fa8c8 100644 --- a/server/pytest.ini +++ b/server/pytest.ini @@ -1,5 +1,7 @@ [pytest] -testpaths = . +testpaths = tests +rootdir=server +consider_namespace_packages = True python_files = test_*.py cov=com cov-report=xml,html diff --git a/server/requirements.txt b/server/requirements.txt index 5e29c17d..ffe09298 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -4,9 +4,8 @@ python-dotenv==1.0.0 openai mangum langserve -langchain_community -langchain -langchain-openai +langchain_community>=0.2.11 +langchain>=0.2.12 PyGithub GitPython python-multipart diff --git a/server/routers/__init__.py b/server/routers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/routers/auth.py b/server/routers/auth.py index d0ce0a26..851b4949 100644 --- a/server/routers/auth.py +++ b/server/routers/auth.py @@ -4,7 +4,7 @@ import httpx from petercat_utils import get_client, get_env_variable -from ..auth.get_user_info import generateAnonymousUser, getAnonymousUserInfoByToken, getUserInfoByToken +from auth.get_user_info import generateAnonymousUser, getAnonymousUserInfoByToken, getUserInfoByToken AUTH0_DOMAIN = get_env_variable("AUTH0_DOMAIN") diff --git a/server/routers/bot.py b/server/routers/bot.py index f674994a..7482fe33 100644 --- a/server/routers/bot.py +++ b/server/routers/bot.py @@ -3,9 +3,9 @@ from petercat_utils import get_client from typing import Annotated, Optional -from ..auth.get_user_info import get_user_id -from ..bot.builder import bot_builder, bot_info_generator -from ..type_class.bot import BotUpdateRequest, BotCreateRequest +from auth.get_user_info import get_user_id +from bot.builder import bot_builder, bot_info_generator +from type_class.bot import BotUpdateRequest, BotCreateRequest router = APIRouter( prefix="/api/bot", diff --git a/server/routers/chat.py b/server/routers/chat.py index 5ebd0114..ed865b11 100644 --- a/server/routers/chat.py +++ b/server/routers/chat.py @@ -3,9 +3,9 @@ from fastapi.responses import StreamingResponse from petercat_utils.data_class import ChatData -from ..agent import qa_chat, bot_builder -from ..verify.rate_limit import verify_rate_limit -from ..auth.get_user_info import get_user_access_token, get_user_id +from agent import qa_chat, bot_builder +from verify.rate_limit import verify_rate_limit +from auth.get_user_info import get_user_access_token, get_user_id router = APIRouter( diff --git a/server/routers/github.py b/server/routers/github.py index 79349885..33097bad 100644 --- a/server/routers/github.py +++ b/server/routers/github.py @@ -9,12 +9,12 @@ from petercat_utils import get_env_variable from jwt import JWT, jwk_from_pem -from ..auth.get_user_info import get_user_access_token -from ..dao.authorizationDAO import AuthorizationDAO -from ..dao.repositoryConfigDAO import RepositoryConfigDAO -from ..models.repository import RepositoryConfig -from ..models.authorization import Authorization -from ..utils.github import get_handler, get_private_key +from auth.get_user_info import get_user_access_token +from dao.authorizationDAO import AuthorizationDAO +from dao.repositoryConfigDAO import RepositoryConfigDAO +from models.repository import RepositoryConfig +from models.authorization import Authorization +from utils.github import get_handler, get_private_key APP_ID = get_env_variable("X_GITHUB_APP_ID") WEB_URL = get_env_variable("WEB_URL") @@ -59,7 +59,7 @@ def get_app_installations_access_token(installation_id: str, jwt: str): return resp.json() def get_installation_repositories(access_token: str): - url = f"https://api.github.com/installation/repositories" + url = "https://api.github.com/installation/repositories" print("get_installation_repositories", url) resp = requests.get(url, headers={ 'X-GitHub-Api-Version': '2022-11-28', diff --git a/server/routers/rag.py b/server/routers/rag.py index 04fe4510..a919f11c 100644 --- a/server/routers/rag.py +++ b/server/routers/rag.py @@ -13,7 +13,7 @@ git_issue_task, ) -from ..verify.rate_limit import verify_rate_limit +from verify.rate_limit import verify_rate_limit router = APIRouter( diff --git a/server/tests/__init__.py b/server/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/tests/conftest.py b/server/tests/conftest.py new file mode 100644 index 00000000..6db85294 --- /dev/null +++ b/server/tests/conftest.py @@ -0,0 +1,5 @@ +# tests/conftest.py +import sys +import os + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) diff --git a/server/test_main.py b/server/tests/test_main.py similarity index 91% rename from server/test_main.py rename to server/tests/test_main.py index 9f3561cf..1612d400 100644 --- a/server/test_main.py +++ b/server/tests/test_main.py @@ -1,5 +1,5 @@ from fastapi.testclient import TestClient -from .main import app +from main import app client = TestClient(app) diff --git a/server/tools/bot_builder.py b/server/tools/bot_builder.py index 3b86dd4c..9f39079f 100644 --- a/server/tools/bot_builder.py +++ b/server/tools/bot_builder.py @@ -1,11 +1,10 @@ -import json from typing import List, Optional from fastapi.responses import JSONResponse from langchain.tools import tool from github import Github from petercat_utils import get_client -from ..bot.builder import bot_builder +from bot.builder import bot_builder g = Github() diff --git a/server/tools/issue.py b/server/tools/issue.py index b937cbc2..4e44e09a 100644 --- a/server/tools/issue.py +++ b/server/tools/issue.py @@ -3,7 +3,7 @@ from github import Auth, Github from langchain.tools import tool -from ..tools.helper import need_github_login +from tools.helper import need_github_login DEFAULT_REPO_NAME = "ant-design/ant-design" diff --git a/server/utils/github.py b/server/utils/github.py index ce2df27e..71a51c83 100644 --- a/server/utils/github.py +++ b/server/utils/github.py @@ -4,9 +4,9 @@ from petercat_utils import get_env_variable from github import Auth -from ..event_handler.pull_request import PullRequestEventHandler -from ..event_handler.discussion import DiscussionEventHandler -from ..event_handler.issue import IssueEventHandler +from event_handler.pull_request import PullRequestEventHandler +from event_handler.discussion import DiscussionEventHandler +from event_handler.issue import IssueEventHandler APP_ID = get_env_variable("X_GITHUB_APP_ID") diff --git a/server/verify/rate_limit.py b/server/verify/rate_limit.py index 14db30f6..cb820378 100644 --- a/server/verify/rate_limit.py +++ b/server/verify/rate_limit.py @@ -4,7 +4,7 @@ from petercat_utils import get_client, get_env_variable -from ..auth.get_user_info import getUserInfoByToken +from auth.get_user_info import getUserInfoByToken RATE_LIMIT_ENABLED = get_env_variable("RATE_LIMIT_ENABLED", "False") == 'True' RATE_LIMIT_REQUESTS = get_env_variable("RATE_LIMIT_REQUESTS") or 100