From 1dd7dafe719c0d7f1787f1d1421f41c0c2a979e5 Mon Sep 17 00:00:00 2001 From: "raoha.rh" Date: Wed, 21 Aug 2024 12:37:34 +0800 Subject: [PATCH 1/4] feat: init pytest workflow --- .github/workflows/aws-preview.yml | 11 +++---- .github/workflows/server-test.yml | 38 ++++++++++++++++++++++++ client/tests/github/pull_request_test.py | 13 -------- server/agent/bot_builder.py | 7 +++-- server/agent/qa_chat.py | 7 +++-- server/bot/builder.py | 3 +- server/dao/authorizationDAO.py | 9 +++--- server/event_handler/discussion.py | 3 +- server/event_handler/issue.py | 3 +- server/main.py | 13 ++++---- server/pytest.ini | 6 ++-- server/routers/auth.py | 5 ++-- server/routers/bot.py | 7 +++-- server/routers/chat.py | 10 +++---- server/routers/github.py | 2 +- server/routers/health_checker.py | 8 +---- server/routers/rag.py | 4 ++- server/test_main.py | 9 ++++++ server/tools/bot_builder.py | 3 +- server/tools/issue.py | 2 +- server/utils/github.py | 6 ++-- server/verify/rate_limit.py | 3 +- 22 files changed, 104 insertions(+), 68 deletions(-) create mode 100644 .github/workflows/server-test.yml delete mode 100644 client/tests/github/pull_request_test.py create mode 100644 server/test_main.py diff --git a/.github/workflows/aws-preview.yml b/.github/workflows/aws-preview.yml index 46b6cae8..0526abba 100644 --- a/.github/workflows/aws-preview.yml +++ b/.github/workflows/aws-preview.yml @@ -1,13 +1,10 @@ name: Deploy Backend to Preview ECS on: - pull_request: - branches: [ "main" ] - paths: - - .github/workflows/aws-preview.yml - - server/** - - petercat_utils/** - - subscriber/** + workflow_run: + workflows: ["Build And Test"] + types: + - completed env: AWS_REGION: ap-northeast-1 diff --git a/.github/workflows/server-test.yml b/.github/workflows/server-test.yml new file mode 100644 index 00000000..13123ad9 --- /dev/null +++ b/.github/workflows/server-test.yml @@ -0,0 +1,38 @@ +name: Build And Test + +on: + pull_request: + branches: [ "main" ] + paths: + - .github/workflows/aws-preview.yml + - server/** + - petercat_utils/** + - subscriber/** + +jobs: + build: + runs-on: ubuntu-latest + environment: production + strategy: + fail-fast: true + + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + run: | + cd server + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Test with pytest + run: | + pip install pytest pytest-cov + pytest \ No newline at end of file diff --git a/client/tests/github/pull_request_test.py b/client/tests/github/pull_request_test.py deleted file mode 100644 index d60b0e69..00000000 --- a/client/tests/github/pull_request_test.py +++ /dev/null @@ -1,13 +0,0 @@ -import pytest -import json -import os - -from server.event_handler.pull_request import PullRequestEventHandler - -def test_event_handler(): - filepath = os.path.join(os.path.dirname(__file__), 'pull_request_event.json') - with open(filepath) as ev: - event = json.load(ev) - handler = PullRequestEventHandler(payload = event, access_token="123") - result = handler.execute() - assert result["success"] == True diff --git a/server/agent/bot_builder.py b/server/agent/bot_builder.py index fbeb9b76..a7b94598 100644 --- a/server/agent/bot_builder.py +++ b/server/agent/bot_builder.py @@ -1,8 +1,9 @@ from typing import AsyncIterator, Optional from petercat_utils.data_class import ChatData -from agent.base import AgentBuilder -from prompts.bot_builder import generate_prompt_by_user_id -from tools import bot_builder + +from ..agent.base import AgentBuilder +from ..prompts.bot_builder import generate_prompt_by_user_id +from ..tools import bot_builder TOOL_MAPPING = { diff --git a/server/agent/qa_chat.py b/server/agent/qa_chat.py index c55ba53b..52717a72 100644 --- a/server/agent/qa_chat.py +++ b/server/agent/qa_chat.py @@ -1,9 +1,10 @@ from typing import AsyncIterator, Optional -from agent.base import AgentBuilder -from prompts.bot_template import generate_prompt_by_repo_name from petercat_utils import get_client from petercat_utils.data_class import ChatData -from tools import issue, sourcecode, knowledge, git_info + +from ..agent.base import AgentBuilder +from ..prompts.bot_template import generate_prompt_by_repo_name +from ..tools import issue, sourcecode, knowledge, git_info def get_tools(bot_id: str, token: Optional[str]): diff --git a/server/bot/builder.py b/server/bot/builder.py index 075e2776..6c270835 100644 --- a/server/bot/builder.py +++ b/server/bot/builder.py @@ -4,7 +4,8 @@ from petercat_utils import get_client from petercat_utils.data_class import RAGGitDocConfig from petercat_utils import git_doc_task -from prompts.bot_template import generate_prompt_by_repo_name + +from ..prompts.bot_template import generate_prompt_by_repo_name g = Github() diff --git a/server/dao/authorizationDAO.py b/server/dao/authorizationDAO.py index 90dbe98f..44f752ef 100644 --- a/server/dao/authorizationDAO.py +++ b/server/dao/authorizationDAO.py @@ -1,11 +1,10 @@ - -import json -from dao.BaseDAO import BaseDAO -from models.authorization import Authorization -from supabase.client import Client, create_client +from supabase.client import Client from petercat_utils.db.client.supabase import get_client +from ..dao.BaseDAO import BaseDAO +from ..models.authorization import Authorization + class AuthorizationDAO(BaseDAO): client: Client diff --git a/server/event_handler/discussion.py b/server/event_handler/discussion.py index f5061d63..c13f43fd 100644 --- a/server/event_handler/discussion.py +++ b/server/event_handler/discussion.py @@ -2,10 +2,11 @@ from typing import Any from github import Github, Auth from github import GithubException -from agent.qa_chat import agent_chat from petercat_utils.data_class import ChatData, Message, TextContentBlock +from ..agent.qa_chat import agent_chat + class DiscussionEventHandler: event: Any diff --git a/server/event_handler/issue.py b/server/event_handler/issue.py index 322c4689..a40f7851 100644 --- a/server/event_handler/issue.py +++ b/server/event_handler/issue.py @@ -1,10 +1,11 @@ from typing import Any from github import Github, Auth from github import GithubException -from agent.qa_chat import agent_chat from petercat_utils.data_class import ChatData, Message, TextContentBlock +from ..agent.qa_chat import agent_chat + class IssueEventHandler: event: Any diff --git a/server/main.py b/server/main.py index fb151903..90beb75d 100644 --- a/server/main.py +++ b/server/main.py @@ -9,7 +9,8 @@ # Import fastapi routers -from routers import bot, health_checker, github, rag, auth, chat, task +from .routers import bot, health_checker, github, rag, auth, chat, task + AUTH0_DOMAIN = get_env_variable("AUTH0_DOMAIN") API_AUDIENCE = get_env_variable("API_IDENTIFIER") CLIENT_ID = get_env_variable("AUTH0_CLIENT_ID") @@ -50,8 +51,8 @@ app.include_router(task.router) -if __name__ == "__main__": - if is_dev: - uvicorn.run("main:app", host="0.0.0.0", port=int(os.environ.get("PORT", "8080")), reload=True) - else: - uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", "8080"))) +# if __name__ == "__main__": +# if is_dev: +# uvicorn.run("main:app", host="0.0.0.0", port=int(os.environ.get("PORT", "8080")), reload=True) +# else: +# uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", "8080"))) diff --git a/server/pytest.ini b/server/pytest.ini index 94a50dc0..7c1d8c1a 100644 --- a/server/pytest.ini +++ b/server/pytest.ini @@ -1,3 +1,5 @@ [pytest] -testpaths = tests -python_files = test_*.py \ No newline at end of file +testpaths = . +python_files = test_*.py +cov=com +cov-report=xml,html diff --git a/server/routers/auth.py b/server/routers/auth.py index 770c4647..d0ce0a26 100644 --- a/server/routers/auth.py +++ b/server/routers/auth.py @@ -1,11 +1,10 @@ from typing import Annotated from fastapi import APIRouter, Cookie, Request, HTTPException, status, Response - from fastapi.responses import RedirectResponse import httpx - from petercat_utils import get_client, get_env_variable -from auth.get_user_info import generateAnonymousUser, getAnonymousUserInfoByToken, getUserAccessToken, getUserInfoByToken + +from ..auth.get_user_info import generateAnonymousUser, getAnonymousUserInfoByToken, getUserInfoByToken AUTH0_DOMAIN = get_env_variable("AUTH0_DOMAIN") diff --git a/server/routers/bot.py b/server/routers/bot.py index 755ecb58..f674994a 100644 --- a/server/routers/bot.py +++ b/server/routers/bot.py @@ -1,11 +1,12 @@ from fastapi import APIRouter, Depends, status, Query, Path from fastapi.responses import JSONResponse -from auth.get_user_info import get_user_id from petercat_utils import get_client -from bot.builder import bot_builder, bot_info_generator -from type_class.bot import BotUpdateRequest, BotCreateRequest from typing import Annotated, Optional +from ..auth.get_user_info import get_user_id +from ..bot.builder import bot_builder, bot_info_generator +from ..type_class.bot import BotUpdateRequest, BotCreateRequest + router = APIRouter( prefix="/api/bot", tags=["bot"], diff --git a/server/routers/chat.py b/server/routers/chat.py index 0baaf94b..5ebd0114 100644 --- a/server/routers/chat.py +++ b/server/routers/chat.py @@ -1,11 +1,11 @@ -import asyncio from typing import Annotated, Optional -from fastapi import APIRouter, Cookie, Depends +from fastapi import APIRouter, Depends from fastapi.responses import StreamingResponse -from auth.get_user_info import get_user_access_token, get_user_id from petercat_utils.data_class import ChatData -from agent import qa_chat, bot_builder -from verify.rate_limit import verify_rate_limit + +from ..agent import qa_chat, bot_builder +from ..verify.rate_limit import verify_rate_limit +from ..auth.get_user_info import get_user_access_token, get_user_id router = APIRouter( diff --git a/server/routers/github.py b/server/routers/github.py index 06b95661..0dc4e8e8 100644 --- a/server/routers/github.py +++ b/server/routers/github.py @@ -59,7 +59,7 @@ def get_app_installations_access_token(installation_id: str, jwt: str): return resp.json() def get_installation_repositories(access_token: str): - url = f"https://api.github.com/installation/repositories" + url = "https://api.github.com/installation/repositories" print("get_installation_repositories", url) resp = requests.get(url, headers={ 'X-GitHub-Api-Version': '2022-11-28', diff --git a/server/routers/health_checker.py b/server/routers/health_checker.py index 4b8ca9a2..cfd90517 100644 --- a/server/routers/health_checker.py +++ b/server/routers/health_checker.py @@ -1,6 +1,4 @@ -from fastapi import APIRouter, Depends - -from verify.rate_limit import verify_rate_limit +from fastapi import APIRouter router = APIRouter( prefix="/api", @@ -11,7 +9,3 @@ @router.get("/health_checker") def health_checker(): return { "Hello": "World" } - -@router.get("/login_checker", dependencies=[Depends(verify_rate_limit)]) -def login_checker(): - return { "Hello": "World" } \ No newline at end of file diff --git a/server/routers/rag.py b/server/routers/rag.py index 1806d1f4..04fe4510 100644 --- a/server/routers/rag.py +++ b/server/routers/rag.py @@ -3,7 +3,6 @@ from fastapi import APIRouter, Depends from petercat_utils.db.client.supabase import get_client -from verify.rate_limit import verify_rate_limit from petercat_utils.data_class import RAGGitDocConfig, RAGGitIssueConfig, TaskType from petercat_utils.rag_helper import ( @@ -14,6 +13,9 @@ git_issue_task, ) +from ..verify.rate_limit import verify_rate_limit + + router = APIRouter( prefix="/api", tags=["rag"], diff --git a/server/test_main.py b/server/test_main.py new file mode 100644 index 00000000..9f3561cf --- /dev/null +++ b/server/test_main.py @@ -0,0 +1,9 @@ +from fastapi.testclient import TestClient +from .main import app + +client = TestClient(app) + +def test_health_checker(): + response = client.get("/api/health_checker") + assert response.status_code == 200 + assert response.json() == { "Hello": "World" } diff --git a/server/tools/bot_builder.py b/server/tools/bot_builder.py index d4ec5dcb..3b86dd4c 100644 --- a/server/tools/bot_builder.py +++ b/server/tools/bot_builder.py @@ -4,7 +4,8 @@ from langchain.tools import tool from github import Github from petercat_utils import get_client -from bot.builder import bot_builder + +from ..bot.builder import bot_builder g = Github() diff --git a/server/tools/issue.py b/server/tools/issue.py index 4e44e09a..b937cbc2 100644 --- a/server/tools/issue.py +++ b/server/tools/issue.py @@ -3,7 +3,7 @@ from github import Auth, Github from langchain.tools import tool -from tools.helper import need_github_login +from ..tools.helper import need_github_login DEFAULT_REPO_NAME = "ant-design/ant-design" diff --git a/server/utils/github.py b/server/utils/github.py index cee13286..ce2df27e 100644 --- a/server/utils/github.py +++ b/server/utils/github.py @@ -1,12 +1,12 @@ from typing import Union import boto3 from botocore.exceptions import ClientError -from event_handler.pull_request import PullRequestEventHandler -from event_handler.discussion import DiscussionEventHandler -from event_handler.issue import IssueEventHandler from petercat_utils import get_env_variable from github import Auth +from ..event_handler.pull_request import PullRequestEventHandler +from ..event_handler.discussion import DiscussionEventHandler +from ..event_handler.issue import IssueEventHandler APP_ID = get_env_variable("X_GITHUB_APP_ID") diff --git a/server/verify/rate_limit.py b/server/verify/rate_limit.py index 2a49e826..14db30f6 100644 --- a/server/verify/rate_limit.py +++ b/server/verify/rate_limit.py @@ -2,9 +2,10 @@ from fastapi import Cookie, HTTPException from datetime import datetime, timedelta -from auth.get_user_info import getUserInfoByToken from petercat_utils import get_client, get_env_variable +from ..auth.get_user_info import getUserInfoByToken + RATE_LIMIT_ENABLED = get_env_variable("RATE_LIMIT_ENABLED", "False") == 'True' RATE_LIMIT_REQUESTS = get_env_variable("RATE_LIMIT_REQUESTS") or 100 RATE_LIMIT_DURATION = timedelta(minutes=int(get_env_variable("RATE_LIMIT_DURATION") or 1)) From ecc3ae2ed8f2bbab8ad17215c7f4c0acbf29723b Mon Sep 17 00:00:00 2001 From: "raoha.rh" Date: Wed, 21 Aug 2024 12:40:05 +0800 Subject: [PATCH 2/4] feat: init pytest workflow --- .github/workflows/server-test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/server-test.yml b/.github/workflows/server-test.yml index 13123ad9..34365c55 100644 --- a/.github/workflows/server-test.yml +++ b/.github/workflows/server-test.yml @@ -19,14 +19,16 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - run: | - cd server - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.x' + - name: Setup Server + run: | + cd server + - name: Install dependencies run: | python -m pip install --upgrade pip From f86aa3b55dfd8307994cb238d95c85f83c3bda5c Mon Sep 17 00:00:00 2001 From: "raoha.rh" Date: Wed, 21 Aug 2024 12:41:58 +0800 Subject: [PATCH 3/4] feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow feat: init pytest workflow --- .github/workflows/after-success.yml | 14 ++++++++++ .github/workflows/aws-preview.yml | 5 ++-- .github/workflows/pr-tests.yml | 16 ++++++++++++ .github/workflows/server-test.yml | 40 ----------------------------- .gitignore | 1 + Makefile | 25 ------------------ docs/guides/self_hosting_aws.md | 25 ++++++++++++++++++ petercat_utils/rag_helper/task.py | 10 ++++---- pyproject.toml | 15 +++++++++++ server/agent/bot_builder.py | 6 ++--- server/agent/qa_chat.py | 6 ++--- server/auth/get_user_info.py | 9 ++++--- server/bot/builder.py | 2 +- server/dao/authorizationDAO.py | 4 +-- server/event_handler/discussion.py | 2 +- server/event_handler/issue.py | 2 +- server/main.py | 4 +-- server/models/authorization.py | 2 +- server/pytest.ini | 4 ++- server/requirements.txt | 5 ++-- server/routers/__init__.py | 0 server/routers/auth.py | 2 +- server/routers/bot.py | 6 ++--- server/routers/chat.py | 6 ++--- server/routers/rag.py | 2 +- server/tests/__init__.py | 0 server/tests/conftest.py | 5 ++++ server/{ => tests}/test_main.py | 2 +- server/tools/bot_builder.py | 3 +-- server/tools/issue.py | 2 +- server/utils/github.py | 6 ++--- server/verify/rate_limit.py | 2 +- 32 files changed, 122 insertions(+), 111 deletions(-) create mode 100644 .github/workflows/after-success.yml create mode 100644 .github/workflows/pr-tests.yml delete mode 100644 .github/workflows/server-test.yml delete mode 100644 Makefile create mode 100644 server/routers/__init__.py create mode 100644 server/tests/__init__.py create mode 100644 server/tests/conftest.py rename server/{ => tests}/test_main.py (91%) diff --git a/.github/workflows/after-success.yml b/.github/workflows/after-success.yml new file mode 100644 index 00000000..c05bb6d1 --- /dev/null +++ b/.github/workflows/after-success.yml @@ -0,0 +1,14 @@ +name: After Success + +on: + workflow_run: + workflows: ["PR Tests"] + types: + - completed + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Confirm Success + run: echo "Previous workflow was successful!" diff --git a/.github/workflows/aws-preview.yml b/.github/workflows/aws-preview.yml index 0526abba..5654fa83 100644 --- a/.github/workflows/aws-preview.yml +++ b/.github/workflows/aws-preview.yml @@ -2,7 +2,7 @@ name: Deploy Backend to Preview ECS on: workflow_run: - workflows: ["Build And Test"] + workflows: ["PR Tests"] types: - completed @@ -14,11 +14,12 @@ env: permissions: id-token: write # This is required for requesting the JWT contents: read # This is required for actions/checkout + actions: write jobs: deploy: runs-on: ubuntu-latest - environment: production + environment: Preview strategy: fail-fast: true diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml new file mode 100644 index 00000000..88342daf --- /dev/null +++ b/.github/workflows/pr-tests.yml @@ -0,0 +1,16 @@ +name: PR Tests + +on: + pull_request: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Run a simple test + run: echo "Running tests..." diff --git a/.github/workflows/server-test.yml b/.github/workflows/server-test.yml deleted file mode 100644 index 34365c55..00000000 --- a/.github/workflows/server-test.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Build And Test - -on: - pull_request: - branches: [ "main" ] - paths: - - .github/workflows/aws-preview.yml - - server/** - - petercat_utils/** - - subscriber/** - -jobs: - build: - runs-on: ubuntu-latest - environment: production - strategy: - fail-fast: true - - steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Setup Server - run: | - cd server - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - - name: Test with pytest - run: | - pip install pytest pytest-cov - pytest \ No newline at end of file diff --git a/.gitignore b/.gitignore index b30bf697..717b791f 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ venv .next/ out/ +.ruff_cache/ # production build diff --git a/Makefile b/Makefile deleted file mode 100644 index 9c30432a..00000000 --- a/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -test: - pytest backend/tests - -dev: - docker compose -f docker/docker-compose.yml build backend-core - docker compose -f docker/docker-compose.yml up --build -d - -dev-ps: - docker compose -f docker/docker-compose.yml ps - -dev-init: - rm -rf docker/volumes/db/data - docker compose -f docker/docker-compose.yml build backend-core - docker compose -f docker/docker-compose.yml up --build - -prod: - docker compose build backend-core - docker compose -f docker-compose.yml up --build - -test-type: - @if command -v python3 &>/dev/null; then \ - python3 -m pyright; \ - else \ - python -m pyright; \ - fi diff --git a/docs/guides/self_hosting_aws.md b/docs/guides/self_hosting_aws.md index e69de29b..e1517df3 100644 --- a/docs/guides/self_hosting_aws.md +++ b/docs/guides/self_hosting_aws.md @@ -0,0 +1,25 @@ +## Supabase Budget + +|Resources| Instances | Pricing | +|---------|------|------| +| Supabase | 1 | $0 + + +## Infrastructure Budget + +|Resources| Instances | Pricing | +|---------|------|------| +| EC2 Container Registry (ECR) | | https://aws.amazon.com/cn/ecr/pricing/ +| Route 53 | 1 | $0.53 +| Secrets Manager | 1 | $0.40 +| S3 | Very Few | https://aws.amazon.com/cn/s3/pricing/ +| CloudFront | 2 | $0 +| Lambda | 4 | https://aws.amazon.com/cn/lambda/pricing/ + + +## LLM Budget + +|Resources| Instances | Pricing | +|---------|------|------| +| OpenAI | - | https://openai.com/pricing/ +| Gemini flash | - | $0 diff --git a/petercat_utils/rag_helper/task.py b/petercat_utils/rag_helper/task.py index fd26995b..e73bee7f 100644 --- a/petercat_utils/rag_helper/task.py +++ b/petercat_utils/rag_helper/task.py @@ -1,5 +1,6 @@ import json from typing import Optional +from github import Github import boto3 @@ -7,15 +8,14 @@ from .git_issue_task import GitIssueTask from .git_task import GitTask -# Create SQS client -sqs = boto3.client("sqs") - -from github import Github - from ..utils.env import get_env_variable from ..data_class import TaskStatus, TaskType from ..db.client.supabase import get_client +# Create SQS client +sqs = boto3.client("sqs") + + g = Github() TABLE_NAME = "rag_tasks" diff --git a/pyproject.toml b/pyproject.toml index 808ee806..ee604801 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,11 +6,26 @@ authors = ["raoha.rh "] readme = "README.md" packages = [{include = "petercat_utils"}] +[tool.ruff] +builtins = ["_"] + +[pytest] +testpaths = ["tests"] +pythonpath = "." +consider_namespace_packages = "True" +python_files = "test_*.py" +cov="com" +cov-report=["xml","html"] +md_report = true +md_report_verbose = 0 +md_report_color = "auto" + [tool.poetry.dependencies] python = "^3.8" langchain_community = "^0.2.11" langchain_openai = "^0.1.20" langchain_core = "0.2.28" +langchain = "^0.2.12" supabase = "2.6.0" pydantic = "2.7.0" PyGithub = "2.3.0" diff --git a/server/agent/bot_builder.py b/server/agent/bot_builder.py index a7b94598..339873ba 100644 --- a/server/agent/bot_builder.py +++ b/server/agent/bot_builder.py @@ -1,9 +1,9 @@ from typing import AsyncIterator, Optional from petercat_utils.data_class import ChatData -from ..agent.base import AgentBuilder -from ..prompts.bot_builder import generate_prompt_by_user_id -from ..tools import bot_builder +from agent.base import AgentBuilder +from prompts.bot_builder import generate_prompt_by_user_id +from tools import bot_builder TOOL_MAPPING = { diff --git a/server/agent/qa_chat.py b/server/agent/qa_chat.py index 52717a72..1be7d8f9 100644 --- a/server/agent/qa_chat.py +++ b/server/agent/qa_chat.py @@ -2,9 +2,9 @@ from petercat_utils import get_client from petercat_utils.data_class import ChatData -from ..agent.base import AgentBuilder -from ..prompts.bot_template import generate_prompt_by_repo_name -from ..tools import issue, sourcecode, knowledge, git_info +from agent.base import AgentBuilder +from prompts.bot_template import generate_prompt_by_repo_name +from tools import issue, sourcecode, knowledge, git_info def get_tools(bot_id: str, token: Optional[str]): diff --git a/server/auth/get_user_info.py b/server/auth/get_user_info.py index 92aa3aad..7198ad98 100644 --- a/server/auth/get_user_info.py +++ b/server/auth/get_user_info.py @@ -1,5 +1,5 @@ from typing import Annotated -from fastapi import Cookie, HTTPException +from fastapi import Cookie import httpx import secrets import random @@ -8,7 +8,8 @@ from .get_oauth_token import get_oauth_token from petercat_utils import get_client, get_env_variable -random_str = lambda N: ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(N)) +def random_str(N): + return ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(N)) AUTH0_DOMAIN = get_env_variable("AUTH0_DOMAIN") @@ -70,7 +71,7 @@ async def get_user_id(petercat_user_token: Annotated[str | None, Cookie()] = Non user_info = await getUserInfoByToken(petercat_user_token) return user_info['id'] - except Exception as e: + except Exception: return None async def get_user_access_token(petercat_user_token: Annotated[str | None, Cookie()] = None): @@ -83,5 +84,5 @@ async def get_user_access_token(petercat_user_token: Annotated[str | None, Cooki access_token = await getUserAccessToken(user_id=user_info['id']) print(f"get_user_access_token: user_info={user_info}, access_token={access_token}") return access_token - except Exception as e: + except Exception: return None diff --git a/server/bot/builder.py b/server/bot/builder.py index 6c270835..35603ac8 100644 --- a/server/bot/builder.py +++ b/server/bot/builder.py @@ -5,7 +5,7 @@ from petercat_utils.data_class import RAGGitDocConfig from petercat_utils import git_doc_task -from ..prompts.bot_template import generate_prompt_by_repo_name +from prompts.bot_template import generate_prompt_by_repo_name g = Github() diff --git a/server/dao/authorizationDAO.py b/server/dao/authorizationDAO.py index 44f752ef..9fff5100 100644 --- a/server/dao/authorizationDAO.py +++ b/server/dao/authorizationDAO.py @@ -2,8 +2,8 @@ from petercat_utils.db.client.supabase import get_client -from ..dao.BaseDAO import BaseDAO -from ..models.authorization import Authorization +from dao.BaseDAO import BaseDAO +from models.authorization import Authorization class AuthorizationDAO(BaseDAO): client: Client diff --git a/server/event_handler/discussion.py b/server/event_handler/discussion.py index c13f43fd..d0f8648d 100644 --- a/server/event_handler/discussion.py +++ b/server/event_handler/discussion.py @@ -5,7 +5,7 @@ from petercat_utils.data_class import ChatData, Message, TextContentBlock -from ..agent.qa_chat import agent_chat +from agent.qa_chat import agent_chat class DiscussionEventHandler: diff --git a/server/event_handler/issue.py b/server/event_handler/issue.py index a40f7851..3485fde3 100644 --- a/server/event_handler/issue.py +++ b/server/event_handler/issue.py @@ -4,7 +4,7 @@ from petercat_utils.data_class import ChatData, Message, TextContentBlock -from ..agent.qa_chat import agent_chat +from agent.qa_chat import agent_chat class IssueEventHandler: diff --git a/server/main.py b/server/main.py index 90beb75d..80748ea4 100644 --- a/server/main.py +++ b/server/main.py @@ -1,6 +1,4 @@ -import os -import uvicorn from fastapi import FastAPI from starlette.middleware.sessions import SessionMiddleware from fastapi.middleware.cors import CORSMiddleware @@ -9,7 +7,7 @@ # Import fastapi routers -from .routers import bot, health_checker, github, rag, auth, chat, task +from routers import bot, health_checker, github, rag, auth, chat, task AUTH0_DOMAIN = get_env_variable("AUTH0_DOMAIN") API_AUDIENCE = get_env_variable("API_IDENTIFIER") diff --git a/server/models/authorization.py b/server/models/authorization.py index c75423bd..f517c947 100644 --- a/server/models/authorization.py +++ b/server/models/authorization.py @@ -1,7 +1,7 @@ from datetime import datetime import json from pydantic import BaseModel, field_serializer -from typing import Any, Dict +from typing import Dict class Authorization(BaseModel): token: str diff --git a/server/pytest.ini b/server/pytest.ini index 7c1d8c1a..993fa8c8 100644 --- a/server/pytest.ini +++ b/server/pytest.ini @@ -1,5 +1,7 @@ [pytest] -testpaths = . +testpaths = tests +rootdir=server +consider_namespace_packages = True python_files = test_*.py cov=com cov-report=xml,html diff --git a/server/requirements.txt b/server/requirements.txt index a2c0e7c3..a8ccc8dd 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -4,9 +4,8 @@ python-dotenv==1.0.0 openai mangum langserve -langchain_community -langchain -langchain-openai +langchain_community>=0.2.11 +langchain>=0.2.12 PyGithub GitPython python-multipart diff --git a/server/routers/__init__.py b/server/routers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/routers/auth.py b/server/routers/auth.py index d0ce0a26..851b4949 100644 --- a/server/routers/auth.py +++ b/server/routers/auth.py @@ -4,7 +4,7 @@ import httpx from petercat_utils import get_client, get_env_variable -from ..auth.get_user_info import generateAnonymousUser, getAnonymousUserInfoByToken, getUserInfoByToken +from auth.get_user_info import generateAnonymousUser, getAnonymousUserInfoByToken, getUserInfoByToken AUTH0_DOMAIN = get_env_variable("AUTH0_DOMAIN") diff --git a/server/routers/bot.py b/server/routers/bot.py index f674994a..7482fe33 100644 --- a/server/routers/bot.py +++ b/server/routers/bot.py @@ -3,9 +3,9 @@ from petercat_utils import get_client from typing import Annotated, Optional -from ..auth.get_user_info import get_user_id -from ..bot.builder import bot_builder, bot_info_generator -from ..type_class.bot import BotUpdateRequest, BotCreateRequest +from auth.get_user_info import get_user_id +from bot.builder import bot_builder, bot_info_generator +from type_class.bot import BotUpdateRequest, BotCreateRequest router = APIRouter( prefix="/api/bot", diff --git a/server/routers/chat.py b/server/routers/chat.py index 5ebd0114..ed865b11 100644 --- a/server/routers/chat.py +++ b/server/routers/chat.py @@ -3,9 +3,9 @@ from fastapi.responses import StreamingResponse from petercat_utils.data_class import ChatData -from ..agent import qa_chat, bot_builder -from ..verify.rate_limit import verify_rate_limit -from ..auth.get_user_info import get_user_access_token, get_user_id +from agent import qa_chat, bot_builder +from verify.rate_limit import verify_rate_limit +from auth.get_user_info import get_user_access_token, get_user_id router = APIRouter( diff --git a/server/routers/rag.py b/server/routers/rag.py index 04fe4510..a919f11c 100644 --- a/server/routers/rag.py +++ b/server/routers/rag.py @@ -13,7 +13,7 @@ git_issue_task, ) -from ..verify.rate_limit import verify_rate_limit +from verify.rate_limit import verify_rate_limit router = APIRouter( diff --git a/server/tests/__init__.py b/server/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/tests/conftest.py b/server/tests/conftest.py new file mode 100644 index 00000000..6db85294 --- /dev/null +++ b/server/tests/conftest.py @@ -0,0 +1,5 @@ +# tests/conftest.py +import sys +import os + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) diff --git a/server/test_main.py b/server/tests/test_main.py similarity index 91% rename from server/test_main.py rename to server/tests/test_main.py index 9f3561cf..1612d400 100644 --- a/server/test_main.py +++ b/server/tests/test_main.py @@ -1,5 +1,5 @@ from fastapi.testclient import TestClient -from .main import app +from main import app client = TestClient(app) diff --git a/server/tools/bot_builder.py b/server/tools/bot_builder.py index 3b86dd4c..9f39079f 100644 --- a/server/tools/bot_builder.py +++ b/server/tools/bot_builder.py @@ -1,11 +1,10 @@ -import json from typing import List, Optional from fastapi.responses import JSONResponse from langchain.tools import tool from github import Github from petercat_utils import get_client -from ..bot.builder import bot_builder +from bot.builder import bot_builder g = Github() diff --git a/server/tools/issue.py b/server/tools/issue.py index b937cbc2..4e44e09a 100644 --- a/server/tools/issue.py +++ b/server/tools/issue.py @@ -3,7 +3,7 @@ from github import Auth, Github from langchain.tools import tool -from ..tools.helper import need_github_login +from tools.helper import need_github_login DEFAULT_REPO_NAME = "ant-design/ant-design" diff --git a/server/utils/github.py b/server/utils/github.py index ce2df27e..71a51c83 100644 --- a/server/utils/github.py +++ b/server/utils/github.py @@ -4,9 +4,9 @@ from petercat_utils import get_env_variable from github import Auth -from ..event_handler.pull_request import PullRequestEventHandler -from ..event_handler.discussion import DiscussionEventHandler -from ..event_handler.issue import IssueEventHandler +from event_handler.pull_request import PullRequestEventHandler +from event_handler.discussion import DiscussionEventHandler +from event_handler.issue import IssueEventHandler APP_ID = get_env_variable("X_GITHUB_APP_ID") diff --git a/server/verify/rate_limit.py b/server/verify/rate_limit.py index 14db30f6..cb820378 100644 --- a/server/verify/rate_limit.py +++ b/server/verify/rate_limit.py @@ -4,7 +4,7 @@ from petercat_utils import get_client, get_env_variable -from ..auth.get_user_info import getUserInfoByToken +from auth.get_user_info import getUserInfoByToken RATE_LIMIT_ENABLED = get_env_variable("RATE_LIMIT_ENABLED", "False") == 'True' RATE_LIMIT_REQUESTS = get_env_variable("RATE_LIMIT_REQUESTS") or 100 From 41ca1bf82d8f1847dbd46e2fcb23e7371bd98382 Mon Sep 17 00:00:00 2001 From: "raoha.rh" Date: Wed, 21 Aug 2024 15:47:23 +0800 Subject: [PATCH 4/4] feat: init pytest workflow --- .github/workflows/after-success.yml | 14 ------- .github/workflows/aws-preview.yml | 16 ++++++-- .github/workflows/pr-tests.yml | 64 +++++++++++++++++++++++++---- 3 files changed, 69 insertions(+), 25 deletions(-) delete mode 100644 .github/workflows/after-success.yml diff --git a/.github/workflows/after-success.yml b/.github/workflows/after-success.yml deleted file mode 100644 index c05bb6d1..00000000 --- a/.github/workflows/after-success.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: After Success - -on: - workflow_run: - workflows: ["PR Tests"] - types: - - completed - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - name: Confirm Success - run: echo "Previous workflow was successful!" diff --git a/.github/workflows/aws-preview.yml b/.github/workflows/aws-preview.yml index 5654fa83..40992023 100644 --- a/.github/workflows/aws-preview.yml +++ b/.github/workflows/aws-preview.yml @@ -1,10 +1,18 @@ name: Deploy Backend to Preview ECS +# on: +# workflow_run: +# workflows: ["PR Tests"] +# types: +# - completed on: - workflow_run: - workflows: ["PR Tests"] - types: - - completed + pull_request: + branches: [ "main" ] + paths: + - .github/workflows/aws-preview.yml + - server/** + - petercat_utils/** + - subscriber/** env: AWS_REGION: ap-northeast-1 diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 88342daf..d145c8b9 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -2,15 +2,65 @@ name: PR Tests on: pull_request: - branches: - - main + branches: [ "main" ] + paths: + - .github/workflows/aws-preview.yml + - server/** + - petercat_utils/** + - subscriber/** + +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + pull-requests: write + actions: write + +env: + AWS_REGION: ap-northeast-1 + REPORT_FILE: md_report.md jobs: - test: + build: runs-on: ubuntu-latest + environment: Preview + strategy: + fail-fast: true + defaults: + run: + working-directory: ./server steps: - - name: Check out code - uses: actions/checkout@v3 + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::654654285942:role/Github-OIDC + audience: sts.amazonaws.com + aws-region: ${{ env.AWS_REGION }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12.0' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install ruff + pip install pytest pytest-cov pytest-md-report + + - name: Lint with Ruff + run: | + ruff check --output-format=github . - - name: Run a simple test - run: echo "Running tests..." + - name: Test with pytest + run: | + pytest -v --md-report --md-report-output ${{ env.REPORT_FILE }} + cat ${{ env.REPORT_FILE }} + + - name: Comment PR + uses: thollander/actions-comment-pull-request@v2 + with: + filePath: ./server/${{ env.REPORT_FILE }} \ No newline at end of file