feat: add the img uploader API (#266)

* add new env params:
 * AWS_SECRET_NAME
 * AWS_REGION_NAME
 * S3_BUCKET_NAME
 * STATIC_URL

.github/workflows/aws-preview.yml

@@ -44,7 +44,7 @@ jobs:
       # Sync .env from remote         
       - run: |
           pip install toml pyyaml boto3
-          python scripts/envs.py build -t .aws/petercat-preview.toml --silence
+          python server/scripts/envs.py build -t .aws/petercat-preview.toml --silence
       # Build inside Docker containers
       - run: sam build --use-container --config-file .aws/petercat-preview.toml
 

.github/workflows/aws-prod.yml

@@ -39,7 +39,7 @@ jobs:
       # Sync .env from remote    
       - run: |
           pip install toml pyyaml boto3
-          python scripts/envs.py build -t .aws/petercat-prod.toml --silence
+          python server/scripts/envs.py build -t .aws/petercat-prod.toml --silence
       # Build inside Docker containers
       - run: sam build --use-container --config-file .aws/petercat-prod.toml
 
@@ -48,4 +48,4 @@ jobs:
           sam deploy \
             --no-confirm-changeset \
             --no-fail-on-empty-changeset \
-            --config-file .aws/petercat-prod.toml
+            --config-file .aws/petercat-prod.toml

.gitignore

@@ -37,6 +37,7 @@ yarn-error.log*
 .env*.pre
 .env
 *.env
+*.env.local
 # vercel
 .vercel
 
@@ -57,3 +58,4 @@ next-env.d.ts
 
 dist/
 lui/src/style.css
+

package.json

@@ -7,8 +7,8 @@
     "client": "cd client && yarn run dev",
     "lui": "cd lui && yarn run dev",
     "server": "cd server && ./venv/bin/python3 -m uvicorn main:app --reload",
-    "env:pull": "python3 scripts/envs.py pull",
-    "env:push": "python3 scripts/envs.py push",
+    "env:pull": "cd server && ./venv/bin/python3 scripts/envs.py pull",
+    "env:push": "cd server && ./venv/bin/python3 scripts/envs.py push",
     "client:server": "concurrently \"yarn run server\" \"yarn run client\"",
     "lui:server": "concurrently \"yarn run server\" \"yarn run lui\"",
     "build:docker": "docker build -t petercat .",

server/.env.example

@@ -28,3 +28,11 @@ WEB_URL=web_url
 SQS_QUEUE_URL=https://sqs.ap-northeast-1.amazonaws.com/{your_aws_user}/{your_aws_sqs_message}
 
 GITHUB_TOKEN=github_token # https://github.com/settings/tokens?type=beta
+
+# AWS Configures
+AWS_SECRET_NAME=AWS_SECRET_NAME
+AWS_REGION_NAME=AWS_REGION_NAME
+S3_BUCKET_NAME=S3_BUCKET_NAME
+
+# you need to redirect your static domain to your s3 bucket domain
+STATIC_URL=STATIC_URL

server/README.zh-CN.md

@@ -3,6 +3,11 @@
 # 介绍
 PeterCat 服务端，采用 FastAPI 框架开发。使用了 supabase 作为数据存储方案。
 
+## 代码目录结构参考
+
+fastapi 最佳实践请参考 https://github.com/zhanymkanov/fastapi-best-practices
+
+
 # 功能模块
 ## 存储
 采用 [supabase](https://supabase.com) 作为数据库进行存储。
@@ -165,4 +170,4 @@ export AWS_PROFILE=my-profile
 export AWS_REGION=ap-northeast-1
 # 生效
 source ～/.zshrc
-```
+```

server/aws/constants.py

@@ -0,0 +1,7 @@
+from petercat_utils.utils.env import get_env_variable
+
+SUCCESS_CODE = "UPLOAD_SUCCESS"
+ERROR_CODES = {"credentials_error": "CREDENTIALS_ERROR", "upload_error": "UPLOAD_ERROR"}
+S3_BUCKET_NAME = get_env_variable("S3_BUCKET_NAME")
+STATIC_URL = get_env_variable("STATIC_URL")
+AWS_REGION_NAME = get_env_variable("AWS_REGION_NAME")

server/aws/dependencies.py

@@ -0,0 +1,8 @@
+from .constants import AWS_REGION_NAME
+import boto3
+
+
+def get_s3_client():
+    session = boto3.session.Session()
+    client = session.client(service_name="s3", region_name=AWS_REGION_NAME)
+    return client

server/aws/exceptions.py

@@ -0,0 +1,6 @@
+from fastapi import HTTPException
+
+
+class UploadError(HTTPException):
+    def __init__(self, detail: str):
+        super().__init__(status_code=500, detail=detail)

server/aws/router.py

@@ -0,0 +1,22 @@
+from fastapi import APIRouter, Depends, File, UploadFile, Form
+from .schemas import ImageMetaData
+from .dependencies import get_s3_client
+from .service import upload_image_to_s3
+
+router = APIRouter(
+    prefix="/api/aws",
+    tags=["aws"],
+    responses={404: {"description": "Not found"}},
+)
+
+
+@router.post("/upload")
+async def upload_image(
+    file: UploadFile = File(...),
+    title: str = Form(None),
+    description: str = Form(None),
+    s3_client=Depends(get_s3_client),
+):
+    metadata = ImageMetaData(title=title, description=description)
+    result = upload_image_to_s3(file, metadata, s3_client)
+    return {"status": "success", "data": result}

server/aws/schemas.py

@@ -0,0 +1,7 @@
+from pydantic import BaseModel
+from typing import Optional
+
+
+class ImageMetaData(BaseModel):
+    title: Optional[str] = None
+    description: Optional[str] = None

server/aws/service.py

@@ -0,0 +1,28 @@
+from .schemas import ImageMetaData
+from .constants import S3_BUCKET_NAME, STATIC_URL
+from .exceptions import UploadError
+
+
+def upload_image_to_s3(file, metadata: ImageMetaData, s3_client):
+    try:
+        file_content = file.file.read()
+
+        s3_key = f"{file.filename}"
+
+        custom_metadata = {
+            "title": metadata.title if metadata.title else "",
+            "description": metadata.description if metadata.description else "",
+        }
+
+        s3_client.put_object(
+            Bucket=S3_BUCKET_NAME,
+            Key=s3_key,
+            Body=file_content,
+            ContentType=file.content_type,
+            Metadata=custom_metadata,
+        )
+        # you need to redirect your static domain to your s3 bucket domain
+        s3_url = f"{STATIC_URL}/{s3_key}"
+        return {"message": "File uploaded successfully", "url": s3_url}
+    except Exception as e:
+        raise UploadError(detail=str(e))

server/github_app/utils.py

@@ -10,61 +10,64 @@
 from petercat_utils.utils.env import get_env_variable
 
 APP_ID = get_env_variable("X_GITHUB_APP_ID")
+SECRET_NAME = get_env_variable("AWS_SECRET_NAME")
+REGIN_NAME = get_env_variable("AWS_REGION_NAME")
+
 
 def get_private_key():
-    secret_name = "prod/githubapp/petercat/pem"
-    region_name = "ap-northeast-1"
     session = boto3.session.Session()
-    client = session.client(
-        service_name='secretsmanager',
-        region_name=region_name
-    )
+    client = session.client(service_name="secretsmanager", region_name=REGIN_NAME)
     try:
-        get_secret_value_response = client.get_secret_value(
-            SecretId=secret_name
-        )
+        get_secret_value_response = client.get_secret_value(SecretId=SECRET_NAME)
     except ClientError as e:
         # For a list of exceptions thrown, see
         # https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html
         raise e
 
-    return get_secret_value_response['SecretString']
+    return get_secret_value_response["SecretString"]
+
 
 def get_jwt():
     payload = {
         # Issued at time
-        'iat': int(time.time()),
+        "iat": int(time.time()),
         # JWT expiration time (10 minutes maximum)
-        'exp': int(time.time()) + 600,
+        "exp": int(time.time()) + 600,
         # GitHub App's identifier
-        'iss': APP_ID
+        "iss": APP_ID,
     }
 
     pem = get_private_key()
     private_key = serialization.load_pem_private_key(
         pem.encode("utf-8"), password=None, backend=default_backend()
     )
-    return jwt.encode(payload, private_key, algorithm='RS256')
+    return jwt.encode(payload, private_key, algorithm="RS256")
+
 
 def get_app_installations_access_token(installation_id: str, jwt: str):
     url = f"https://api.github.com/app/installations/{installation_id}/access_tokens"
     print("get_app_installations_access_token", url, jwt)
-    resp = requests.post(url,
+    resp = requests.post(
+        url,
         headers={
-            'X-GitHub-Api-Version': '2022-11-28',
-            'Accept': 'application/vnd.github+json',
-            'Authorization': f"Bearer {jwt}"
-        }
+            "X-GitHub-Api-Version": "2022-11-28",
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {jwt}",
+        },
     )
 
     return resp.json()
 
+
 def get_installation_repositories(access_token: str):
     url = "https://api.github.com/installation/repositories"
     print("get_installation_repositories", url)
-    resp = requests.get(url, headers={
-        'X-GitHub-Api-Version': '2022-11-28',
-        'Accept': 'application/vnd.github+json',
-        'Authorization': f"Bearer {access_token}"
-    })
-    return resp.json()
+    resp = requests.get(
+        url,
+        headers={
+            "X-GitHub-Api-Version": "2022-11-28",
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {access_token}",
+        },
+    )
+    return resp.json()

server/main.py

@@ -16,36 +16,35 @@
 from rag import router as rag_router
 from task import router as task_router
 from github_app import router as github_app_router
+from aws import router as aws_router
 
 AUTH0_DOMAIN = get_env_variable("AUTH0_DOMAIN")
 API_AUDIENCE = get_env_variable("API_IDENTIFIER")
 CLIENT_ID = get_env_variable("AUTH0_CLIENT_ID")
-API_URL =  get_env_variable("API_URL")
-WEB_URL =  get_env_variable("WEB_URL")
+API_URL = get_env_variable("API_URL")
+WEB_URL = get_env_variable("WEB_URL")
 CALLBACK_URL = f"{API_URL}/api/auth/callback"
 
 is_dev = bool(get_env_variable("IS_DEV"))
 session_secret_key = get_env_variable("FASTAPI_SECRET_KEY")
 cors_origins_whitelist = get_env_variable("CORS_ORIGIN_WHITELIST") or None
-app = FastAPI( 
-    title="Bo-meta Server",
-    version="1.0",
-    description="Agent Chat APIs"
-)
+app = FastAPI(title="Bo-meta Server", version="1.0", description="Agent Chat APIs")
 
 app.add_middleware(
     SessionMiddleware,
-    secret_key = session_secret_key,
+    secret_key=session_secret_key,
 )
 
-cors_origins = ["*"] if cors_origins_whitelist is None else cors_origins_whitelist.split(',')
+cors_origins = (
+    ["*"] if cors_origins_whitelist is None else cors_origins_whitelist.split(",")
+)
 
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=cors_origins, 
+    allow_origins=cors_origins,
     allow_credentials=True,
     allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
-    allow_headers=["Content-Type", "Authorization"], 
+    allow_headers=["Content-Type", "Authorization"],
 )
 
 
@@ -55,19 +54,25 @@
 app.include_router(chat_router.router)
 app.include_router(task_router.router)
 app.include_router(github_app_router.router)
+app.include_router(aws_router.router)
 
 
 @app.get("/api/health_checker")
 def health_checker():
     return {
-       "API_URL": API_URL,
-       "WEB_URL": WEB_URL,
-       "CALLBACK_URL": CALLBACK_URL,
+        "API_URL": API_URL,
+        "WEB_URL": WEB_URL,
+        "CALLBACK_URL": CALLBACK_URL,
     }
 
 
 if __name__ == "__main__":
     if is_dev:
-        uvicorn.run("main:app", host="0.0.0.0", port=int(os.environ.get("PORT", "8080")), reload=True)
+        uvicorn.run(
+            "main:app",
+            host="0.0.0.0",
+            port=int(os.environ.get("PORT", "8080")),
+            reload=True,
+        )
     else:
         uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", "8080")))

server/requirements.txt

@@ -15,6 +15,7 @@ load_dotenv
 supabase
 authlib==0.14.3
 boto3>=1.34.84
+pytest-cov
 PyJWT
 pydantic>=2.7.0
 unstructured[md]