diff --git a/.github/workflows/aws-preview.yml b/.github/workflows/aws-preview.yml index e9394818..2ad44c10 100644 --- a/.github/workflows/aws-preview.yml +++ b/.github/workflows/aws-preview.yml @@ -53,7 +53,9 @@ jobs: --parameter-overrides APIUrl="https://api-pre.petercat.ai" \ WebUrl="https://www.petercat.ai" \ StaticUrl="https://static.petercat.ai" \ - AWSSecretName=${{ vars.AWS_SECRET_NAME }} \ + AWSGithubSecretName=${{ secrets.AWS_GITHUB_SECRET_NAME }} \ + AWSStaticSecretName=${{ secrets.AWS_STATIC_SECRET_NAME }} \ + AWSStaticKeyPairId=${{ secrets.AWS_STATIC_KEYPAIR_ID }} \ S3TempBucketName=${{ vars.S3_TEMP_BUCKET_NAME }} \ GitHubAppID=${{ secrets.X_GITHUB_APP_ID }} \ GithubAppsClientId=${{ secrets.X_GITHUB_APPS_CLIENT_ID }} \ diff --git a/.github/workflows/aws-prod.yml b/.github/workflows/aws-prod.yml index 486a5444..59c559e4 100644 --- a/.github/workflows/aws-prod.yml +++ b/.github/workflows/aws-prod.yml @@ -47,7 +47,9 @@ jobs: --parameter-overrides APIUrl="https://api.petercat.ai" \ WebUrl="https://www.petercat.ai" \ StaticUrl="https://static.petercat.ai" \ - AWSSecretName=${{ vars.AWS_SECRET_NAME }} \ + AWSGithubSecretName=${{ vars.AWS_GITHUB_SECRET_NAME }} \ + AWSStaticSecretName=${{ vars.AWS_STATIC_SECRET_NAME }} \ + AWSStaticKeyPairId=${{ secrets.AWS_STATIC_KEYPAIR_ID }} \ S3TempBucketName=${{ vars.S3_TEMP_BUCKET_NAME }} \ GitHubAppID=${{ secrets.X_GITHUB_APP_ID }} \ GithubAppsClientId=${{ secrets.X_GITHUB_APPS_CLIENT_ID }} \ diff --git a/README.en-US.md b/README.en-US.md index d68fb130..e2bcfb25 100644 --- a/README.en-US.md +++ b/README.en-US.md @@ -86,8 +86,10 @@ The project requires environment variables to be set: | `WEB_URL` | Required | Domain of the frontend web service | `https://petercat.ai` | | `STATIC_URL` | Required | Static resource domain | `https://static.petercat.ai` | | **AWS Related Environment Variables** | -| `AWS_SECRET_NAME` | Required | AWS secret file name | `prod/githubapp/petercat/pem` | -| `S3_TEMP_BUCKET_NAME` | Required | AWS S3 bucket for temporary image files | `xxx-temp` | +| `AWS_GITHUB_SECRET_NAME` | Required | AWS secret file name | `prod/githubapp/petercat/pem` | +| `AWS_STATIC_SECRET_NAME` | Optional | The name of the AWS-managed CloudFront private key. If configured, CloudFront signed URLs will be used to protect your resources. For more information, see the [AWS documentation](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/private-content-trusted-signers.html). | `prod/petercat/static` | +| `AWS_STATIC_KEYPAIR_ID` | Optional | The Key Pair ID for AWS CloudFront. If configured, CloudFront signed URLs will be used to protect your resources. For more information, see the [AWS documentation](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/private-content-trusted-signers.html). | `APKxxxxxxxx` | +| `S3_TEMP_BUCKET_NAME` | Required | AWS S3 bucket for temporary image files | `xxx-temp` | | `SQS_QUEUE_URL` | Required | AWS SQS queue URL | `https://sqs.ap-northeast-1.amazonaws.com/xxx/petercat-task-queue` | | **Supabase Related Environment Variables** | | `SUPABASE_URL` | Required | Supabase service URL, found [here](https://supabase.com/dashboard/project/_/settings/database) | `https://***.supabase.co` | diff --git a/README.md b/README.md index 806f3e16..2d3a72e5 100644 --- a/README.md +++ b/README.md @@ -90,13 +90,15 @@ | `WEB_URL` | 必选 | 前端 Web 服务的域名 | `https://petercat.ai` | `STATIC_URL` | 必选 | 静态资源域名 | `https://static.petercat.ai` | **AWS 相关环境变量** | -| `AWS_SECRET_NAME` | 必选 | AWS 托管的私钥文件名 | `prod/githubapp/petercat/pem` -| `S3_TEMP_BUCKET_NAME` | 必选 | 用于托管 AWS 临时图片文件 S3 的 bucket | `xxx-temp` +| `AWS_GITHUB_SECRET_NAME` | 必选 | AWS 托管的 Github 私钥文件名 | `prod/githubapp/petercat/pem` +| `AWS_STATIC_SECRET_NAME` | 可选 | AWS 托管的 CloudFront 签名私钥名称。如果配置了该项，将使用 CloudFront 签名 URL 来保护你的资源。更多信息请参阅 [AWS 文档](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/private-content-trusted-signers.html)。 | `prod/petercat/static` | +| `AWS_STATIC_KEYPAIR_ID` | 可选 | AWS CloudFront 的 Key Pair ID。如果配置了该项，将使用 CloudFront 签名 URL 来保护你的资源。更多信息请参阅 [AWS 文档](https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/private-content-trusted-signers.html)。 | `APKxxxxxxxx` | +| `S3_TEMP_BUCKET_NAME` | 可选 | 用于托管 AWS 临时图片文件 S3 的 bucket | `xxx-temp` | `SQS_QUEUE_URL`| 必选 | AWS SQS 消息队列 URL | `https://sqs.ap-northeast-1.amazonaws.com/xxx/petercat-task-queue` | **SUPABASE 相关 env** | | `SUPABASE_URL` | 必选 | supabase 服务的 URL，可以在[这里](https://supabase.com/dashboard/project/_/settings/database)找到 | `https://***.supabase.co` | | `SUPABASE_SERVICE_KEY` | 必选 | supabase 服务密钥，可以在[这里](https://supabase.com/dashboard/project/_/settings/database)找到 | `{{SUPABASE_SERVICE_KEY}}` | -| **Auth0 相关 env **| +| **Auth0 相关 env**| | `AUTH0_DOMAIN` | 必选 | auth0 服务域名，从 auth0 / Application / Basic Information 下获取 | `petercat.us.auth0.com` | `AUTH0_CLIENT_ID` | 必选 | auth0 客户端 ID，从 auth0 / Application / Basic Information 下获取 | `artfiUxxxx` | `AUTH0_CLIENT_SECRET` | 必选 | auth0 客户端密钥， 从 auth0 / Application / Basic Information 下获取 | `xxxx-xxxx-xxx` diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 111b8ed8..f3377f10 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -17,7 +17,7 @@ services: ports: - 8080:8080 environment: - AWS_SECRET_NAME: ${AWS_SECRET_NAME} + AWS_GITHUB_SECRET_NAME: ${AWS_GITHUB_SECRET_NAME} S3_TEMP_BUCKET_NAME: ${S3_TEMP_BUCKET_NAME} API_URL: ${API_URL} WEB_URL: ${WEB_URL} diff --git a/docs/guides/self_hosted_aws.md b/docs/guides/self_hosted_aws.md index ad63c51e..45ba8710 100644 --- a/docs/guides/self_hosted_aws.md +++ b/docs/guides/self_hosted_aws.md @@ -98,7 +98,7 @@ sam deploy \ --config-file .aws/petercat-ap-southeast.toml \ --parameter-overrides APIUrl=$API_URL \ WebUrl=$WEB_URL \ - AWSSecretName=$AWS_SECRET_NAME \ + AWSSecretName=$AWS_GITHUB_SECRET_NAME \ S3TempBucketName=$S3_TEMP_BUCKET_NAME \ GitHubAppID=$X_GITHUB_APP_ID \ GithubAppsClientId=$X_GITHUB_APPS_CLIENT_ID \ diff --git a/docs/guides/self_hosted_aws_cn.md b/docs/guides/self_hosted_aws_cn.md index 7daddc1f..86c37de1 100644 --- a/docs/guides/self_hosted_aws_cn.md +++ b/docs/guides/self_hosted_aws_cn.md @@ -98,7 +98,7 @@ sam deploy \ --config-file .aws/petercat-ap-southeast.toml \ --parameter-overrides APIUrl=$API_URL \ WebUrl=$WEB_URL \ - AWSSecretName=$AWS_SECRET_NAME \ + AWSSecretName=$AWS_GITHUB_SECRET_NAME \ S3TempBucketName=$S3_TEMP_BUCKET_NAME \ GitHubAppID=$X_GITHUB_APP_ID \ GithubAppsClientId=$X_GITHUB_APPS_CLIENT_ID \ diff --git a/server/.env.example b/server/.env.example index b04b6255..d1898ed5 100644 --- a/server/.env.example +++ b/server/.env.example @@ -32,5 +32,7 @@ AUTH0_CLIENT_SECRET=auth0_client_secret # OPTIONAL - AWS Configures SQS_QUEUE_URL=https://sqs.ap-northeast-1.amazonaws.com/{your_aws_user}/{your_aws_sqs_message} -AWS_SECRET_NAME=AWS_SECRET_NAME +AWS_GITHUB_SECRET_NAME="prod/githubapp/petercat/pem" +AWS_STATIC_SECRET_NAME="prod/petercat/static" +AWS_STATIC_KEYPAIR_ID="xxxxxx" S3_TEMP_BUCKET_NAME=S3_TEMP_BUCKET_NAME diff --git a/server/aws/service.py b/server/aws/service.py index c1108cd4..c125b061 100644 --- a/server/aws/service.py +++ b/server/aws/service.py @@ -1,9 +1,37 @@ import base64 import hashlib +from botocore.signers import CloudFrontSigner +from petercat_utils import get_env_variable +import rsa +from datetime import datetime, timedelta + +from utils.get_private_key import get_private_key from .schemas import ImageMetaData from .constants import S3_TEMP_BUCKET_NAME, STATIC_URL from .exceptions import UploadError +REGIN_NAME = get_env_variable("AWS_REGION") +AWS_STATIC_SECRET_NAME = get_env_variable("AWS_STATIC_SECRET_NAME") +AWS_STATIC_KEYPAIR_ID = get_env_variable("AWS_STATIC_KEYPAIR_ID") + +def rsa_signer(message): + private_key_str = get_private_key(REGIN_NAME, AWS_STATIC_SECRET_NAME) + private_key = rsa.PrivateKey.load_pkcs1(private_key_str.encode('utf-8')) + return rsa.sign(message, private_key, 'SHA-1') + +def create_signed_url(url, expire_minutes=60) -> str: + cloudfront_signer = CloudFrontSigner(AWS_STATIC_KEYPAIR_ID, rsa_signer) + + # 设置过期时间 + expire_date = datetime.now() + timedelta(minutes=expire_minutes) + + # 创建签名 URL + signed_url = cloudfront_signer.generate_presigned_url( + url=url, + date_less_than=expire_date + ) + + return signed_url def upload_image_to_s3(file, metadata: ImageMetaData, s3_client): try: @@ -36,6 +64,9 @@ def upload_image_to_s3(file, metadata: ImageMetaData, s3_client): ) # you need to redirect your static domain to your s3 bucket domain s3_url = f"{STATIC_URL}/{s3_key}" - return {"message": "File uploaded successfully", "url": s3_url} + signed_url = create_signed_url(url=s3_url, expire_minutes=60) \ + if (AWS_STATIC_SECRET_NAME and AWS_STATIC_KEYPAIR_ID) \ + else s3_url + return {"message": "File uploaded successfully", "url": signed_url } except Exception as e: raise UploadError(detail=str(e)) diff --git a/server/github_app/utils.py b/server/github_app/utils.py index cef56ccc..2b727fed 100644 --- a/server/github_app/utils.py +++ b/server/github_app/utils.py @@ -1,32 +1,19 @@ -import boto3 import jwt import requests -from botocore.exceptions import ClientError + import time from cryptography.hazmat.primitives import serialization from cryptography.hazmat.backends import default_backend from petercat_utils.utils.env import get_env_variable +from utils.get_private_key import get_private_key APP_ID = get_env_variable("X_GITHUB_APP_ID") -SECRET_NAME = get_env_variable("AWS_SECRET_NAME") +AWS_GITHUB_SECRET_NAME = get_env_variable("AWS_GITHUB_SECRET_NAME") REGIN_NAME = get_env_variable("AWS_REGION") -def get_private_key(): - session = boto3.session.Session() - client = session.client(service_name="secretsmanager", region_name=REGIN_NAME) - try: - get_secret_value_response = client.get_secret_value(SecretId=SECRET_NAME) - except ClientError as e: - # For a list of exceptions thrown, see - # https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html - raise e - - return get_secret_value_response["SecretString"] - - def get_jwt(): payload = { # Issued at time @@ -37,7 +24,7 @@ def get_jwt(): "iss": APP_ID, } - pem = get_private_key() + pem = get_private_key(region_name=REGIN_NAME, secret_id=AWS_GITHUB_SECRET_NAME) private_key = serialization.load_pem_private_key( pem.encode("utf-8"), password=None, backend=default_backend() ) diff --git a/server/utils/get_private_key.py b/server/utils/get_private_key.py new file mode 100644 index 00000000..710d4260 --- /dev/null +++ b/server/utils/get_private_key.py @@ -0,0 +1,14 @@ +import boto3 +from botocore.exceptions import ClientError + +def get_private_key(region_name: str, secret_id: str): + session = boto3.session.Session() + client = session.client(service_name="secretsmanager", region_name=region_name) + try: + get_secret_value_response = client.get_secret_value(SecretId=secret_id) + except ClientError as e: + # For a list of exceptions thrown, see + # https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html + raise e + + return get_secret_value_response["SecretString"] diff --git a/template.yml b/template.yml index 81e2bd88..243bff8b 100644 --- a/template.yml +++ b/template.yml @@ -84,9 +84,17 @@ Parameters: Type: String Description: Auth0 Clientt Secret Default: 1 - AWSSecretName: + AWSGithubSecretName: Type: String - Description: AWS Secret Name + Description: Github Secret Name Store in AWS + Default: 1 + AWSStaticSecretName: + Type: String + Description: Static Secret Name Store in AWS + Default: 1 + AWSStaticKeyPairId: + Type: String + Description: Static Key Pair Id Default: 1 S3TempBucketName: Type: String @@ -101,7 +109,9 @@ Resources: Environment: Variables: AWS_LWA_INVOKE_MODE: RESPONSE_STREAM - AWS_SECRET_NAME: !Ref AWSSecretName + AWS_GITHUB_SECRET_NAME: !Ref AWSGithubSecretName + AWS_STATIC_SECRET_NAME: !Ref AWSStaticSecretName + AWS_STATIC_KEYPAIR_ID: !Ref AWSStaticKeyPairId S3_TEMP_BUCKET_NAME: !Ref S3TempBucketName API_URL: !Ref APIUrl WEB_URL: !Ref WebUrl @@ -129,16 +139,37 @@ Resources: - Sid: BedrockInvokePolicy Effect: Allow Action: - - bedrock:InvokeModelWithResponseStream + - bedrock:InvokeModelWithResponseStream Resource: '*' - Sid: AllObjectActions Effect: Allow Action: - - s3:PutObject - - s3:GetObject - - s3:DeleteObject + - s3:PutObject + - s3:GetObject + - s3:DeleteObject Resource: - - !Sub 'arn:aws:s3:::${S3TempBucketName}/*' + - !Sub 'arn:aws:s3:::${S3TempBucketName}/*' + - Sid: LimitUploadsBySize + Effect: Deny + Action: s3:PutObject + Resource: + - !Sub 'arn:aws:s3:::${S3TempBucketName}/*' + Condition: + NumericGreaterThan: + s3:ContentLength: 5242880 # 5mb + - Sid: "AllowOnlyImageUploads" + Effect: "Deny" + Action: "s3:PutObject" + Resource: + - !Sub 'arn:aws:s3:::${S3TempBucketName}/*' + Condition: + StringNotLike: + "s3:ContentType": + - "image/jpeg" + - "image/png" + - "image/gif" + - "image/webp" + - "image/bmp" Tracing: Active Metadata: DockerContext: server