From d053f2b4f717fdc01aafdce956f0e518a667a57b Mon Sep 17 00:00:00 2001 From: Dev Aggarwal Date: Fri, 2 Aug 2024 05:51:29 +0530 Subject: [PATCH] - Update `serve_static_file` logic to handle redirects, HTML responses. - Show current list of gcs static files on the webflow page - parallel zip upload - Enhance unknown error template with detailed error reporting and GitHub link - Add `CustomAPIRouter` class and refactor routes to standardize trailing slashes handling - replace healthcheck / with /status to support index.html static page --- Dockerfile | 3 + daras_ai/image_input.py | 37 +++++-- daras_ai_v2/base.py | 2 +- daras_ai_v2/bot_integration_widgets.py | 2 +- daras_ai_v2/bots.py | 2 +- daras_ai_v2/gpu_server.py | 4 +- daras_ai_v2/settings.py | 2 + glossary_resources/tests.py | 4 +- routers/account.py | 4 +- routers/api.py | 56 ++-------- routers/bots_api.py | 21 +--- routers/broadcast_api.py | 4 +- routers/custom_api_router.py | 14 +++ routers/facebook_api.py | 4 +- routers/root.py | 101 +++++++---------- routers/static_pages.py | 148 +++++++++++++++++++++++++ server.py | 50 +++++++-- static_pages.py | 111 ------------------- templates/errors/unknown.html | 28 ++++- templates/login_container.html | 2 +- tests/test_public_endpoints.py | 4 +- url_shortener/routers.py | 6 +- 22 files changed, 331 insertions(+), 278 deletions(-) create mode 100644 routers/custom_api_router.py create mode 100644 routers/static_pages.py delete mode 100644 static_pages.py diff --git a/Dockerfile b/Dockerfile index 929e01ae5..83052f5ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,6 +54,9 @@ COPY . . ENV FORWARDED_ALLOW_IPS='*' ENV PYTHONUNBUFFERED=1 +ARG CAPROVER_GIT_COMMIT_SHA=${CAPROVER_GIT_COMMIT_SHA} +ENV CAPROVER_GIT_COMMIT_SHA=${CAPROVER_GIT_COMMIT_SHA} + EXPOSE 8000 EXPOSE 8501 diff --git a/daras_ai/image_input.py b/daras_ai/image_input.py index 5e61fcba9..4ee0ab5f9 100644 --- a/daras_ai/image_input.py +++ b/daras_ai/image_input.py @@ -1,10 +1,11 @@ +import math import mimetypes import os import re +import typing import uuid from pathlib import Path -import math import numpy as np import requests from PIL import Image, ImageOps @@ -13,6 +14,9 @@ from daras_ai_v2 import settings from daras_ai_v2.exceptions import UserError +if typing.TYPE_CHECKING: + from google.cloud.storage import Blob, Bucket + def resize_img_pad(img_bytes: bytes, size: tuple[int, int]) -> bytes: img_cv2 = bytes_to_cv2_img(img_bytes) @@ -57,25 +61,38 @@ def upload_file_from_bytes( data: bytes, content_type: str = None, ) -> str: - if not content_type: - content_type = mimetypes.guess_type(filename)[0] - content_type = content_type or "application/octet-stream" - blob = storage_blob_for(filename) - blob.upload_from_string(data, content_type=content_type) + blob = gcs_blob_for(filename) + upload_gcs_blob_from_bytes(blob, data, content_type) return blob.public_url -def storage_blob_for(filename: str) -> "storage.storage.Blob": - from firebase_admin import storage - +def gcs_blob_for(filename: str) -> "Blob": filename = safe_filename(filename) - bucket = storage.bucket(settings.GS_BUCKET_NAME) + bucket = gcs_bucket() blob = bucket.blob( os.path.join(settings.GS_MEDIA_PATH, str(uuid.uuid1()), filename) ) return blob +def upload_gcs_blob_from_bytes( + blob: "Blob", + data: bytes, + content_type: str = None, +) -> str: + if not content_type: + content_type = mimetypes.guess_type(blob.path)[0] + content_type = content_type or "application/octet-stream" + blob.upload_from_string(data, content_type=content_type) + return blob.public_url + + +def gcs_bucket() -> "Bucket": + from firebase_admin import storage + + return storage.bucket(settings.GS_BUCKET_NAME) + + def cv2_img_to_bytes(img: np.ndarray) -> bytes: import cv2 diff --git a/daras_ai_v2/base.py b/daras_ai_v2/base.py index a90f2504b..8e3610a28 100644 --- a/daras_ai_v2/base.py +++ b/daras_ai_v2/base.py @@ -161,7 +161,7 @@ def __init__( @classmethod @property def endpoint(cls) -> str: - return f"/v2/{cls.slug_versions[0]}/" + return f"/v2/{cls.slug_versions[0]}" @classmethod def current_app_url( diff --git a/daras_ai_v2/bot_integration_widgets.py b/daras_ai_v2/bot_integration_widgets.py index 2db423ed1..4958f389d 100644 --- a/daras_ai_v2/bot_integration_widgets.py +++ b/daras_ai_v2/bot_integration_widgets.py @@ -318,7 +318,7 @@ def get_web_widget_embed_code(bi: BotIntegration) -> str: integration_id=bi.api_integration_id(), integration_name=slugify(bi.name) or "untitled", ), - ) + ).rstrip("/") return dedent( f"""
diff --git a/daras_ai_v2/bots.py b/daras_ai_v2/bots.py index e7721a6fe..58a154590 100644 --- a/daras_ai_v2/bots.py +++ b/daras_ai_v2/bots.py @@ -497,7 +497,7 @@ def build_run_vars(convo: Conversation, user_msg_id: str): bi = convo.bot_integration if bi.platform == Platform.WEB: - user_msg_id = user_msg_id.lstrip(MSG_ID_PREFIX) + user_msg_id = user_msg_id.removeprefix(MSG_ID_PREFIX) variables = dict( platform=Platform(bi.platform).name, integration_id=bi.api_integration_id(), diff --git a/daras_ai_v2/gpu_server.py b/daras_ai_v2/gpu_server.py index 03ce72855..f4ceab7b3 100644 --- a/daras_ai_v2/gpu_server.py +++ b/daras_ai_v2/gpu_server.py @@ -4,7 +4,7 @@ import typing from time import time -from daras_ai.image_input import storage_blob_for +from daras_ai.image_input import gcs_blob_for from daras_ai_v2 import settings from daras_ai_v2.exceptions import GPUError, UserError from gooeysite.bg_db_conn import get_celery_result_db_safe @@ -43,7 +43,7 @@ def call_celery_task_outfile( filename: str, num_outputs: int = 1, ): - blobs = [storage_blob_for(filename) for i in range(num_outputs)] + blobs = [gcs_blob_for(filename) for i in range(num_outputs)] pipeline["upload_urls"] = [ blob.generate_signed_url( version="v4", diff --git a/daras_ai_v2/settings.py b/daras_ai_v2/settings.py index 8ea6091d4..acfc1e6a2 100644 --- a/daras_ai_v2/settings.py +++ b/daras_ai_v2/settings.py @@ -230,6 +230,8 @@ GS_BUCKET_NAME = config("GS_BUCKET_NAME", default=f"{GCP_PROJECT}.appspot.com") GS_MEDIA_PATH = config("GS_MEDIA_PATH", default="daras_ai/media") +GS_STATIC_PATH = config("GS_STATIC_PATH", default="gooeyai/static") + GOOGLE_CLIENT_ID = config("GOOGLE_CLIENT_ID", default="") FIREBASE_CONFIG = config("FIREBASE_CONFIG", default="") diff --git a/glossary_resources/tests.py b/glossary_resources/tests.py index 4a410b866..0a5d05219 100644 --- a/glossary_resources/tests.py +++ b/glossary_resources/tests.py @@ -1,6 +1,6 @@ import pytest -from daras_ai.image_input import storage_blob_for +from daras_ai.image_input import gcs_blob_for from daras_ai_v2 import settings from daras_ai_v2.crypto import get_random_doc_id from glossary_resources.models import GlossaryResource @@ -64,7 +64,7 @@ def glossary_url(): import pandas as pd df = pd.DataFrame.from_records(GLOSSARY) - blob = storage_blob_for("test glossary.csv") + blob = gcs_blob_for("test glossary.csv") blob.upload_from_string(df.to_csv(index=False).encode(), content_type="text/csv") try: diff --git a/routers/account.py b/routers/account.py index 552e193d9..89fc1aff2 100644 --- a/routers/account.py +++ b/routers/account.py @@ -20,7 +20,9 @@ from payments.webhooks import PaypalWebhookHandler from routers.root import page_wrapper, get_og_url_path -app = APIRouter() +from routers.custom_api_router import CustomAPIRouter + +app = CustomAPIRouter() @gui.route(app, "/payment-processing/") diff --git a/routers/api.py b/routers/api.py index 88d837a94..b9da2d1f1 100644 --- a/routers/api.py +++ b/routers/api.py @@ -43,7 +43,9 @@ from functions.models import CalledFunctionResponse from gooeysite.bg_db_conn import get_celery_result_db_safe -app = APIRouter() +from routers.custom_api_router import CustomAPIRouter + +app = CustomAPIRouter() O = typing.TypeVar("O") @@ -140,7 +142,7 @@ def script_to_api(page_cls: typing.Type[BasePage]): } @app.post( - os.path.join(endpoint, ""), + endpoint, response_model=response_model, responses={ HTTP_500_INTERNAL_SERVER_ERROR: {"model": FailedReponseModelV2}, @@ -150,15 +152,6 @@ def script_to_api(page_cls: typing.Type[BasePage]): tags=[page_cls.title], name=page_cls.title + " (v2 sync)", ) - @app.post( - endpoint, - response_model=response_model, - responses={ - HTTP_500_INTERNAL_SERVER_ERROR: {"model": FailedReponseModelV2}, - **common_errs, - }, - include_in_schema=False, - ) def run_api_json( request: Request, page_request: request_model, @@ -174,16 +167,6 @@ def run_api_json( ) return build_sync_api_response(page=page, result=result, run_id=run_id, uid=uid) - @app.post( - os.path.join(endpoint, "form/"), - response_model=response_model, - responses={ - HTTP_500_INTERNAL_SERVER_ERROR: {"model": FailedReponseModelV2}, - HTTP_400_BAD_REQUEST: {"model": GenericErrorResponse}, - **common_errs, - }, - include_in_schema=False, - ) @app.post( os.path.join(endpoint, "form"), response_model=response_model, @@ -209,7 +192,7 @@ def run_api_form( response_model = AsyncApiResponseModelV3 @app.post( - os.path.join(endpoint, "async/"), + os.path.join(endpoint, "async"), response_model=response_model, responses=common_errs, operation_id="async__" + page_cls.slug_versions[0], @@ -217,13 +200,6 @@ def run_api_form( tags=[page_cls.title], status_code=202, ) - @app.post( - os.path.join(endpoint, "async"), - response_model=response_model, - responses=common_errs, - include_in_schema=False, - status_code=202, - ) def run_api_json_async( request: Request, response: Response, @@ -243,15 +219,6 @@ def run_api_json_async( response.headers["Access-Control-Expose-Headers"] = "Location" return ret - @app.post( - os.path.join(endpoint, "async/form/"), - response_model=response_model, - responses={ - HTTP_400_BAD_REQUEST: {"model": GenericErrorResponse}, - **common_errs, - }, - include_in_schema=False, - ) @app.post( os.path.join(endpoint, "async/form"), response_model=response_model, @@ -281,19 +248,13 @@ def run_api_form_async( ) @app.get( - os.path.join(endpoint, "status/"), + os.path.join(endpoint, "status"), response_model=response_model, responses=common_errs, operation_id="status__" + page_cls.slug_versions[0], tags=[page_cls.title], name=page_cls.title + " (v3 status)", ) - @app.get( - os.path.join(endpoint, "status"), - response_model=response_model, - responses=common_errs, - include_in_schema=False, - ) def get_run_status( run_id: str, user: AppUser = Depends(api_auth_header), @@ -471,3 +432,8 @@ class BalanceResponse(BaseModel): @app.get("/v1/balance/", response_model=BalanceResponse, tags=["Misc"]) def get_balance(user: AppUser = Depends(api_auth_header)): return BalanceResponse(balance=user.balance) + + +@app.get("/status") +async def health(): + return "OK" diff --git a/routers/bots_api.py b/routers/bots_api.py index 87add064c..780a7b918 100644 --- a/routers/bots_api.py +++ b/routers/bots_api.py @@ -5,7 +5,7 @@ from typing import Any import hashids -from fastapi import APIRouter, HTTPException +from fastapi import HTTPException from furl import furl from pydantic import BaseModel, Field from starlette.responses import StreamingResponse, Response @@ -22,8 +22,9 @@ AsyncStatusResponseModelV3, build_async_api_response, ) +from routers.custom_api_router import CustomAPIRouter -app = APIRouter() +app = CustomAPIRouter() api_hashids = hashids.Hashids(salt=settings.HASHIDS_API_SALT) MSG_ID_PREFIX = "web-" @@ -78,19 +79,13 @@ class CreateStreamResponse(BaseModel): @app.post( - "/v3/integrations/stream/", + "/v3/integrations/stream", response_model=CreateStreamResponse, responses={402: {}}, operation_id=VideoBotsPage.slug_versions[0] + "__stream_create", tags=["Copilot Integrations"], name="Copilot Integrations Create Stream", ) -@app.post( - "/v3/integrations/stream/", - response_model=CreateStreamResponse, - responses={402: {}}, - include_in_schema=False, -) def stream_create(request: CreateStreamRequest, response: Response): request_id = str(uuid.uuid4()) get_redis_cache().set( @@ -172,19 +167,13 @@ class StreamError(BaseModel): @app.get( - "/v3/integrations/stream/{request_id}/", + "/v3/integrations/stream/{request_id}", response_model=StreamEvent, responses={402: {}}, operation_id=VideoBotsPage.slug_versions[0] + "__stream", tags=["Copilot Integrations"], name="Copilot integrations Stream Response", ) -@app.get( - "/v3/integrations/stream/{request_id}", - response_model=StreamEvent, - responses={402: {}}, - include_in_schema=False, -) def stream_response(request_id: str): r = get_redis_cache().getdel(f"gooey/stream-init/v1/{request_id}") if not r: diff --git a/routers/broadcast_api.py b/routers/broadcast_api.py index 3b88afaa3..cbf1ca07c 100644 --- a/routers/broadcast_api.py +++ b/routers/broadcast_api.py @@ -1,7 +1,6 @@ import typing from django.db.models import Q -from fastapi import APIRouter from fastapi import Depends from fastapi import HTTPException from pydantic import BaseModel, Field @@ -11,8 +10,9 @@ from bots.models import BotIntegration from bots.tasks import send_broadcast_msgs_chunked from recipes.VideoBots import ReplyButton, VideoBotsPage +from routers.custom_api_router import CustomAPIRouter -app = APIRouter() +app = CustomAPIRouter() class BotBroadcastFilters(BaseModel): diff --git a/routers/custom_api_router.py b/routers/custom_api_router.py new file mode 100644 index 000000000..84025f52e --- /dev/null +++ b/routers/custom_api_router.py @@ -0,0 +1,14 @@ +from fastapi import APIRouter + + +class CustomAPIRouter(APIRouter): + def add_api_route(self, path: str, *args, **kwargs) -> None: + super().add_api_route(path, *args, **kwargs) + if path.endswith("/"): + path = path[:-1] + else: + path += "/" + kwargs["include_in_schema"] = False + kwargs.pop("name", None) + kwargs.pop("tags", None) + super().add_api_route(path, *args, **kwargs) diff --git a/routers/facebook_api.py b/routers/facebook_api.py index bf90f4e7c..4e396f466 100644 --- a/routers/facebook_api.py +++ b/routers/facebook_api.py @@ -1,5 +1,4 @@ import requests -from fastapi import APIRouter from fastapi.responses import RedirectResponse from furl import furl from starlette.background import BackgroundTasks @@ -13,8 +12,9 @@ from daras_ai_v2.facebook_bots import WhatsappBot, FacebookBot from daras_ai_v2.fastapi_tricks import fastapi_request_json from daras_ai_v2.functional import map_parallel +from routers.custom_api_router import CustomAPIRouter -app = APIRouter() +app = CustomAPIRouter() @app.get("/__/fb/connect_whatsapp/") diff --git a/routers/root.py b/routers/root.py index 0fa3c159f..33763b539 100644 --- a/routers/root.py +++ b/routers/root.py @@ -6,10 +6,10 @@ from enum import Enum from time import time +import gooey_gui as gui from fastapi import Depends from fastapi import HTTPException from fastapi.responses import RedirectResponse -from fastapi.routing import APIRouter from firebase_admin import auth, exceptions from furl import furl from loguru import logger @@ -21,7 +21,6 @@ FileResponse, ) -import gooey_gui as gui from app_users.models import AppUser from bots.models import Workflow, BotIntegration from daras_ai.image_input import upload_file_from_bytes, safe_filename @@ -43,8 +42,10 @@ from daras_ai_v2.query_params_util import extract_query_params from daras_ai_v2.settings import templates from handles.models import Handle +from routers.custom_api_router import CustomAPIRouter +from routers.static_pages import serve_static_file -app = APIRouter() +app = CustomAPIRouter() DEFAULT_LOGIN_REDIRECT = "/explore/" DEFAULT_LOGOUT_REDIRECT = "/" @@ -76,9 +77,7 @@ async def get_sitemap(): @app.get("/favicon") -@app.get("/favicon/") @app.get("/favicon.ico") -@app.get("/favicon.ico/") async def favicon(): return FileResponse("static/favicon.ico") @@ -188,30 +187,12 @@ def file_upload(form_data: FormData = fastapi_request_form): img.format = "png" content_type = "image/png" filename += ".png" - img.transform(resize=form_data.get("resize", f"{1024**2}@>")) + img.transform(resize=form_data.get("resize", f"{1024 ** 2}@>")) data = img.make_blob() return {"url": upload_file_from_bytes(filename, data, content_type)} -@gui.route(app, "/internal/file-upload/") -def zip_file_upload(request: Request): - from static_pages import StaticPageUpload - - uploader = StaticPageUpload(request=request) - - with page_wrapper(request): - uploader.render_file_upload() - - return { - "meta": raw_build_meta_tags( - url=get_og_url_path(request), - title="Upload ZIP File", - description="Internal Page: Upload a ZIP file to extract its contents, to google cloud", - ), - } - - @gui.route(app, "/GuiComponents/") def component_page(request: Request): import components_doc @@ -357,7 +338,7 @@ def _api_docs_page(request): def examples_route( request: Request, page_slug: str, run_slug: str = None, example_id: str = None ): - return render_page(request, page_slug, RecipeTabs.examples, example_id) + return render_recipe_page(request, page_slug, RecipeTabs.examples, example_id) @gui.route( @@ -369,7 +350,7 @@ def examples_route( def api_route( request: Request, page_slug: str, run_slug: str = None, example_id: str = None ): - return render_page(request, page_slug, RecipeTabs.run_as_api, example_id) + return render_recipe_page(request, page_slug, RecipeTabs.run_as_api, example_id) @gui.route( @@ -381,7 +362,7 @@ def api_route( def history_route( request: Request, page_slug: str, run_slug: str = None, example_id: str = None ): - return render_page(request, page_slug, RecipeTabs.history, example_id) + return render_recipe_page(request, page_slug, RecipeTabs.history, example_id) @gui.route( @@ -393,7 +374,7 @@ def history_route( def save_route( request: Request, page_slug: str, run_slug: str = None, example_id: str = None ): - return render_page(request, page_slug, RecipeTabs.saved, example_id) + return render_recipe_page(request, page_slug, RecipeTabs.saved, example_id) @gui.route( @@ -409,7 +390,7 @@ def add_integrations_route( example_id: str = None, ): gui.session_state["--add-integration"] = True - return render_page(request, page_slug, RecipeTabs.integrations, example_id) + return render_recipe_page(request, page_slug, RecipeTabs.integrations, example_id) @gui.route( @@ -431,7 +412,7 @@ def integrations_stats_route( gui.session_state.setdefault("bi_id", api_hashids.decode(integration_id)[0]) except IndexError: raise HTTPException(status_code=404) - return render_page(request, "stats", RecipeTabs.integrations, example_id) + return render_recipe_page(request, "stats", RecipeTabs.integrations, example_id) @gui.route( @@ -494,7 +475,7 @@ def integrations_route( gui.session_state.setdefault("bi_id", api_hashids.decode(integration_id)[0]) except IndexError: raise HTTPException(status_code=404) - return render_page(request, page_slug, RecipeTabs.integrations, example_id) + return render_recipe_page(request, page_slug, RecipeTabs.integrations, example_id) @gui.route( @@ -544,7 +525,6 @@ def chat_route( @app.get("/chat/{integration_name}-{integration_id}/lib.js") -@app.get("/chat/{integration_name}-{integration_id}/lib.js/") def chat_lib_route(request: Request, integration_id: str, integration_name: str = None): from routers.bots_api import api_hashids @@ -591,41 +571,36 @@ def chat_lib_route(request: Request, integration_id: str, integration_name: str @gui.route( app, + "/{path:path}", "/{page_slug}/", "/{page_slug}/{run_slug}/", - "/{page_slug}/{path:path}", "/{page_slug}/{run_slug}-{example_id}/", ) -def recipe_page_or_handle( - request: Request, - page_slug: str, - run_slug: str = None, - path: str = None, - example_id: str = None, +def recipe_or_handle_or_static( + request: Request, page_slug=None, run_slug=None, example_id=None, path=None ): - try: - handle = Handle.objects.get_by_name(page_slug) - except Handle.DoesNotExist: - import static_pages - - static_content = static_pages.serve(page_slug, path) - if not static_content: - # render recipe page - return render_page(request, page_slug, RecipeTabs.run, example_id) + path = furl(request.url).pathstr.lstrip("/") - if static_content.get("redirectUrl"): - return RedirectResponse(static_content.get("redirectUrl")) - - if static_content.get("content"): - return Response( - content=static_content["content"], - ) + parts = path.strip("/").split("/") + if len(parts) in {1, 2}: + try: + example_id = parts[1].split("-")[-1] or None + except IndexError: + example_id = None + try: + return render_recipe_page(request, parts[0], RecipeTabs.run, example_id) + except RecipePageNotFound: + pass + try: + return render_handle_page(request, parts[0]) + except Handle.DoesNotExist: + pass - else: - return render_page_for_handle(request, handle) + return serve_static_file(request, path) -def render_page_for_handle(request: Request, handle: Handle): +def render_handle_page(request: Request, name: str): + handle = Handle.objects.get_by_name(name) if handle.has_user: with page_wrapper(request): user_profile_page(request, handle.user) @@ -639,7 +614,11 @@ def render_page_for_handle(request: Request, handle: Handle): raise HTTPException(status_code=404) -def render_page( +class RecipePageNotFound(Exception): + pass + + +def render_recipe_page( request: Request, page_slug: str, tab: "RecipeTabs", example_id: str | None ): from daras_ai_v2.all_pages import normalize_slug, page_slug_map @@ -648,7 +627,7 @@ def render_page( try: page_cls = page_slug_map[normalize_slug(page_slug)] except KeyError: - raise HTTPException(status_code=404) + raise RecipePageNotFound # ensure the latest slug is used latest_slug = page_cls.slug_versions[-1] @@ -745,7 +724,7 @@ class RecipeTabs(TabData, Enum): run = TabData( title=f"{icons.run} Run", label="", - route=recipe_page_or_handle, + route=recipe_or_handle_or_static, ) examples = TabData( title=f"{icons.example} Examples", diff --git a/routers/static_pages.py b/routers/static_pages.py new file mode 100644 index 000000000..3969dbf8c --- /dev/null +++ b/routers/static_pages.py @@ -0,0 +1,148 @@ +import io +import os.path +from zipfile import ZipFile, is_zipfile, ZipInfo + +import gooey_gui as gui +import requests +from starlette.requests import Request +from starlette.responses import ( + Response, + RedirectResponse, + HTMLResponse, + PlainTextResponse, +) +from starlette.status import HTTP_308_PERMANENT_REDIRECT, HTTP_401_UNAUTHORIZED + +from daras_ai.image_input import gcs_bucket, upload_gcs_blob_from_bytes +from daras_ai.text_format import format_number_with_suffix +from daras_ai_v2 import settings +from daras_ai_v2.exceptions import raise_for_status +from daras_ai_v2.functional import map_parallel +from daras_ai_v2.user_date_widgets import render_local_dt_attrs +from routers.custom_api_router import CustomAPIRouter + +app = CustomAPIRouter() + + +def serve_static_file(request: Request, path: str): + bucket = gcs_bucket() + + if path.endswith("/"): + # relative css/js paths dont work with trailing slashes + return RedirectResponse( + os.path.join("/", path.strip("/")), status_code=HTTP_308_PERMANENT_REDIRECT + ) + + path = path or "index" + gcs_path = os.path.join(settings.GS_STATIC_PATH, path) + + # if the path has no extension, try to serve a .html file + if not os.path.splitext(gcs_path)[1]: + html_url = bucket.blob(gcs_path + ".html").public_url + r = requests.get(html_url) + if r.ok: + html = r.content.decode() + # replace sign in button with user's name if logged in + if request.user and not request.user.is_anonymous: + html = html.replace( + ">Sign in<", + f">Hi, {request.user.first_name() or request.user.email or request.user.phone_number or 'Anon'}<", + 1, + ) + return HTMLResponse(html, status_code=r.status_code) + + url = bucket.blob(gcs_path).public_url + r = requests.head(url) + if r.ok: + return RedirectResponse(url, status_code=HTTP_308_PERMANENT_REDIRECT) + + return Response(status_code=r.status_code) + + +@gui.route(app, "/internal/webflow-upload/") +def webflow_upload(request: Request): + from daras_ai_v2.base import BasePage + from routers.root import page_wrapper + + if not (request.user and BasePage.is_user_admin(request.user)): + return PlainTextResponse("Not authorized", status_code=HTTP_401_UNAUTHORIZED) + + with page_wrapper(request), gui.div( + className="d-flex align-items-center justify-content-center flex-column" + ): + render_webflow_upload() + + +def render_webflow_upload(): + zip_file = gui.file_uploader(label="##### Upload ZIP File", accept=[".zip"]) + pressed_save = gui.button( + "Extract ZIP File", + key="zip_file", + type="primary", + disabled=not zip_file, + ) + if pressed_save: + extract_zip_to_gcloud(zip_file) + + gui.caption( + "After successful upload, files will be displayed below.", + className="my-4 text-muted", + ) + + bucket = gcs_bucket() + blobs = list(bucket.list_blobs(prefix=settings.GS_STATIC_PATH)) + blobs.sort(key=lambda b: (b.name.count("/"), b.name)) + + with ( + gui.tag("table", className="my-4 table table-striped table-sm"), + gui.tag("tbody"), + ): + for blob in blobs: + with gui.tag("tr"): + with gui.tag("td"): + gui.html("...", **render_local_dt_attrs(blob.updated)) + with gui.tag("td"), gui.tag("code"): + gui.html(format_number_with_suffix(blob.size) + "B") + with gui.tag("td"), gui.tag("code"): + gui.html(blob.content_type) + with ( + gui.tag("td"), + gui.tag("a", href=blob.public_url), + ): + gui.html(blob.name.removeprefix(settings.GS_STATIC_PATH)) + + +def extract_zip_to_gcloud(url: str): + r = requests.get(url) + try: + raise_for_status(r) + except requests.HTTPError as e: + gui.error(str(e)) + return + f = io.BytesIO(r.content) + if not (f and is_zipfile(f)): + gui.error("Invalid ZIP file.") + return + + bucket = gcs_bucket() + with ZipFile(f) as zipfile: + files = [ + file_info for file_info in zipfile.infolist() if not file_info.is_dir() + ] + uploaded = set( + map_parallel(lambda file_info: upload_zipfile(zipfile, file_info), files) + ) + + # clear old files + for blob in bucket.list_blobs(prefix=settings.GS_STATIC_PATH): + if blob.name not in uploaded: + blob.delete() + + +def upload_zipfile(zipfile: ZipFile, file_info: ZipInfo): + filename = file_info.filename + bucket = gcs_bucket() + blob = bucket.blob(os.path.join(settings.GS_STATIC_PATH, filename)) + data = zipfile.read(file_info) + upload_gcs_blob_from_bytes(blob, data) + return blob.name diff --git a/server.py b/server.py index 6d187eb63..64ca3e626 100644 --- a/server.py +++ b/server.py @@ -1,10 +1,15 @@ +import os +import traceback + from fastapi.exception_handlers import ( http_exception_handler, request_validation_exception_handler, ) from fastapi.exceptions import RequestValidationError +from furl import furl from starlette.exceptions import HTTPException from starlette.requests import Request +from starlette.responses import JSONResponse from starlette.status import ( HTTP_404_NOT_FOUND, HTTP_405_METHOD_NOT_ALLOWED, @@ -50,6 +55,7 @@ broadcast_api, bots_api, twilio_api, + static_pages, ) import url_shortener.routers as url_shortener @@ -67,6 +73,7 @@ app.include_router(paypal.router, include_in_schema=False) app.include_router(stripe.router, include_in_schema=False) app.include_router(twilio_api.router, include_in_schema=False) +app.include_router(static_pages.app, include_in_schema=False) app.include_router(root.app, include_in_schema=False) # this has a catch-all route app.add_middleware( @@ -92,11 +99,6 @@ async def startup(): limiter.total_tokens = config("MAX_THREADS", default=limiter.total_tokens, cast=int) -@app.get("/", tags=["Misc"]) -async def health(): - return "OK" - - @app.add_middleware def request_time_middleware(app): logger = logging.getLogger("uvicorn.time") @@ -125,17 +127,43 @@ async def not_found_exception_handler(request: Request, exc: HTTPException): return await _exc_handler(request, exc, "errors/404.html") -@app.exception_handler(HTTPException) -async def server_error_exception_handler(request: Request, exc: HTTPException): +@app.exception_handler(Exception) +async def server_error_exception_handler(request: Request, exc: Exception): return await _exc_handler(request, exc, "errors/unknown.html") -async def _exc_handler(request: Request, exc: HTTPException, template_name: str): +async def _exc_handler(request: Request, exc: Exception, template_name: str): + from celeryapp.tasks import err_msg_for_exc + if request.headers.get("accept", "").startswith("text/html"): return templates.TemplateResponse( template_name, - context=dict(request=request, settings=settings), - status_code=exc.status_code, + context=dict( + request=request, + settings=settings, + error=err_msg_for_exc(exc), + github_url=github_url_for_exc(exc), + traceback=traceback.format_exc(), + ), + status_code=getattr(exc, "status_code", 500), ) - else: + elif isinstance(exc, HTTPException): return await http_exception_handler(request, exc) + else: + return JSONResponse(dict(detail=err_msg_for_exc(exc)), status_code=500) + + +GITHUB_REPO = "https://github.com/GooeyAI/gooey-server/" + + +def github_url_for_exc(exc: Exception) -> str | None: + base_dir = str(settings.BASE_DIR) + ref = os.environ.get("CAPROVER_GIT_COMMIT_SHA") or "master" + for frame in reversed(traceback.extract_tb(exc.__traceback__)): + if not frame.filename.startswith(base_dir): + continue + path = os.path.relpath(frame.filename, base_dir) + return str( + furl(GITHUB_REPO, fragment_path=f"L{frame.lineno}") / "blob" / ref / path + ) + return GITHUB_REPO diff --git a/static_pages.py b/static_pages.py deleted file mode 100644 index c26bffdbc..000000000 --- a/static_pages.py +++ /dev/null @@ -1,111 +0,0 @@ -import gooey_gui as gui -from daras_ai_v2 import settings -import io -import mimetypes -from starlette.requests import Request -from fastapi import HTTPException -from zipfile import ZipFile, is_zipfile -import urllib.request -from daras_ai_v2.base import BasePage - - -def gcs_bucket() -> "storage.storage.Bucket": - from firebase_admin import storage - - return storage.bucket(settings.GS_BUCKET_NAME) - - -WEBSITE_FOLDER_PATH = "gooey-website" - - -def serve(page_slug: str, file_path: str): - - bucket = gcs_bucket() - - blob_path = "" - if page_slug and not file_path: - # If page_slug is provided, then it's a page - blob_path = f"{WEBSITE_FOLDER_PATH}/{page_slug}.html" - elif file_path: - blob_path = f"{WEBSITE_FOLDER_PATH}/{file_path}" - - if not (blob_path.endswith(".html")): - return dict( - redirectUrl=f"https://storage.googleapis.com/{settings.GS_BUCKET_NAME}/{WEBSITE_FOLDER_PATH}/{file_path}" - ) - - static_file = bucket.get_blob(blob_path) - if not static_file: - return None - - blob = static_file.download_as_string() - blob = blob.decode("utf-8") - content = io.StringIO(blob).read() - - return dict(content=content) - - -class StaticPageUpload(BasePage): - def __init__(self, request: Request): - self.zip_file = None - self.extracted_files = [] - self.request = request - self.is_uploading = False - - def extract_zip_to_gcloud(self): - bucket = gcs_bucket() - if not self.zip_file: - return - - # download zip file from gcloud (uppy) - zip_file = urllib.request.urlopen(self.zip_file) - archive = io.BytesIO() - archive.write(zip_file.read()) - - if archive and is_zipfile(archive): - with ZipFile(archive, "r") as z: - for file_info in z.infolist(): - if not file_info.is_dir(): - file_data = z.read(file_info) - file_name = file_info.filename # Maintain directory structure - blob_path = f"{WEBSITE_FOLDER_PATH}/{file_name}" - blob = bucket.blob(blob_path) - content_type = ( - mimetypes.guess_type(file_name)[0] or "text/plain" - ) - blob.upload_from_string(file_data, content_type=content_type) - self.extracted_files.append(blob.public_url) - - def render_file_upload(self) -> None: - if not BasePage.is_current_user_admin(self): - raise HTTPException(status_code=404, detail="Not authorized") - - with gui.div( - className="container d-flex align-items-center justify-content-center flex-column" - ): - self.zip_file = gui.file_uploader( - label="Upload ZIP File", - accept=[".zip"], - value=self.zip_file, - ) - pressed_save = gui.button( - "Extract ZIP File", - key="zip_file", - type="primary", - disabled=not self.zip_file, - ) - gui.caption( - "After successful upload, extracted files will be displayed below.", - className="my-4 text-muted", - ) - if pressed_save: - self.extract_zip_to_gcloud() - - # render extracted files if any - if self.extracted_files and len(self.extracted_files) > 0: - gui.write("Extracted Files:") - with gui.tag("div", className="my-4 d-flex flex-column"): - for extracted_file in self.extracted_files: - with gui.tag("div", className="bg-light p-2 my-2"): - with gui.tag("a", href=extracted_file): - gui.html(extracted_file) diff --git a/templates/errors/unknown.html b/templates/errors/unknown.html index 6e55404d0..07003555f 100644 --- a/templates/errors/unknown.html +++ b/templates/errors/unknown.html @@ -3,10 +3,26 @@ {% block title %}Doh!{% endblock title %} {% block content %} -
-

- Uh oh... Something went wrong -

- 404 -
+
+

+ Uh oh... Something went wrong +

+ {% if error %} +
+ Stats For Nerds +
+ {% if github_url %} + You look smart! Help us fix this issue on + + + GitHub + . + {% endif %} + 
{{ error }}
+ 
{{ traceback }}
+
+
+ {% endif %} + 404 +
{% endblock %} diff --git a/templates/login_container.html b/templates/login_container.html index fcc0ddc78..0c063ced4 100644 --- a/templates/login_container.html +++ b/templates/login_container.html @@ -1,6 +1,6 @@ {% if request.user and not request.user.is_anonymous %} - Hi, {{ request.user.first_name() or request.user.email or request.user.phone_number }} + Hi, {{ request.user.first_name() or request.user.email or request.user.phone_number or "Anon" }} {% else %} {% if block_incognito %} diff --git a/tests/test_public_endpoints.py b/tests/test_public_endpoints.py index 0fd705497..6802a5fed 100644 --- a/tests/test_public_endpoints.py +++ b/tests/test_public_endpoints.py @@ -5,7 +5,7 @@ from bots.models import PublishedRun, Workflow from daras_ai_v2.all_pages import all_api_pages, all_hidden_pages from routers import facebook_api -from routers.root import RecipeTabs +from routers.root import RecipeTabs, webflow_upload from routers.slack_api import slack_connect_redirect_shortcuts, slack_connect_redirect from server import app @@ -17,7 +17,7 @@ slack_connect_redirect_shortcuts.__name__, "get_run_status", # needs query params "get_balance", # needs authentication - "internal/file-upload", # needs admin authentication + webflow_upload.__name__, # needs admin authentication ] route_paths = [ diff --git a/url_shortener/routers.py b/url_shortener/routers.py index 1b3991cf8..dfa2c25cc 100644 --- a/url_shortener/routers.py +++ b/url_shortener/routers.py @@ -1,12 +1,12 @@ from django.db.models import F -from fastapi import APIRouter, Request +from fastapi import Request from fastapi.responses import RedirectResponse from fastapi.responses import Response +from routers.custom_api_router import CustomAPIRouter from url_shortener.models import ShortenedURL -from url_shortener.tasks import save_click_info -app = APIRouter() +app = CustomAPIRouter() @app.api_route("/2/{hashid}", methods=["GET", "POST"])