From 5afdf8b850183d7e7d1f86af523a903e6c62daec Mon Sep 17 00:00:00 2001
From: Alexander Metzger <alexander@metzger.dk>
Date: Wed, 20 Sep 2023 13:54:11 -0700
Subject: [PATCH 01/36] multiple asr models

---
 daras_ai_v2/asr.py                | 79 ++++++++++++++++++++++++-------
 daras_ai_v2/text_output_widget.py |  3 ++
 recipes/asr.py                    | 56 ++++++++++++++++------
 3 files changed, 107 insertions(+), 31 deletions(-)

diff --git a/daras_ai_v2/asr.py b/daras_ai_v2/asr.py
index 5e88fec58..9870affe1 100644
--- a/daras_ai_v2/asr.py
+++ b/daras_ai_v2/asr.py
@@ -117,7 +117,7 @@ def google_translate_languages() -> dict[str, str]:
     parent = f"projects/{project}/locations/global"
     client = translate.TranslationServiceClient()
     supported_languages = client.get_supported_languages(
-        parent, display_language_code="en"
+        parent=parent, display_language_code="en"
     )
     return {
         lang.language_code: lang.display_name
@@ -126,18 +126,38 @@ def google_translate_languages() -> dict[str, str]:
     }
 
 
+def _get_asr_languages(selected_model: AsrModels) -> set[str]:
+    forced_lang = forced_asr_languages.get(selected_model)
+    if forced_lang:
+        return {forced_lang}
+
+    return asr_supported_languages.get(selected_model, set())
+
+
 def asr_language_selector(
-    selected_model: AsrModels,
+    selected_model: AsrModels | list[AsrModels],
     label="##### Spoken Language",
     key="language",
 ):
-    # don't show language selector for models with forced language
-    forced_lang = forced_asr_languages.get(selected_model)
-    if forced_lang:
-        st.session_state[key] = forced_lang
-        return forced_lang
+    if not isinstance(selected_model, list):
+        selected_model = [selected_model]
+    languages = set.intersection(
+        *[
+            set(
+                map(lambda l: langcodes.Language.get(l).language, _get_asr_languages(m))
+            )
+            for m in selected_model
+        ]
+    )
+
+    if len(languages) < 1:
+        st.session_state[key] = None
+        return
+    elif len(languages) == 1:
+        st.session_state[key] = languages.pop()
+        return
 
-    options = [None, *asr_supported_languages.get(selected_model, [])]
+    options = [None, *languages]
 
     # handle non-canonical language codes
     old_val = st.session_state.get(key)
@@ -264,26 +284,25 @@ def get_google_auth_session():
 
 def run_asr(
     audio_url: str,
-    selected_model: str,
+    selected_model: str | list[str],
     language: str = None,
     output_format: str = "text",
-) -> str | AsrOutputJson:
+) -> str | AsrOutputJson | list[str | AsrOutputJson]:
     """
     Run ASR on audio.
     Args:
         audio_url (str): url of audio to be transcribed.
-        selected_model (str): ASR model to use.
+        selected_model (str): ASR model(s) to use.
         language: language of the audio
         output_format: format of the output
     Returns:
         str: Transcribed text.
     """
-    import google.cloud.speech_v2 as cloud_speech
-    from google.api_core.client_options import ClientOptions
-    from google.cloud.texttospeech_v1 import AudioEncoding
 
-    selected_model = AsrModels[selected_model]
-    output_format = AsrOutputFormat[output_format]
+    if not isinstance(selected_model, list):
+        selected_model = [selected_model]
+    selected_models = [AsrModels[m] for m in selected_model]
+    output_format: AsrOutputFormat = AsrOutputFormat[output_format]
     is_youtube_url = "youtube" in audio_url or "youtu.be" in audio_url
     if is_youtube_url:
         audio_url, size = download_youtube_to_wav(audio_url)
@@ -291,6 +310,34 @@ def run_asr(
         audio_url, size = audio_url_to_wav(audio_url)
     is_short = size < SHORT_FILE_CUTOFF
 
+    outputs = map_parallel(
+        lambda model: _run_asr_one_model(
+            model,
+            output_format,
+            audio_url,
+            language,
+            is_short,
+        ),
+        selected_models,
+    )
+
+    if len(outputs) == 1:
+        return outputs[0]
+    else:
+        return outputs
+
+
+def _run_asr_one_model(
+    selected_model: AsrModels,
+    output_format: AsrOutputFormat,
+    audio_url: str,
+    language: str | None,
+    is_short: bool,
+) -> str | AsrOutputJson:
+    import google.cloud.speech_v2 as cloud_speech
+    from google.api_core.client_options import ClientOptions
+    from google.cloud.texttospeech_v1 import AudioEncoding
+
     if selected_model == AsrModels.deepgram:
         r = requests.post(
             "https://api.deepgram.com/v1/listen",
diff --git a/daras_ai_v2/text_output_widget.py b/daras_ai_v2/text_output_widget.py
index 9c2e164ca..8d1762e09 100644
--- a/daras_ai_v2/text_output_widget.py
+++ b/daras_ai_v2/text_output_widget.py
@@ -9,6 +9,7 @@ def text_outputs(
     key: str = None,
     value: str = None,
     height: int = 200,
+    captions: list[str] | None = None,
 ):
     value = value or st.session_state.get(key)
     match value:
@@ -24,6 +25,8 @@ def text_outputs(
                     idx=idx,
                     value=str(text),
                 )
+                if captions:
+                    st.caption(captions[idx])
         case _:
             st.div()
 
diff --git a/recipes/asr.py b/recipes/asr.py
index ade2fbebe..3ae720af2 100644
--- a/recipes/asr.py
+++ b/recipes/asr.py
@@ -12,14 +12,13 @@
     run_google_translate,
     AsrOutputFormat,
     AsrOutputJson,
-    forced_asr_languages,
     asr_language_selector,
 )
 from daras_ai_v2.base import BasePage
 from daras_ai_v2.doc_search_settings_widgets import (
     document_uploader,
 )
-from daras_ai_v2.enum_selector_widget import enum_selector
+from daras_ai_v2.enum_selector_widget import enum_selector, enum_multiselect
 from daras_ai_v2.functional import map_parallel
 from daras_ai_v2.text_output_widget import text_outputs
 from recipes.DocSearch import render_documents
@@ -34,7 +33,9 @@ class AsrPage(BasePage):
 
     class RequestModel(BaseModel):
         documents: list[str]
-        selected_model: typing.Literal[tuple(e.name for e in AsrModels)] | None
+        selected_model: typing.Literal[tuple(e.name for e in AsrModels)] | list[
+            typing.Literal[tuple(e.name for e in AsrModels)]
+        ] | None
         language: str | None
         google_translate_target: str | None
         output_format: typing.Literal[tuple(e.name for e in AsrOutputFormat)] | None
@@ -79,14 +80,19 @@ def render_form_v2(self):
         )
         col1, col2 = st.columns(2, responsive=False)
         with col1:
-            selected_model = enum_selector(
+            if not isinstance(st.session_state.get("selected_model"), list):
+                st.session_state["selected_model"] = [
+                    st.session_state["selected_model"]
+                ]
+            selected_model = enum_multiselect(
                 AsrModels,
-                label="##### ASR Model",
+                label="##### ASR Models",
+                checkboxes=False,
                 key="selected_model",
-                use_selectbox=True,
+                allow_none=False,
             )
         with col2:
-            asr_language_selector(AsrModels[selected_model])
+            asr_language_selector([AsrModels[m] for m in selected_model])
 
     def render_settings(self):
         google_translate_language_selector()
@@ -99,7 +105,18 @@ def validate_form_v2(self):
         assert st.session_state.get("documents"), "Please provide at least 1 Audio File"
 
     def render_output(self):
-        text_outputs("**Transcription**", key="output_text", height=300)
+        text_outputs(
+            "**Transcription**",
+            key="output_text",
+            height=300,
+            captions=[
+                "Transcribed with " + AsrModels[model].value
+                for _ in st.session_state.get("documents", [])
+                for model in st.session_state.get(
+                    "selected_model", [AsrModels.whisper_large_v2.name]
+                )
+            ],
+        )
 
     def render_example(self, state: dict):
         render_documents(state)
@@ -120,17 +137,28 @@ def run(self, state: dict):
         request: AsrPage.RequestModel = self.RequestModel.parse_obj(state)
 
         # Run ASR
-        selected_model = AsrModels[request.selected_model]
-        yield f"Running {selected_model.value}..."
+        selected_models: list[str] = request.selected_model or [
+            AsrModels.whisper_large_v2.name
+        ]
+        if not isinstance(selected_models, list):
+            selected_models = [selected_models]
+        yield f"Running {', '.join([AsrModels[m].value for m in selected_models])}..."
         asr_output = map_parallel(
             lambda audio: run_asr(
                 audio_url=audio,
-                selected_model=request.selected_model,
+                selected_model=selected_models,
                 language=request.language,
                 output_format=request.output_format,
             ),
             request.documents,
         )
+        if len(selected_models) != 1:
+            # flatten
+            asr_output = [out for model_out in asr_output for out in model_out]
+        str_asr_output: list[str] = [
+            out if not isinstance(out, AsrModels) else out.get("text", "").strip()
+            for out in asr_output
+        ]
 
         # Run Translation
         if request.google_translate_target:
@@ -138,11 +166,9 @@ def run(self, state: dict):
             state["raw_output_text"] = asr_output
             # Run Translation
             state["output_text"] = run_google_translate(
-                asr_output,
+                str_asr_output,
                 target_language=request.google_translate_target,
-                source_language=forced_asr_languages.get(
-                    selected_model, request.language
-                ),
+                source_language=request.language,
             )
         else:
             # Save the raw ASR text for details view

From 3e66d595b65c204a77b5c652d294564836cb6b21 Mon Sep 17 00:00:00 2001
From: Kaustubh Maske Patil <37668193+nikochiko@users.noreply.github.com>
Date: Wed, 20 Sep 2023 23:13:00 +0530
Subject: [PATCH 02/36] Fix: add trailing slash at the end of canonical URLs

---
 routers/root.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/routers/root.py b/routers/root.py
index 7f5936235..386cc9813 100644
--- a/routers/root.py
+++ b/routers/root.py
@@ -250,6 +250,7 @@ def st_page(
         )
         / latest_slug
         / tab
+        / "/"  # preserve trailing slash
     )
 
     ret |= {

From 8f767508ee5ed3457419a694818292b67eb115a1 Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Fri, 22 Sep 2023 04:34:57 +0530
Subject: [PATCH 03/36] if the user submits an api call with only a run id,
 assume their uid to be the default

---
 routers/api.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/routers/api.py b/routers/api.py
index f23c34402..75c1ba0b3 100644
--- a/routers/api.py
+++ b/routers/api.py
@@ -328,6 +328,7 @@ def submit_api_call(
     self = page_cls(request=SimpleNamespace(user=user))
 
     # get saved state from db
+    query_params.setdefault("uid", user.uid)
     state = self.get_sr_from_query_params_dict(query_params).to_dict()
     if state is None:
         raise HTTPException(status_code=404)

From 80ecccb9d925be445bf658ee824fc1099e27784f Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Fri, 22 Sep 2023 04:38:18 +0530
Subject: [PATCH 04/36] fix type error

---
 routers/api.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/routers/api.py b/routers/api.py
index 75c1ba0b3..47b7989a1 100644
--- a/routers/api.py
+++ b/routers/api.py
@@ -122,7 +122,7 @@ def run_api_json(
             page_cls=page_cls,
             user=user,
             request_body=page_request.dict(),
-            query_params=request.query_params,
+            query_params=dict(request.query_params),
         )
 
     @app.post(
@@ -176,7 +176,7 @@ def run_api_json_async(
             page_cls=page_cls,
             user=user,
             request_body=page_request.dict(),
-            query_params=request.query_params,
+            query_params=dict(request.query_params),
             run_async=True,
         )
         response.headers["Location"] = ret["status_url"]

From 630411ddced274da876c598bfbcaacbceda0caf3 Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Fri, 22 Sep 2023 18:20:33 +0530
Subject: [PATCH 05/36] force create personal channels button

---
 bots/admin.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/bots/admin.py b/bots/admin.py
index 391d28ae5..4872b7bd1 100644
--- a/bots/admin.py
+++ b/bots/admin.py
@@ -26,6 +26,7 @@
     Conversation,
     BotIntegration,
 )
+from bots.tasks import create_personal_channels_for_all_members
 from gooeysite.custom_widgets import JSONEditorWidget
 
 
@@ -84,6 +85,15 @@ class Media:
         ]
 
 
+def create_personal_channels(modeladmin, request, queryset):
+    for bi in queryset:
+        create_personal_channels_for_all_members.delay(bi.id)
+    modeladmin.message_user(
+        request,
+        f"Started creating personal channels for {queryset.count()} bots in the background.",
+    )
+
+
 @admin.register(BotIntegration)
 class BotIntegrationAdmin(admin.ModelAdmin):
     search_fields = [
@@ -190,6 +200,8 @@ class BotIntegrationAdmin(admin.ModelAdmin):
         ),
     ]
 
+    actions = [create_personal_channels]
+
     @admin.display(description="Messages")
     def view_messsages(self, bi: BotIntegration):
         return list_related_html_url(

From cb69ab623f76974a10977046d540acb5e129836a Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Fri, 22 Sep 2023 18:45:22 +0530
Subject: [PATCH 06/36] handle user_team_not_in_channel

---
 daras_ai_v2/slack_bot.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/daras_ai_v2/slack_bot.py b/daras_ai_v2/slack_bot.py
index 20481e15a..af346dd90 100644
--- a/daras_ai_v2/slack_bot.py
+++ b/daras_ai_v2/slack_bot.py
@@ -334,7 +334,11 @@ def create_personal_channel(
         )
     except SlackAPIError as e:
         # skip if the user is restricted
-        if e.error in ["user_is_ultra_restricted", "user_is_restricted"]:
+        if e.error in [
+            "user_is_ultra_restricted",
+            "user_is_restricted",
+            "user_team_not_in_channel",
+        ]:
             return
         else:
             raise

From 80b42a347a408ebedd3380a47bddce318aaf6e0a Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Sat, 23 Sep 2023 01:42:47 +0530
Subject: [PATCH 07/36] slack personal channel support for shortcuts

---
 routers/slack.py | 81 ++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 68 insertions(+), 13 deletions(-)

diff --git a/routers/slack.py b/routers/slack.py
index c0a2537f4..5f70d400e 100644
--- a/routers/slack.py
+++ b/routers/slack.py
@@ -9,7 +9,7 @@
 from starlette.background import BackgroundTasks
 from starlette.responses import RedirectResponse, HTMLResponse
 
-from bots.models import BotIntegration, Platform
+from bots.models import BotIntegration, Platform, Conversation
 from bots.tasks import create_personal_channels_for_all_members
 from daras_ai_v2 import settings
 from daras_ai_v2.bots import _on_msg, request_json, request_urlencoded_body
@@ -225,9 +225,17 @@ def _handle_slack_event(event: dict, background_tasks: BackgroundTasks):
         capture_exception(e)
 
 
+@router.get("/__/slack/oauth/shortcuts/")
+def slack_oauth_shortcuts(dm: bool = False):
+    return RedirectResponse(str(get_slack_shortcuts_connect_url(dm)))
+
+
 @router.get("/__/slack/redirect/shortcuts/")
-def slack_connect_redirect_shortcuts(request: Request):
-    retry_button = f'<a href="{slack_connect_url}">Retry</a>'
+def slack_connect_redirect_shortcuts(
+    request: Request,
+    dm: bool = False,
+):
+    retry_button = f'<a href="{get_slack_shortcuts_connect_url(dm)}">Retry</a>'
 
     code = request.query_params.get("code")
     if not code:
@@ -239,19 +247,53 @@ def slack_connect_redirect_shortcuts(request: Request):
     res = requests.post(
         furl(
             "https://slack.com/api/oauth.v2.access",
-            query_params=dict(code=code, redirect_uri=slack_shortcuts_redirect_uri),
+            query_params=dict(
+                code=code, redirect_uri=get_slack_shortcuts_redirect_uri(dm)
+            ),
         ).url,
         auth=HTTPBasicAuth(settings.SLACK_CLIENT_ID, settings.SLACK_CLIENT_SECRET),
     )
     res.raise_for_status()
-    print(res.text)
     res = res.json()
+    print("> slack_connect_redirect_shortcuts:", res)
+
+    channel_id = res["incoming_webhook"]["channel_id"]
+    channel_name = res["incoming_webhook"]["channel"].strip("#")
+    team_id = res["team"]["id"]
+    user_id = res["authed_user"]["id"]
+    access_token = res["authed_user"]["access_token"]
+
+    if dm:
+        try:
+            convo = Conversation.objects.get(
+                bot_integration__slack_channel_id=channel_id,
+                bot_integration__slack_team_id=team_id,
+                slack_user_id=user_id,
+                slack_team_id=team_id,
+                slack_channel_is_personal=True,
+            )
+        except Conversation.DoesNotExist:
+            bi = BotIntegration.objects.get(
+                slack_channel_id=channel_id,
+                slack_team_id=team_id,
+            )
+            user = fetch_user_info(user_id, bi.slack_access_token)
+            convo = create_personal_channel(bi=bi, user=user)
+        if not convo:
+            return HTMLResponse(
+                f"<p>Oh No! Something went wrong here.</p><p>Error: Could not create personal channel.</p>"
+                + retry_button,
+                status_code=400,
+            )
+        channel_id = convo.slack_channel_id
+        channel_name = convo.slack_channel_name
+
     payload = json.dumps(
         dict(
-            slack_channel=res["incoming_webhook"]["channel"].strip("#"),
-            slack_channel_id=res["incoming_webhook"]["channel_id"],
-            slack_user_access_token=res["authed_user"]["access_token"],
-            slack_team_id=res["team"]["id"],
+            slack_channel=channel_name,
+            slack_channel_id=channel_id,
+            slack_user_access_token=access_token,
+            slack_team_id=team_id,
         ),
         indent=2,
     )
@@ -284,7 +326,20 @@ def slack_connect_redirect_shortcuts(request: Request):
     )
 
 
-slack_shortcuts_redirect_uri = (
-    furl(settings.APP_BASE_URL)
-    / router.url_path_for(slack_connect_redirect_shortcuts.__name__)
-).url
+def get_slack_shortcuts_connect_url(dm: bool = False):
+    return furl(
+        "https://slack.com/oauth/v2/authorize",
+        query_params=dict(
+            client_id=settings.SLACK_CLIENT_ID,
+            scope=",".join(["incoming-webhook"]),
+            user_scope=",".join(["chat:write"]),
+            redirect_uri=get_slack_shortcuts_redirect_uri(dm),
+        ),
+    )
+
+
+def get_slack_shortcuts_redirect_uri(dm: bool = False) -> str:
+    return (
+        furl(settings.APP_BASE_URL, query_params=dict(dm=dm))
+        / router.url_path_for(slack_connect_redirect_shortcuts.__name__)
+    ).url

From 0d2e55a03647f19dc934c55769af47cc3e440d66 Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Sat, 23 Sep 2023 02:03:05 +0530
Subject: [PATCH 08/36] slack personal channel support for shortcuts

---
 routers/slack.py | 42 ++++++++++++++----------------------------
 1 file changed, 14 insertions(+), 28 deletions(-)

diff --git a/routers/slack.py b/routers/slack.py
index 5f70d400e..ea6be9a67 100644
--- a/routers/slack.py
+++ b/routers/slack.py
@@ -258,40 +258,26 @@ def slack_connect_redirect_shortcuts(
     print("> slack_connect_redirect_shortcuts:", res)
 
     channel_id = res["incoming_webhook"]["channel_id"]
-    channel_name = res["incoming_webhook"]["channel"].strip("#")
-    team_id = res["team"]["id"]
     user_id = res["authed_user"]["id"]
+    team_id = res["team"]["id"]
     access_token = res["authed_user"]["access_token"]
 
-    if dm:
-        try:
-            convo = Conversation.objects.get(
-                bot_integration__slack_channel_id=channel_id,
-                bot_integration__slack_team_id=team_id,
-                slack_user_id=user_id,
-                slack_team_id=team_id,
-                slack_channel_is_personal=True,
-            )
-        except Conversation.DoesNotExist:
-            bi = BotIntegration.objects.get(
-                slack_channel_id=channel_id,
-                slack_team_id=team_id,
-            )
-            user = fetch_user_info(user_id, bi.slack_access_token)
-            convo = create_personal_channel(bi=bi, user=user)
-        if not convo:
-            return HTMLResponse(
-                f"<p>Oh No! Something went wrong here.</p><p>Error: Could not create personal channel.</p>"
-                + retry_button,
-                status_code=400,
-            )
-        channel_id = convo.slack_channel_id
-        channel_name = convo.slack_channel_name
+    try:
+        convo = Conversation.objects.get(
+            slack_channel_id=channel_id, slack_user_id=user_id, slack_team_id=team_id
+        )
+    except Conversation.DoesNotExist:
+        return HTMLResponse(
+            "<p>Oh No! Something went wrong here.</p>"
+            "<p>Conversation not found. Please make sure this channel is connected to the gooey bot</p>"
+            + retry_button,
+            status_code=400,
+        )
 
     payload = json.dumps(
         dict(
-            slack_channel=channel_name,
-            slack_channel_id=channel_id,
+            slack_channel=convo.slack_channel_name,
+            slack_channel_id=convo.slack_channel_id,
             slack_user_access_token=access_token,
             slack_team_id=team_id,
         ),

From 2d4b4c01359a49e5e01dedb2411c7f39bc1ed5c8 Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Sat, 23 Sep 2023 02:08:21 +0530
Subject: [PATCH 09/36] remove the dm query param

---
 routers/slack.py | 44 ++++++++++++++++++--------------------------
 1 file changed, 18 insertions(+), 26 deletions(-)

diff --git a/routers/slack.py b/routers/slack.py
index ea6be9a67..323107291 100644
--- a/routers/slack.py
+++ b/routers/slack.py
@@ -226,16 +226,13 @@ def _handle_slack_event(event: dict, background_tasks: BackgroundTasks):
 
 
 @router.get("/__/slack/oauth/shortcuts/")
-def slack_oauth_shortcuts(dm: bool = False):
-    return RedirectResponse(str(get_slack_shortcuts_connect_url(dm)))
+def slack_oauth_shortcuts():
+    return RedirectResponse(str(slack_shortcuts_connect_url))
 
 
 @router.get("/__/slack/redirect/shortcuts/")
-def slack_connect_redirect_shortcuts(
-    request: Request,
-    dm: bool = False,
-):
-    retry_button = f'<a href="{get_slack_shortcuts_connect_url(dm)}">Retry</a>'
+def slack_connect_redirect_shortcuts(request: Request):
+    retry_button = f'<a href="{slack_shortcuts_connect_url}">Retry</a>'
 
     code = request.query_params.get("code")
     if not code:
@@ -247,9 +244,7 @@ def slack_connect_redirect_shortcuts(
     res = requests.post(
         furl(
             "https://slack.com/api/oauth.v2.access",
-            query_params=dict(
-                code=code, redirect_uri=get_slack_shortcuts_redirect_uri(dm)
-            ),
+            query_params=dict(code=code, redirect_uri=slack_shortcuts_redirect_uri),
         ).url,
         auth=HTTPBasicAuth(settings.SLACK_CLIENT_ID, settings.SLACK_CLIENT_SECRET),
     )
@@ -312,20 +307,17 @@ def slack_connect_redirect_shortcuts(
     )
 
 
-def get_slack_shortcuts_connect_url(dm: bool = False):
-    return furl(
-        "https://slack.com/oauth/v2/authorize",
-        query_params=dict(
-            client_id=settings.SLACK_CLIENT_ID,
-            scope=",".join(["incoming-webhook"]),
-            user_scope=",".join(["chat:write"]),
-            redirect_uri=get_slack_shortcuts_redirect_uri(dm),
-        ),
-    )
-
+slack_shortcuts_redirect_uri = (
+    furl(settings.APP_BASE_URL)
+    / router.url_path_for(slack_connect_redirect_shortcuts.__name__)
+).url
 
-def get_slack_shortcuts_redirect_uri(dm: bool = False) -> str:
-    return (
-        furl(settings.APP_BASE_URL, query_params=dict(dm=dm))
-        / router.url_path_for(slack_connect_redirect_shortcuts.__name__)
-    ).url
+slack_shortcuts_connect_url = furl(
+    "https://slack.com/oauth/v2/authorize",
+    query_params=dict(
+        client_id=settings.SLACK_CLIENT_ID,
+        scope=",".join(["incoming-webhook"]),
+        user_scope=",".join(["chat:write"]),
+        redirect_uri=slack_shortcuts_redirect_uri,
+    ),
+)

From d637105ff5f3834f27c86e4bf6766aa937ed98bc Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Sat, 23 Sep 2023 02:13:38 +0530
Subject: [PATCH 10/36] send back the connected example_id and run_id with
 slack shortcuts redirect

---
 routers/slack.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/routers/slack.py b/routers/slack.py
index 323107291..fdeee8511 100644
--- a/routers/slack.py
+++ b/routers/slack.py
@@ -266,8 +266,17 @@ def slack_connect_redirect_shortcuts(request: Request):
             "<p>Oh No! Something went wrong here.</p>"
             "<p>Conversation not found. Please make sure this channel is connected to the gooey bot</p>"
             + retry_button,
+            status_code=404,
+        )
+
+    if not convo.bot_integration.saved_run_id:
+        return HTMLResponse(
+            "<p>Oh No! Something went wrong here.</p>"
+            "<p>Please make sure this bot is connected to a gooey run or example</p>"
+            + retry_button,
             status_code=400,
         )
+    sr = convo.bot_integration.saved_run
 
     payload = json.dumps(
         dict(
@@ -275,6 +284,8 @@ def slack_connect_redirect_shortcuts(request: Request):
             slack_channel_id=convo.slack_channel_id,
             slack_user_access_token=access_token,
             slack_team_id=team_id,
+            gooey_example_id=sr.example_id,
+            gooey_run_id=sr.run_id,
         ),
         indent=2,
     )

From f0241a872843459e238422f90b66d236d10f5ab0 Mon Sep 17 00:00:00 2001
From: Kaustubh Maske Patil <kaustubh@Kaustubhs-MacBook-Air.local>
Date: Wed, 20 Sep 2023 13:20:34 +0530
Subject: [PATCH 11/36] Add Procfile to start all services with one command

Starting all services in different terminals is tedious. This PR adds
a Procfile and a honcho dependency that will start up all required
processes in a single command.
---
 Procfile       |  26 +++++++++++++
 README.md      |  42 ++++++++++++---------
 poetry.lock    | 100 +++++++++++++++++++++++++++----------------------
 pyproject.toml |   1 +
 4 files changed, 107 insertions(+), 62 deletions(-)
 create mode 100644 Procfile

diff --git a/Procfile b/Procfile
new file mode 100644
index 000000000..c4032fdb5
--- /dev/null
+++ b/Procfile
@@ -0,0 +1,26 @@
+# this Procfile can be run with `honcho`, and it can start multiple processes
+# with a single command. Handy for development. All of the below commands are
+# setup to run in dev mode only, and not in prod.
+#
+# The assumptions here are that:
+# - you have redis installed but not running as a background service
+# - you have rabbitmq installed but not running as a background service
+# - your local gooey-ui repo is at ../gooey-ui/
+#
+# You can comment any of the processes if you have background services running
+# for them. You can also change the path for the `ui` process from `../gooey-ui/`
+# to wherever your local gooey-ui directory is.
+
+api: poetry run uvicorn server:app --host 127.0.0.1 --port 8080 --reload
+
+admin: poetry run python manage.py runserver 127.0.0.1:8000
+
+dashboard: poetry run streamlit run Home.py --server.port 8501 --server.headless true
+
+rabbitmq: rabbitmq-server
+
+redis: redis-server
+
+celery: poetry run celery -A celeryapp worker
+
+ui: /bin/zsh -c "cd ../gooey-ui/; PORT=3000 npm run dev"
diff --git a/README.md b/README.md
index 9159ddf37..28048f250 100644
--- a/README.md
+++ b/README.md
@@ -1,37 +1,43 @@
 ## Setup
+
 * Install [pyenv](https://github.com/pyenv/pyenv) & install the same python version as in our [Dockerfile](Dockerfile)
 * Install [poetry](https://python-poetry.org/docs/)
 * Create & active a virtualenv (e.g. `poetry shell`)
 * Run `poetry install --with dev`
 * Create an `.env` file from `.env.example` (Read [12factor.net/config](https://12factor.net/config))
 * Run `./manage.py migrate`
+* Install the zbar shared library (`brew install zbar`)
+* Install [redis](https://redis.io/docs/getting-started/installation/install-redis-on-mac-os/) and [rabbitmq](https://www.rabbitmq.com/install-homebrew.html)
 
 ## Run
 
-### API + GUI server
-
-```bash
-uvicorn server:app --host 0.0.0.0 --port 8080 --reload
-```
-
-Open [localhost:8080](localhost:8080) in your browser
+You can start all required processes in one command with Honcho:
 
-### Admin Site
-
-```bash
-python3 manage.py runserver 0.0.0.0:8000
+```shell
+$ poetry run honcho start
 ```
 
-Open [localhost:8000](localhost:8000) in your browser
-
+The processes that it starts are defined in [`Procfile`](Procfile).
+Currently they are these:
 
-### Usage Dashboard
+| Service          | Port |
+| -------          | ---- |
+| API + GUI Server | 8080 |
+| Admin site       | 8000 |
+| Usage dashboard  | 8501 |
+| Redis            | 6379 |
+| RabbitMQ         | 5672 |
+| Celery           | -    |
+| UI               | 3000 |
 
-```
-streamlit run Home.py --server.port 8501
-```
+This default startup assumes that Redis and RabbitMQ are installed, but not
+running as system services (e.g. with `brew services`) already. It also assumes
+that the gooey-ui repo can be found at `../gooey-ui/` (adjacent to where the
+gooey-server repo sits). You can open the Procfile and comment any of these
+if you want to run it in some other way.
 
-Open [localhost:8501](localhost:8501) in your browser
+**Note:** the Celery worker must be manually restarted on code changes. You
+can do this by stopping and starting Honcho.
 
 ## To run any recipe 
 
diff --git a/poetry.lock b/poetry.lock
index 1e1f23de3..3886e97da 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand.
 
 [[package]]
 name = "absl-py"
@@ -888,6 +888,7 @@ files = [
     {file = "contourpy-1.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18a64814ae7bce73925131381603fff0116e2df25230dfc80d6d690aa6e20b37"},
     {file = "contourpy-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90c81f22b4f572f8a2110b0b741bb64e5a6427e0a198b2cdc1fbaf85f352a3aa"},
     {file = "contourpy-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:53cc3a40635abedbec7f1bde60f8c189c49e84ac180c665f2cd7c162cc454baa"},
+    {file = "contourpy-1.1.0-cp310-cp310-win32.whl", hash = "sha256:9b2dd2ca3ac561aceef4c7c13ba654aaa404cf885b187427760d7f7d4c57cff8"},
     {file = "contourpy-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:1f795597073b09d631782e7245016a4323cf1cf0b4e06eef7ea6627e06a37ff2"},
     {file = "contourpy-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0b7b04ed0961647691cfe5d82115dd072af7ce8846d31a5fac6c142dcce8b882"},
     {file = "contourpy-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:27bc79200c742f9746d7dd51a734ee326a292d77e7d94c8af6e08d1e6c15d545"},
@@ -896,6 +897,7 @@ files = [
     {file = "contourpy-1.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5cec36c5090e75a9ac9dbd0ff4a8cf7cecd60f1b6dc23a374c7d980a1cd710e"},
     {file = "contourpy-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f0cbd657e9bde94cd0e33aa7df94fb73c1ab7799378d3b3f902eb8eb2e04a3a"},
     {file = "contourpy-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:181cbace49874f4358e2929aaf7ba84006acb76694102e88dd15af861996c16e"},
+    {file = "contourpy-1.1.0-cp311-cp311-win32.whl", hash = "sha256:edb989d31065b1acef3828a3688f88b2abb799a7db891c9e282df5ec7e46221b"},
     {file = "contourpy-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fb3b7d9e6243bfa1efb93ccfe64ec610d85cfe5aec2c25f97fbbd2e58b531256"},
     {file = "contourpy-1.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bcb41692aa09aeb19c7c213411854402f29f6613845ad2453d30bf421fe68fed"},
     {file = "contourpy-1.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5d123a5bc63cd34c27ff9c7ac1cd978909e9c71da12e05be0231c608048bb2ae"},
@@ -904,6 +906,7 @@ files = [
     {file = "contourpy-1.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:317267d915490d1e84577924bd61ba71bf8681a30e0d6c545f577363157e5e94"},
     {file = "contourpy-1.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d551f3a442655f3dcc1285723f9acd646ca5858834efeab4598d706206b09c9f"},
     {file = "contourpy-1.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e7a117ce7df5a938fe035cad481b0189049e8d92433b4b33aa7fc609344aafa1"},
+    {file = "contourpy-1.1.0-cp38-cp38-win32.whl", hash = "sha256:108dfb5b3e731046a96c60bdc46a1a0ebee0760418951abecbe0fc07b5b93b27"},
     {file = "contourpy-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:d4f26b25b4f86087e7d75e63212756c38546e70f2a92d2be44f80114826e1cd4"},
     {file = "contourpy-1.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc00bb4225d57bff7ebb634646c0ee2a1298402ec10a5fe7af79df9a51c1bfd9"},
     {file = "contourpy-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:189ceb1525eb0655ab8487a9a9c41f42a73ba52d6789754788d1883fb06b2d8a"},
@@ -912,6 +915,7 @@ files = [
     {file = "contourpy-1.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:143dde50520a9f90e4a2703f367cf8ec96a73042b72e68fcd184e1279962eb6f"},
     {file = "contourpy-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e94bef2580e25b5fdb183bf98a2faa2adc5b638736b2c0a4da98691da641316a"},
     {file = "contourpy-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ed614aea8462735e7d70141374bd7650afd1c3f3cb0c2dbbcbe44e14331bf002"},
+    {file = "contourpy-1.1.0-cp39-cp39-win32.whl", hash = "sha256:71551f9520f008b2950bef5f16b0e3587506ef4f23c734b71ffb7b89f8721999"},
     {file = "contourpy-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:438ba416d02f82b692e371858143970ed2eb6337d9cdbbede0d8ad9f3d7dd17d"},
     {file = "contourpy-1.1.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a698c6a7a432789e587168573a864a7ea374c6be8d4f31f9d87c001d5a843493"},
     {file = "contourpy-1.1.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:397b0ac8a12880412da3551a8cb5a187d3298a72802b45a3bd1805e204ad8439"},
@@ -1469,33 +1473,6 @@ grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio-status (>=1.33.2,<2.0dev)"]
 grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"]
 grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"]
 
-[[package]]
-name = "google-api-core"
-version = "2.11.0"
-description = "Google API client core library"
-optional = false
-python-versions = ">=3.7"
-files = [
-    {file = "google-api-core-2.11.0.tar.gz", hash = "sha256:4b9bb5d5a380a0befa0573b302651b8a9a89262c1730e37bf423cec511804c22"},
-    {file = "google_api_core-2.11.0-py3-none-any.whl", hash = "sha256:ce222e27b0de0d7bc63eb043b956996d6dccab14cc3b690aaea91c9cc99dc16e"},
-]
-
-[package.dependencies]
-google-auth = ">=2.14.1,<3.0dev"
-googleapis-common-protos = ">=1.56.2,<2.0dev"
-grpcio = [
-    {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""},
-    {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""},
-]
-grpcio-status = {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""}
-protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev"
-requests = ">=2.18.0,<3.0.0dev"
-
-[package.extras]
-grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0dev)", "grpcio-status (>=1.49.1,<2.0dev)"]
-grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"]
-grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"]
-
 [[package]]
 name = "google-api-core"
 version = "2.11.1"
@@ -1510,11 +1487,8 @@ files = [
 [package.dependencies]
 google-auth = ">=2.14.1,<3.0.dev0"
 googleapis-common-protos = ">=1.56.2,<2.0.dev0"
-grpcio = [
-    {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""},
-    {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""},
-]
-grpcio-status = {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "extra == \"grpc\""}
+grpcio = {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}
+grpcio-status = {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}
 protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0"
 requests = ">=2.18.0,<3.0.0.dev0"
 
@@ -1631,8 +1605,8 @@ files = [
 google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]}
 google-cloud-core = ">=1.4.1,<3.0.0dev"
 proto-plus = [
-    {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""},
     {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""},
+    {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""},
 ]
 protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev"
 
@@ -1650,8 +1624,8 @@ files = [
 [package.dependencies]
 google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]}
 proto-plus = [
-    {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""},
     {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""},
+    {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""},
 ]
 protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev"
 
@@ -1690,8 +1664,8 @@ files = [
 [package.dependencies]
 google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]}
 proto-plus = [
-    {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""},
     {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""},
+    {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""},
 ]
 protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev"
 
@@ -1854,6 +1828,7 @@ files = [
     {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"},
     {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"},
     {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"},
+    {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d967650d3f56af314b72df7089d96cda1083a7fc2da05b375d2bc48c82ab3f3c"},
     {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"},
     {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"},
     {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"},
@@ -1862,6 +1837,7 @@ files = [
     {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"},
     {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"},
     {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"},
+    {file = "greenlet-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d4606a527e30548153be1a9f155f4e283d109ffba663a15856089fb55f933e47"},
     {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"},
     {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"},
     {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"},
@@ -1891,6 +1867,7 @@ files = [
     {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"},
     {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"},
     {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"},
+    {file = "greenlet-2.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1087300cf9700bbf455b1b97e24db18f2f77b55302a68272c56209d5587c12d1"},
     {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"},
     {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"},
     {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"},
@@ -1899,6 +1876,7 @@ files = [
     {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"},
     {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"},
     {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"},
+    {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8512a0c38cfd4e66a858ddd1b17705587900dd760c6003998e9472b77b56d417"},
     {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"},
     {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"},
     {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"},
@@ -2048,6 +2026,23 @@ files = [
 [package.extras]
 test = ["pytest (>=2.1.0)"]
 
+[[package]]
+name = "honcho"
+version = "1.1.0"
+description = "Honcho: a Python clone of Foreman. For managing Procfile-based applications."
+optional = false
+python-versions = "*"
+files = [
+    {file = "honcho-1.1.0-py2.py3-none-any.whl", hash = "sha256:a4d6e3a88a7b51b66351ecfc6e9d79d8f4b87351db9ad7e923f5632cc498122f"},
+    {file = "honcho-1.1.0.tar.gz", hash = "sha256:c5eca0bded4bef6697a23aec0422fd4f6508ea3581979a3485fc4b89357eb2a9"},
+]
+
+[package.dependencies]
+colorama = {version = "*", markers = "sys_platform == \"win32\""}
+
+[package.extras]
+export = ["jinja2 (>=2.7,<3)"]
+
 [[package]]
 name = "html-sanitizer"
 version = "1.9.3"
@@ -2845,6 +2840,16 @@ files = [
     {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"},
     {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"},
     {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"},
+    {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"},
     {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"},
     {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"},
     {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"},
@@ -3405,12 +3410,9 @@ files = [
 
 [package.dependencies]
 numpy = [
-    {version = ">=1.21.2", markers = "python_version >= \"3.10\""},
-    {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\""},
-    {version = ">=1.19.3", markers = "python_version >= \"3.6\" and platform_system == \"Linux\" and platform_machine == \"aarch64\" or python_version >= \"3.9\""},
-    {version = ">=1.17.0", markers = "python_version >= \"3.7\""},
-    {version = ">=1.17.3", markers = "python_version >= \"3.8\""},
     {version = ">=1.23.5", markers = "python_version >= \"3.11\""},
+    {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\" and python_version < \"3.11\""},
+    {version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\" and python_version < \"3.11\""},
 ]
 
 [[package]]
@@ -3468,8 +3470,8 @@ files = [
 
 [package.dependencies]
 numpy = [
-    {version = ">=1.22.4", markers = "python_version < \"3.11\""},
     {version = ">=1.23.2", markers = "python_version >= \"3.11\""},
+    {version = ">=1.22.4", markers = "python_version < \"3.11\""},
 ]
 python-dateutil = ">=2.8.2"
 pytz = ">=2020.1"
@@ -4357,6 +4359,7 @@ files = [
     {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"},
     {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"},
     {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"},
+    {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"},
     {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"},
     {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"},
     {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"},
@@ -4364,8 +4367,15 @@ files = [
     {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"},
     {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"},
     {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"},
+    {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"},
     {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"},
     {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
+    {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
+    {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
+    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
+    {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
+    {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
+    {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"},
     {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"},
     {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"},
     {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"},
@@ -4382,6 +4392,7 @@ files = [
     {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"},
     {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"},
     {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"},
+    {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"},
     {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"},
     {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"},
     {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"},
@@ -4389,6 +4400,7 @@ files = [
     {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"},
     {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"},
     {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"},
+    {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"},
     {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"},
     {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"},
     {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"},
@@ -5214,7 +5226,7 @@ files = [
 ]
 
 [package.dependencies]
-greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\")"}
+greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"}
 
 [package.extras]
 aiomysql = ["aiomysql", "greenlet (!=0.4.17)"]
@@ -6139,4 +6151,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.10,<3.13"
-content-hash = "d6562e5c259fa3bde5ae541012cfe4576413c97b8304756ecab8c49d2059ee5f"
+content-hash = "e1aa198ba112e95195815327669b1e7687428ab0510c5485f057442a207b982e"
diff --git a/pyproject.toml b/pyproject.toml
index 3169f751c..3ea6240be 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -79,6 +79,7 @@ rank-bm25 = "^0.2.2"
 [tool.poetry.group.dev.dependencies]
 watchdog = "^2.1.9"
 ipython = "^8.5.0"
+honcho = "^1.1.0"
 
 [build-system]
 requires = ["poetry-core"]

From 053837d31b3535fcd49f645593c8526833119408 Mon Sep 17 00:00:00 2001
From: Kaustubh Maske Patil <kaustubh@Kaustubhs-MacBook-Air.local>
Date: Wed, 20 Sep 2023 14:11:54 +0530
Subject: [PATCH 12/36] Procfile: aoivd specifying an explicit shell from
 gooey-ui's command

This makes it compatible with devs that use other shells
---
 Procfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Procfile b/Procfile
index c4032fdb5..1e540a406 100644
--- a/Procfile
+++ b/Procfile
@@ -23,4 +23,4 @@ redis: redis-server
 
 celery: poetry run celery -A celeryapp worker
 
-ui: /bin/zsh -c "cd ../gooey-ui/; PORT=3000 npm run dev"
+ui: cd ../gooey-ui/; PORT=3000 npm run dev

From 719efc23ae6a18348bf364641fecb7a5292b8354 Mon Sep 17 00:00:00 2001
From: Kaustubh Maske Patil <37668193+nikochiko@users.noreply.github.com>
Date: Wed, 20 Sep 2023 16:06:33 +0530
Subject: [PATCH 13/36] Add instructions for postgresql installation, prefer
 background services for redis/rabbitmq over honcho

The local dev setup fails with fixtures when run with SQLite, but works
fine with postgresql. This could be due to some constraints that postgres
enforces but not sqlite. It's also better to use postgres for local dev
too because that would be consistent with production environment.
The other change is preferring background services for redis/rabbitmq
over starting them through honcho. This is because starting/stopping them
takes longer, especially for rabbitmq. Honcho should be quick to start and
stop. This also makes sure that task results persist in Redis and unprocessed
tasks persist in RabbitMQ over honcho restarts.
---
 .env.example |  5 +++++
 Procfile     |  4 ----
 README.md    | 29 +++++++++++++----------------
 3 files changed, 18 insertions(+), 20 deletions(-)

diff --git a/.env.example b/.env.example
index 8ed136da7..dc3c29b99 100644
--- a/.env.example
+++ b/.env.example
@@ -1,3 +1,8 @@
 APP_BASE_URL=http://localhost:3000
 API_BASE_URL=http://localhost:8080
 GS_BUCKET_NAME=dara-c1b52.appspot.com
+PGHOST=127.0.0.1
+PGPORT=5432
+PGUSER=gooey
+PGDATABASE=gooey
+PGPASSWORD=gooey
diff --git a/Procfile b/Procfile
index 1e540a406..a32d9aafc 100644
--- a/Procfile
+++ b/Procfile
@@ -17,10 +17,6 @@ admin: poetry run python manage.py runserver 127.0.0.1:8000
 
 dashboard: poetry run streamlit run Home.py --server.port 8501 --server.headless true
 
-rabbitmq: rabbitmq-server
-
-redis: redis-server
-
 celery: poetry run celery -A celeryapp worker
 
 ui: cd ../gooey-ui/; PORT=3000 npm run dev
diff --git a/README.md b/README.md
index 28048f250..41acb3328 100644
--- a/README.md
+++ b/README.md
@@ -2,12 +2,17 @@
 
 * Install [pyenv](https://github.com/pyenv/pyenv) & install the same python version as in our [Dockerfile](Dockerfile)
 * Install [poetry](https://python-poetry.org/docs/)
-* Create & active a virtualenv (e.g. `poetry shell`)
+* Create & activate a virtualenv (e.g. `poetry shell`)
 * Run `poetry install --with dev`
+* Install [redis](https://redis.io/docs/getting-started/installation/install-redis-on-mac-os/), [rabbitmq](https://www.rabbitmq.com/install-homebrew.html), and [postgresql](https://formulae.brew.sh/formula/postgresql@15) (e.g. `brew install redis rabbitmq postgresql@15`)
+* Enable background services for `redis`, `rabbitmq`, and `postgresql` (e.g. with `brew services start redis` and similar for `rabbitmq` and `postgresql`)
+* Create a user and database for gooey in PostgreSQL:
+  * `createuser gooey --pwprompt` (when prompted for password, enter `gooey`)
+  * `createdb gooey -O gooey`
+  * make sure you are able to access the database with `psql -W -U gooey gooey` (and when prompted for password, entering `gooey`)
 * Create an `.env` file from `.env.example` (Read [12factor.net/config](https://12factor.net/config))
 * Run `./manage.py migrate`
-* Install the zbar shared library (`brew install zbar`)
-* Install [redis](https://redis.io/docs/getting-started/installation/install-redis-on-mac-os/) and [rabbitmq](https://www.rabbitmq.com/install-homebrew.html)
+* Install the zbar library (`brew install zbar`)
 
 ## Run
 
@@ -25,16 +30,14 @@ Currently they are these:
 | API + GUI Server | 8080 |
 | Admin site       | 8000 |
 | Usage dashboard  | 8501 |
-| Redis            | 6379 |
-| RabbitMQ         | 5672 |
 | Celery           | -    |
 | UI               | 3000 |
 
-This default startup assumes that Redis and RabbitMQ are installed, but not
-running as system services (e.g. with `brew services`) already. It also assumes
-that the gooey-ui repo can be found at `../gooey-ui/` (adjacent to where the
-gooey-server repo sits). You can open the Procfile and comment any of these
-if you want to run it in some other way.
+This default startup assumes that Redis, RabbitMQ, and PostgreSQL are installed and running
+as background services on ports 6379, 5672, and 5432 respectively. 
+It also assumes that the gooey-ui repo can be found at `../gooey-ui/` (adjacent to where the
+gooey-server repo sits). You can open the Procfile and comment this out if you don't need
+to run it.
 
 **Note:** the Celery worker must be manually restarted on code changes. You
 can do this by stopping and starting Honcho.
@@ -42,12 +45,6 @@ can do this by stopping and starting Honcho.
 ## To run any recipe 
 
 * Save `serviceAccountKey.json` to project root
-* Install & start [redis](https://redis.io/docs/getting-started/installation/install-redis-on-mac-os/)
-* Install & start [rabbitmq](https://www.rabbitmq.com/install-homebrew.html)
-* Run the celery worker (**Note:** you must manually restart it on code changes)
-```bash
-celery -A celeryapp worker
-```
 
 ## To connect to our GPU cluster 
 

From d3eaa4cef0e1073d76708323db28a5ca336f0ff2 Mon Sep 17 00:00:00 2001
From: Kaustubh Maske Patil <37668193+nikochiko@users.noreply.github.com>
Date: Thu, 21 Sep 2023 13:28:05 +0530
Subject: [PATCH 14/36] Use sqlcreate helper for creating user/db

---
 README.md | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 41acb3328..3df6d00bc 100644
--- a/README.md
+++ b/README.md
@@ -6,9 +6,8 @@
 * Run `poetry install --with dev`
 * Install [redis](https://redis.io/docs/getting-started/installation/install-redis-on-mac-os/), [rabbitmq](https://www.rabbitmq.com/install-homebrew.html), and [postgresql](https://formulae.brew.sh/formula/postgresql@15) (e.g. `brew install redis rabbitmq postgresql@15`)
 * Enable background services for `redis`, `rabbitmq`, and `postgresql` (e.g. with `brew services start redis` and similar for `rabbitmq` and `postgresql`)
-* Create a user and database for gooey in PostgreSQL:
-  * `createuser gooey --pwprompt` (when prompted for password, enter `gooey`)
-  * `createdb gooey -O gooey`
+* Use `sqlcreate` helper to create a user and database for gooey:
+  * `./manage.py sqlcreate | psql postgres`
   * make sure you are able to access the database with `psql -W -U gooey gooey` (and when prompted for password, entering `gooey`)
 * Create an `.env` file from `.env.example` (Read [12factor.net/config](https://12factor.net/config))
 * Run `./manage.py migrate`

From 3f7b9d9555d91dff0aa0ddc23c80359a8a8908d4 Mon Sep 17 00:00:00 2001
From: Ravi Shankar <42587315+ravishankar63@users.noreply.github.com>
Date: Fri, 15 Sep 2023 12:40:24 +0530
Subject: [PATCH 15/36] Added metadata images for workflows

---
 recipes/DocExtract.py      | 5 +++++
 recipes/DocSummary.py      | 5 ++++-
 recipes/Lipsync.py         | 5 +++++
 recipes/QRCodeGenerator.py | 4 ++++
 recipes/RelatedQnA.py      | 5 ++++-
 recipes/RelatedQnADoc.py   | 5 ++++-
 recipes/SmartGPT.py        | 5 +++++
 recipes/Text2Audio.py      | 5 +++++
 recipes/VideoBots.py       | 5 +++++
 recipes/asr.py             | 5 +++++
 10 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/recipes/DocExtract.py b/recipes/DocExtract.py
index 5e098c7f1..e8c1c97ce 100644
--- a/recipes/DocExtract.py
+++ b/recipes/DocExtract.py
@@ -38,6 +38,8 @@
 from daras_ai_v2.vector_search import doc_url_to_metadata, DocMetadata
 from recipes.DocSearch import render_documents
 
+DEFAULT_YOUTUBE_BOT_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/6c8f6876-538c-11ee-bea7-02420a000195/youtube%20bot%201.png.png"
+
 
 class Columns(IntegerChoices):
     webpage_url = 1, "Source"
@@ -82,6 +84,9 @@ class RequestModel(BaseModel):
     class ResponseModel(BaseModel):
         pass
 
+    def preview_image(self, state: dict) -> str | None:
+        return DEFAULT_YOUTUBE_BOT_META_IMG
+
     def render_form_v2(self):
         document_uploader(
             "##### 🤖 Youtube URLS",
diff --git a/recipes/DocSummary.py b/recipes/DocSummary.py
index 63fb0ed71..f49025c7f 100644
--- a/recipes/DocSummary.py
+++ b/recipes/DocSummary.py
@@ -27,7 +27,7 @@
 )
 from recipes.GoogleGPT import render_output_with_refs, GoogleGPTPage
 
-DEFAULT_DOC_SEARCH_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/assets/DOC%20SEARCH.gif"
+DEFAULT_DOC_SUMMARY_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/4bce6718-538c-11ee-a837-02420a000190/doc%20summary%201.gif.png"
 
 
 class CombineDocumentsChains(Enum):
@@ -78,6 +78,9 @@ class ResponseModel(BaseModel):
         prompt_tree: PromptTree | None
         final_prompt: str
 
+    def preview_image(self, state: dict) -> str | None:
+        return DEFAULT_DOC_SUMMARY_META_IMG
+
     def render_form_v2(self):
         document_uploader("##### 📎 Documents")
         st.text_area("##### 👩‍💻 Instructions", key="task_instructions", height=150)
diff --git a/recipes/Lipsync.py b/recipes/Lipsync.py
index 9c56e2a1b..4959ccdd0 100644
--- a/recipes/Lipsync.py
+++ b/recipes/Lipsync.py
@@ -15,6 +15,8 @@
 
 CREDITS_PER_MB = 2
 
+DEFAULT_LIPSYNC_GIF = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/13a80d08-538c-11ee-9c77-02420a000193/lipsync%20audio%201.gif.png"
+
 
 class LipsyncPage(BasePage):
     title = "Lip Syncing"
@@ -33,6 +35,9 @@ class RequestModel(BaseModel):
     class ResponseModel(BaseModel):
         output_video: str
 
+    def preview_image(self, state: dict) -> str | None:
+        return DEFAULT_LIPSYNC_GIF
+
     def render_form_v2(self) -> bool:
         st.file_uploader(
             """
diff --git a/recipes/QRCodeGenerator.py b/recipes/QRCodeGenerator.py
index 64fe12d30..56d907220 100644
--- a/recipes/QRCodeGenerator.py
+++ b/recipes/QRCodeGenerator.py
@@ -35,6 +35,7 @@
 from daras_ai_v2.loom_video_widget import youtube_video
 
 ATTEMPTS = 1
+DEFAULT_QR_CODE_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/f09c8cfa-5393-11ee-a837-02420a000190/ai%20art%20qr%20codes1%201.png.png"
 
 
 class QRCodeGeneratorPage(BasePage):
@@ -89,6 +90,9 @@ class ResponseModel(BaseModel):
         shortened_url: str | None
         cleaned_qr_code: str
 
+    def preview_image(self, state: dict) -> str | None:
+        return DEFAULT_QR_CODE_META_IMG
+
     def related_workflows(self) -> list:
         from recipes.CompareText2Img import CompareText2ImgPage
         from recipes.CompareUpscaler import CompareUpscalerPage
diff --git a/recipes/RelatedQnA.py b/recipes/RelatedQnA.py
index 6ad08fb04..1b9aaf392 100644
--- a/recipes/RelatedQnA.py
+++ b/recipes/RelatedQnA.py
@@ -16,7 +16,7 @@
 from recipes.GoogleGPT import GoogleGPTPage
 from recipes.RelatedQnADoc import render_qna_outputs
 
-DEFAULT_GOOGLE_GPT_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/assets/WEBSEARCH%20%2B%20CHATGPT.jpg"
+DEFAULT_SEO_CONTENT_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/9b415768-5393-11ee-a837-02420a000190/RQnA%20SEO%20content%201.png.png"
 
 
 class RelatedGoogleGPTResponse(GoogleGPTPage.ResponseModel):
@@ -46,6 +46,9 @@ class ResponseModel(BaseModel):
         output_queries: list[RelatedGoogleGPTResponse]
         serp_results: dict
 
+    def preview_image(self, state: dict) -> str | None:
+        return DEFAULT_SEO_CONTENT_META_IMG
+
     def render_description(self) -> str:
         return "This workflow gets the related queries for your Google search, searches your custom domain and builds answers using the results and GPT."
 
diff --git a/recipes/RelatedQnADoc.py b/recipes/RelatedQnADoc.py
index 046f9571b..9f3f3d11e 100644
--- a/recipes/RelatedQnADoc.py
+++ b/recipes/RelatedQnADoc.py
@@ -15,7 +15,7 @@
 from recipes.DocSearch import DocSearchPage, render_doc_search_step, EmptySearchResults
 from recipes.GoogleGPT import render_output_with_refs, GoogleSearchMixin
 
-DEFAULT_GOOGLE_GPT_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/assets/WEBSEARCH%20%2B%20CHATGPT.jpg"
+DEFAULT_QNA_DOC_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/bab3dd2a-538c-11ee-920f-02420a00018e/RQnA-doc%20search%201.png.png"
 
 
 class RelatedDocSearchResponse(DocSearchPage.ResponseModel):
@@ -43,6 +43,9 @@ class ResponseModel(BaseModel):
         output_queries: list[RelatedDocSearchResponse]
         serp_results: dict
 
+    def preview_image(self, state: dict) -> str | None:
+        return DEFAULT_QNA_DOC_META_IMG
+
     def render_description(self) -> str:
         return "This workflow gets the related queries for your Google search, searches your custom domain and builds answers using the results and GPT."
 
diff --git a/recipes/SmartGPT.py b/recipes/SmartGPT.py
index fb97bee69..55e2a9f5d 100644
--- a/recipes/SmartGPT.py
+++ b/recipes/SmartGPT.py
@@ -17,6 +17,8 @@
 from daras_ai_v2.language_model_settings_widgets import language_model_settings
 from daras_ai_v2.pt import PromptTree
 
+DEFAULT_SMARTGPT_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/e02d1582-538a-11ee-9d7b-02420a000194/smartgpt%201.png.png"
+
 
 class SmartGPTPage(BasePage):
     title = "SmartGPT"
@@ -45,6 +47,9 @@ class ResponseModel(BaseModel):
 
         prompt_tree: PromptTree | None
 
+    def preview_image(self, state: dict) -> str | None:
+        return DEFAULT_SMARTGPT_META_IMG
+
     def render_form_v2(self):
         st.text_area(
             """
diff --git a/recipes/Text2Audio.py b/recipes/Text2Audio.py
index 05350c815..54435751f 100644
--- a/recipes/Text2Audio.py
+++ b/recipes/Text2Audio.py
@@ -17,6 +17,8 @@
     num_outputs_setting,
 )
 
+DEFAULT_TEXT2AUDIO_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/ddc6e894-538b-11ee-a837-02420a000190/text2audio1%201.png.png"
+
 
 class Text2AudioModels(Enum):
     audio_ldm = "AudioLDM (CVSSP)"
@@ -58,6 +60,9 @@ class ResponseModel(BaseModel):
             typing.Literal[tuple(e.name for e in Text2AudioModels)], list[str]
         ]
 
+    def preview_image(self, state: dict) -> str | None:
+        return DEFAULT_TEXT2AUDIO_META_IMG
+
     def render_form_v2(self):
         st.text_area(
             """
diff --git a/recipes/VideoBots.py b/recipes/VideoBots.py
index 59697a0c6..4c8895901 100644
--- a/recipes/VideoBots.py
+++ b/recipes/VideoBots.py
@@ -59,6 +59,8 @@
 from recipes.TextToSpeech import TextToSpeechPage
 from url_shortener.models import ShortenedURL
 
+DEFAULT_COPILOT_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/c8b24b0c-538a-11ee-a1a3-02420a00018d/meta%20tags1%201.png.png"
+
 BOT_SCRIPT_RE = re.compile(
     # start of line
     r"^"
@@ -255,6 +257,9 @@ class ResponseModel(BaseModel):
         final_search_query: str | None
         final_keyword_query: str | None
 
+    def preview_image(self, state: dict) -> str | None:
+        return DEFAULT_COPILOT_META_IMG
+
     def related_workflows(self):
         from recipes.LipsyncTTS import LipsyncTTSPage
         from recipes.CompareText2Img import CompareText2ImgPage
diff --git a/recipes/asr.py b/recipes/asr.py
index 3ae720af2..289ba91e8 100644
--- a/recipes/asr.py
+++ b/recipes/asr.py
@@ -23,6 +23,8 @@
 from daras_ai_v2.text_output_widget import text_outputs
 from recipes.DocSearch import render_documents
 
+DEFAULT_ASR_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/3b98d906-538b-11ee-9c77-02420a000193/Speech1%201.png.png"
+
 
 class AsrPage(BasePage):
     title = "Speech Recognition & Translation"
@@ -44,6 +46,9 @@ class ResponseModel(BaseModel):
         raw_output_text: list[str] | None
         output_text: list[str | AsrOutputJson]
 
+    def preview_image(self, state: dict) -> str | None:
+        return DEFAULT_ASR_META_IMG
+
     def preview_description(self, state: dict):
         return "Transcribe mp3, WhatsApp audio + wavs with OpenAI's Whisper or AI4Bharat / Bhashini ASR models. Optionally translate to any language too."
 

From c89524c62b99b72dfb20b8d32ce9a06bda59d963 Mon Sep 17 00:00:00 2001
From: Ravi Shankar <42587315+ravishankar63@users.noreply.github.com>
Date: Thu, 21 Sep 2023 14:15:02 +0530
Subject: [PATCH 16/36] added docsummary and lipsync images

---
 recipes/DocSummary.py | 2 +-
 recipes/Lipsync.py    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/recipes/DocSummary.py b/recipes/DocSummary.py
index f49025c7f..5e121a6ab 100644
--- a/recipes/DocSummary.py
+++ b/recipes/DocSummary.py
@@ -27,7 +27,7 @@
 )
 from recipes.GoogleGPT import render_output_with_refs, GoogleGPTPage
 
-DEFAULT_DOC_SUMMARY_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/4bce6718-538c-11ee-a837-02420a000190/doc%20summary%201.gif.png"
+DEFAULT_DOC_SUMMARY_META_IMG = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/db70c56e-585a-11ee-990b-02420a00018f/doc%20summary.png.png"
 
 
 class CombineDocumentsChains(Enum):
diff --git a/recipes/Lipsync.py b/recipes/Lipsync.py
index 4959ccdd0..2e17d8e41 100644
--- a/recipes/Lipsync.py
+++ b/recipes/Lipsync.py
@@ -15,7 +15,7 @@
 
 CREDITS_PER_MB = 2
 
-DEFAULT_LIPSYNC_GIF = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/13a80d08-538c-11ee-9c77-02420a000193/lipsync%20audio%201.gif.png"
+DEFAULT_LIPSYNC_GIF = "https://storage.googleapis.com/dara-c1b52.appspot.com/daras_ai/media/91acbbde-5857-11ee-920a-02420a000194/lipsync%20audio.png.png"
 
 
 class LipsyncPage(BasePage):

From 5beb24fe1f60c92ca7a11de01e6ae323e30af8e8 Mon Sep 17 00:00:00 2001
From: Alexander Metzger <alexander@metzger.dk>
Date: Mon, 25 Sep 2023 22:24:44 -0700
Subject: [PATCH 17/36] done

---
 daras_ai_v2/bots.py      |  6 ++++++
 daras_ai_v2/slack_bot.py | 10 +++++++---
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/daras_ai_v2/bots.py b/daras_ai_v2/bots.py
index c4410be86..17a5aaf8f 100644
--- a/daras_ai_v2/bots.py
+++ b/daras_ai_v2/bots.py
@@ -89,6 +89,9 @@ def _unpack_bot_integration(self):
     def get_interactive_msg_info(self) -> tuple[str, str]:
         raise NotImplementedError("This bot does not support interactive messages.")
 
+    def get_recieved_msg_id(self) -> str | None:
+        return None
+
 
 PAGE_NOT_CONNECTED_ERROR = (
     "💔 Looks like you haven't connected this page to a gooey.ai workflow. "
@@ -290,6 +293,9 @@ def _process_and_send_msg(
     # save the message id for the sent message
     if msg_id:
         msgs_to_save[-1].platform_msg_id = msg_id
+    # save the message id for the received message
+    if bot.get_recieved_msg_id():
+        msgs_to_save[0].platform_msg_id = bot.get_recieved_msg_id()
     # save the messages
     for msg in msgs_to_save:
         msg.save()
diff --git a/daras_ai_v2/slack_bot.py b/daras_ai_v2/slack_bot.py
index af346dd90..7d5a447b2 100644
--- a/daras_ai_v2/slack_bot.py
+++ b/daras_ai_v2/slack_bot.py
@@ -33,6 +33,7 @@ class SlackBot(BotInterface):
     platform = Platform.SLACK
 
     _read_rcpt_ts: str | None = None
+    _bot_res_ts: str | None = None
 
     def __init__(
         self,
@@ -125,6 +126,9 @@ def get_interactive_msg_info(self) -> tuple[str, str]:
         button_id = self._actions[0]["value"]
         return button_id, self._msg_ts
 
+    def get_recieved_msg_id(self) -> str:
+        return self._msg_ts
+
     def send_msg(
         self,
         *,
@@ -149,7 +153,7 @@ def send_msg(
 
         splits = text_splitter(text, chunk_size=SLACK_MAX_SIZE, length_function=len)
         for doc in splits[:-1]:
-            self._msg_ts = chat_post_message(
+            self._bot_res_ts = chat_post_message(
                 text=doc.text,
                 channel=self.bot_id,
                 channel_is_personal=self.convo.slack_channel_is_personal,
@@ -157,7 +161,7 @@ def send_msg(
                 username=self.convo.bot_integration.name,
                 token=self._access_token,
             )
-        self._msg_ts = chat_post_message(
+        self._bot_res_ts = chat_post_message(
             text=splits[-1].text,
             audio=audio,
             video=video,
@@ -168,7 +172,7 @@ def send_msg(
             token=self._access_token,
             buttons=buttons or [],
         )
-        return self._msg_ts
+        return self._bot_res_ts
 
     def mark_read(self):
         text = self.convo.bot_integration.slack_read_receipt_msg.strip()

From 48b1e7d2f02308df1395fe2e8c3b0bb2844baf02 Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Wed, 27 Sep 2023 08:55:45 +0530
Subject: [PATCH 18/36] fix slack text split

---
 daras_ai_v2/bots.py      |  8 +++-----
 daras_ai_v2/slack_bot.py | 12 ++++--------
 2 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/daras_ai_v2/bots.py b/daras_ai_v2/bots.py
index 17a5aaf8f..ffc251c2a 100644
--- a/daras_ai_v2/bots.py
+++ b/daras_ai_v2/bots.py
@@ -44,6 +44,7 @@ class BotInterface:
     language: str
     show_feedback_buttons: bool = False
     convo: Conversation
+    recieved_msg_id: str | None
 
     def send_msg(
         self,
@@ -89,9 +90,6 @@ def _unpack_bot_integration(self):
     def get_interactive_msg_info(self) -> tuple[str, str]:
         raise NotImplementedError("This bot does not support interactive messages.")
 
-    def get_recieved_msg_id(self) -> str | None:
-        return None
-
 
 PAGE_NOT_CONNECTED_ERROR = (
     "💔 Looks like you haven't connected this page to a gooey.ai workflow. "
@@ -294,8 +292,8 @@ def _process_and_send_msg(
     if msg_id:
         msgs_to_save[-1].platform_msg_id = msg_id
     # save the message id for the received message
-    if bot.get_recieved_msg_id():
-        msgs_to_save[0].platform_msg_id = bot.get_recieved_msg_id()
+    if bot.recieved_msg_id:
+        msgs_to_save[0].platform_msg_id = bot.recieved_msg_id
     # save the messages
     for msg in msgs_to_save:
         msg.save()
diff --git a/daras_ai_v2/slack_bot.py b/daras_ai_v2/slack_bot.py
index 7d5a447b2..ef5986b9d 100644
--- a/daras_ai_v2/slack_bot.py
+++ b/daras_ai_v2/slack_bot.py
@@ -33,7 +33,6 @@ class SlackBot(BotInterface):
     platform = Platform.SLACK
 
     _read_rcpt_ts: str | None = None
-    _bot_res_ts: str | None = None
 
     def __init__(
         self,
@@ -46,7 +45,7 @@ def __init__(
         files: list[dict] = None,
         actions: list[dict] = None,
     ):
-        self._msg_ts = message_ts
+        self.recieved_msg_id = self._msg_ts = message_ts
         self._team_id = team_id
         self.bot_id = channel_id
         self.user_id = user_id
@@ -126,9 +125,6 @@ def get_interactive_msg_info(self) -> tuple[str, str]:
         button_id = self._actions[0]["value"]
         return button_id, self._msg_ts
 
-    def get_recieved_msg_id(self) -> str:
-        return self._msg_ts
-
     def send_msg(
         self,
         *,
@@ -153,7 +149,7 @@ def send_msg(
 
         splits = text_splitter(text, chunk_size=SLACK_MAX_SIZE, length_function=len)
         for doc in splits[:-1]:
-            self._bot_res_ts = chat_post_message(
+            self._msg_ts = chat_post_message(
                 text=doc.text,
                 channel=self.bot_id,
                 channel_is_personal=self.convo.slack_channel_is_personal,
@@ -161,7 +157,7 @@ def send_msg(
                 username=self.convo.bot_integration.name,
                 token=self._access_token,
             )
-        self._bot_res_ts = chat_post_message(
+        self._msg_ts = chat_post_message(
             text=splits[-1].text,
             audio=audio,
             video=video,
@@ -172,7 +168,7 @@ def send_msg(
             token=self._access_token,
             buttons=buttons or [],
         )
-        return self._bot_res_ts
+        return self._msg_ts
 
     def mark_read(self):
         text = self.convo.bot_integration.slack_read_receipt_msg.strip()

From 4e8b85425ee738230497c8cb66febcd7fbb3cb6f Mon Sep 17 00:00:00 2001
From: Alexander Metzger <alexander@metzger.dk>
Date: Fri, 22 Sep 2023 17:40:35 -0700
Subject: [PATCH 19/36] use member join event instead of message event

---
 routers/slack.py | 54 +++++++++++++++++++++++++-----------------------
 1 file changed, 28 insertions(+), 26 deletions(-)

diff --git a/routers/slack.py b/routers/slack.py
index fdeee8511..07c8055d0 100644
--- a/routers/slack.py
+++ b/routers/slack.py
@@ -177,14 +177,12 @@ def _handle_slack_event(event: dict, background_tasks: BackgroundTasks):
     if event["type"] != "event_callback":
         return
     message = event["event"]
-    if message["type"] != "message":
-        return
     try:
-        match message.get("subtype", "any"):
-            case "channel_join":
+        match message.get("type", "any"):
+            case "member_joined_channel":
                 bi = BotIntegration.objects.get(
                     slack_channel_id=message["channel"],
-                    slack_team_id=event["team_id"],
+                    slack_team_id=event["team"],
                 )
                 if not bi.slack_create_personal_channels:
                     return
@@ -198,27 +196,31 @@ def _handle_slack_event(event: dict, background_tasks: BackgroundTasks):
                         raise
                 else:
                     create_personal_channel(bi, user)
-
-            case "any" | "slack_audio" | "file_share":
-                files = message.get("files", [])
-                if not files:
-                    message.get("messsage", {}).get("files", [])
-                if not files:
-                    attachments = message.get("attachments", [])
-                    files = [
-                        file
-                        for attachment in attachments
-                        for file in attachment.get("files", [])
-                    ]
-                bot = SlackBot(
-                    message_ts=message["ts"],
-                    team_id=message.get("team", event["team_id"]),
-                    channel_id=message["channel"],
-                    user_id=message["user"],
-                    text=message.get("text", ""),
-                    files=files,
-                )
-                background_tasks.add_task(_on_msg, bot)
+            case "message":
+                if message.get("subtype", "any") in [
+                    "any",
+                    "slack_audio",
+                    "file_share",
+                ]:
+                    files = message.get("files", [])
+                    if not files:
+                        message.get("message", {}).get("files", [])
+                    if not files:
+                        attachments = message.get("attachments", [])
+                        files = [
+                            file
+                            for attachment in attachments
+                            for file in attachment.get("files", [])
+                        ]
+                    bot = SlackBot(
+                        message_ts=message["ts"],
+                        team_id=message.get("team", event["team_id"]),
+                        channel_id=message["channel"],
+                        user_id=message["user"],
+                        text=message.get("text", ""),
+                        files=files,
+                    )
+                    background_tasks.add_task(_on_msg, bot)
 
     except BotIntegration.DoesNotExist as e:
         print(f"Error: contacted from an unknown channel - {e!r}")

From 72531bf73a4cd0f7ae0c11b9881777378391fa17 Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Wed, 27 Sep 2023 09:05:05 +0530
Subject: [PATCH 20/36] smol refactor: invert if-condition

---
 routers/slack.py | 51 ++++++++++++++++++++++++------------------------
 1 file changed, 26 insertions(+), 25 deletions(-)

diff --git a/routers/slack.py b/routers/slack.py
index 07c8055d0..ac221fa46 100644
--- a/routers/slack.py
+++ b/routers/slack.py
@@ -178,7 +178,7 @@ def _handle_slack_event(event: dict, background_tasks: BackgroundTasks):
         return
     message = event["event"]
     try:
-        match message.get("type", "any"):
+        match message.get("type"):
             case "member_joined_channel":
                 bi = BotIntegration.objects.get(
                     slack_channel_id=message["channel"],
@@ -196,31 +196,32 @@ def _handle_slack_event(event: dict, background_tasks: BackgroundTasks):
                         raise
                 else:
                     create_personal_channel(bi, user)
+
             case "message":
-                if message.get("subtype", "any") in [
-                    "any",
-                    "slack_audio",
-                    "file_share",
-                ]:
-                    files = message.get("files", [])
-                    if not files:
-                        message.get("message", {}).get("files", [])
-                    if not files:
-                        attachments = message.get("attachments", [])
-                        files = [
-                            file
-                            for attachment in attachments
-                            for file in attachment.get("files", [])
-                        ]
-                    bot = SlackBot(
-                        message_ts=message["ts"],
-                        team_id=message.get("team", event["team_id"]),
-                        channel_id=message["channel"],
-                        user_id=message["user"],
-                        text=message.get("text", ""),
-                        files=files,
-                    )
-                    background_tasks.add_task(_on_msg, bot)
+                # Ignore subtypes other than slack_audio and file_share. If there's no subtype, assume text
+                subtype = message.get("subtype")
+                if subtype and subtype not in ["slack_audio", "file_share"]:
+                    return
+
+                files = message.get("files", [])
+                if not files:
+                    message.get("message", {}).get("files", [])
+                if not files:
+                    attachments = message.get("attachments", [])
+                    files = [
+                        file
+                        for attachment in attachments
+                        for file in attachment.get("files", [])
+                    ]
+                bot = SlackBot(
+                    message_ts=message["ts"],
+                    team_id=message.get("team", event["team_id"]),
+                    channel_id=message["channel"],
+                    user_id=message["user"],
+                    text=message.get("text", ""),
+                    files=files,
+                )
+                background_tasks.add_task(_on_msg, bot)
 
     except BotIntegration.DoesNotExist as e:
         print(f"Error: contacted from an unknown channel - {e!r}")

From 0f9fddf5440072115fe7e207fe11fbe162c1a213 Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Wed, 27 Sep 2023 09:14:09 +0530
Subject: [PATCH 21/36] fix attr error

---
 daras_ai_v2/bots.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/daras_ai_v2/bots.py b/daras_ai_v2/bots.py
index ffc251c2a..4170188b2 100644
--- a/daras_ai_v2/bots.py
+++ b/daras_ai_v2/bots.py
@@ -44,7 +44,7 @@ class BotInterface:
     language: str
     show_feedback_buttons: bool = False
     convo: Conversation
-    recieved_msg_id: str | None
+    recieved_msg_id: str = None
 
     def send_msg(
         self,

From 2accf944eaee64e76baa117f478ef75857e83ec0 Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Wed, 27 Sep 2023 11:30:38 +0530
Subject: [PATCH 22/36] add /__/slack/get-response-for-msg/{msg_id}/ for apple
 shortcuts

---
 poetry.lock         | 54 +++++++++++++-----------------------------
 routers/slack.py    | 57 +++++++++++++++++++++++++++++++++++++++++++--
 tests/test_slack.py | 45 +++++++++++++++++++++++++++++++++++
 3 files changed, 116 insertions(+), 40 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 3886e97da..32a70bdde 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand.
 
 [[package]]
 name = "absl-py"
@@ -888,7 +888,6 @@ files = [
     {file = "contourpy-1.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18a64814ae7bce73925131381603fff0116e2df25230dfc80d6d690aa6e20b37"},
     {file = "contourpy-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90c81f22b4f572f8a2110b0b741bb64e5a6427e0a198b2cdc1fbaf85f352a3aa"},
     {file = "contourpy-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:53cc3a40635abedbec7f1bde60f8c189c49e84ac180c665f2cd7c162cc454baa"},
-    {file = "contourpy-1.1.0-cp310-cp310-win32.whl", hash = "sha256:9b2dd2ca3ac561aceef4c7c13ba654aaa404cf885b187427760d7f7d4c57cff8"},
     {file = "contourpy-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:1f795597073b09d631782e7245016a4323cf1cf0b4e06eef7ea6627e06a37ff2"},
     {file = "contourpy-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0b7b04ed0961647691cfe5d82115dd072af7ce8846d31a5fac6c142dcce8b882"},
     {file = "contourpy-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:27bc79200c742f9746d7dd51a734ee326a292d77e7d94c8af6e08d1e6c15d545"},
@@ -897,7 +896,6 @@ files = [
     {file = "contourpy-1.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5cec36c5090e75a9ac9dbd0ff4a8cf7cecd60f1b6dc23a374c7d980a1cd710e"},
     {file = "contourpy-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f0cbd657e9bde94cd0e33aa7df94fb73c1ab7799378d3b3f902eb8eb2e04a3a"},
     {file = "contourpy-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:181cbace49874f4358e2929aaf7ba84006acb76694102e88dd15af861996c16e"},
-    {file = "contourpy-1.1.0-cp311-cp311-win32.whl", hash = "sha256:edb989d31065b1acef3828a3688f88b2abb799a7db891c9e282df5ec7e46221b"},
     {file = "contourpy-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fb3b7d9e6243bfa1efb93ccfe64ec610d85cfe5aec2c25f97fbbd2e58b531256"},
     {file = "contourpy-1.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bcb41692aa09aeb19c7c213411854402f29f6613845ad2453d30bf421fe68fed"},
     {file = "contourpy-1.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5d123a5bc63cd34c27ff9c7ac1cd978909e9c71da12e05be0231c608048bb2ae"},
@@ -906,7 +904,6 @@ files = [
     {file = "contourpy-1.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:317267d915490d1e84577924bd61ba71bf8681a30e0d6c545f577363157e5e94"},
     {file = "contourpy-1.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d551f3a442655f3dcc1285723f9acd646ca5858834efeab4598d706206b09c9f"},
     {file = "contourpy-1.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e7a117ce7df5a938fe035cad481b0189049e8d92433b4b33aa7fc609344aafa1"},
-    {file = "contourpy-1.1.0-cp38-cp38-win32.whl", hash = "sha256:108dfb5b3e731046a96c60bdc46a1a0ebee0760418951abecbe0fc07b5b93b27"},
     {file = "contourpy-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:d4f26b25b4f86087e7d75e63212756c38546e70f2a92d2be44f80114826e1cd4"},
     {file = "contourpy-1.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc00bb4225d57bff7ebb634646c0ee2a1298402ec10a5fe7af79df9a51c1bfd9"},
     {file = "contourpy-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:189ceb1525eb0655ab8487a9a9c41f42a73ba52d6789754788d1883fb06b2d8a"},
@@ -915,7 +912,6 @@ files = [
     {file = "contourpy-1.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:143dde50520a9f90e4a2703f367cf8ec96a73042b72e68fcd184e1279962eb6f"},
     {file = "contourpy-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e94bef2580e25b5fdb183bf98a2faa2adc5b638736b2c0a4da98691da641316a"},
     {file = "contourpy-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ed614aea8462735e7d70141374bd7650afd1c3f3cb0c2dbbcbe44e14331bf002"},
-    {file = "contourpy-1.1.0-cp39-cp39-win32.whl", hash = "sha256:71551f9520f008b2950bef5f16b0e3587506ef4f23c734b71ffb7b89f8721999"},
     {file = "contourpy-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:438ba416d02f82b692e371858143970ed2eb6337d9cdbbede0d8ad9f3d7dd17d"},
     {file = "contourpy-1.1.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a698c6a7a432789e587168573a864a7ea374c6be8d4f31f9d87c001d5a843493"},
     {file = "contourpy-1.1.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:397b0ac8a12880412da3551a8cb5a187d3298a72802b45a3bd1805e204ad8439"},
@@ -1487,8 +1483,11 @@ files = [
 [package.dependencies]
 google-auth = ">=2.14.1,<3.0.dev0"
 googleapis-common-protos = ">=1.56.2,<2.0.dev0"
-grpcio = {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}
-grpcio-status = {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}
+grpcio = [
+    {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""},
+    {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""},
+]
+grpcio-status = {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "extra == \"grpc\""}
 protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0"
 requests = ">=2.18.0,<3.0.0.dev0"
 
@@ -1605,8 +1604,8 @@ files = [
 google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]}
 google-cloud-core = ">=1.4.1,<3.0.0dev"
 proto-plus = [
-    {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""},
     {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""},
+    {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""},
 ]
 protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev"
 
@@ -1624,8 +1623,8 @@ files = [
 [package.dependencies]
 google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]}
 proto-plus = [
-    {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""},
     {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""},
+    {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""},
 ]
 protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev"
 
@@ -1664,8 +1663,8 @@ files = [
 [package.dependencies]
 google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]}
 proto-plus = [
-    {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""},
     {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""},
+    {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""},
 ]
 protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev"
 
@@ -1828,7 +1827,6 @@ files = [
     {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"},
     {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"},
     {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"},
-    {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d967650d3f56af314b72df7089d96cda1083a7fc2da05b375d2bc48c82ab3f3c"},
     {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"},
     {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"},
     {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"},
@@ -1837,7 +1835,6 @@ files = [
     {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"},
     {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"},
     {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"},
-    {file = "greenlet-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d4606a527e30548153be1a9f155f4e283d109ffba663a15856089fb55f933e47"},
     {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"},
     {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"},
     {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"},
@@ -1867,7 +1864,6 @@ files = [
     {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"},
     {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"},
     {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"},
-    {file = "greenlet-2.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1087300cf9700bbf455b1b97e24db18f2f77b55302a68272c56209d5587c12d1"},
     {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"},
     {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"},
     {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"},
@@ -1876,7 +1872,6 @@ files = [
     {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"},
     {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"},
     {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"},
-    {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8512a0c38cfd4e66a858ddd1b17705587900dd760c6003998e9472b77b56d417"},
     {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"},
     {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"},
     {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"},
@@ -2840,16 +2835,6 @@ files = [
     {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"},
     {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"},
     {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"},
-    {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"},
     {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"},
     {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"},
     {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"},
@@ -3410,9 +3395,12 @@ files = [
 
 [package.dependencies]
 numpy = [
+    {version = ">=1.21.2", markers = "python_version >= \"3.10\""},
+    {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\""},
     {version = ">=1.23.5", markers = "python_version >= \"3.11\""},
-    {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\" and python_version < \"3.11\""},
-    {version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\" and python_version < \"3.11\""},
+    {version = ">=1.19.3", markers = "python_version >= \"3.6\" and platform_system == \"Linux\" and platform_machine == \"aarch64\" or python_version >= \"3.9\""},
+    {version = ">=1.17.0", markers = "python_version >= \"3.7\""},
+    {version = ">=1.17.3", markers = "python_version >= \"3.8\""},
 ]
 
 [[package]]
@@ -3470,8 +3458,8 @@ files = [
 
 [package.dependencies]
 numpy = [
-    {version = ">=1.23.2", markers = "python_version >= \"3.11\""},
     {version = ">=1.22.4", markers = "python_version < \"3.11\""},
+    {version = ">=1.23.2", markers = "python_version >= \"3.11\""},
 ]
 python-dateutil = ">=2.8.2"
 pytz = ">=2020.1"
@@ -4359,7 +4347,6 @@ files = [
     {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"},
     {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"},
     {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"},
-    {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"},
     {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"},
     {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"},
     {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"},
@@ -4367,15 +4354,8 @@ files = [
     {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"},
     {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"},
     {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"},
-    {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"},
     {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"},
     {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
-    {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
-    {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
-    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
-    {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
-    {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
-    {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"},
     {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"},
     {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"},
     {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"},
@@ -4392,7 +4372,6 @@ files = [
     {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"},
     {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"},
     {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"},
-    {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"},
     {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"},
     {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"},
     {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"},
@@ -4400,7 +4379,6 @@ files = [
     {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"},
     {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"},
     {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"},
-    {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"},
     {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"},
     {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"},
     {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"},
@@ -5226,7 +5204,7 @@ files = [
 ]
 
 [package.dependencies]
-greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"}
+greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\")"}
 
 [package.extras]
 aiomysql = ["aiomysql", "greenlet (!=0.4.17)"]
diff --git a/routers/slack.py b/routers/slack.py
index ac221fa46..53684fc40 100644
--- a/routers/slack.py
+++ b/routers/slack.py
@@ -2,24 +2,29 @@
 
 import requests
 from django.db import transaction
-from fastapi import APIRouter, HTTPException, Depends, Request, Response
+from fastapi import APIRouter, HTTPException, Depends, Request, Response, Header
 from furl import furl
+from requests import HTTPError
 from requests.auth import HTTPBasicAuth
 from sentry_sdk import capture_exception
 from starlette.background import BackgroundTasks
 from starlette.responses import RedirectResponse, HTMLResponse
 
-from bots.models import BotIntegration, Platform, Conversation
+from bots.models import BotIntegration, Platform, Conversation, Message
 from bots.tasks import create_personal_channels_for_all_members
 from daras_ai_v2 import settings
 from daras_ai_v2.bots import _on_msg, request_json, request_urlencoded_body
+from daras_ai_v2.search_ref import parse_refs
 from daras_ai_v2.slack_bot import (
     SlackBot,
     invite_bot_account_to_channel,
     create_personal_channel,
     SlackAPIError,
     fetch_user_info,
+    parse_slack_response,
 )
+from gooey_token_authentication1.token_authentication import auth_keyword
+from recipes.VideoBots import VideoBotsPage
 
 router = APIRouter()
 
@@ -335,3 +340,51 @@ def slack_connect_redirect_shortcuts(request: Request):
         redirect_uri=slack_shortcuts_redirect_uri,
     ),
 )
+
+
+def slack_auth_header(
+    authorization: str = Header(
+        alias="Authorization",
+        description=f"Bearer $SLACK_AUTH_TOKEN",
+    ),
+) -> dict:
+    r = requests.post(
+        "https://slack.com/api/auth.test", headers={"Authorization": authorization}
+    )
+    try:
+        return parse_slack_response(r)
+    except HTTPError:
+        raise HTTPException(
+            status_code=r.status_code, detail=r.content, headers=r.headers
+        )
+    except SlackAPIError as e:
+        raise HTTPException(500, {"error": e.error})
+
+
+@router.get("/__/slack/get-response-for-msg/{msg_id}/")
+def slack_get_response_for_msg_id(
+    msg_id: str,
+    remove_refs: bool = True,
+    slack_user: dict = Depends(slack_auth_header),
+):
+    try:
+        msg = Message.objects.get(platform_msg_id=msg_id)
+        response_msg = msg.get_next_by_created_at()
+    except Message.DoesNotExist:
+        return {"status": "not_found"}
+
+    if msg.conversation.slack_team_id != slack_user.get(
+        "team_id",
+    ) and msg.conversation.slack_user_id != slack_user.get(
+        "user_id",
+    ):
+        raise HTTPException(403, "Not authorized")
+
+    output_text = response_msg.saved_run.state.get("output_text")
+    if not output_text:
+        return {"status": "no_output"}
+
+    if remove_refs:
+        output_text = "".join(snippet for snippet, _ in parse_refs(output_text, []))
+
+    return {"status": "ok", "output_text": output_text}
diff --git a/tests/test_slack.py b/tests/test_slack.py
index 6a82b1113..dd83d35a2 100644
--- a/tests/test_slack.py
+++ b/tests/test_slack.py
@@ -1,6 +1,51 @@
+from decouple import config
+from starlette.testclient import TestClient
+
+from bots.models import Message, SavedRun, BotIntegration, Conversation, Platform
 from daras_ai_v2.slack_bot import safe_channel_name
+from recipes.VideoBots import VideoBotsPage
+from server import app
 
 
 def test_slack_safe_channel_name():
     assert safe_channel_name("hello world!") == "hello-world"
     assert safe_channel_name("My, Awesome, Channel %") == "my-awesome-channel"
+
+
+def test_slack_get_response_for_msg_id(transactional_db):
+    team_id = config("TEST_SLACK_TEAM_ID")
+    user_id = config("TEST_SLACK_USER_ID")
+    auth_token = config("TEST_SLACK_AUTH_TOKEN")
+
+    convo = Conversation.objects.create(
+        bot_integration=BotIntegration.objects.create(
+            platform=Platform.SLACK,
+        ),
+        slack_team_id=team_id,
+        slack_user_id=user_id,
+    )
+    incoming_msg = Message.objects.create(
+        conversation=convo,
+        platform_msg_id="incoming-msg",
+    )
+    Message.objects.create(
+        conversation=convo,
+        platform_msg_id="response-msg",
+        saved_run=SavedRun.objects.create(
+            state=VideoBotsPage.ResponseModel(
+                output_text=["hello, world! [2]"],
+                final_prompt="",
+                output_audio=[],
+                output_video=[],
+                references=[],
+            ).dict(),
+        ),
+    )
+
+    client = TestClient(app)
+    r = client.get(
+        f"/__/slack/get-response-for-msg/{incoming_msg.platform_msg_id}/",
+        headers={"Authorization": f"Bearer {auth_token}"},
+    )
+    assert r.status_code == 200
+    assert r.json().get("output_text") == ["hello, world!"]

From 6d494bbf76c8660a86424a691adb9fa7fba0bdd5 Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Wed, 27 Sep 2023 11:41:54 +0530
Subject: [PATCH 23/36] fix TypeError

---
 routers/slack.py    | 6 +++---
 tests/test_slack.py | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/routers/slack.py b/routers/slack.py
index 53684fc40..d845d9f75 100644
--- a/routers/slack.py
+++ b/routers/slack.py
@@ -384,7 +384,7 @@ def slack_get_response_for_msg_id(
     if not output_text:
         return {"status": "no_output"}
 
+    content = output_text[0]
     if remove_refs:
-        output_text = "".join(snippet for snippet, _ in parse_refs(output_text, []))
-
-    return {"status": "ok", "output_text": output_text}
+        content = "".join(snippet for snippet, _ in parse_refs(content, []))
+    return {"status": "ok", "content": content}
diff --git a/tests/test_slack.py b/tests/test_slack.py
index dd83d35a2..ebe72ca96 100644
--- a/tests/test_slack.py
+++ b/tests/test_slack.py
@@ -48,4 +48,4 @@ def test_slack_get_response_for_msg_id(transactional_db):
         headers={"Authorization": f"Bearer {auth_token}"},
     )
     assert r.status_code == 200
-    assert r.json().get("output_text") == ["hello, world!"]
+    assert r.json().get("content") == "hello, world!"

From b24e3b434a8308dd3be989eeb61b0b66730c8a19 Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Wed, 27 Sep 2023 11:58:57 +0530
Subject: [PATCH 24/36] fix KeyError 'team'

---
 routers/slack.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/routers/slack.py b/routers/slack.py
index d845d9f75..fb1bf51a5 100644
--- a/routers/slack.py
+++ b/routers/slack.py
@@ -187,7 +187,7 @@ def _handle_slack_event(event: dict, background_tasks: BackgroundTasks):
             case "member_joined_channel":
                 bi = BotIntegration.objects.get(
                     slack_channel_id=message["channel"],
-                    slack_team_id=event["team"],
+                    slack_team_id=message["team"],
                 )
                 if not bi.slack_create_personal_channels:
                     return

From fdadabb22a6ee6c47fcdc47e68f45c6e8ce6c87b Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Wed, 27 Sep 2023 13:00:05 +0530
Subject: [PATCH 25/36] invoke shortcut via the shortcuts://run-shortcuts url

---
 routers/slack.py | 52 ++++++++++++++++++++++++++----------------------
 1 file changed, 28 insertions(+), 24 deletions(-)

diff --git a/routers/slack.py b/routers/slack.py
index fb1bf51a5..75ed8465e 100644
--- a/routers/slack.py
+++ b/routers/slack.py
@@ -239,7 +239,9 @@ def slack_oauth_shortcuts():
 
 
 @router.get("/__/slack/redirect/shortcuts/")
-def slack_connect_redirect_shortcuts(request: Request):
+def slack_connect_redirect_shortcuts(
+    request: Request, shortcut_name: str = "Start Copilot"
+):
     retry_button = f'<a href="{slack_shortcuts_connect_url}">Retry</a>'
 
     code = request.query_params.get("code")
@@ -286,17 +288,22 @@ def slack_connect_redirect_shortcuts(request: Request):
         )
     sr = convo.bot_integration.saved_run
 
-    payload = json.dumps(
-        dict(
-            slack_channel=convo.slack_channel_name,
-            slack_channel_id=convo.slack_channel_id,
-            slack_user_access_token=access_token,
-            slack_team_id=team_id,
-            gooey_example_id=sr.example_id,
-            gooey_run_id=sr.run_id,
-        ),
-        indent=2,
+    slack_creds = dict(
+        slack_channel=convo.slack_channel_name,
+        slack_channel_id=convo.slack_channel_id,
+        slack_user_access_token=access_token,
+        slack_team_id=team_id,
+        gooey_example_id=sr.example_id,
+        gooey_run_id=sr.run_id,
     )
+    shortcut_url = furl(
+        "shortcuts://run-shortcut",
+        query_params=dict(
+            name=shortcut_name,
+            input=json.dumps({"slack_creds": slack_creds}),
+        ),
+    ).tostr(query_quote_plus=False)
+
     return HTMLResponse(
         # language=HTML
         """
@@ -305,24 +312,21 @@ def slack_connect_redirect_shortcuts(request: Request):
         </head>
         <body style="font-size: 1.2rem; text-align: center; font-family: sans-serif">
             <p>
-                <textarea style="font-family: monospace; width: 100%%;" rows=5 id="foo">%(payload)s</textarea>
+                <textarea style="font-family: monospace; width: 100%%;" rows=5 id="foo">%(slack_creds)s</textarea>
             </p>
-
             <p>
-                <button id="new-copy" style="padding: 2rem; font-size: 1.5rem;">
-                    Click here to Complete Setup
-                </button>
+                <a href="%(shortcut_url)s">
+                    <button id="new-copy" style="padding: 2rem; font-size: 1.5rem;">
+                        Click here to Complete Setup
+                    </button>
+                </a>
             </p>
-
-            <script>
-                document.getElementById("new-copy").addEventListener("click", event => {
-                    navigator.clipboard.writeText(%(payload)r);
-                    document.body.innerText = "Setup Complete! You can close this window now."
-                });
-            </script>
         </body>
         """
-        % dict(payload=payload)
+        % dict(
+            slack_creds=json.dumps(slack_creds, indent=2),
+            shortcut_url=shortcut_url,
+        )
     )
 
 

From e9ba0b89ea39954e2faf2ba0f55d0abfd8a461a1 Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Wed, 27 Sep 2023 13:22:38 +0530
Subject: [PATCH 26/36] return raw_tts_text from videobots &
 get-response-for-msg api

---
 recipes/VideoBots.py | 12 ++++++------
 routers/slack.py     | 13 +++++++------
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/recipes/VideoBots.py b/recipes/VideoBots.py
index 4c8895901..d5f405ed6 100644
--- a/recipes/VideoBots.py
+++ b/recipes/VideoBots.py
@@ -244,6 +244,7 @@ class ResponseModel(BaseModel):
         final_prompt: str
         raw_input_text: str | None
         raw_output_text: list[str] | None
+        raw_tts_text: list[str] | None
         output_text: list[str]
 
         # tts
@@ -734,11 +735,10 @@ def run(self, state: dict) -> typing.Iterator[str | None]:
                 source_language="en",
                 target_language=request.user_language,
             )
-
-        tts_text = [
-            "".join(snippet for snippet, _ in parse_refs(text, references))
-            for text in output_text
-        ]
+            state["raw_tts_text"] = [
+                "".join(snippet for snippet, _ in parse_refs(text, references))
+                for text in output_text
+            ]
 
         if references:
             citation_style = (
@@ -754,7 +754,7 @@ def run(self, state: dict) -> typing.Iterator[str | None]:
         if not request.tts_provider:
             return
         tts_state = dict(state)
-        for text in tts_text:
+        for text in state.get("raw_tts_text", state["raw_output_text"]):
             tts_state["text_prompt"] = text
             yield from TextToSpeechPage().run(tts_state)
             state["output_audio"].append(tts_state["audio_url"])
diff --git a/routers/slack.py b/routers/slack.py
index 75ed8465e..b9669ec1d 100644
--- a/routers/slack.py
+++ b/routers/slack.py
@@ -368,7 +368,6 @@ def slack_auth_header(
 @router.get("/__/slack/get-response-for-msg/{msg_id}/")
 def slack_get_response_for_msg_id(
     msg_id: str,
-    remove_refs: bool = True,
     slack_user: dict = Depends(slack_auth_header),
 ):
     try:
@@ -384,11 +383,13 @@ def slack_get_response_for_msg_id(
     ):
         raise HTTPException(403, "Not authorized")
 
-    output_text = response_msg.saved_run.state.get("output_text")
+    state = response_msg.saved_run.state
+    output_text = (
+        state.get("raw_tts_text")
+        or state.get("raw_output_text")
+        or state.get("output_text")
+    )
     if not output_text:
         return {"status": "no_output"}
 
-    content = output_text[0]
-    if remove_refs:
-        content = "".join(snippet for snippet, _ in parse_refs(content, []))
-    return {"status": "ok", "content": content}
+    return {"status": "ok", "content": output_text[0]}

From bda8f220ba7984013ca8b28dc63cd58aeccf2ed5 Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Wed, 27 Sep 2023 14:07:17 +0530
Subject: [PATCH 27/36] citation symbols

---
 daras_ai_v2/search_ref.py | 96 +++++++++++++++++++++++++++++++++------
 tests/test_search_refs.py | 22 ++++++++-
 2 files changed, 102 insertions(+), 16 deletions(-)

diff --git a/daras_ai_v2/search_ref.py b/daras_ai_v2/search_ref.py
index 657d2898c..09f3714c7 100644
--- a/daras_ai_v2/search_ref.py
+++ b/daras_ai_v2/search_ref.py
@@ -19,17 +19,23 @@ class CitationStyles(Enum):
     number = "Numbers ( [1] [2] [3] ..)"
     title = "Source Title ( [Source 1] [Source 2] [Source 3] ..)"
     url = "Source URL ( [https://source1.com] [https://source2.com] [https://source3.com] ..)"
+    symbol = "Symbols ( [*] [†] [‡] ..)"
 
     markdown = "Markdown ( [Source 1](https://source1.com) [Source 2](https://source2.com) [Source 3](https://source3.com) ..)"
     html = "HTML ( <a href='https://source1.com'>Source 1</a> <a href='https://source2.com'>Source 2</a> <a href='https://source3.com'>Source 3</a> ..)"
     slack_mrkdwn = "Slack mrkdwn ( <https://source1.com|Source 1> <https://source2.com|Source 2> <https://source3.com|Source 3> ..)"
     plaintext = "Plain Text / WhatsApp ( [Source 1 https://source1.com] [Source 2 https://source2.com] [Source 3 https://source3.com] ..)"
 
-    number_markdown = " Markdown Numbers + Footnotes"
+    number_markdown = "Markdown Numbers + Footnotes"
     number_html = "HTML Numbers + Footnotes"
     number_slack_mrkdwn = "Slack mrkdown Numbers + Footnotes"
     number_plaintext = "Plain Text / WhatsApp Numbers + Footnotes"
 
+    symbol_markdown = "Markdown Symbols + Footnotes"
+    symbol_html = "HTML Symbols + Footnotes"
+    symbol_slack_mrkdwn = "Slack mrkdown Symbols + Footnotes"
+    symbol_plaintext = "Plain Text / WhatsApp Symbols + Footnotes"
+
 
 def remove_quotes(snippet: str) -> str:
     return re.sub(r"[\"\']+", r'"', snippet).strip()
@@ -63,36 +69,65 @@ def apply_response_template(
             match citation_style:
                 case CitationStyles.number | CitationStyles.number_plaintext:
                     cites = " ".join(f"[{ref_num}]" for ref_num in ref_map.keys())
-                case CitationStyles.number_html:
+                case CitationStyles.title:
+                    cites = " ".join(f"[{ref['title']}]" for ref in ref_map.values())
+                case CitationStyles.url:
+                    cites = " ".join(f"[{ref['url']}]" for ref in ref_map.values())
+                case CitationStyles.symbol | CitationStyles.symbol_plaintext:
                     cites = " ".join(
-                        html_link(f"[{ref_num}]", ref["url"])
+                        f"[{generate_footnote_symbol(ref_num - 1)}]"
+                        for ref_num in ref_map.keys()
+                    )
+
+                case CitationStyles.markdown:
+                    cites = " ".join(ref_to_markdown(ref) for ref in ref_map.values())
+                case CitationStyles.html:
+                    cites = " ".join(ref_to_html(ref) for ref in ref_map.values())
+                case CitationStyles.slack_mrkdwn:
+                    cites = " ".join(
+                        ref_to_slack_mrkdwn(ref) for ref in ref_map.values()
+                    )
+                case CitationStyles.plaintext:
+                    cites = " ".join(
+                        f'[{ref["title"]} {ref["url"]}]'
                         for ref_num, ref in ref_map.items()
                     )
+
                 case CitationStyles.number_markdown:
                     cites = " ".join(
                         markdown_link(f"[{ref_num}]", ref["url"])
                         for ref_num, ref in ref_map.items()
                     )
+                case CitationStyles.number_html:
+                    cites = " ".join(
+                        html_link(f"[{ref_num}]", ref["url"])
+                        for ref_num, ref in ref_map.items()
+                    )
                 case CitationStyles.number_slack_mrkdwn:
                     cites = " ".join(
                         slack_mrkdwn_link(f"[{ref_num}]", ref["url"])
                         for ref_num, ref in ref_map.items()
                     )
-                case CitationStyles.title:
-                    cites = " ".join(f"[{ref['title']}]" for ref in ref_map.values())
-                case CitationStyles.url:
-                    cites = " ".join(f"[{ref['url']}]" for ref in ref_map.values())
-                case CitationStyles.markdown:
-                    cites = " ".join(ref_to_markdown(ref) for ref in ref_map.values())
-                case CitationStyles.html:
-                    cites = " ".join(ref_to_html(ref) for ref in ref_map.values())
-                case CitationStyles.slack_mrkdwn:
+
+                case CitationStyles.symbol_markdown:
                     cites = " ".join(
-                        ref_to_slack_mrkdwn(ref) for ref in ref_map.values()
+                        markdown_link(
+                            f"[{generate_footnote_symbol(ref_num - 1)}]", ref["url"]
+                        )
+                        for ref_num, ref in ref_map.items()
                     )
-                case CitationStyles.plaintext:
+                case CitationStyles.symbol_html:
                     cites = " ".join(
-                        f'[{ref["title"]} {ref["url"]}]'
+                        html_link(
+                            f"[{generate_footnote_symbol(ref_num - 1)}]", ref["url"]
+                        )
+                        for ref_num, ref in ref_map.items()
+                    )
+                case CitationStyles.symbol_slack_mrkdwn:
+                    cites = " ".join(
+                        slack_mrkdwn_link(
+                            f"[{generate_footnote_symbol(ref_num - 1)}]", ref["url"]
+                        )
                         for ref_num, ref in ref_map.items()
                     )
                 case None:
@@ -128,6 +163,31 @@ def apply_response_template(
                     for ref_num, ref in sorted(all_refs.items())
                 )
 
+            case CitationStyles.symbol_markdown:
+                formatted += "\n\n"
+                formatted += "\n".join(
+                    f"{generate_footnote_symbol(ref_num - 1)} {ref_to_markdown(ref)}"
+                    for ref_num, ref in sorted(all_refs.items())
+                )
+            case CitationStyles.symbol_html:
+                formatted += "<br><br>"
+                formatted += "<br>".join(
+                    f"{generate_footnote_symbol(ref_num - 1)} {ref_to_html(ref)}"
+                    for ref_num, ref in sorted(all_refs.items())
+                )
+            case CitationStyles.symbol_slack_mrkdwn:
+                formatted += "\n\n"
+                formatted += "\n".join(
+                    f"{generate_footnote_symbol(ref_num - 1)} {ref_to_slack_mrkdwn(ref)}"
+                    for ref_num, ref in sorted(all_refs.items())
+                )
+            case CitationStyles.symbol_plaintext:
+                formatted += "\n\n"
+                formatted += "\n".join(
+                    f'{generate_footnote_symbol(ref_num - 1)}. {ref["title"]} {ref["url"]}'
+                    for ref_num, ref in sorted(all_refs.items())
+                )
+
         for ref_num, ref in all_refs.items():
             try:
                 template = ref["response_template"]
@@ -205,3 +265,9 @@ def render_output_with_refs(state, height):
     for text in output_text:
         html = render_text_with_refs(text, state.get("references", []))
         scrollable_html(html, height=height)
+
+
+FOOTNOTE_SYMBOLS = ["*", "†", "‡", "§", "¶", "#", "♠", "♥", "♦", "♣", "✠", "☮", "☯", "✡"]  # fmt: skip
+def generate_footnote_symbol(idx: int) -> str:
+    quotient, remainder = divmod(idx, len(FOOTNOTE_SYMBOLS))
+    return FOOTNOTE_SYMBOLS[remainder] * (quotient + 1)
diff --git a/tests/test_search_refs.py b/tests/test_search_refs.py
index 5374db260..00698bf88 100644
--- a/tests/test_search_refs.py
+++ b/tests/test_search_refs.py
@@ -1,4 +1,6 @@
-from daras_ai_v2.search_ref import parse_refs
+import pytest
+
+from daras_ai_v2.search_ref import parse_refs, generate_footnote_symbol
 
 
 def test_ref_parser():
@@ -126,3 +128,21 @@ def test_ref_parser():
             },
         ),
     ]
+
+
+def test_generate_footnote_symbol():
+    assert generate_footnote_symbol(0) == "*"
+    assert generate_footnote_symbol(1) == "†"
+    assert generate_footnote_symbol(13) == "✡"
+    assert generate_footnote_symbol(14) == "**"
+    assert generate_footnote_symbol(15) == "††"
+    assert generate_footnote_symbol(27) == "✡✡"
+    assert generate_footnote_symbol(28) == "***"
+    assert generate_footnote_symbol(29) == "†††"
+    assert generate_footnote_symbol(41) == "✡✡✡"
+    assert generate_footnote_symbol(70) == "******"
+    assert generate_footnote_symbol(71) == "††††††"
+
+    # testing with non-integer index
+    with pytest.raises(TypeError):
+        generate_footnote_symbol(1.5)

From 363b35445851203219bf9c390881c3dd9e58c159 Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Tue, 3 Oct 2023 10:32:03 +0530
Subject: [PATCH 28/36] better tests that run faster bulk runner

---
 auth_backend.py => auth/auth_backend.py       |  29 +-
 .../token_authentication.py                   |   9 +-
 bots/models.py                                |   3 +-
 bots/tests.py                                 |   4 +-
 conftest.py                                   | 147 +++++++-
 daras_ai/image_input.py                       |  10 +-
 daras_ai_v2/all_pages.py                      |   2 +
 daras_ai_v2/api_examples_widget.py            |   2 +-
 daras_ai_v2/base.py                           |  28 +-
 daras_ai_v2/doc_search_settings_widgets.py    |  17 +-
 daras_ai_v2/search_ref.py                     |   3 +-
 gooey_ui/components.py                        |   6 +-
 pyproject.toml                                |  10 +-
 pytest.ini                                    |   4 +-
 recipes/BulkRunner.py                         | 344 ++++++++++++++++++
 recipes/DeforumSD.py                          |   3 +-
 routers/api.py                                |   2 +-
 routers/root.py                               |   4 +-
 server.py                                     |  10 +-
 tests/test_apis.py                            |  88 ++++-
 tests/test_checkout.py                        |   4 +-
 tests/test_public_endpoints.py                |  41 ++-
 tests/test_translation.py                     |  16 +-
 url_shortener/tests.py                        |  10 +-
 24 files changed, 688 insertions(+), 108 deletions(-)
 rename auth_backend.py => auth/auth_backend.py (76%)
 rename {gooey_token_authentication1 => auth}/token_authentication.py (94%)
 create mode 100644 recipes/BulkRunner.py

diff --git a/auth_backend.py b/auth/auth_backend.py
similarity index 76%
rename from auth_backend.py
rename to auth/auth_backend.py
index 24dded058..d197ef213 100644
--- a/auth_backend.py
+++ b/auth/auth_backend.py
@@ -6,36 +6,37 @@
 from starlette.concurrency import run_in_threadpool
 
 from app_users.models import AppUser
-from daras_ai_v2.crypto import get_random_string, get_random_doc_id
+from daras_ai_v2.crypto import get_random_doc_id
 from daras_ai_v2.db import FIREBASE_SESSION_COOKIE, ANONYMOUS_USER_COOKIE
 from gooeysite.bg_db_conn import db_middleware
 
 # quick and dirty way to bypass authentication for testing
-_forced_auth_user = []
+authlocal = []
 
 
 @contextmanager
 def force_authentication():
-    user = AppUser.objects.create(
-        is_anonymous=True, uid=get_random_doc_id(), balance=10**9
+    authlocal.append(
+        AppUser.objects.get_or_create(
+            email="tests@pytest.org",
+            defaults=dict(is_anonymous=True, uid=get_random_doc_id(), balance=10**9),
+        )[0]
     )
     try:
-        _forced_auth_user.append(user)
-        yield
+        yield authlocal[0]
     finally:
-        _forced_auth_user.clear()
+        authlocal.clear()
 
 
 class SessionAuthBackend(AuthenticationBackend):
     async def authenticate(self, conn):
-        return await run_in_threadpool(authenticate, conn)
+        if authlocal:
+            return AuthCredentials(["authenticated"]), authlocal[0]
+        return await run_in_threadpool(_authenticate, conn)
 
 
 @db_middleware
-def authenticate(conn):
-    if _forced_auth_user:
-        return AuthCredentials(["authenticated"]), _forced_auth_user[0]
-
+def _authenticate(conn):
     session_cookie = conn.session.get(FIREBASE_SESSION_COOKIE)
     if not session_cookie:
         # Session cookie is unavailable. Check if anonymous user is available.
@@ -51,7 +52,7 @@ def authenticate(conn):
         # Session cookie is unavailable. Force user to login.
         return AuthCredentials(), None
 
-    user = verify_session_cookie(session_cookie)
+    user = _verify_session_cookie(session_cookie)
     if not user:
         # Session cookie was invalid
         conn.session.pop(FIREBASE_SESSION_COOKIE, None)
@@ -60,7 +61,7 @@ def authenticate(conn):
     return AuthCredentials(["authenticated"]), user
 
 
-def verify_session_cookie(firebase_cookie: str) -> UserRecord | None:
+def _verify_session_cookie(firebase_cookie: str) -> UserRecord | None:
     # Verify the session cookie. In this case an additional check is added to detect
     # if the user's Firebase session was revoked, user deleted/disabled, etc.
     try:
diff --git a/gooey_token_authentication1/token_authentication.py b/auth/token_authentication.py
similarity index 94%
rename from gooey_token_authentication1/token_authentication.py
rename to auth/token_authentication.py
index 4e81c42b0..b33bbbbd0 100644
--- a/gooey_token_authentication1/token_authentication.py
+++ b/auth/token_authentication.py
@@ -1,8 +1,10 @@
+import threading
+
 from fastapi import Header
 from fastapi.exceptions import HTTPException
 
 from app_users.models import AppUser
-from auth_backend import _forced_auth_user
+from auth.auth_backend import authlocal
 from daras_ai_v2 import db
 from daras_ai_v2.crypto import PBKDF2PasswordHasher
 
@@ -15,9 +17,8 @@ def api_auth_header(
         description=f"{auth_keyword} $GOOEY_API_KEY",
     ),
 ) -> AppUser:
-    if _forced_auth_user:
-        return _forced_auth_user[0]
-
+    if authlocal:
+        return authlocal[0]
     return authenticate(authorization)
 
 
diff --git a/bots/models.py b/bots/models.py
index b4eb715c2..0f0ddce50 100644
--- a/bots/models.py
+++ b/bots/models.py
@@ -68,6 +68,7 @@ class Workflow(models.IntegerChoices):
     RELATED_QNA_MAKER = (27, "Related QnA Maker")
     RELATED_QNA_MAKER_DOC = (28, "Related QnA Maker Doc")
     EMBEDDINGS = (29, "Embeddings")
+    BULK_RUNNER = (30, "Bulk Runner")
 
     @property
     def short_slug(self):
@@ -239,7 +240,7 @@ def submit_api_call(
                 kwds=dict(
                     page_cls=Workflow(self.workflow).page_cls,
                     query_params=dict(
-                        example_id=self.example_id, run_id=self.id, uid=self.uid
+                        example_id=self.example_id, run_id=self.run_id, uid=self.uid
                     ),
                     user=current_user,
                     request_body=request_body,
diff --git a/bots/tests.py b/bots/tests.py
index 0bc887389..1fb66e16b 100644
--- a/bots/tests.py
+++ b/bots/tests.py
@@ -14,7 +14,7 @@
 CHATML_ROLE_ASSISSTANT = "assistant"
 
 
-def test_add_balance_direct():
+def test_add_balance_direct(transactional_db):
     pk = AppUser.objects.create(balance=0, is_anonymous=False).pk
     amounts = [[random.randint(-100, 10_000) for _ in range(100)] for _ in range(5)]
 
@@ -28,7 +28,7 @@ def worker(amts):
     assert AppUser.objects.get(pk=pk).balance == sum(map(sum, amounts))
 
 
-def test_create_bot_integration_conversation_message():
+def test_create_bot_integration_conversation_message(transactional_db):
     # Create a new BotIntegration with WhatsApp as the platform
     bot_integration = BotIntegration.objects.create(
         name="My Bot Integration",
diff --git a/conftest.py b/conftest.py
index 7bea1a865..ff61671aa 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,7 +1,17 @@
+import typing
+from functools import wraps
+from threading import Thread
+from unittest.mock import patch
+
 import pytest
+from pytest_subtests import subtests
 
+from auth import auth_backend
+from celeryapp import app
+from daras_ai_v2.base import BasePage
 
-@pytest.fixture
+
+@pytest.fixture(scope="session")
 def django_db_setup(django_db_setup, django_db_blocker):
     with django_db_blocker.unblock():
         from django.core.management import call_command
@@ -9,12 +19,131 @@ def django_db_setup(django_db_setup, django_db_blocker):
         call_command("loaddata", "fixture.json")
 
 
-@pytest.fixture(
-    # add this fixture to all tests
-    autouse=True
-)
-def enable_db_access_for_all_tests(
-    # enable transactional db
-    transactional_db,
+@pytest.fixture
+def force_authentication():
+    with auth_backend.force_authentication() as user:
+        yield user
+
+
+app.conf.task_always_eager = True
+
+
+@pytest.fixture
+def mock_gui_runner():
+    with patch("celeryapp.tasks.gui_runner", _mock_gui_runner):
+        yield
+
+
+@app.task
+def _mock_gui_runner(
+    *, page_cls: typing.Type[BasePage], run_id: str, uid: str, **kwargs
 ):
-    pass
+    sr = page_cls.run_doc_sr(run_id, uid)
+    sr.set(sr.parent.to_dict())
+    sr.save()
+
+
+@pytest.fixture
+def threadpool_subtest(subtests, max_workers: int = 8):
+    ts = []
+
+    def submit(fn, *args, **kwargs):
+        msg = "--".join(map(str, args))
+
+        @wraps(fn)
+        def runner(*args, **kwargs):
+            with subtests.test(msg=msg):
+                return fn(*args, **kwargs)
+
+        ts.append(Thread(target=runner, args=args, kwargs=kwargs))
+
+    yield submit
+
+    for i in range(0, len(ts), max_workers):
+        s = slice(i, i + max_workers)
+        for t in ts[s]:
+            t.start()
+        for t in ts[s]:
+            t.join()
+
+
+# class DummyDatabaseBlocker(pytest_django.plugin._DatabaseBlocker):
+#     class _dj_db_wrapper:
+#         def ensure_connection(self):
+#             pass
+#
+#
+# @pytest.mark.tryfirst
+# def pytest_sessionstart(session):
+#     # make the session threadsafe
+#     _pytest.runner.SetupState = ThreadLocalSetupState
+#
+#     # ensure that the fixtures (specifically finalizers) are threadsafe
+#     _pytest.fixtures.FixtureDef = ThreadLocalFixtureDef
+#
+#     django.test.utils._TestState = threading.local()
+#
+#     # make the environment threadsafe
+#     os.environ = ThreadLocalEnviron(os.environ)
+#
+#
+# @pytest.hookimpl
+# def pytest_runtestloop(session: _pytest.main.Session):
+#     if session.testsfailed and not session.config.option.continue_on_collection_errors:
+#         raise session.Interrupted(
+#             "%d error%s during collection"
+#             % (session.testsfailed, "s" if session.testsfailed != 1 else "")
+#         )
+#
+#     if session.config.option.collectonly:
+#         return True
+#
+#     num_threads = 10
+#
+#     threadpool_items = [
+#         item
+#         for item in session.items
+#         if any("run_in_threadpool" in marker.name for marker in item.iter_markers())
+#     ]
+#
+#     manager = pytest_django.plugin._blocking_manager
+#     pytest_django.plugin._blocking_manager = DummyDatabaseBlocker()
+#     try:
+#         with manager.unblock():
+#             for j in range(0, len(threadpool_items), num_threads):
+#                 s = slice(j, j + num_threads)
+#                 futs = []
+#                 for i, item in enumerate(threadpool_items[s]):
+#                     nextitem = (
+#                         threadpool_items[i + 1]
+#                         if i + 1 < len(threadpool_items)
+#                         else None
+#                     )
+#                     p = threading.Thread(
+#                         target=item.config.hook.pytest_runtest_protocol,
+#                         kwargs=dict(item=item, nextitem=nextitem),
+#                     )
+#                     p.start()
+#                     futs.append(p)
+#                 for p in futs:
+#                     p.join()
+#                     if session.shouldfail:
+#                         raise session.Failed(session.shouldfail)
+#                     if session.shouldstop:
+#                         raise session.Interrupted(session.shouldstop)
+#     finally:
+#         pytest_django.plugin._blocking_manager = manager
+#
+#     session.items = [
+#         item
+#         for item in session.items
+#         if not any("run_in_threadpool" in marker.name for marker in item.iter_markers())
+#     ]
+#     for i, item in enumerate(session.items):
+#         nextitem = session.items[i + 1] if i + 1 < len(session.items) else None
+#         item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
+#         if session.shouldfail:
+#             raise session.Failed(session.shouldfail)
+#         if session.shouldstop:
+#             raise session.Interrupted(session.shouldstop)
+#     return True
diff --git a/daras_ai/image_input.py b/daras_ai/image_input.py
index 5b67d0445..96bd7734c 100644
--- a/daras_ai/image_input.py
+++ b/daras_ai/image_input.py
@@ -10,6 +10,9 @@
 
 from daras_ai_v2 import settings
 
+if False:
+    from firebase_admin import storage
+
 
 def resize_img_pad(img_bytes: bytes, size: tuple[int, int]) -> bytes:
     img_cv2 = bytes_to_cv2_img(img_bytes)
@@ -54,15 +57,10 @@ def upload_file_from_bytes(
     data: bytes,
     content_type: str = None,
 ) -> str:
-    from firebase_admin import storage
-
     if not content_type:
         content_type = mimetypes.guess_type(filename)[0]
     content_type = content_type or "application/octet-stream"
-
-    filename = safe_filename(filename)
-    bucket = storage.bucket(settings.GS_BUCKET_NAME)
-    blob = bucket.blob(f"daras_ai/media/{uuid.uuid1()}/{filename}")
+    blob = storage_blob_for(filename)
     blob.upload_from_string(data, content_type=content_type)
     return blob.public_url
 
diff --git a/daras_ai_v2/all_pages.py b/daras_ai_v2/all_pages.py
index 5befa1451..85047e770 100644
--- a/daras_ai_v2/all_pages.py
+++ b/daras_ai_v2/all_pages.py
@@ -3,6 +3,7 @@
 
 from bots.models import Workflow
 from daras_ai_v2.base import BasePage
+from recipes.BulkRunner import BulkRunnerPage
 from recipes.ChyronPlant import ChyronPlantPage
 from recipes.CompareLLM import CompareLLMPage
 from recipes.CompareText2Img import CompareText2ImgPage
@@ -71,6 +72,7 @@
     ImageSegmentationPage,
     CompareUpscalerPage,
     DocExtractPage,
+    BulkRunnerPage,
 ]
 
 # exposed as API
diff --git a/daras_ai_v2/api_examples_widget.py b/daras_ai_v2/api_examples_widget.py
index 7f15b06f4..da9a25a5e 100644
--- a/daras_ai_v2/api_examples_widget.py
+++ b/daras_ai_v2/api_examples_widget.py
@@ -6,7 +6,7 @@
 from furl import furl
 
 from daras_ai_v2.doc_search_settings_widgets import is_user_uploaded_url
-from gooey_token_authentication1.token_authentication import auth_keyword
+from auth.token_authentication import auth_keyword
 
 
 def get_filenames(request_body):
diff --git a/daras_ai_v2/base.py b/daras_ai_v2/base.py
index c0d594dd2..9744110c3 100644
--- a/daras_ai_v2/base.py
+++ b/daras_ai_v2/base.py
@@ -365,30 +365,35 @@ def get_sr_from_query_params_dict(self, query_params) -> SavedRun:
         example_id, run_id, uid = extract_query_params(query_params)
         return self.get_sr_from_query_params(example_id, run_id, uid)
 
-    def get_sr_from_query_params(self, example_id, run_id, uid) -> SavedRun:
+    @classmethod
+    def get_sr_from_query_params(
+        cls, example_id: str, run_id: str, uid: str
+    ) -> SavedRun:
         try:
             if run_id and uid:
-                sr = self.run_doc_sr(run_id, uid)
+                sr = cls.run_doc_sr(run_id, uid)
             elif example_id:
-                sr = self.example_doc_sr(example_id)
+                sr = cls.example_doc_sr(example_id)
             else:
-                sr = self.recipe_doc_sr()
+                sr = cls.recipe_doc_sr()
             return sr
         except SavedRun.DoesNotExist:
             raise HTTPException(status_code=404)
 
-    def recipe_doc_sr(self) -> SavedRun:
+    @classmethod
+    def recipe_doc_sr(cls) -> SavedRun:
         return SavedRun.objects.get_or_create(
-            workflow=self.workflow,
+            workflow=cls.workflow,
             run_id__isnull=True,
             uid__isnull=True,
             example_id__isnull=True,
         )[0]
 
+    @classmethod
     def run_doc_sr(
-        self, run_id: str, uid: str, create: bool = False, parent: SavedRun = None
+        cls, run_id: str, uid: str, create: bool = False, parent: SavedRun = None
     ) -> SavedRun:
-        config = dict(workflow=self.workflow, uid=uid, run_id=run_id)
+        config = dict(workflow=cls.workflow, uid=uid, run_id=run_id)
         if create:
             return SavedRun.objects.get_or_create(
                 **config, defaults=dict(parent=parent)
@@ -396,8 +401,9 @@ def run_doc_sr(
         else:
             return SavedRun.objects.get(**config)
 
-    def example_doc_sr(self, example_id: str, create: bool = False) -> SavedRun:
-        config = dict(workflow=self.workflow, example_id=example_id)
+    @classmethod
+    def example_doc_sr(cls, example_id: str, create: bool = False) -> SavedRun:
+        config = dict(workflow=cls.workflow, example_id=example_id)
         if create:
             return SavedRun.objects.get_or_create(**config)[0]
         else:
@@ -851,7 +857,7 @@ def _render(sr: SavedRun):
             workflow=self.workflow,
             hidden=False,
             example_id__isnull=False,
-        ).exclude()[:50]
+        )[:50]
 
         grid_layout(3, example_runs, _render)
 
diff --git a/daras_ai_v2/doc_search_settings_widgets.py b/daras_ai_v2/doc_search_settings_widgets.py
index 31865022b..5c3c2de59 100644
--- a/daras_ai_v2/doc_search_settings_widgets.py
+++ b/daras_ai_v2/doc_search_settings_widgets.py
@@ -1,3 +1,5 @@
+import typing
+
 import gooey_ui as st
 
 from daras_ai_v2 import settings
@@ -12,8 +14,18 @@ def is_user_uploaded_url(url: str) -> bool:
 
 def document_uploader(
     label: str,
-    key="documents",
-    accept=(".pdf", ".txt", ".docx", ".md", ".html", ".wav", ".ogg", ".mp3", ".aac"),
+    key: str = "documents",
+    accept: typing.Iterable[str] = (
+        ".pdf",
+        ".txt",
+        ".docx",
+        ".md",
+        ".html",
+        ".wav",
+        ".ogg",
+        ".mp3",
+        ".aac",
+    ),
 ):
     st.write(label, className="gui-input")
     documents = st.session_state.get(key) or []
@@ -45,6 +57,7 @@ def document_uploader(
             accept=accept,
             accept_multiple_files=True,
         )
+    return st.session_state.get(key, [])
 
 
 def doc_search_settings(
diff --git a/daras_ai_v2/search_ref.py b/daras_ai_v2/search_ref.py
index 09f3714c7..1baf64b47 100644
--- a/daras_ai_v2/search_ref.py
+++ b/daras_ai_v2/search_ref.py
@@ -3,12 +3,13 @@
 from enum import Enum
 
 import jinja2
+from typing_extensions import TypedDict
 
 import gooey_ui
 from daras_ai_v2.scrollable_html_widget import scrollable_html
 
 
-class SearchReference(typing.TypedDict):
+class SearchReference(TypedDict):
     url: str
     title: str
     snippet: str
diff --git a/gooey_ui/components.py b/gooey_ui/components.py
index f9b366d1c..ea3bbedda 100644
--- a/gooey_ui/components.py
+++ b/gooey_ui/components.py
@@ -241,7 +241,7 @@ def text_area(
         key = md5_values(
             "textarea", label, height, help, value, placeholder, label_visibility
         )
-    value = state.session_state.setdefault(key, value)
+    value = str(state.session_state.setdefault(key, value))
     if label_visibility != "visible":
         label = None
     if disabled:
@@ -462,6 +462,10 @@ def json(value: typing.Any, expanded: bool = False, depth: int = 1):
     ).mount()
 
 
+def data_table(file_url: str):
+    return _node("data-table", fileUrl=file_url)
+
+
 def table(df: "pd.DataFrame"):
     state.RenderTreeNode(
         name="table",
diff --git a/pyproject.toml b/pyproject.toml
index 3ea6240be..6c697412c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -18,12 +18,12 @@ pandas = "^2.0.1"
 google-cloud-firestore = "^2.7.0"
 replicate = "^0.4.0"
 fastapi = "^0.85.0"
-uvicorn = {extras = ["standard"], version = "^0.18.3"}
+uvicorn = { extras = ["standard"], version = "^0.18.3" }
 firebase-admin = "^6.0.0"
 # mediapipe for M1 macs
-mediapipe-silicon = {version = "^0.8.11", markers = "platform_machine == 'arm64'", platform = "darwin"}
+mediapipe-silicon = { version = "^0.8.11", markers = "platform_machine == 'arm64'", platform = "darwin" }
 # mediapipe for others
-mediapipe = {version = "^0.8.11", markers = "platform_machine != 'arm64'"}
+mediapipe = { version = "^0.8.11", markers = "platform_machine != 'arm64'" }
 furl = "^2.1.3"
 itsdangerous = "^2.1.2"
 pytest = "^7.2.0"
@@ -53,7 +53,7 @@ llama-index = "^0.5.27"
 nltk = "^3.8.1"
 Jinja2 = "^3.1.2"
 Django = "^4.2"
-django-phonenumber-field = {extras = ["phonenumberslite"], version = "^7.0.2"}
+django-phonenumber-field = { extras = ["phonenumberslite"], version = "^7.0.2" }
 gunicorn = "^20.1.0"
 psycopg2-binary = "^2.9.6"
 whitenoise = "^6.4.0"
@@ -75,6 +75,8 @@ tabulate = "^0.9.0"
 deepgram-sdk = "^2.11.0"
 scipy = "^1.11.2"
 rank-bm25 = "^0.2.2"
+pytest-subtests = "^0.11.0"
+anyio = "^3.4.0"
 
 [tool.poetry.group.dev.dependencies]
 watchdog = "^2.1.9"
diff --git a/pytest.ini b/pytest.ini
index 55040e469..4a9863462 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,4 +1,4 @@
 [pytest]
-addopts = --tb=native -vv -n 16 --disable-warnings
+addopts = --tb=native --disable-warnings
 DJANGO_SETTINGS_MODULE = daras_ai_v2.settings
-python_files = tests.py test_*.py *_tests.py
\ No newline at end of file
+python_files = tests.py test_*.py *_tests.py
diff --git a/recipes/BulkRunner.py b/recipes/BulkRunner.py
new file mode 100644
index 000000000..92f265aa5
--- /dev/null
+++ b/recipes/BulkRunner.py
@@ -0,0 +1,344 @@
+import io
+import typing
+
+import pandas as pd
+import requests
+from fastapi import HTTPException
+from furl import furl
+from pydantic import BaseModel
+
+import gooey_ui as st
+from bots.models import Workflow
+from daras_ai.image_input import upload_file_from_bytes
+from daras_ai_v2.base import BasePage
+from daras_ai_v2.doc_search_settings_widgets import document_uploader
+from daras_ai_v2.functional import map_parallel
+from daras_ai_v2.query_params_util import extract_query_params
+from recipes.DocSearch import render_documents
+
+CACHED_COLUMNS = "__cached_columns"
+
+
+class BulkRunnerPage(BasePage):
+    title = "Bulk Runner"
+    workflow = Workflow.BULK_RUNNER
+    slug_versions = ["bulk-runner", "bulk"]
+
+    class RequestModel(BaseModel):
+        documents: list[str]
+        run_urls: list[str]
+
+        input_columns: dict[str, str]
+        output_columns: dict[str, str]
+
+    class ResponseModel(BaseModel):
+        output_documents: list[str]
+
+    def fields_to_save(self) -> [str]:
+        return super().fields_to_save() + [CACHED_COLUMNS]
+
+    def render_form_v2(self):
+        from daras_ai_v2.all_pages import page_slug_map, normalize_slug
+
+        run_urls = st.session_state.get("run_urls", "")
+        st.session_state.setdefault("__run_urls", "\n".join(run_urls))
+        run_urls = (
+            st.text_area("##### Run URL(s)", key="__run_urls").strip().splitlines()
+        )
+        st.session_state["run_urls"] = run_urls
+
+        files = document_uploader(
+            "##### Upload a File",
+            accept=(".csv", ".xlsx", ".xls", ".json", ".tsv", ".xml"),
+        )
+
+        if files:
+            st.session_state[CACHED_COLUMNS] = list(
+                {
+                    col: None
+                    for df in map_parallel(_read_df, files)
+                    for col in df.columns
+                    if not col.startswith("Unnamed:")
+                }
+            )
+        else:
+            st.session_state.pop(CACHED_COLUMNS, None)
+
+        required_input_fields = {}
+        optional_input_fields = {}
+        output_fields = {}
+
+        for url in run_urls:
+            f = furl(url)
+            slug = f.path.segments[0]
+            try:
+                page_cls = page_slug_map[normalize_slug(slug)]
+            except KeyError as e:
+                st.error(repr(e))
+                continue
+
+            example_id, run_id, uid = extract_query_params(f.query.params)
+            try:
+                sr = page_cls.get_sr_from_query_params(example_id, run_id, uid)
+            except HTTPException as e:
+                st.error(repr(e))
+                continue
+
+            schema = page_cls.RequestModel.schema(ref_template="{model}")
+            for field, model_field in page_cls.RequestModel.__fields__.items():
+                if model_field.required:
+                    input_fields = required_input_fields
+                else:
+                    input_fields = optional_input_fields
+                field_props = schema["properties"][field]
+                title = field_props["title"]
+                keys = None
+                if field_props["type"] == "array":
+                    try:
+                        ref = field_props["items"]["$ref"]
+                        props = schema["definitions"][ref]["properties"]
+                        keys = {k: prop["title"] for k, prop in props.items()}
+                    except KeyError:
+                        try:
+                            keys = {k: k for k in sr.state[field][0].keys()}
+                        except (KeyError, IndexError, AttributeError):
+                            pass
+                elif field_props["type"] == "object":
+                    try:
+                        keys = {k: k for k in sr.state[field].keys()}
+                    except (KeyError, AttributeError):
+                        pass
+                if keys:
+                    for k, ktitle in keys.items():
+                        input_fields[f"{field}.{k}"] = f"{title}.{ktitle}"
+                else:
+                    input_fields[field] = title
+
+            schema = page_cls.ResponseModel.schema()
+            output_fields |= {
+                field: schema["properties"][field]["title"]
+                for field, model_field in page_cls.ResponseModel.__fields__.items()
+            }
+
+        columns = st.session_state.get(CACHED_COLUMNS, [])
+        if not columns:
+            return
+
+        for file in files:
+            st.data_table(file)
+
+        if not (required_input_fields or optional_input_fields):
+            return
+
+        col1, col2 = st.columns(2)
+
+        with col1:
+            st.write("##### Input Columns")
+
+            input_columns_old = st.session_state.pop("input_columns", {})
+            input_columns_new = st.session_state.setdefault("input_columns", {})
+
+            column_options = [None, *columns]
+            for fields in (required_input_fields, optional_input_fields):
+                for field, title in fields.items():
+                    col = st.selectbox(
+                        label="`" + title + "`",
+                        options=column_options,
+                        key="--input-mapping:" + field,
+                        default_value=input_columns_old.get(field),
+                    )
+                    if col:
+                        input_columns_new[field] = col
+                st.write("---")
+
+        with col2:
+            st.write("##### Output Columns")
+
+            output_columns_old = st.session_state.pop("output_columns", {})
+            output_columns_new = st.session_state.setdefault("output_columns", {})
+
+            fields = {**output_fields, "error_msg": "Error Msg", "run_url": "Run URL"}
+            for field, title in fields.items():
+                col = st.text_input(
+                    label="`" + title + "`",
+                    key="--output-mapping:" + field,
+                    value=output_columns_old.get(field, title),
+                )
+                if col:
+                    output_columns_new[field] = col
+
+    def render_example(self, state: dict):
+        render_documents(state)
+
+    def render_output(self):
+        files = st.session_state.get("output_documents", [])
+        for file in files:
+            st.write(file)
+            st.data_table(file)
+
+    def run_v2(
+        self,
+        request: "BulkRunnerPage.RequestModel",
+        response: "BulkRunnerPage.ResponseModel",
+    ) -> typing.Iterator[str | None]:
+        response.output_documents = []
+
+        for doc_ix, doc in enumerate(request.documents):
+            df = _read_df(doc)
+            in_recs = df.to_dict(orient="records")
+            out_recs = []
+
+            f = upload_file_from_bytes(
+                filename=f"bulk-runner-{doc_ix}-0-0.csv",
+                data=df.to_csv(index=False).encode(),
+                content_type="text/csv",
+            )
+            response.output_documents.append(f)
+
+            df_slices = list(slice_request_df(df, request))
+            for slice_ix, (df_ix, arr_len) in enumerate(df_slices):
+                rec_ix = len(out_recs)
+                out_recs.extend(in_recs[df_ix : df_ix + arr_len])
+
+                for url_ix, f, request_body, page_cls in build_requests_for_df(
+                    df, request, df_ix, arr_len
+                ):
+                    progress = round(
+                        (slice_ix + url_ix)
+                        / (len(df_slices) + len(request.run_urls))
+                        * 100
+                    )
+                    yield f"{progress}%"
+
+                    example_id, run_id, uid = extract_query_params(f.query.params)
+                    sr = page_cls.get_sr_from_query_params(example_id, run_id, uid)
+
+                    result, sr = sr.submit_api_call(
+                        current_user=self.request.user, request_body=request_body
+                    )
+                    result.get(disable_sync_subtasks=False)
+                    sr.refresh_from_db()
+                    state = sr.to_dict()
+                    state["run_url"] = sr.get_app_url()
+                    state["error_msg"] = sr.error_msg
+
+                    for field, col in request.output_columns.items():
+                        if len(request.run_urls) > 1:
+                            col = f"({url_ix + 1}) {col}"
+                        out_val = state.get(field)
+                        if isinstance(out_val, list):
+                            for arr_ix, item in enumerate(out_val):
+                                if len(out_recs) <= rec_ix + arr_ix:
+                                    out_recs.append({})
+                                if isinstance(item, dict):
+                                    for key, val in item.items():
+                                        out_recs[rec_ix + arr_ix][f"{col}.{key}"] = str(
+                                            val
+                                        )
+                                else:
+                                    out_recs[rec_ix + arr_ix][col] = str(item)
+                        elif isinstance(out_val, dict):
+                            for key, val in out_val.items():
+                                if isinstance(val, list):
+                                    for arr_ix, item in enumerate(val):
+                                        if len(out_recs) <= rec_ix + arr_ix:
+                                            out_recs.append({})
+                                        out_recs[rec_ix + arr_ix][f"{col}.{key}"] = str(
+                                            item
+                                        )
+                                else:
+                                    out_recs[rec_ix][f"{col}.{key}"] = str(val)
+                        else:
+                            out_recs[rec_ix][col] = str(out_val)
+
+                    out_df = pd.DataFrame.from_records(out_recs)
+                    f = upload_file_from_bytes(
+                        filename=f"bulk-runner-{doc_ix}-{url_ix}-{df_ix}.csv",
+                        data=out_df.to_csv(index=False).encode(),
+                        content_type="text/csv",
+                    )
+                    response.output_documents[doc_ix] = f
+
+
+def build_requests_for_df(df, request, df_ix, arr_len):
+    from daras_ai_v2.all_pages import page_slug_map, normalize_slug
+
+    for url_ix, url in enumerate(request.run_urls):
+        f = furl(url)
+        slug = f.path.segments[0]
+        page_cls = page_slug_map[normalize_slug(slug)]
+        schema = page_cls.RequestModel.schema()
+        properties = schema["properties"]
+
+        request_body = {}
+        for field, col in request.input_columns.items():
+            parts = field.split(".")
+            field_props = properties.get(parts[0]) or properties.get(parts)
+            if field_props["type"] == "array":
+                arr = request_body.setdefault(parts[0], [])
+                for arr_ix in range(arr_len):
+                    value = df.at[df_ix + arr_ix, col]
+                    if len(parts) > 1:
+                        if len(arr) <= arr_ix:
+                            arr.append({})
+                        arr[arr_ix][parts[1]] = value
+                    else:
+                        if len(arr) <= arr_ix:
+                            arr.append(None)
+                        arr[arr_ix] = value
+            elif len(parts) > 1 and field_props["type"] == "object":
+                obj = request_body.setdefault(parts[0], {})
+                obj[parts[1]] = df.at[df_ix, col]
+            else:
+                request_body[field] = df.at[df_ix, col]
+        # for validation
+        request_body = page_cls.RequestModel.parse_obj(request_body).dict()
+
+        yield url_ix, f, request_body, page_cls
+
+
+def slice_request_df(df, request):
+    from daras_ai_v2.all_pages import page_slug_map, normalize_slug
+
+    non_array_cols = set()
+    for url_ix, url in enumerate(request.run_urls):
+        f = furl(url)
+        slug = f.path.segments[0]
+        page_cls = page_slug_map[normalize_slug(slug)]
+        schema = page_cls.RequestModel.schema()
+        properties = schema["properties"]
+
+        for field, col in request.input_columns.items():
+            if properties.get(field.split(".")[0])["type"] != "array":
+                non_array_cols.add(col)
+    non_array_df = df[list(non_array_cols)]
+
+    df_ix = 0
+    while df_ix < len(df):
+        arr_len = 1
+        while df_ix + arr_len < len(df):
+            if not non_array_df.iloc[df_ix + arr_len].isnull().all():
+                break
+            arr_len += 1
+        yield df_ix, arr_len
+        df_ix += arr_len
+
+
+def _read_df(f: str) -> "pd.DataFrame":
+    import pandas as pd
+
+    r = requests.get(f)
+    r.raise_for_status()
+    if f.endswith(".csv"):
+        df = pd.read_csv(io.StringIO(r.text))
+    elif f.endswith(".xlsx") or f.endswith(".xls"):
+        df = pd.read_excel(io.BytesIO(r.content))
+    elif f.endswith(".json"):
+        df = pd.read_json(io.StringIO(r.text))
+    elif f.endswith(".tsv"):
+        df = pd.read_csv(io.StringIO(r.text), sep="\t")
+    elif f.endswith(".xml"):
+        df = pd.read_xml(io.StringIO(r.text))
+    else:
+        raise ValueError(f"Unsupported file type: {f}")
+    return df.dropna(how="all", axis=1).dropna(how="all", axis=0)
diff --git a/recipes/DeforumSD.py b/recipes/DeforumSD.py
index a30396907..f523ba14e 100644
--- a/recipes/DeforumSD.py
+++ b/recipes/DeforumSD.py
@@ -3,6 +3,7 @@
 
 from django.db.models import TextChoices
 from pydantic import BaseModel
+from typing_extensions import TypedDict
 
 import gooey_ui as st
 from app_users.models import AppUser
@@ -21,7 +22,7 @@ class AnimationModels(TextChoices):
     epicdream = ("epicdream.safetensors", "epiCDream (epinikion)")
 
 
-class _AnimationPrompt(typing.TypedDict):
+class _AnimationPrompt(TypedDict):
     frame: str
     prompt: str
 
diff --git a/routers/api.py b/routers/api.py
index 47b7989a1..62514cc14 100644
--- a/routers/api.py
+++ b/routers/api.py
@@ -29,7 +29,7 @@
     BasePage,
     StateKeys,
 )
-from gooey_token_authentication1.token_authentication import api_auth_header
+from auth.token_authentication import api_auth_header
 
 app = APIRouter()
 
diff --git a/routers/root.py b/routers/root.py
index 386cc9813..42627572e 100644
--- a/routers/root.py
+++ b/routers/root.py
@@ -21,9 +21,6 @@
 
 import gooey_ui as st
 from app_users.models import AppUser
-from auth_backend import (
-    FIREBASE_SESSION_COOKIE,
-)
 from daras_ai.image_input import upload_file_from_bytes, safe_filename
 from daras_ai_v2 import settings
 from daras_ai_v2.all_pages import all_api_pages, normalize_slug, page_slug_map
@@ -32,6 +29,7 @@
     RedirectException,
 )
 from daras_ai_v2.copy_to_clipboard_button_widget import copy_to_clipboard_scripts
+from daras_ai_v2.db import FIREBASE_SESSION_COOKIE
 from daras_ai_v2.meta_content import build_meta_tags
 from daras_ai_v2.query_params_util import extract_query_params
 from daras_ai_v2.settings import templates
diff --git a/server.py b/server.py
index 1e6d03e1e..078b1508b 100644
--- a/server.py
+++ b/server.py
@@ -1,3 +1,5 @@
+import logging
+
 import anyio
 from decouple import config
 
@@ -18,7 +20,7 @@
 from starlette.middleware.authentication import AuthenticationMiddleware
 from starlette.middleware.sessions import SessionMiddleware
 
-from auth_backend import (
+from auth.auth_backend import (
     SessionAuthBackend,
 )
 from daras_ai_v2 import settings
@@ -67,10 +69,14 @@ async def health():
 
 @app.add_middleware
 def request_time_middleware(app):
+    logger = logging.getLogger("uvicorn.time")
+
     async def middleware(scope, receive, send):
         start_time = time()
         await app(scope, receive, send)
         response_time = (time() - start_time) * 1000
-        print(f"{scope.get('method')} {scope.get('path')} - {response_time:.3f} ms")
+        logger.info(
+            f"{scope.get('method')} {scope.get('path')} - {response_time:.3f} ms"
+        )
 
     return middleware
diff --git a/tests/test_apis.py b/tests/test_apis.py
index d3e1d8b7d..6412e9eda 100644
--- a/tests/test_apis.py
+++ b/tests/test_apis.py
@@ -1,10 +1,10 @@
 import typing
 
 import pytest
-from fastapi.testclient import TestClient
+from starlette.testclient import TestClient
 
-from auth_backend import force_authentication
-from daras_ai_v2 import db
+from auth.auth_backend import force_authentication
+from bots.models import SavedRun, Workflow
 from daras_ai_v2.all_pages import all_test_pages
 from daras_ai_v2.base import (
     BasePage,
@@ -12,21 +12,77 @@
 )
 from server import app
 
+MAX_WORKERS = 20
+
 client = TestClient(app)
 
 
-@pytest.mark.parametrize("page_cls", all_test_pages)
-def test_apis_basic(page_cls: typing.Type[BasePage]):
-    page = page_cls()
-    state = db.get_or_create_doc(db.get_doc_ref(page.slug_versions[0])).to_dict()
+@pytest.mark.django_db
+def test_apis_sync(mock_gui_runner, force_authentication, threadpool_subtest):
+    for page_cls in all_test_pages:
+        threadpool_subtest(_test_api_sync, page_cls)
+
+
+def _test_api_sync(page_cls: typing.Type[BasePage]):
+    state = page_cls.recipe_doc_sr().state
+    r = client.post(
+        f"/v2/{page_cls.slug_versions[0]}/",
+        json=get_example_request_body(page_cls.RequestModel, state),
+        headers={"Authorization": f"Token None"},
+        allow_redirects=False,
+    )
+    assert r.status_code == 200, r.text
+
+
+@pytest.mark.django_db
+def test_apis_async(mock_gui_runner, force_authentication, threadpool_subtest):
+    for page_cls in all_test_pages:
+        threadpool_subtest(_test_api_async, page_cls)
+
+
+def _test_api_async(page_cls: typing.Type[BasePage]):
+    state = page_cls.recipe_doc_sr().state
+
+    r = client.post(
+        f"/v3/{page_cls.slug_versions[0]}/async/",
+        json=get_example_request_body(page_cls.RequestModel, state),
+        headers={"Authorization": f"Token None"},
+        allow_redirects=False,
+    )
+    assert r.status_code == 202, r.text
+
+    status_url = r.json()["status_url"]
+
+    r = client.get(
+        status_url,
+        headers={"Authorization": f"Token None"},
+        allow_redirects=False,
+    )
+    assert r.status_code == 200, r.text
+
+    data = r.json()
+    assert data.get("status") == "completed", data
+    assert data.get("output") is not None, data
+
+
+@pytest.mark.django_db
+def test_apis_examples(mock_gui_runner, force_authentication, threadpool_subtest):
+    for page in all_test_pages:
+        for sr in SavedRun.objects.filter(
+            workflow=page.workflow,
+            hidden=False,
+            example_id__isnull=False,
+        ):
+            threadpool_subtest(_test_apis_examples, sr)
 
-    with force_authentication():
-        r = client.post(
-            page.endpoint,
-            json=get_example_request_body(page.RequestModel, state),
-            headers={"Authorization": f"Token None"},
-            allow_redirects=False,
-        )
 
-    print(r.content)
-    assert r.status_code == 200
+def _test_apis_examples(sr: SavedRun):
+    state = sr.state
+    page_cls = Workflow(sr.workflow).page_cls
+    r = client.post(
+        f"/v2/{page_cls.slug_versions[0]}/?example_id={sr.example_id}",
+        json=get_example_request_body(page_cls.RequestModel, state),
+        headers={"Authorization": f"Token None"},
+        allow_redirects=False,
+    )
+    assert r.status_code == 200, r.text
diff --git a/tests/test_checkout.py b/tests/test_checkout.py
index 5ea7c326b..5e822b775 100644
--- a/tests/test_checkout.py
+++ b/tests/test_checkout.py
@@ -1,7 +1,7 @@
 import pytest
 from fastapi.testclient import TestClient
 
-from auth_backend import force_authentication
+from auth.auth_backend import force_authentication
 from routers.billing import available_subscriptions
 from server import app
 
@@ -9,7 +9,7 @@
 
 
 @pytest.mark.parametrize("subscription", available_subscriptions.keys())
-def test_create_checkout_session(subscription: str):
+def test_create_checkout_session(subscription: str, transactional_db):
     with force_authentication():
         form_data = {"lookup_key": subscription}
         if subscription == "addon":
diff --git a/tests/test_public_endpoints.py b/tests/test_public_endpoints.py
index 009fb836c..02006b92f 100644
--- a/tests/test_public_endpoints.py
+++ b/tests/test_public_endpoints.py
@@ -1,20 +1,20 @@
 import pytest
-import requests
-from furl import furl
 from starlette.routing import Route
 from starlette.testclient import TestClient
 
-from auth_backend import force_authentication
-from daras_ai_v2 import settings
+from bots.models import SavedRun
 from daras_ai_v2.all_pages import all_api_pages
 from daras_ai_v2.tabs_widget import MenuTabs
 from routers import facebook
+from routers.slack import slack_connect_redirect_shortcuts, slack_connect_redirect
 from server import app
 
 client = TestClient(app)
 
 excluded_endpoints = [
     facebook.fb_webhook_verify.__name__,  # gives 403
+    slack_connect_redirect.__name__,
+    slack_connect_redirect_shortcuts.__name__,
     "get_run_status",  # needs query params
 ]
 
@@ -30,10 +30,10 @@
 ]
 
 
+@pytest.mark.django_db
 @pytest.mark.parametrize("path", route_paths)
 def test_all_get(path):
     r = client.get(path, allow_redirects=False)
-    print(r.content)
     assert r.ok
 
 
@@ -41,14 +41,29 @@ def test_all_get(path):
 tabs = list(MenuTabs.paths.values())
 
 
-@pytest.mark.parametrize("tab", tabs)
+@pytest.mark.django_db
 @pytest.mark.parametrize("slug", page_slugs)
+@pytest.mark.parametrize("tab", tabs)
 def test_page_slugs(slug, tab):
-    with force_authentication():
-        r = requests.post(
-            str(furl(settings.API_BASE_URL) / slug / tab),
-            json={},
-        )
-        # r = client.post(os.path.join(slug, tab), json={}, allow_redirects=True)
-    print(r.content)
+    r = client.post(
+        f"/{slug}/{tab}",
+        json={},
+        allow_redirects=True,
+    )
     assert r.status_code == 200
+
+
+@pytest.mark.django_db
+def test_example_slugs(subtests):
+    for page_cls in all_api_pages:
+        for tab in tabs:
+            for example_id in SavedRun.objects.filter(
+                workflow=page_cls.workflow,
+                hidden=False,
+                example_id__isnull=False,
+            ).values_list("example_id", flat=True):
+                slug = page_cls.slug_versions[0]
+                url = f"/{slug}/{tab}?example_id={example_id}"
+                with subtests.test(msg=url):
+                    r = client.post(url, json={}, allow_redirects=True)
+                    assert r.status_code == 200
diff --git a/tests/test_translation.py b/tests/test_translation.py
index 1079036dd..95cf2a539 100644
--- a/tests/test_translation.py
+++ b/tests/test_translation.py
@@ -1,21 +1,19 @@
-import pytest
-
 from daras_ai_v2.asr import run_google_translate
 
 TRANSLATION_TESTS = [
     # hindi romanized
     (
         "Hi Sir Mera khet me mircha ke ped me fal gal Kar gir hai to  iske liye  ham kon sa dawa  de please  help me",
-        "Hi sir the fruits of chilli tree in my field have rotted and fallen so what medicine should we give for this please help",
+        "hi sir the fruit of the chilli tree in my field has rotted and fallen so what medicine should we give for this please help",
     ),
     (
         "Mirchi ka ped",
-        "chili tree",
+        "chilli tree",
     ),
     # hindi
     (
         "ान का नर्सरी खेत में रोकने के लिए कितने दिन में तैयार हो जाता है",
-        "in how many days the corn nursery is ready to stop in the field",
+        "in how many days does the seed nursery become ready to be planted in the field?",
     ),
     # telugu
     (
@@ -44,8 +42,12 @@
 ]
 
 
-@pytest.mark.parametrize("text, expected", TRANSLATION_TESTS)
-def test_run_google_translate(text: str, expected: str):
+def test_run_google_translate(threadpool_subtest):
+    for text, expected in TRANSLATION_TESTS:
+        threadpool_subtest(_test_run_google_translate, text, expected)
+
+
+def _test_run_google_translate(text: str, expected: str):
     actual = run_google_translate([text], "en")[0]
     assert (
         actual.replace(".", "").replace(",", "").strip().lower()
diff --git a/url_shortener/tests.py b/url_shortener/tests.py
index 5e028329a..2589434f1 100644
--- a/url_shortener/tests.py
+++ b/url_shortener/tests.py
@@ -9,14 +9,14 @@
 client = TestClient(app)
 
 
-def test_url_shortener():
+def test_url_shortener(transactional_db):
     surl = ShortenedURL.objects.create(url=TEST_URL)
     short_url = surl.shortened_url()
     r = client.get(short_url, allow_redirects=False)
     assert r.is_redirect and r.headers["location"] == TEST_URL
 
 
-def test_url_shortener_max_clicks():
+def test_url_shortener_max_clicks(transactional_db):
     surl = ShortenedURL.objects.create(url=TEST_URL, max_clicks=5)
     short_url = surl.shortened_url()
     for _ in range(5):
@@ -26,14 +26,14 @@ def test_url_shortener_max_clicks():
     assert r.status_code == 410
 
 
-def test_url_shortener_disabled():
+def test_url_shortener_disabled(transactional_db):
     surl = ShortenedURL.objects.create(url=TEST_URL, disabled=True)
     short_url = surl.shortened_url()
     r = client.get(short_url, allow_redirects=False)
     assert r.status_code == 410
 
 
-def test_url_shortener_create_atomic():
+def test_url_shortener_create_atomic(transactional_db):
     def create(_):
         return [
             ShortenedURL.objects.create(url=TEST_URL).shortened_url()
@@ -43,7 +43,7 @@ def create(_):
     assert len(set(flatmap_parallel(create, range(5)))) == 500
 
 
-def test_url_shortener_clicks_decrement_atomic():
+def test_url_shortener_clicks_decrement_atomic(transactional_db):
     surl = ShortenedURL.objects.create(url=TEST_URL)
     short_url = surl.shortened_url()
 

From 98ee8fff01acf471c3a8d29482dca3408c9fd141 Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Tue, 3 Oct 2023 12:21:35 +0530
Subject: [PATCH 29/36] update lock file

---
 poetry.lock | 59 ++++++++++++++++++++++++++++++-----------------------
 1 file changed, 34 insertions(+), 25 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 32a70bdde..de91446e3 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand.
 
 [[package]]
 name = "absl-py"
@@ -173,24 +173,24 @@ vine = ">=5.0.0"
 
 [[package]]
 name = "anyio"
-version = "4.0.0"
+version = "3.7.1"
 description = "High level compatibility layer for multiple asynchronous event loop implementations"
 optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.7"
 files = [
-    {file = "anyio-4.0.0-py3-none-any.whl", hash = "sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f"},
-    {file = "anyio-4.0.0.tar.gz", hash = "sha256:f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a"},
+    {file = "anyio-3.7.1-py3-none-any.whl", hash = "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5"},
+    {file = "anyio-3.7.1.tar.gz", hash = "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780"},
 ]
 
 [package.dependencies]
-exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""}
+exceptiongroup = {version = "*", markers = "python_version < \"3.11\""}
 idna = ">=2.8"
 sniffio = ">=1.1"
 
 [package.extras]
-doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)"]
-test = ["anyio[trio]", "coverage[toml] (>=7)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"]
-trio = ["trio (>=0.22)"]
+doc = ["Sphinx", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme (>=1.2.2)", "sphinxcontrib-jquery"]
+test = ["anyio[trio]", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"]
+trio = ["trio (<0.22)"]
 
 [[package]]
 name = "appdirs"
@@ -1483,11 +1483,8 @@ files = [
 [package.dependencies]
 google-auth = ">=2.14.1,<3.0.dev0"
 googleapis-common-protos = ">=1.56.2,<2.0.dev0"
-grpcio = [
-    {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""},
-    {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""},
-]
-grpcio-status = {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "extra == \"grpc\""}
+grpcio = {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}
+grpcio-status = {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}
 protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0"
 requests = ">=2.18.0,<3.0.0.dev0"
 
@@ -1604,8 +1601,8 @@ files = [
 google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]}
 google-cloud-core = ">=1.4.1,<3.0.0dev"
 proto-plus = [
-    {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""},
     {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""},
+    {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""},
 ]
 protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev"
 
@@ -1623,8 +1620,8 @@ files = [
 [package.dependencies]
 google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]}
 proto-plus = [
-    {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""},
     {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""},
+    {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""},
 ]
 protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev"
 
@@ -1663,8 +1660,8 @@ files = [
 [package.dependencies]
 google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]}
 proto-plus = [
-    {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""},
     {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""},
+    {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""},
 ]
 protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev"
 
@@ -3395,12 +3392,9 @@ files = [
 
 [package.dependencies]
 numpy = [
-    {version = ">=1.21.2", markers = "python_version >= \"3.10\""},
-    {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\""},
     {version = ">=1.23.5", markers = "python_version >= \"3.11\""},
-    {version = ">=1.19.3", markers = "python_version >= \"3.6\" and platform_system == \"Linux\" and platform_machine == \"aarch64\" or python_version >= \"3.9\""},
-    {version = ">=1.17.0", markers = "python_version >= \"3.7\""},
-    {version = ">=1.17.3", markers = "python_version >= \"3.8\""},
+    {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\" and python_version < \"3.11\""},
+    {version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\" and python_version < \"3.11\""},
 ]
 
 [[package]]
@@ -3458,8 +3452,8 @@ files = [
 
 [package.dependencies]
 numpy = [
-    {version = ">=1.22.4", markers = "python_version < \"3.11\""},
     {version = ">=1.23.2", markers = "python_version >= \"3.11\""},
+    {version = ">=1.22.4", markers = "python_version < \"3.11\""},
 ]
 python-dateutil = ">=2.8.2"
 pytz = ">=2020.1"
@@ -4238,6 +4232,21 @@ pytest = ">=5.4.0"
 docs = ["sphinx", "sphinx-rtd-theme"]
 testing = ["Django", "django-configurations (>=2.0)"]
 
+[[package]]
+name = "pytest-subtests"
+version = "0.11.0"
+description = "unittest subTest() support and subtests fixture"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "pytest-subtests-0.11.0.tar.gz", hash = "sha256:51865c88457545f51fb72011942f0a3c6901ee9e24cbfb6d1b9dc1348bafbe37"},
+    {file = "pytest_subtests-0.11.0-py3-none-any.whl", hash = "sha256:453389984952eec85ab0ce0c4f026337153df79587048271c7fd0f49119c07e4"},
+]
+
+[package.dependencies]
+attrs = ">=19.2.0"
+pytest = ">=7.0"
+
 [[package]]
 name = "pytest-xdist"
 version = "3.3.1"
@@ -5204,7 +5213,7 @@ files = [
 ]
 
 [package.dependencies]
-greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\")"}
+greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"}
 
 [package.extras]
 aiomysql = ["aiomysql", "greenlet (!=0.4.17)"]
@@ -6129,4 +6138,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.10,<3.13"
-content-hash = "e1aa198ba112e95195815327669b1e7687428ab0510c5485f057442a207b982e"
+content-hash = "8cb6f5a826bc1bbd06c65a871e027cf967178cd2e8e6c33c262b80a43a772cfe"

From 0b2747ee7f530597a09a69c059e4fe7c92cd3b13 Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Tue, 3 Oct 2023 12:22:38 +0530
Subject: [PATCH 30/36] fix imports

---
 routers/slack.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/routers/slack.py b/routers/slack.py
index b9669ec1d..20e19fec6 100644
--- a/routers/slack.py
+++ b/routers/slack.py
@@ -14,7 +14,6 @@
 from bots.tasks import create_personal_channels_for_all_members
 from daras_ai_v2 import settings
 from daras_ai_v2.bots import _on_msg, request_json, request_urlencoded_body
-from daras_ai_v2.search_ref import parse_refs
 from daras_ai_v2.slack_bot import (
     SlackBot,
     invite_bot_account_to_channel,
@@ -23,8 +22,6 @@
     fetch_user_info,
     parse_slack_response,
 )
-from gooey_token_authentication1.token_authentication import auth_keyword
-from recipes.VideoBots import VideoBotsPage
 
 router = APIRouter()
 

From 88c3afca12f5aa0432dc7b208e6015fd001091cb Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Tue, 3 Oct 2023 12:26:43 +0530
Subject: [PATCH 31/36] fix slack test

---
 tests/test_slack.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_slack.py b/tests/test_slack.py
index ebe72ca96..3a5802c8d 100644
--- a/tests/test_slack.py
+++ b/tests/test_slack.py
@@ -33,6 +33,7 @@ def test_slack_get_response_for_msg_id(transactional_db):
         platform_msg_id="response-msg",
         saved_run=SavedRun.objects.create(
             state=VideoBotsPage.ResponseModel(
+                raw_tts_text=["hello, world!"],
                 output_text=["hello, world! [2]"],
                 final_prompt="",
                 output_audio=[],

From ef13ddbddb3b459569a7702041d20f8c8bc23c37 Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Tue, 3 Oct 2023 15:42:57 +0530
Subject: [PATCH 32/36] fix img2img request spec better fixture creator

---
 recipes/BulkRunner.py     |  5 ++++-
 recipes/Img2Img.py        |  6 +++---
 scripts/create_fixture.py | 23 +++++++++++++++++++----
 3 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/recipes/BulkRunner.py b/recipes/BulkRunner.py
index 92f265aa5..1bf185fd2 100644
--- a/recipes/BulkRunner.py
+++ b/recipes/BulkRunner.py
@@ -157,12 +157,15 @@ def render_form_v2(self):
             output_columns_old = st.session_state.pop("output_columns", {})
             output_columns_new = st.session_state.setdefault("output_columns", {})
 
+            prev_fields = st.session_state.get("--prev-output-fields")
             fields = {**output_fields, "error_msg": "Error Msg", "run_url": "Run URL"}
+            did_change = prev_fields is not None and prev_fields != fields
+            st.session_state["--prev-output-fields"] = fields
             for field, title in fields.items():
                 col = st.text_input(
                     label="`" + title + "`",
                     key="--output-mapping:" + field,
-                    value=output_columns_old.get(field, title),
+                    value=output_columns_old.get(field, title if did_change else None),
                 )
                 if col:
                     output_columns_new[field] = col
diff --git a/recipes/Img2Img.py b/recipes/Img2Img.py
index 3bc6f169c..8562789b2 100644
--- a/recipes/Img2Img.py
+++ b/recipes/Img2Img.py
@@ -43,9 +43,9 @@ class RequestModel(BaseModel):
         text_prompt: str | None
 
         selected_model: typing.Literal[tuple(e.name for e in Img2ImgModels)] | None
-        selected_controlnet_model: list[
-            typing.Literal[tuple(e.name for e in ControlNetModels)]
-        ] | typing.Literal[tuple(e.name for e in ControlNetModels)] | None
+        selected_controlnet_model: typing.Literal[
+            tuple(e.name for e in ControlNetModels)
+        ] | None
         negative_prompt: str | None
 
         num_outputs: int | None
diff --git a/scripts/create_fixture.py b/scripts/create_fixture.py
index d84495221..9482cef64 100644
--- a/scripts/create_fixture.py
+++ b/scripts/create_fixture.py
@@ -1,9 +1,24 @@
-from django.core.management import call_command
+import sys
+
+from django.core import serializers
 
 from bots.models import SavedRun
 
 
 def run():
-    qs = SavedRun.objects.filter(run_id__isnull=True).values_list("pk", flat=True)
-    pks = ",".join(map(str, qs))
-    call_command("dumpdata", "bots.SavedRun", "--pks", pks, "--output", "fixture.json")
+    qs = SavedRun.objects.filter(run_id__isnull=True)
+    with open("fixture.json", "w") as f:
+        serializers.serialize(
+            "json",
+            get_objects(qs),
+            indent=2,
+            stream=f,
+            progress_output=sys.stdout,
+            object_count=qs.count(),
+        )
+
+
+def get_objects(qs):
+    for obj in qs:
+        obj.parent = None
+        yield obj

From 5e040b1cb723f3bad53100649330a053c96b6719 Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Tue, 3 Oct 2023 15:46:32 +0530
Subject: [PATCH 33/36] fix img2img request spec

---
 recipes/Img2Img.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/recipes/Img2Img.py b/recipes/Img2Img.py
index 8562789b2..3bc6f169c 100644
--- a/recipes/Img2Img.py
+++ b/recipes/Img2Img.py
@@ -43,9 +43,9 @@ class RequestModel(BaseModel):
         text_prompt: str | None
 
         selected_model: typing.Literal[tuple(e.name for e in Img2ImgModels)] | None
-        selected_controlnet_model: typing.Literal[
-            tuple(e.name for e in ControlNetModels)
-        ] | None
+        selected_controlnet_model: list[
+            typing.Literal[tuple(e.name for e in ControlNetModels)]
+        ] | typing.Literal[tuple(e.name for e in ControlNetModels)] | None
         negative_prompt: str | None
 
         num_outputs: int | None

From b85965881198dd15bb1a7f6abdd3dac786f6b279 Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Tue, 3 Oct 2023 15:56:57 +0530
Subject: [PATCH 34/36] fix bulk runner for img2img

---
 recipes/BulkRunner.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/recipes/BulkRunner.py b/recipes/BulkRunner.py
index 1bf185fd2..5d5f9b554 100644
--- a/recipes/BulkRunner.py
+++ b/recipes/BulkRunner.py
@@ -93,7 +93,7 @@ def render_form_v2(self):
                 field_props = schema["properties"][field]
                 title = field_props["title"]
                 keys = None
-                if field_props["type"] == "array":
+                if is_arr(field_props):
                     try:
                         ref = field_props["items"]["$ref"]
                         props = schema["definitions"][ref]["properties"]
@@ -103,7 +103,7 @@ def render_form_v2(self):
                             keys = {k: k for k in sr.state[field][0].keys()}
                         except (KeyError, IndexError, AttributeError):
                             pass
-                elif field_props["type"] == "object":
+                elif field_props.get("type") == "object":
                     try:
                         keys = {k: k for k in sr.state[field].keys()}
                     except (KeyError, AttributeError):
@@ -277,7 +277,7 @@ def build_requests_for_df(df, request, df_ix, arr_len):
         for field, col in request.input_columns.items():
             parts = field.split(".")
             field_props = properties.get(parts[0]) or properties.get(parts)
-            if field_props["type"] == "array":
+            if is_arr(field_props):
                 arr = request_body.setdefault(parts[0], [])
                 for arr_ix in range(arr_len):
                     value = df.at[df_ix + arr_ix, col]
@@ -289,7 +289,7 @@ def build_requests_for_df(df, request, df_ix, arr_len):
                         if len(arr) <= arr_ix:
                             arr.append(None)
                         arr[arr_ix] = value
-            elif len(parts) > 1 and field_props["type"] == "object":
+            elif len(parts) > 1 and field_props.get("type") == "object":
                 obj = request_body.setdefault(parts[0], {})
                 obj[parts[1]] = df.at[df_ix, col]
             else:
@@ -312,7 +312,7 @@ def slice_request_df(df, request):
         properties = schema["properties"]
 
         for field, col in request.input_columns.items():
-            if properties.get(field.split(".")[0])["type"] != "array":
+            if is_arr(properties.get(field.split(".")[0])):
                 non_array_cols.add(col)
     non_array_df = df[list(non_array_cols)]
 
@@ -327,6 +327,16 @@ def slice_request_df(df, request):
         df_ix += arr_len
 
 
+def is_arr(field_props: dict) -> bool:
+    try:
+        return field_props["type"] == "array"
+    except KeyError:
+        for props in field_props.get("anyOf", []):
+            if props["type"] == "array":
+                return True
+    return False
+
+
 def _read_df(f: str) -> "pd.DataFrame":
     import pandas as pd
 

From 319327fb8baad885cac244f1637229ecff6be8d9 Mon Sep 17 00:00:00 2001
From: Kaustubh Maske Patil <37668193+nikochiko@users.noreply.github.com>
Date: Wed, 4 Oct 2023 20:47:59 +0530
Subject: [PATCH 35/36] bugfix: strip qr_code_data *before* checking for
 none/blank

---
 recipes/QRCodeGenerator.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/recipes/QRCodeGenerator.py b/recipes/QRCodeGenerator.py
index 56d907220..be1b9970f 100644
--- a/recipes/QRCodeGenerator.py
+++ b/recipes/QRCodeGenerator.py
@@ -399,9 +399,11 @@ def generate_and_upload_qr_code(
     qr_code_data = request.qr_code_data
     if request.qr_code_input_image:
         qr_code_data = download_qr_code_data(request.qr_code_input_image)
+
+    if isinstance(qr_code_data, str):
+        qr_code_data = qr_code_data.strip()
     if not qr_code_data:
         raise ValueError("Please provide QR Code URL, text content, or an image")
-    qr_code_data = qr_code_data.strip()
 
     shortened = request.use_url_shortener and is_url(qr_code_data)
     if shortened:

From 762965936353132b6dd326e296fe451b553b151f Mon Sep 17 00:00:00 2001
From: Dev Aggarwal <devxpy@gmail.com>
Date: Thu, 5 Oct 2023 19:36:34 +0530
Subject: [PATCH 36/36] refactor asr code

---
 daras_ai_v2/asr.py           | 68 ++++++++++++------------------------
 daras_ai_v2/vector_search.py |  4 ++-
 recipes/DocExtract.py        |  2 +-
 recipes/asr.py               | 26 ++++++--------
 4 files changed, 36 insertions(+), 64 deletions(-)

diff --git a/daras_ai_v2/asr.py b/daras_ai_v2/asr.py
index 9870affe1..1d18757a7 100644
--- a/daras_ai_v2/asr.py
+++ b/daras_ai_v2/asr.py
@@ -126,37 +126,12 @@ def google_translate_languages() -> dict[str, str]:
     }
 
 
-def _get_asr_languages(selected_model: AsrModels) -> set[str]:
-    forced_lang = forced_asr_languages.get(selected_model)
-    if forced_lang:
-        return {forced_lang}
-
-    return asr_supported_languages.get(selected_model, set())
-
-
 def asr_language_selector(
-    selected_model: AsrModels | list[AsrModels],
+    selected_models: list[AsrModels],
     label="##### Spoken Language",
     key="language",
 ):
-    if not isinstance(selected_model, list):
-        selected_model = [selected_model]
-    languages = set.intersection(
-        *[
-            set(
-                map(lambda l: langcodes.Language.get(l).language, _get_asr_languages(m))
-            )
-            for m in selected_model
-        ]
-    )
-
-    if len(languages) < 1:
-        st.session_state[key] = None
-        return
-    elif len(languages) == 1:
-        st.session_state[key] = languages.pop()
-        return
-
+    languages = set.intersection(*map(_get_asr_languages, selected_models))
     options = [None, *languages]
 
     # handle non-canonical language codes
@@ -178,6 +153,15 @@ def asr_language_selector(
     )
 
 
+def _get_asr_languages(selected_model: AsrModels) -> set[str]:
+    forced_lang = forced_asr_languages.get(selected_model)
+    if forced_lang:
+        languages = {forced_lang}
+    else:
+        languages = asr_supported_languages.get(selected_model, set())
+    return {langcodes.Language.get(lang).language for lang in languages}
+
+
 def run_google_translate(
     texts: list[str],
     target_language: str,
@@ -284,25 +268,22 @@ def get_google_auth_session():
 
 def run_asr(
     audio_url: str,
-    selected_model: str | list[str],
+    selected_models: list[str],
     language: str = None,
     output_format: str = "text",
-) -> str | AsrOutputJson | list[str | AsrOutputJson]:
+) -> list[str] | list[AsrOutputJson]:
     """
     Run ASR on audio.
     Args:
         audio_url (str): url of audio to be transcribed.
-        selected_model (str): ASR model(s) to use.
+        selected_models (str): ASR model(s) to use.
         language: language of the audio
         output_format: format of the output
     Returns:
         str: Transcribed text.
     """
-
-    if not isinstance(selected_model, list):
-        selected_model = [selected_model]
-    selected_models = [AsrModels[m] for m in selected_model]
-    output_format: AsrOutputFormat = AsrOutputFormat[output_format]
+    selected_models = [AsrModels[m] for m in selected_models]
+    output_format = AsrOutputFormat[output_format]
     is_youtube_url = "youtube" in audio_url or "youtu.be" in audio_url
     if is_youtube_url:
         audio_url, size = download_youtube_to_wav(audio_url)
@@ -310,22 +291,17 @@ def run_asr(
         audio_url, size = audio_url_to_wav(audio_url)
     is_short = size < SHORT_FILE_CUTOFF
 
-    outputs = map_parallel(
+    return map_parallel(
         lambda model: _run_asr_one_model(
-            model,
-            output_format,
-            audio_url,
-            language,
-            is_short,
+            selected_model=model,
+            output_format=output_format,
+            audio_url=audio_url,
+            language=language,
+            is_short=is_short,
         ),
         selected_models,
     )
 
-    if len(outputs) == 1:
-        return outputs[0]
-    else:
-        return outputs
-
 
 def _run_asr_one_model(
     selected_model: AsrModels,
diff --git a/daras_ai_v2/vector_search.py b/daras_ai_v2/vector_search.py
index bf6f6e61c..76668717a 100644
--- a/daras_ai_v2/vector_search.py
+++ b/daras_ai_v2/vector_search.py
@@ -482,7 +482,9 @@ def doc_url_to_text_pages(
                 f_url = upload_file_from_bytes(
                     f_name, f_bytes, content_type=doc_meta.mime_type
                 )
-            pages = [run_asr(f_url, selected_model=selected_asr_model, language="en")]
+            pages = [
+                run_asr(f_url, [selected_asr_model], language="en")[0],
+            ]
         case ".csv" | ".xlsx" | ".tsv" | ".ods":
             import pandas as pd
 
diff --git a/recipes/DocExtract.py b/recipes/DocExtract.py
index e8c1c97ce..cf147feba 100644
--- a/recipes/DocExtract.py
+++ b/recipes/DocExtract.py
@@ -382,7 +382,7 @@ def process_source(
             or "audio/" in doc_meta.mime_type
         ):
             yield "Transcribing"
-            transcript = run_asr(content_url, request.selected_asr_model)
+            transcript = run_asr(content_url, [request.selected_asr_model])[0]
         elif "application/pdf" in doc_meta.mime_type:
             yield "Extracting PDF"
             transcript = str(azure_doc_extract_pages(content_url)[0])
diff --git a/recipes/asr.py b/recipes/asr.py
index 289ba91e8..5bd7b88f0 100644
--- a/recipes/asr.py
+++ b/recipes/asr.py
@@ -19,7 +19,7 @@
     document_uploader,
 )
 from daras_ai_v2.enum_selector_widget import enum_selector, enum_multiselect
-from daras_ai_v2.functional import map_parallel
+from daras_ai_v2.functional import flatmap_parallel
 from daras_ai_v2.text_output_widget import text_outputs
 from recipes.DocSearch import render_documents
 
@@ -44,7 +44,7 @@ class RequestModel(BaseModel):
 
     class ResponseModel(BaseModel):
         raw_output_text: list[str] | None
-        output_text: list[str | AsrOutputJson]
+        output_text: list[str] | list[AsrOutputJson]
 
     def preview_image(self, state: dict) -> str | None:
         return DEFAULT_ASR_META_IMG
@@ -85,10 +85,9 @@ def render_form_v2(self):
         )
         col1, col2 = st.columns(2, responsive=False)
         with col1:
-            if not isinstance(st.session_state.get("selected_model"), list):
-                st.session_state["selected_model"] = [
-                    st.session_state["selected_model"]
-                ]
+            selected_model = st.session_state.get("selected_model")
+            if isinstance(selected_model, str):
+                st.session_state["selected_model"] = [selected_model]
             selected_model = enum_multiselect(
                 AsrModels,
                 label="##### ASR Models",
@@ -142,26 +141,21 @@ def run(self, state: dict):
         request: AsrPage.RequestModel = self.RequestModel.parse_obj(state)
 
         # Run ASR
-        selected_models: list[str] = request.selected_model or [
-            AsrModels.whisper_large_v2.name
-        ]
-        if not isinstance(selected_models, list):
+        selected_models = request.selected_model
+        if isinstance(selected_models, str):
             selected_models = [selected_models]
         yield f"Running {', '.join([AsrModels[m].value for m in selected_models])}..."
-        asr_output = map_parallel(
+        asr_output = flatmap_parallel(
             lambda audio: run_asr(
                 audio_url=audio,
-                selected_model=selected_models,
+                selected_models=selected_models,
                 language=request.language,
                 output_format=request.output_format,
             ),
             request.documents,
         )
-        if len(selected_models) != 1:
-            # flatten
-            asr_output = [out for model_out in asr_output for out in model_out]
         str_asr_output: list[str] = [
-            out if not isinstance(out, AsrModels) else out.get("text", "").strip()
+            out.get("text", "").strip() if isinstance(out, dict) else out
             for out in asr_output
         ]