Merge remote-tracking branch 'origin/master' into glossary_new

.env.example

@@ -1,3 +1,8 @@
 APP_BASE_URL=http://localhost:3000
 API_BASE_URL=http://localhost:8080
 GS_BUCKET_NAME=dara-c1b52.appspot.com
+PGHOST=127.0.0.1
+PGPORT=5432
+PGUSER=gooey
+PGDATABASE=gooey
+PGPASSWORD=gooey

Procfile

@@ -0,0 +1,22 @@
+# this Procfile can be run with `honcho`, and it can start multiple processes
+# with a single command. Handy for development. All of the below commands are
+# setup to run in dev mode only, and not in prod.
+#
+# The assumptions here are that:
+# - you have redis installed but not running as a background service
+# - you have rabbitmq installed but not running as a background service
+# - your local gooey-ui repo is at ../gooey-ui/
+#
+# You can comment any of the processes if you have background services running
+# for them. You can also change the path for the `ui` process from `../gooey-ui/`
+# to wherever your local gooey-ui directory is.
+
+api: poetry run uvicorn server:app --host 127.0.0.1 --port 8080 --reload
+
+admin: poetry run python manage.py runserver 127.0.0.1:8000
+
+dashboard: poetry run streamlit run Home.py --server.port 8501 --server.headless true
+
+celery: poetry run celery -A celeryapp worker
+
+ui: cd ../gooey-ui/; PORT=3000 npm run dev

README.md

@@ -1,47 +1,49 @@
 ## Setup
+
 * Install [pyenv](https://github.com/pyenv/pyenv) & install the same python version as in our [Dockerfile](Dockerfile)
 * Install [poetry](https://python-poetry.org/docs/)
-* Create & active a virtualenv (e.g. `poetry shell`)
+* Create & activate a virtualenv (e.g. `poetry shell`)
 * Run `poetry install --with dev`
+* Install [redis](https://redis.io/docs/getting-started/installation/install-redis-on-mac-os/), [rabbitmq](https://www.rabbitmq.com/install-homebrew.html), and [postgresql](https://formulae.brew.sh/formula/postgresql@15) (e.g. `brew install redis rabbitmq postgresql@15`)
+* Enable background services for `redis`, `rabbitmq`, and `postgresql` (e.g. with `brew services start redis` and similar for `rabbitmq` and `postgresql`)
+* Use `sqlcreate` helper to create a user and database for gooey:
+  * `./manage.py sqlcreate | psql postgres`
+  * make sure you are able to access the database with `psql -W -U gooey gooey` (and when prompted for password, entering `gooey`)
 * Create an `.env` file from `.env.example` (Read [12factor.net/config](https://12factor.net/config))
 * Run `./manage.py migrate`
+* Install the zbar library (`brew install zbar`)
 
 ## Run
 
-### API + GUI server
-
-```bash
-uvicorn server:app --host 0.0.0.0 --port 8080 --reload
-```
-
-Open [localhost:8080](localhost:8080) in your browser
-
-### Admin Site
+You can start all required processes in one command with Honcho:
 
-```bash
-python3 manage.py runserver 0.0.0.0:8000
+```shell
+$ poetry run honcho start
 ```
 
-Open [localhost:8000](localhost:8000) in your browser
-
+The processes that it starts are defined in [`Procfile`](Procfile).
+Currently they are these:
 
-### Usage Dashboard
+| Service          | Port |
+| -------          | ---- |
+| API + GUI Server | 8080 |
+| Admin site       | 8000 |
+| Usage dashboard  | 8501 |
+| Celery           | -    |
+| UI               | 3000 |
 
-```
-streamlit run Home.py --server.port 8501
-```
+This default startup assumes that Redis, RabbitMQ, and PostgreSQL are installed and running
+as background services on ports 6379, 5672, and 5432 respectively. 
+It also assumes that the gooey-ui repo can be found at `../gooey-ui/` (adjacent to where the
+gooey-server repo sits). You can open the Procfile and comment this out if you don't need
+to run it.
 
-Open [localhost:8501](localhost:8501) in your browser
+**Note:** the Celery worker must be manually restarted on code changes. You
+can do this by stopping and starting Honcho.
 
 ## To run any recipe 
 
 * Save `serviceAccountKey.json` to project root
-* Install & start [redis](https://redis.io/docs/getting-started/installation/install-redis-on-mac-os/)
-* Install & start [rabbitmq](https://www.rabbitmq.com/install-homebrew.html)
-* Run the celery worker (**Note:** you must manually restart it on code changes)
-```bash
-celery -A celeryapp worker
-```
 
 ## To connect to our GPU cluster 
 

bots/admin.py

@@ -26,6 +26,7 @@
     Conversation,
     BotIntegration,
 )
+from bots.tasks import create_personal_channels_for_all_members
 from gooeysite.custom_widgets import JSONEditorWidget
 
 
@@ -84,16 +85,34 @@ class Media:
         ]
 
 
+def create_personal_channels(modeladmin, request, queryset):
+    for bi in queryset:
+        create_personal_channels_for_all_members.delay(bi.id)
+    modeladmin.message_user(
+        request,
+        f"Started creating personal channels for {queryset.count()} bots in the background.",
+    )
+
+
 @admin.register(BotIntegration)
 class BotIntegrationAdmin(admin.ModelAdmin):
     search_fields = [
         "name",
+        "billing_account_uid",
+        "user_language",
+        "fb_page_id",
+        "fb_page_name",
+        "fb_page_access_token",
         "ig_account_id",
         "ig_username",
-        "fb_page_id",
         "wa_phone_number",
+        "wa_phone_number_id",
+        "slack_team_id",
+        "slack_team_name",
         "slack_channel_id",
-        "billing_account_uid",
+        "slack_channel_name",
+        "slack_channel_hook_url",
+        "slack_access_token",
     ]
     list_display = [
         "name",
@@ -181,6 +200,8 @@ class BotIntegrationAdmin(admin.ModelAdmin):
         ),
     ]
 
+    actions = [create_personal_channels]
+
     @admin.display(description="Messages")
     def view_messsages(self, bi: BotIntegration):
         return list_related_html_url(
@@ -343,7 +364,17 @@ class ConversationAdmin(admin.ModelAdmin):
     list_filter = ["bot_integration", "created_at", LastActiveDeltaFilter]
     autocomplete_fields = ["bot_integration"]
     search_fields = [
+        "fb_page_id",
+        "fb_page_name",
+        "fb_page_access_token",
+        "ig_account_id",
+        "ig_username",
         "wa_phone_number",
+        "slack_user_id",
+        "slack_team_id",
+        "slack_user_name",
+        "slack_channel_id",
+        "slack_channel_name",
     ] + [f"bot_integration__{field}" for field in BotIntegrationAdmin.search_fields]
     actions = [export_to_csv, export_to_excel]
 
@@ -395,10 +426,11 @@ class MessageAdmin(admin.ModelAdmin):
         "created_at",
     ]
     search_fields = [
-        "analysis_result",
         "role",
         "content",
         "display_content",
+        "platform_msg_id",
+        "analysis_result",
     ] + [f"conversation__{field}" for field in ConversationAdmin.search_fields]
     list_display = [
         "__str__",

bots/migrations/0042_alter_message_platform_msg_id.py

@@ -0,0 +1,41 @@
+# Generated by Django 4.2.3 on 2023-09-19 20:51
+
+from django.db import migrations, models
+
+
+def forwards_func(apps, schema_editor):
+    Message = apps.get_model("bots", "Message")
+    db_alias = schema_editor.connection.alias
+    objects = Message.objects.using(db_alias)
+    objects.filter(platform_msg_id="").update(platform_msg_id=None)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("bots", "0041_alter_botintegration_slack_create_personal_channels"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="message",
+            name="platform_msg_id",
+            field=models.TextField(
+                blank=True,
+                default=None,
+                help_text="The platform's delivered message id",
+                null=True,
+            ),
+        ),
+        migrations.RunPython(forwards_func),
+        migrations.AlterField(
+            model_name="message",
+            name="platform_msg_id",
+            field=models.TextField(
+                blank=True,
+                default=None,
+                help_text="The platform's delivered message id",
+                null=True,
+                unique=True,
+            ),
+        ),
+    ]

bots/migrations/0043_alter_savedrun_workflow.py

@@ -0,0 +1,50 @@
+# Generated by Django 4.2.5 on 2023-09-20 09:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("bots", "0042_alter_message_platform_msg_id"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="savedrun",
+            name="workflow",
+            field=models.IntegerField(
+                choices=[
+                    (1, "Doc Search"),
+                    (2, "Doc Summary"),
+                    (3, "Google GPT"),
+                    (4, "Copilot"),
+                    (5, "Lipysnc + TTS"),
+                    (6, "Text to Speech"),
+                    (7, "Speech Recognition"),
+                    (8, "Lipsync"),
+                    (9, "Deforum Animation"),
+                    (10, "Compare Text2Img"),
+                    (11, "Text2Audio"),
+                    (12, "Img2Img"),
+                    (13, "Face Inpainting"),
+                    (14, "Google Image Gen"),
+                    (15, "Compare AI Upscalers"),
+                    (16, "SEO Summary"),
+                    (17, "Email Face Inpainting"),
+                    (18, "Social Lookup Email"),
+                    (19, "Object Inpainting"),
+                    (20, "Image Segmentation"),
+                    (21, "Compare LLM"),
+                    (22, "Chyron Plant"),
+                    (23, "Letter Writer"),
+                    (24, "Smart GPT"),
+                    (25, "AI QR Code"),
+                    (26, "Doc Extract"),
+                    (27, "Related QnA Maker"),
+                    (28, "Related QnA Maker Doc"),
+                    (29, "Embeddings"),
+                ],
+                default=4,
+            ),
+        ),
+    ]

bots/models.py

@@ -53,7 +53,7 @@ class Workflow(models.IntegerChoices):
     IMG_2_IMG = (12, "Img2Img")
     FACE_INPAINTING = (13, "Face Inpainting")
     GOOGLE_IMAGE_GEN = (14, "Google Image Gen")
-    COMPARE_UPSCALER = (15, "Comapre AI Upscalers")
+    COMPARE_UPSCALER = (15, "Compare AI Upscalers")
     SEO_SUMMARY = (16, "SEO Summary")
     EMAIL_FACE_INPAINTING = (17, "Email Face Inpainting")
     SOCIAL_LOOKUP_EMAIL = (18, "Social Lookup Email")
@@ -704,7 +704,9 @@ class Message(models.Model):
 
     platform_msg_id = models.TextField(
         blank=True,
-        default="",
+        null=True,
+        default=None,
+        unique=True,
         help_text="The platform's delivered message id",
     )
 

daras_ai_v2/base.py

@@ -167,7 +167,7 @@ def render(self):
 
         root_url = self.app_url(example_id=example_id)
         st.write(
-            f'## <a style="text-decoration: none;" target="_top" href="{root_url}">{st.session_state.get(StateKeys.page_title)}</a>',
+            f'# <a style="text-decoration: none;" target="_top" href="{root_url}">{st.session_state.get(StateKeys.page_title)}</a>',
             unsafe_allow_html=True,
         )
         st.write(st.session_state.get(StateKeys.page_notes))

daras_ai_v2/bots.py

@@ -69,7 +69,8 @@ def nice_filename(self, mime_type: str) -> str:
         ext = mimetypes.guess_extension(mime_type) or ""
         return f"{self.platform}_{self.input_type}_from_{self.user_id}_to_{self.bot_id}{ext}"
 
-    def _unpack_bot_integration(self, bi: BotIntegration):
+    def _unpack_bot_integration(self):
+        bi = self.convo.bot_integration
         if bi.saved_run:
             self.page_cls = Workflow(bi.saved_run.workflow).page_cls
             self.query_params = self.page_cls.clean_query_params(
@@ -156,8 +157,9 @@ def _on_msg(bot: BotInterface):
     if not bot.page_cls:
         bot.send_msg(text=PAGE_NOT_CONNECTED_ERROR)
         return
-    # mark message as read
-    bot.mark_read()
+    if bot.input_type != "interactive":
+        # mark message as read
+        bot.mark_read()
     # get the attached billing account
     billing_account_user = AppUser.objects.get_or_create_from_uid(
         bot.billing_account_uid
@@ -305,6 +307,9 @@ def _handle_interactive_msg(bot: BotInterface):
             try:
                 context_msg = Message.objects.get(platform_msg_id=context_msg_id)
             except Message.DoesNotExist as e:
+                traceback.print_exc()
+                capture_exception(e)
+                # send error msg as repsonse
                 bot.send_msg(text=ERROR_MSG.format(e))
                 return
             if button_id == ButtonIds.feedback_thumbs_up:

daras_ai_v2/facebook_bots.py

@@ -35,7 +35,7 @@ def __init__(self, message: dict, metadata: dict):
             bot_integration=bi,
             wa_phone_number="+" + self.user_id,
         )[0]
-        self._unpack_bot_integration(bi)
+        self._unpack_bot_integration()
 
     def get_input_text(self) -> str | None:
         try:
@@ -325,7 +325,7 @@ def __init__(self, object_name: str, messaging: dict):
                 fb_page_id=self.user_id,
                 ig_account_id=self.user_id,
             )[0]
-        self._unpack_bot_integration(bi)
+        self._unpack_bot_integration()
         self.bot_id = bi.fb_page_id
 
         self._access_token = bi.fb_page_access_token