Merge branch 'main' into cli-doc

.github/workflows/test.yml

@@ -6,7 +6,7 @@ on:
       - ".github/**"
       - "ragna/**"
       - "tests/**"
-      - "environment.yaml"
+      - "environment.yml"
       - "pyproject.toml"
 
 jobs:

examples/local_llm/local_llm.ipynb

@@ -127,7 +127,7 @@
      "output_type": "stream",
      "text": [
       "User: What is Ragna?\n",
-      "Assistant: Ragna is an OSS app for RAG workflows that offers a Python and REST API as well as web UI.\n"
+      "Assistant: Ragna is an open-source application (OSS) for RAG workflows. It offers a Python and REST API as well as a web UI.\n"
      ]
     }
    ],

examples/python_api/python_api.ipynb

@@ -21,7 +21,7 @@
     {
      "data": {
       "text/plain": [
-       "'0.1.dev21+g7f8ec2d.d20230925071852'"
+       "'0.1.dev29+g03721c7.d20231009084220'"
       ]
      },
      "execution_count": 2,
@@ -44,7 +44,7 @@
     {
      "data": {
       "text/plain": [
-       "Config(local_cache_root=PosixPath('/home/philip/.cache/ragna'), state_database_url='sqlite://', queue_database_url='memory', ragna_api_url='http://127.0.0.1:31476', ragna_ui_url='http://127.0.0.1:31477', document_class=<class 'ragna.core.LocalDocument'>, upload_token_secret='245be1b4c5656eefec1ac16d4a856f189e9e35d35aa014eb7426573e5b86c03f', upload_token_ttl=30, registered_source_storage_classes={'Ragna/DemoSourceStorage': <class 'ragna.source_storage._demo.RagnaDemoSourceStorage'>}, registered_assistant_classes={'Ragna/DemoAssistant': <class 'ragna.assistant._demo.RagnaDemoAssistant'>})"
+       "Config(local_cache_root=PosixPath('/home/philip/.cache/ragna'), state_database_url='sqlite://', queue_database_url='memory', ragna_api_url='http://127.0.0.1:31476', ragna_ui_url='http://127.0.0.1:31477', document_class=<class 'ragna.core.LocalDocument'>, upload_token_secret='a9e79b896f90c136820e70ced28f30e6cbbaca6d19f52e41c35b154df21a657a', upload_token_ttl=30, registered_source_storage_classes={'Ragna/DemoSourceStorage': <class 'ragna.source_storage._demo.RagnaDemoSourceStorage'>}, registered_assistant_classes={'Ragna/DemoAssistant': <class 'ragna.assistant._demo.RagnaDemoAssistant'>})"
       ]
      },
      "execution_count": 3,
@@ -98,7 +98,11 @@
     "    OpenaiGpt35Turbo16kAssistant,\n",
     "    OpenaiGpt4Assistant,\n",
     ")\n",
-    "from ragna.source_storage import ChromaSourceStorage, RagnaDemoSourceStorage\n",
+    "from ragna.source_storage import (\n",
+    "    ChromaSourceStorage,\n",
+    "    RagnaDemoSourceStorage,\n",
+    "    LanceDBSourceStorage,\n",
+    ")\n",
     "\n",
     "rag = Rag(demo_config)\n",
     "\n",
@@ -123,8 +127,10 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{('Chroma', 'OpenAI/gpt-3.5-turbo-16k'): <coroutine object answer_prompt at 0x7f73bfd15f40>,\n",
-      " ('Chroma', 'OpenAI/gpt-4'): <coroutine object answer_prompt at 0x7f740774a040>}\n"
+      "{('Chroma', 'OpenAI/gpt-3.5-turbo-16k'): <coroutine object answer_prompt at 0x7fc40c89c640>,\n",
+      " ('Chroma', 'OpenAI/gpt-4'): <coroutine object answer_prompt at 0x7fc40c77f4c0>,\n",
+      " ('LanceDB', 'OpenAI/gpt-3.5-turbo-16k'): <coroutine object answer_prompt at 0x7fc40c77f840>,\n",
+      " ('LanceDB', 'OpenAI/gpt-4'): <coroutine object answer_prompt at 0x7fc40c77f9c0>}\n"
      ]
     }
    ],
@@ -133,7 +139,7 @@
     "import asyncio\n",
     "from pprint import pprint\n",
     "\n",
-    "source_storages = [ChromaSourceStorage]\n",
+    "source_storages = [ChromaSourceStorage, LanceDBSourceStorage]\n",
     "assistants = [OpenaiGpt35Turbo16kAssistant, OpenaiGpt4Assistant]\n",
     "\n",
     "\n",
@@ -167,8 +173,22 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
+      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
+      "To disable this warning, you can either:\n",
+      "\t- Avoid using `tokenizers` before the fork if possible\n",
+      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
+      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
+      "To disable this warning, you can either:\n",
+      "\t- Avoid using `tokenizers` before the fork if possible\n",
+      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
+      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
+      "To disable this warning, you can either:\n",
+      "\t- Avoid using `tokenizers` before the fork if possible\n",
+      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
       "{('Chroma', 'OpenAI/gpt-3.5-turbo-16k'): Ragna is an open-source RAG (Response Analysis Graph) orchestration app. It is designed to help users create conversational AI applications by providing a framework for managing and orchestrating the flow of conversations. Ragna allows developers to define conversation flows, handle user inputs, and generate dynamic responses based on predefined rules and logic. It is built on top of the Rasa framework and provides additional features and functionalities to simplify the development process.,\n",
-      " ('Chroma', 'OpenAI/gpt-4'): Ragna is an open-source RAG orchestration app.}\n"
+      " ('Chroma', 'OpenAI/gpt-4'): Ragna is an open-source RAG orchestration app.,\n",
+      " ('LanceDB', 'OpenAI/gpt-3.5-turbo-16k'): Ragna is an open-source rag orchestration app. It is a software application that allows users to create and arrange musical compositions using ragtime music. It is designed to be accessible and customizable for musicians and composers.,\n",
+      " ('LanceDB', 'OpenAI/gpt-4'): Ragna is an open-source rag orchestration app.}\n"
      ]
     }
    ],