Add support for a web crawler (superagent-ai#419)

* Add support for a web crawler * Small tweaks
mgunnin · Sep 25, 2023 · 7cccd1f · 7cccd1f
1 parent 459cf1d
commit 7cccd1f
Show file tree

Hide file tree

Showing 4 changed files with 32 additions and 9 deletions.
diff --git a/libs/superagent/app/datasource/loader.py b/libs/superagent/app/datasource/loader.py
@@ -5,10 +5,12 @@
 from urllib.parse import urlparse
 
 import requests
+from bs4 import BeautifulSoup as Soup
 from langchain.docstore.document import Document
 from langchain.document_loaders import (
     GitLoader,
     PyPDFLoader,
+    RecursiveUrlLoader,
     TextLoader,
     UnstructuredMarkdownLoader,
     UnstructuredWordDocumentLoader,
@@ -120,9 +122,16 @@ def load_github(self):
             return loader.load_and_split()
 
     def load_webpage(self):
-        RemoteDepthReader = download_loader("RemoteDepthReader")
-        loader = RemoteDepthReader(depth=0)
-        return loader.load_langchain_documents(url=self.datasource.url)
+        loader = RecursiveUrlLoader(
+            url=self.datasource.url,
+            max_depth=2,
+            extractor=lambda x: Soup(x, "html.parser").text,
+        )
+        chunks = loader.load_and_split()
+        for chunk in chunks:
+            if "language" in chunk.metadata:
+                del chunk.metadata["language"]
+        return chunks
 
     def load_notion(self):
         metadata = json.loads(self.datasource.metadata)

diff --git a/libs/superagent/poetry.lock b/libs/superagent/poetry.lock
diff --git a/libs/superagent/pyproject.toml b/libs/superagent/pyproject.toml
@@ -46,6 +46,7 @@ langsmith = "^0.0.39"
 requests = "^2.31.0"
 wolframalpha = "^5.0.0"
 langchain = "^0.0.300"
+bs4 = "^0.0.1"
 
 
 [build-system]

diff --git a/libs/ui/app/datasources/data-table.tsx b/libs/ui/app/datasources/data-table.tsx
@@ -280,10 +280,10 @@ export function DataTable<TData, TValue>({
                           size="sm"
                           onClick={() => {
                             setSelectedType("webpage")
-                            form.setValue("type", "URL")
+                            form.setValue("type", "WEBPAGE")
                           }}
                         >
-                          Add URLs
+                          Add webpage
                         </Button>
                       </Alert>
                       <Alert className="flex items-center justify-between">
@@ -355,10 +355,10 @@ export function DataTable<TData, TValue>({
                       name="url"
                       render={({ field }) => (
                         <FormItem>
-                          <FormLabel>URLs</FormLabel>
+                          <FormLabel>URL</FormLabel>
                           <FormControl>
-                            <Textarea
-                              placeholder="Comma separeted list of URLs."
+                            <Input
+                              placeholder="E.g https://www.superagent.sh"
                               {...field}
                             />
                           </FormControl>