Feat: Hivemind include url for documents

dags/hivemind_etl_helpers/src/db/discord/utils/transform_discord_raw_messges.py

@@ -95,6 +95,10 @@ def prepare_document(
     reactions = message["reactions"]
     raw_content = message["content"]
 
+    message_id = message["messageId"]
+    channel_id = message["channelId"]
+    thread_id = message["threadId"]
+
     reaction_ids = prepare_raction_ids(reactions)
 
     mention_names: list[str]
@@ -161,10 +165,16 @@ def prepare_document(
     # always has length 1
     assert len(author_name) == 1, "Either None or multiple authors!"
 
+    if thread_id is None:
+        url = f"https://discord.com/channels/{guild_id}/{channel_id}/{message_id}"
+    else:
+        url = f"https://discord.com/channels/{guild_id}/{thread_id}/{message_id}"
+
     msg_meta_data = {
         "channel": message["channelName"],
         "date": message["createdDate"].strftime(DATE_FORMAT),
         "author_username": author_name[0],
+        "url": url,
         # always including the thread_name, if `None`, then it was a channel message
         "thread": message["threadName"],
     }
@@ -234,6 +244,7 @@ def prepare_document(
             "replier_global_name",
             "replier_nickname",
             "role_mentions",
+            "url",
         ]
         doc.excluded_llm_metadata_keys = [
             "author_nickname",
@@ -250,6 +261,7 @@ def prepare_document(
             "replier_global_name",
             "replier_nickname",
             "role_mentions",
+            "url",
         ]
     else:
         doc = Document(text=content_url_updated)

dags/hivemind_etl_helpers/src/db/github/extract/commit.py

@@ -78,7 +78,7 @@ def _fetch_raw_commits(
                 user_commiter.login AS committer_name,
                 co.`commit.message` AS message,
                 co.`commit.url` AS api_url,
-                co.`parents.0.html_url` AS html_url,
+                co.`parents.0.html_url` AS url,
                 co.repository_id AS repository_id,
                 repo.full_name AS repository_name,
                 co.sha AS sha,

dags/hivemind_etl_helpers/src/db/github/schema/commit.py

@@ -8,7 +8,7 @@ def __init__(
         committer_name: str,
         message: str,
         api_url: str,
-        html_url: str,
+        url: str,
         repository_id: int,
         repository_name: str,
         sha: str,
@@ -26,7 +26,7 @@ def __init__(
         self.committer_name = committer_name
         self.message = message
         self.api_url = api_url
-        self.html_url = html_url
+        self.url = url
         self.repository_id = repository_id
         self.repository_name = repository_name
         self.sha = sha
@@ -42,7 +42,7 @@ def from_dict(cls, data: dict[str, str | int | None]) -> "GitHubCommit":
             committer_name=data["committer_name"],  # type: ignore
             message=data["message"],  # type: ignore
             api_url=data["api_url"],  # type: ignore
-            html_url=data["html_url"],  # type: ignore
+            url=data["url"],  # type: ignore
             repository_id=data["repository_id"],  # type: ignore
             repository_name=data["repository_name"],  # type: ignore
             sha=data["sha"],  # type: ignore
@@ -58,7 +58,7 @@ def to_dict(self) -> dict[str, str | int | None]:
             "committer_name": self.committer_name,
             "message": self.message,
             "api_url": self.api_url,
-            "html_url": self.html_url,
+            "url": self.url,
             "repository_id": self.repository_id,
             "repository_name": self.repository_name,
             "sha": self.sha,

dags/hivemind_etl_helpers/src/db/github/transform/commits.py

@@ -27,7 +27,7 @@ def transform_commits(data: list[GitHubCommit]) -> list[Document]:
             metadata=metadata,
             # all metadata to be excluded from embedding model
             excluded_embed_metadata_keys=list(metadata.keys()),
-            excluded_llm_metadata_keys=["sha", "api_url", "html_url", "verification"],
+            excluded_llm_metadata_keys=["sha", "api_url", "url", "verification"],
         )
         transformed_commits.append(document)
 

dags/hivemind_etl_helpers/tests/integration/test_discord_prepare_llama.py

@@ -168,6 +168,7 @@ def test_transform_two_data(self):
             "author_username": "user1",
             "author_global_name": "user1_GlobalName",
             "author_nickname": "user1_nickname",
+            "url": f"https://discord.com/channels/{guild_id}/1313130/1111111110",
             "thread": None,
         }
 
@@ -180,6 +181,7 @@ def test_transform_two_data(self):
             "mention_global_names": ["user3_GlobalName", "user4_GlobalName"],
             "replier_username": "user4",
             "replier_global_name": "user4_GlobalName",
+            "url": f"https://discord.com/channels/{guild_id}/1313131/1111111111",
             "thread": None,
         }
 
@@ -192,6 +194,7 @@ def test_transform_two_data(self):
             "mention_global_names": ["user3_GlobalName", "user4_GlobalName"],
             "replier_username": "user4",
             "replier_global_name": "user4_GlobalName",
+            "url": f"https://discord.com/channels/{guild_id}/88888/1111111112",
             "thread": "example_thread1",
             "role_mentions": ["role1"],
         }
@@ -203,6 +206,7 @@ def test_transform_two_data(self):
             "author_global_name": "user1_GlobalName",
             "author_nickname": "user1_nickname",
             "url_reference": {"[URL0]": "https://www.google.com"},
+            "url": f"https://discord.com/channels/{guild_id}/1313133/1111111113",
             "thread": None,
         }
 

dags/hivemind_etl_helpers/tests/integration/test_github_etl_fetch_commits.py

@@ -62,7 +62,7 @@ def test_get_single_commit_single_repo_no_from_date(self):
         self.assertEqual(commits[0].committer_name, "author #1")
         self.assertEqual(commits[0].message, "Issue #1 is resolved!")
         self.assertEqual(commits[0].api_url, "https://api.sample_url_for_commit.html")
-        self.assertEqual(commits[0].html_url, "https://sample_url_for_commit.html")
+        self.assertEqual(commits[0].url, "https://sample_url_for_commit.html")
         self.assertEqual(commits[0].repository_id, 123)
         self.assertEqual(commits[0].repository_name, "Org/SampleRepo")
         self.assertEqual(commits[0].sha, "sha#1111")
@@ -105,7 +105,7 @@ def test_get_single_commit_single_repo_no_from_date_no_commiter(self):
         self.assertEqual(commits[0].committer_name, None)
         self.assertEqual(commits[0].message, "Issue #1 is resolved!")
         self.assertEqual(commits[0].api_url, "https://api.sample_url_for_commit.html")
-        self.assertEqual(commits[0].html_url, "https://sample_url_for_commit.html")
+        self.assertEqual(commits[0].url, "https://sample_url_for_commit.html")
         self.assertEqual(commits[0].repository_id, 123)
         self.assertEqual(commits[0].repository_name, "Org/SampleRepo")
         self.assertEqual(commits[0].sha, "sha#1111")
@@ -150,7 +150,7 @@ def test_get_single_commit_single_repo_with_from_date(self):
         self.assertEqual(commits[0].author_name, "author #2")
         self.assertEqual(commits[0].message, "Issue #1 is resolved!")
         self.assertEqual(commits[0].api_url, "https://api.sample_url_for_commit.html")
-        self.assertEqual(commits[0].html_url, "https://sample_url_for_commit.html")
+        self.assertEqual(commits[0].url, "https://sample_url_for_commit.html")
         self.assertEqual(commits[0].repository_id, 123)
         self.assertEqual(commits[0].repository_name, "Org/SampleRepo2")
         self.assertEqual(commits[0].sha, "sha#1111")
@@ -228,7 +228,7 @@ def test_get_multiple_commit_multi_repo_with_from_date_filter(self):
         self.assertEqual(commits[0].committer_name, "author #1")
         self.assertEqual(commits[0].message, "Issue #1 is resolved!")
         self.assertEqual(commits[0].api_url, "https://api.sample_url_for_commit.html")
-        self.assertEqual(commits[0].html_url, "https://sample_url_for_commit.html")
+        self.assertEqual(commits[0].url, "https://sample_url_for_commit.html")
         self.assertEqual(commits[0].repository_id, 123)
         self.assertEqual(commits[0].repository_name, "Org/SampleRepo2")
         self.assertEqual(commits[0].sha, "sha#1111")
@@ -272,7 +272,7 @@ def test_fetch_commit_no_related_pr(self):
         self.assertEqual(commits[0].committer_name, None)
         self.assertEqual(commits[0].message, "Issue #1 is resolved!")
         self.assertEqual(commits[0].api_url, "https://api.sample_url_for_commit.html")
-        self.assertEqual(commits[0].html_url, "https://sample_url_for_commit.html")
+        self.assertEqual(commits[0].url, "https://sample_url_for_commit.html")
         self.assertEqual(commits[0].repository_id, 123)
         self.assertEqual(commits[0].repository_name, "Org/SampleRepo")
         self.assertEqual(commits[0].sha, "sha#1111")

dags/hivemind_etl_helpers/tests/integration/test_github_transformation.py

@@ -83,7 +83,7 @@ def test_commit_transformation(self):
                 sha="sha#1000000",
                 created_at=datetime(2023, 11, 1).strftime("%Y-%m-%dT%H:%M:%SZ"),
                 latest_saved_at=datetime(2023, 12, 1, 1).strftime("%Y-%m-%dT%H:%M:%SZ"),
-                html_url="https://github.com/repo/commit/1",
+                url="https://github.com/repo/commit/1",
                 api_url="https://api.github.com/repo/commit/1",
                 repository_id=123,
                 repository_name="SampleRepo",
@@ -104,7 +104,7 @@ def test_commit_transformation(self):
                 "sha": "sha#1000000",
                 "created_at": "2023-11-01 00:00:00",
                 "latest_saved_at": "2023-12-01 01:00:00",
-                "html_url": "https://github.com/repo/commit/1",
+                "url": "https://github.com/repo/commit/1",
                 "api_url": "https://api.github.com/repo/commit/1",
                 "repository_id": 123,
                 "repository_name": "SampleRepo",

dags/hivemind_etl_helpers/tests/unit/test_github_transform_commits.py

@@ -20,7 +20,7 @@ def test_github_single_document(self):
                 sha="sha#1000000",
                 created_at=datetime(2023, 11, 1).strftime("%Y-%m-%dT%H:%M:%SZ"),
                 latest_saved_at=datetime(2023, 12, 1, 1).strftime("%Y-%m-%dT%H:%M:%SZ"),
-                html_url="https://github.com/repo/commit/1",
+                url="https://github.com/repo/commit/1",
                 api_url="https://api.github.com/repo/commit/1",
                 repository_id=123,
                 repository_name="SampleRepo",
@@ -42,7 +42,7 @@ def test_github_single_document(self):
                 "sha": "sha#1000000",
                 "created_at": "2023-11-01 00:00:00",
                 "latest_saved_at": "2023-12-01 01:00:00",
-                "html_url": "https://github.com/repo/commit/1",
+                "url": "https://github.com/repo/commit/1",
                 "api_url": "https://api.github.com/repo/commit/1",
                 "repository_id": 123,
                 "repository_name": "SampleRepo",
@@ -61,7 +61,7 @@ def test_multiple_documents(self):
                 sha="sha#1000000",
                 created_at=datetime(2023, 11, 1).strftime("%Y-%m-%dT%H:%M:%SZ"),
                 latest_saved_at=datetime(2023, 12, 1, 1).strftime("%Y-%m-%dT%H:%M:%SZ"),
-                html_url="https://github.com/repo/commit/1",
+                url="https://github.com/repo/commit/1",
                 api_url="https://api.github.com/repo/commit/1",
                 repository_id=123,
                 repository_name="SampleRepo",
@@ -75,7 +75,7 @@ def test_multiple_documents(self):
                 sha="sha#1000001",
                 created_at=datetime(2023, 11, 2).strftime("%Y-%m-%dT%H:%M:%SZ"),
                 latest_saved_at=datetime(2023, 12, 2, 1).strftime("%Y-%m-%dT%H:%M:%SZ"),
-                html_url="https://github.com/repo/commit/2",
+                url="https://github.com/repo/commit/2",
                 api_url="https://api.github.com/repo/commit/2",
                 repository_id=123,
                 repository_name="SampleRepo",
@@ -88,7 +88,7 @@ def test_multiple_documents(self):
                 sha="sha#1000002",
                 created_at=datetime(2023, 11, 3).strftime("%Y-%m-%dT%H:%M:%SZ"),
                 latest_saved_at=datetime(2023, 12, 3, 1).strftime("%Y-%m-%dT%H:%M:%SZ"),
-                html_url="https://github.com/repo/commit/3",
+                url="https://github.com/repo/commit/3",
                 api_url="https://api.github.com/repo/commit/3",
                 repository_id=126,
                 repository_name="SampleRepo#6",
@@ -114,7 +114,7 @@ def test_multiple_documents(self):
                 "sha": "sha#1000000",
                 "created_at": "2023-11-01 00:00:00",
                 "latest_saved_at": "2023-12-01 01:00:00",
-                "html_url": "https://github.com/repo/commit/1",
+                "url": "https://github.com/repo/commit/1",
                 "api_url": "https://api.github.com/repo/commit/1",
                 "repository_id": 123,
                 "repository_name": "SampleRepo",
@@ -132,7 +132,7 @@ def test_multiple_documents(self):
                 "sha": "sha#1000001",
                 "created_at": "2023-11-02 00:00:00",
                 "latest_saved_at": "2023-12-02 01:00:00",
-                "html_url": "https://github.com/repo/commit/2",
+                "url": "https://github.com/repo/commit/2",
                 "api_url": "https://api.github.com/repo/commit/2",
                 "repository_id": 123,
                 "repository_name": "SampleRepo",
@@ -150,7 +150,7 @@ def test_multiple_documents(self):
                 "sha": "sha#1000002",
                 "created_at": "2023-11-03 00:00:00",
                 "latest_saved_at": "2023-12-03 01:00:00",
-                "html_url": "https://github.com/repo/commit/3",
+                "url": "https://github.com/repo/commit/3",
                 "api_url": "https://api.github.com/repo/commit/3",
                 "repository_id": 126,
                 "repository_name": "SampleRepo#6",