From ae80bddcf50c9736396efde7a40c828492d17a2b Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Mon, 16 Sep 2024 09:45:17 +0330 Subject: [PATCH] feat: updated to use platform agnostic structure! --- utils/mongo_base.py | 23 ++++++++------- utils/mongo_utils.py | 58 +++++++++++++++---------------------- utils/process_guild_data.py | 6 ++-- 3 files changed, 39 insertions(+), 48 deletions(-) diff --git a/utils/mongo_base.py b/utils/mongo_base.py index 214993b..7792f72 100644 --- a/utils/mongo_base.py +++ b/utils/mongo_base.py @@ -1,24 +1,25 @@ from datetime import datetime from typing import Any +from bson import ObjectId from utils.mongo import MongoSingleton class MongoBase: - def __init__(self, guild_id: str) -> None: - self.guild_id = guild_id + def __init__(self, platform_id: str) -> None: + self.platform_id = platform_id - def get_guild_raw_data_count(self, from_date: datetime): + def get_raw_data_count(self, from_date: datetime): client = MongoSingleton.get_instance().client - raw_data_count = client[self.guild_id]["rawinfos"].count_documents( - {"createdDate": {"$gte": from_date}} - ) + raw_data_count = client[self.platform_id][ + "rawmemberactivities" + ].count_documents({"createdDate": {"$gte": from_date}}) return raw_data_count def get_guild_members_count(self) -> int: client = MongoSingleton.get_instance().client - doc_count = client[self.guild_id]["guildmembers"].count_documents({}) + doc_count = client[self.platform_id]["rawmembers"].count_documents({}) return doc_count def get_latest_document( @@ -38,7 +39,9 @@ def get_latest_document( def get_distinct_channels(self) -> list[str]: client = MongoSingleton.get_instance().client - channels = client[self.guild_id]["rawinfos"].distinct("channelId") + channels = client[self.platform_id]["rawmemberactivities"].distinct( + "metadata.channel_id" + ) return channels def get_guild_platform_id(self) -> str: @@ -53,11 +56,11 @@ def get_guild_platform_id(self) -> str: mongo_client = MongoSingleton.get_instance().client guild_info = mongo_client["Core"]["platforms"].find_one( - {"metadata.id": self.guild_id}, {"_id": 1} + {"_id": ObjectId(self.platform_id)}, {"_id": 1} ) if guild_info is not None: platform_id = str(guild_info["_id"]) else: - raise ValueError(f"No available guild with id {self.guild_id}") + raise ValueError(f"No available platform with id {self.platform_id}") return platform_id diff --git a/utils/mongo_utils.py b/utils/mongo_utils.py index e476850..ee361f5 100644 --- a/utils/mongo_utils.py +++ b/utils/mongo_utils.py @@ -4,20 +4,22 @@ class MongoUtils(MongoBase): - def __init__(self, guild_id: str) -> None: - super().__init__(guild_id) + def __init__(self, platform_id: str) -> None: + super().__init__(platform_id) def get_latest_discord_raw_info_date(self) -> datetime | None: - date_field = "createdDate" + date_field = "date" latest_document = self.get_latest_document( - db_name=self.guild_id, collection_name="rawinfos", date_field=date_field + db_name=self.platform_id, + collection_name="rawmemberactivities", + date_field=date_field, ) return self.get_latest_date(latest_document, date_field) def get_latest_memberactivities_date(self) -> str: date_field = "date" latest_document = self.get_latest_document( - db_name=self.guild_id, + db_name=self.platform_id, collection_name="memberactivities", date_field=date_field, ) @@ -26,39 +28,25 @@ def get_latest_memberactivities_date(self) -> str: def get_latest_heatmaps_date(self) -> str: date_field = "date" latest_document = self.get_latest_document( - db_name=self.guild_id, collection_name="heatmaps", date_field=date_field + db_name=self.platform_id, collection_name="heatmaps", date_field=date_field ) return self.get_latest_date(latest_document, date_field) # type: ignore - def get_latest_fired_saga( - self, guild_id: str | None = None, platform_id: str | None = None - ) -> datetime | None: - date_field = "createdAt" - if platform_id: - latest_document = self.get_latest_document( - db_name="Saga", - collection_name="sagas", - date_field=date_field, - filters={ - "data.platformId": platform_id, - "choreography.name": "DISCORD_SCHEDULED_JOB", - }, - ) - elif guild_id: - fetched_platform_id = self.get_guild_platform_id() - latest_document = self.get_latest_document( - db_name="Saga", - collection_name="sagas", - date_field=date_field, - filters={ - "data.platformId": fetched_platform_id, - "choreography.name": "DISCORD_SCHEDULED_JOB", - }, - ) - else: - raise ValueError("One of guild_id or platform_id should be given") - - return self.get_latest_date(latest_document, date_field) + # def get_latest_fired_saga( + # self, platform_id: str | None = None + # ) -> datetime | None: + # date_field = "createdAt" + # latest_document = self.get_latest_document( + # db_name="Saga", + # collection_name="sagas", + # date_field=date_field, + # filters={ + # "data.platformId": platform_id, + # "choreography.name": "DISCORD_SCHEDULED_JOB", + # }, + # ) + + # return self.get_latest_date(latest_document, date_field) def get_latest_date(self, document: dict, date_field: str) -> datetime | None: latest_date: datetime | None diff --git a/utils/process_guild_data.py b/utils/process_guild_data.py index d9f1e4c..7e63da3 100644 --- a/utils/process_guild_data.py +++ b/utils/process_guild_data.py @@ -23,14 +23,14 @@ def process_guild_data( # getting the latest dates raw_infos_date = utils.get_latest_discord_raw_info_date() - fired_sage_date = utils.get_latest_fired_saga(platform_id=platform_id) + # fired_sage_date = utils.get_latest_fired_saga(platform_id=platform_id) heatmaps_date = utils.get_latest_heatmaps_date() memberactivities_date = utils.get_latest_memberactivities_date() extracted_channels = utils.get_distinct_channels() guild_members_count = utils.get_guild_members_count() # 30 days before - raw_data_count = utils.get_guild_raw_data_count( + raw_data_count = utils.get_raw_data_count( from_date=datetime.now() - timedelta(days=31) ) @@ -43,7 +43,7 @@ def process_guild_data( "selected_channels_count": selected_channel_count, "extracted_channel_counts": len(extracted_channels), "latest_raw_info": raw_infos_date, - "latest_analyzer_run_fired_saga": fired_sage_date, + # "latest_analyzer_run_fired_saga": fired_sage_date, "latest_heatmaps": heatmaps_date, "latest_memberactivities": memberactivities_date, "raw_data_count_30days": raw_data_count,