diff --git a/.github/workflows/production.yml b/.github/workflows/production.yml new file mode 100644 index 0000000..a1be27b --- /dev/null +++ b/.github/workflows/production.yml @@ -0,0 +1,12 @@ +name: Production CI/CD Pipeline + +on: + push: + branches: + - main + +jobs: + ci: + uses: TogetherCrew/operations/.github/workflows/ci.yml@main + secrets: + CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} \ No newline at end of file diff --git a/.github/workflows/start.staging.yml b/.github/workflows/start.staging.yml new file mode 100644 index 0000000..842e3bd --- /dev/null +++ b/.github/workflows/start.staging.yml @@ -0,0 +1,9 @@ +name: Staging CI/CD Pipeline + +on: pull_request + +jobs: + ci: + uses: TogetherCrew/operations/.github/workflows/ci.yml@main + secrets: + CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }} \ No newline at end of file diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/app.py b/app.py index a5f67f2..868b654 100644 --- a/app.py +++ b/app.py @@ -1,14 +1,8 @@ +import logging from datetime import datetime -from pandas.api.types import ( - is_categorical_dtype, - is_datetime64_any_dtype, - is_numeric_dtype, - is_object_dtype, -) + import pandas as pd import streamlit as st -import logging - from utils.mongo import MongoSingleton from utils.process_guild_data import process_guild_data @@ -43,6 +37,6 @@ def load_guilds_latest_date_df(): logging.basicConfig(level=logging.INFO) st.subheader("MongoDB data Analytics") -df = load_guilds_latest_date_df() +load_guilds_latest_date_df() # df = process_df(df) # st.dataframe(df, use_container_width=True, hide_index=True) diff --git a/tests/integration/test_credentials_load.py b/tests/integration/test_credentials_load.py new file mode 100644 index 0000000..c1907c9 --- /dev/null +++ b/tests/integration/test_credentials_load.py @@ -0,0 +1,14 @@ +from unittest import TestCase + +from utils.creds import get_mongo_credentials + + +class TestLoadCredentials(TestCase): + def test_creds_non_empty(self): + creds = get_mongo_credentials() + + self.assertIsNotNone(creds["user"]) + self.assertIsNotNone(creds["password"]) + self.assertIsNotNone(creds["host"]) + self.assertIsNotNone(creds["port"]) + self.assertNotEqual(creds["connection_str"], "mongodb://None:None@None:None") diff --git a/tests/unit/test_sample.py b/tests/unit/test_sample.py index 26a96e6..4b2d323 100644 --- a/tests/unit/test_sample.py +++ b/tests/unit/test_sample.py @@ -3,4 +3,4 @@ class TestSample(TestCase): def test_sample(self): - self.assertEqual(True, True) \ No newline at end of file + self.assertEqual(True, True) diff --git a/utils/creds.py b/utils/creds.py index 6f6d3a2..eebade7 100644 --- a/utils/creds.py +++ b/utils/creds.py @@ -1,6 +1,7 @@ -from dotenv import load_dotenv import os +from dotenv import load_dotenv + def get_mongo_credentials(): """ diff --git a/utils/mongo_base.py b/utils/mongo_base.py index ab3fe25..214993b 100644 --- a/utils/mongo_base.py +++ b/utils/mongo_base.py @@ -24,7 +24,6 @@ def get_guild_members_count(self) -> int: def get_latest_document( self, db_name: str, collection_name: str, date_field: str, **kwargs ) -> dict[str, Any]: - filters = kwargs.get("filters", None) client = MongoSingleton.get_instance().client if filters is None: diff --git a/utils/mongo_utils.py b/utils/mongo_utils.py index 9dcb8e4..e476850 100644 --- a/utils/mongo_utils.py +++ b/utils/mongo_utils.py @@ -1,4 +1,5 @@ from datetime import datetime + from utils.mongo_base import MongoBase @@ -6,7 +7,7 @@ class MongoUtils(MongoBase): def __init__(self, guild_id: str) -> None: super().__init__(guild_id) - def get_latest_discord_raw_info_date(self) -> datetime: + def get_latest_discord_raw_info_date(self) -> datetime | None: date_field = "createdDate" latest_document = self.get_latest_document( db_name=self.guild_id, collection_name="rawinfos", date_field=date_field @@ -20,18 +21,18 @@ def get_latest_memberactivities_date(self) -> str: collection_name="memberactivities", date_field=date_field, ) - return self.get_latest_date(latest_document, date_field) + return self.get_latest_date(latest_document, date_field) # type: ignore def get_latest_heatmaps_date(self) -> str: date_field = "date" latest_document = self.get_latest_document( db_name=self.guild_id, collection_name="heatmaps", date_field=date_field ) - return self.get_latest_date(latest_document, date_field) + return self.get_latest_date(latest_document, date_field) # type: ignore def get_latest_fired_saga( self, guild_id: str | None = None, platform_id: str | None = None - ) -> datetime: + ) -> datetime | None: date_field = "createdAt" if platform_id: latest_document = self.get_latest_document( @@ -44,7 +45,7 @@ def get_latest_fired_saga( }, ) elif guild_id: - fetched_platform_id = self.get_guild_platform_id(guild_id) + fetched_platform_id = self.get_guild_platform_id() latest_document = self.get_latest_document( db_name="Saga", collection_name="sagas", diff --git a/utils/process_guild_data.py b/utils/process_guild_data.py index db231ea..d9f1e4c 100644 --- a/utils/process_guild_data.py +++ b/utils/process_guild_data.py @@ -3,9 +3,12 @@ from utils.mongo_utils import MongoUtils -def process_guild_data(platform_document: dict) -> dict[str, str | datetime | None]: - data: dict[str, str | datetime | None] = {} - +def process_guild_data( + platform_document: dict, +) -> dict[str, str | int | datetime | None]: + """ + process each platform and prepare the analytics for it + """ platform_id = str(platform_document["_id"]) guild_id = platform_document["metadata"]["id"] utils = MongoUtils(guild_id) @@ -18,12 +21,6 @@ def process_guild_data(platform_document: dict) -> dict[str, str | datetime | No else: selected_channel_count = -1 - data["guild_id"] = guild_id - data["guild_name"] = guild_name - data["connected_at"] = connected_at - data["platform_id"] = platform_id - data["disconnected_at"] = disconnected_at - # getting the latest dates raw_infos_date = utils.get_latest_discord_raw_info_date() fired_sage_date = utils.get_latest_fired_saga(platform_id=platform_id) @@ -37,13 +34,20 @@ def process_guild_data(platform_document: dict) -> dict[str, str | datetime | No from_date=datetime.now() - timedelta(days=31) ) - data["selected_channels_count"] = selected_channel_count - data["extracted_channel_counts"] = len(extracted_channels) - data["latest_raw_info"] = raw_infos_date - data["latest_analyzer_run_fired_saga"] = fired_sage_date - data["latest_heatmaps"] = heatmaps_date - data["latest_memberactivities"] = memberactivities_date - data["raw_data_count_30days"] = raw_data_count - data["guild_members_count"] = guild_members_count + data: dict[str, str | int | datetime | None] = { + "guild_id": guild_id, + "guild_name": guild_name, + "connected_at": connected_at, + "platform_id": platform_id, + "disconnected_at": disconnected_at, + "selected_channels_count": selected_channel_count, + "extracted_channel_counts": len(extracted_channels), + "latest_raw_info": raw_infos_date, + "latest_analyzer_run_fired_saga": fired_sage_date, + "latest_heatmaps": heatmaps_date, + "latest_memberactivities": memberactivities_date, + "raw_data_count_30days": raw_data_count, + "guild_members_count": guild_members_count, + } return data