diff --git a/dags/analyzer_helper/discourse/extract_raw_members.py b/dags/analyzer_helper/discourse/extract_raw_members.py index 2c6c9e39..3ce69b2e 100644 --- a/dags/analyzer_helper/discourse/extract_raw_members.py +++ b/dags/analyzer_helper/discourse/extract_raw_members.py @@ -36,7 +36,7 @@ def fetch_member_details(self, start_date: datetime = None): query = """ MATCH (forum:DiscourseForum {endpoint: $forum_endpoint}) MATCH (user:DiscourseUser)-[:HAS_JOINED]->(forum) - WHERE user.id IS NOT NULL + WHERE user.username IS NOT NULL """ parameters = {"forum_endpoint": self.forum_endpoint} @@ -46,7 +46,12 @@ def fetch_member_details(self, start_date: datetime = None): parameters["start_date"] = start_date query += """ - RETURN user.id AS id, user.createdAt AS joined_at + RETURN + user.id AS id, + user.createdAt AS joined_at, + user.name as name, + user.avatarTemplate as avatar, + user.username as username ORDER BY joined_at ASC """ with self.driver.session() as session: diff --git a/dags/analyzer_helper/discourse/transform_raw_members.py b/dags/analyzer_helper/discourse/transform_raw_members.py index fd4646d8..c201bb60 100644 --- a/dags/analyzer_helper/discourse/transform_raw_members.py +++ b/dags/analyzer_helper/discourse/transform_raw_members.py @@ -9,8 +9,9 @@ class TransformRawMembers(TransformRawMembersBase): - def __init__(self): + def __init__(self, endpoint: str): self.converter = DateTimeFormatConverter() + self.endpoint = endpoint def transform(self, raw_members: list) -> list: """ @@ -32,12 +33,28 @@ def transform_member(self, member: dict) -> dict: Transform a single member's data to the rawmember structure. """ discourse_id = member.get("id") + avatar_template: str = member["avatar"] + + avatar: str + if avatar_template.startswith(("http://", "https://")): + avatar = avatar_template + elif avatar_template is not None: + avatar = "https://" + self.endpoint + avatar_template + else: + avatar = None + + avatar = avatar.replace("{size}", "128") if avatar else None + member = { "id": str(discourse_id) if discourse_id is not None else None, "is_bot": member.get("isBot", False), "left_at": None, "joined_at": self.converter.from_date_string(member.get("joined_at")), - "options": {}, + "options": { + "name": member["name"], + "username": member["username"], + "avatar": avatar, + }, } return member diff --git a/dags/analyzer_helper/tests/integration/test_discourse_extract_raw_members.py b/dags/analyzer_helper/tests/integration/test_discourse_extract_raw_members.py index 43db4bf0..0b2a8a0f 100644 --- a/dags/analyzer_helper/tests/integration/test_discourse_extract_raw_members.py +++ b/dags/analyzer_helper/tests/integration/test_discourse_extract_raw_members.py @@ -46,7 +46,15 @@ def setUp(self): result1 = session.run( """ MATCH (f:DiscourseForum {endpoint: $forum_endpoint}) - CREATE (u:DiscourseUser {id: 'user1', avatarTemplate: 'avatar1', createdAt: '2023-07-01'}) + CREATE ( + u:DiscourseUser { + id: 'user1', + avatarTemplate: 'avatar1', + createdAt: '2023-07-01', + name: 'user1name', + username: 'username1' + } + ) -[:HAS_JOINED]->(f) CREATE (u)-[:HAS_BADGE]->(:Badge {id: 'badge1'}) RETURN id(u) AS id @@ -58,7 +66,14 @@ def setUp(self): result2 = session.run( """ MATCH (f:DiscourseForum {endpoint: $forum_endpoint}) - CREATE (u:DiscourseUser {id: 'user2', avatarTemplate: 'avatar2', createdAt: '2023-07-02'}) + CREATE (u:DiscourseUser { + id: 'user2', + avatarTemplate: 'avatar2', + createdAt: '2023-07-02', + name: 'user2name', + username: 'username2' + } + ) -[:HAS_JOINED]->(f) CREATE (u)-[:HAS_BADGE]->(:Badge {id: 'badge2'}) RETURN id(u) AS id @@ -81,10 +96,16 @@ def test_fetch_member_details(self): { "id": "user1", "joined_at": "2023-07-01", + "name": "user1name", + "username": "username1", + "avatar": "avatar1", }, { "id": "user2", "joined_at": "2023-07-02", + "name": "user2name", + "username": "username2", + "avatar": "avatar2", }, ] self.assertEqual(len(result), 2) @@ -97,10 +118,16 @@ def test_extract_recompute(self): { "id": "user1", "joined_at": "2023-07-01", + "name": "user1name", + "username": "username1", + "avatar": "avatar1", }, { "id": "user2", "joined_at": "2023-07-02", + "name": "user2name", + "username": "username2", + "avatar": "avatar2", }, ] self.assertEqual(len(result), 2) @@ -115,6 +142,9 @@ def test_extract_without_recompute(self): { "id": "user2", "joined_at": "2023-07-02", + "name": "user2name", + "username": "username2", + "avatar": "avatar2", } ] self.assertEqual(result, expected_result) diff --git a/dags/analyzer_helper/tests/unit/test_discourse_transform_raw_members.py b/dags/analyzer_helper/tests/unit/test_discourse_transform_raw_members.py index 418b995d..2f3afb38 100644 --- a/dags/analyzer_helper/tests/unit/test_discourse_transform_raw_members.py +++ b/dags/analyzer_helper/tests/unit/test_discourse_transform_raw_members.py @@ -6,24 +6,27 @@ class TestDiscordTransformRawMembers(unittest.TestCase): def setUp(self): - self.transformer = TransformRawMembers() + self.endpoint = "sample.endpoint.gov" + self.transformer = TransformRawMembers(endpoint=self.endpoint) def test_transform_empty_list(self): """ Tests that transform returns an empty list for an empty members list """ - transformer = TransformRawMembers() - result = transformer.transform([]) + result = self.transformer.transform([]) self.assertEqual(result, []) def test_transform_single_member(self): """ Tests that transform correctly transforms a single member """ - transformer = TransformRawMembers() raw_member = { "id": 85149, "joined_at": "2023-07-02", + "isBot": False, + "name": "member1", + "username": "memberUser1", + "avatar": "/path1/{size}/endpoint", } expected_result = [ @@ -32,44 +35,59 @@ def test_transform_single_member(self): "is_bot": False, "left_at": None, "joined_at": datetime(2023, 7, 2, 0, 0, 0, 0, tzinfo=timezone.utc), - "options": {}, + "options": { + "name": "member1", + "username": "memberUser1", + "avatar": "https://" + self.endpoint + "/path1/128/endpoint", + }, } ] - result = transformer.transform([raw_member]) + result = self.transformer.transform([raw_member]) self.assertEqual(result, expected_result) def test_transform_multiple_members(self): """ Tests that transform correctly transforms multiple members """ - transformer = TransformRawMembers() raw_member1 = { "id": "85159", - "avatar": "avatar1", "joined_at": "2023-07-01", - "badgeIds": ["badge1"], + "isBot": True, + "name": "member1", + "username": "memberUser1", + "avatar": "https://cdn.com/path1/endpoint", } raw_member2 = { "id": "85161", - "avatar": "avatar2", "joined_at": "2023-07-02", - "badgeIds": ["badge2"], + "name": "member2", + "username": "memberUser2", + "avatar": "/path1/endpoint2", } expected_result1 = { "id": "85159", - "is_bot": False, + "is_bot": True, "left_at": None, "joined_at": datetime(2023, 7, 1, 0, 0, 0, 0, tzinfo=timezone.utc), - "options": {}, + "options": { + "name": "member1", + "username": "memberUser1", + "avatar": "https://cdn.com/path1/endpoint", + }, } expected_result2 = { "id": "85161", "is_bot": False, "left_at": None, "joined_at": datetime(2023, 7, 2, 0, 0, 0, 0, tzinfo=timezone.utc), - "options": {}, + "options": { + "name": "member2", + "username": "memberUser2", + "avatar": "https://" + self.endpoint + "/path1/endpoint2", + }, } - result = transformer.transform([raw_member1, raw_member2]) + result = self.transformer.transform([raw_member1, raw_member2]) + print(result) self.assertEqual(result, [expected_result1, expected_result2])