Skip to content

Commit

Permalink
Merge pull request #277 from TogetherCrew/feat/276-add-discourse-memb…
Browse files Browse the repository at this point in the history
…ers-data

feat: new fields for each discourse user!
  • Loading branch information
amindadgar authored Sep 10, 2024
2 parents 9ca1db0 + df1678c commit c94565e
Show file tree
Hide file tree
Showing 4 changed files with 91 additions and 21 deletions.
9 changes: 7 additions & 2 deletions dags/analyzer_helper/discourse/extract_raw_members.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def fetch_member_details(self, start_date: datetime = None):
query = """
MATCH (forum:DiscourseForum {endpoint: $forum_endpoint})
MATCH (user:DiscourseUser)-[:HAS_JOINED]->(forum)
WHERE user.id IS NOT NULL
WHERE user.username IS NOT NULL
"""

parameters = {"forum_endpoint": self.forum_endpoint}
Expand All @@ -46,7 +46,12 @@ def fetch_member_details(self, start_date: datetime = None):
parameters["start_date"] = start_date

query += """
RETURN user.id AS id, user.createdAt AS joined_at
RETURN
user.id AS id,
user.createdAt AS joined_at,
user.name as name,
user.avatarTemplate as avatar,
user.username as username
ORDER BY joined_at ASC
"""
with self.driver.session() as session:
Expand Down
21 changes: 19 additions & 2 deletions dags/analyzer_helper/discourse/transform_raw_members.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,9 @@


class TransformRawMembers(TransformRawMembersBase):
def __init__(self):
def __init__(self, endpoint: str):
self.converter = DateTimeFormatConverter()
self.endpoint = endpoint

def transform(self, raw_members: list) -> list:
"""
Expand All @@ -32,12 +33,28 @@ def transform_member(self, member: dict) -> dict:
Transform a single member's data to the rawmember structure.
"""
discourse_id = member.get("id")
avatar_template: str = member["avatar"]

avatar: str
if avatar_template.startswith(("http://", "https://")):
avatar = avatar_template
elif avatar_template is not None:
avatar = "https://" + self.endpoint + avatar_template
else:
avatar = None

avatar = avatar.replace("{size}", "128") if avatar else None

member = {
"id": str(discourse_id) if discourse_id is not None else None,
"is_bot": member.get("isBot", False),
"left_at": None,
"joined_at": self.converter.from_date_string(member.get("joined_at")),
"options": {},
"options": {
"name": member["name"],
"username": member["username"],
"avatar": avatar,
},
}

return member
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,15 @@ def setUp(self):
result1 = session.run(
"""
MATCH (f:DiscourseForum {endpoint: $forum_endpoint})
CREATE (u:DiscourseUser {id: 'user1', avatarTemplate: 'avatar1', createdAt: '2023-07-01'})
CREATE (
u:DiscourseUser {
id: 'user1',
avatarTemplate: 'avatar1',
createdAt: '2023-07-01',
name: 'user1name',
username: 'username1'
}
)
-[:HAS_JOINED]->(f)
CREATE (u)-[:HAS_BADGE]->(:Badge {id: 'badge1'})
RETURN id(u) AS id
Expand All @@ -58,7 +66,14 @@ def setUp(self):
result2 = session.run(
"""
MATCH (f:DiscourseForum {endpoint: $forum_endpoint})
CREATE (u:DiscourseUser {id: 'user2', avatarTemplate: 'avatar2', createdAt: '2023-07-02'})
CREATE (u:DiscourseUser {
id: 'user2',
avatarTemplate: 'avatar2',
createdAt: '2023-07-02',
name: 'user2name',
username: 'username2'
}
)
-[:HAS_JOINED]->(f)
CREATE (u)-[:HAS_BADGE]->(:Badge {id: 'badge2'})
RETURN id(u) AS id
Expand All @@ -81,10 +96,16 @@ def test_fetch_member_details(self):
{
"id": "user1",
"joined_at": "2023-07-01",
"name": "user1name",
"username": "username1",
"avatar": "avatar1",
},
{
"id": "user2",
"joined_at": "2023-07-02",
"name": "user2name",
"username": "username2",
"avatar": "avatar2",
},
]
self.assertEqual(len(result), 2)
Expand All @@ -97,10 +118,16 @@ def test_extract_recompute(self):
{
"id": "user1",
"joined_at": "2023-07-01",
"name": "user1name",
"username": "username1",
"avatar": "avatar1",
},
{
"id": "user2",
"joined_at": "2023-07-02",
"name": "user2name",
"username": "username2",
"avatar": "avatar2",
},
]
self.assertEqual(len(result), 2)
Expand All @@ -115,6 +142,9 @@ def test_extract_without_recompute(self):
{
"id": "user2",
"joined_at": "2023-07-02",
"name": "user2name",
"username": "username2",
"avatar": "avatar2",
}
]
self.assertEqual(result, expected_result)
Original file line number Diff line number Diff line change
Expand Up @@ -6,24 +6,27 @@

class TestDiscordTransformRawMembers(unittest.TestCase):
def setUp(self):
self.transformer = TransformRawMembers()
self.endpoint = "sample.endpoint.gov"
self.transformer = TransformRawMembers(endpoint=self.endpoint)

def test_transform_empty_list(self):
"""
Tests that transform returns an empty list for an empty members list
"""
transformer = TransformRawMembers()
result = transformer.transform([])
result = self.transformer.transform([])
self.assertEqual(result, [])

def test_transform_single_member(self):
"""
Tests that transform correctly transforms a single member
"""
transformer = TransformRawMembers()
raw_member = {
"id": 85149,
"joined_at": "2023-07-02",
"isBot": False,
"name": "member1",
"username": "memberUser1",
"avatar": "/path1/{size}/endpoint",
}

expected_result = [
Expand All @@ -32,44 +35,59 @@ def test_transform_single_member(self):
"is_bot": False,
"left_at": None,
"joined_at": datetime(2023, 7, 2, 0, 0, 0, 0, tzinfo=timezone.utc),
"options": {},
"options": {
"name": "member1",
"username": "memberUser1",
"avatar": "https://" + self.endpoint + "/path1/128/endpoint",
},
}
]

result = transformer.transform([raw_member])
result = self.transformer.transform([raw_member])
self.assertEqual(result, expected_result)

def test_transform_multiple_members(self):
"""
Tests that transform correctly transforms multiple members
"""
transformer = TransformRawMembers()
raw_member1 = {
"id": "85159",
"avatar": "avatar1",
"joined_at": "2023-07-01",
"badgeIds": ["badge1"],
"isBot": True,
"name": "member1",
"username": "memberUser1",
"avatar": "https://cdn.com/path1/endpoint",
}
raw_member2 = {
"id": "85161",
"avatar": "avatar2",
"joined_at": "2023-07-02",
"badgeIds": ["badge2"],
"name": "member2",
"username": "memberUser2",
"avatar": "/path1/endpoint2",
}
expected_result1 = {
"id": "85159",
"is_bot": False,
"is_bot": True,
"left_at": None,
"joined_at": datetime(2023, 7, 1, 0, 0, 0, 0, tzinfo=timezone.utc),
"options": {},
"options": {
"name": "member1",
"username": "memberUser1",
"avatar": "https://cdn.com/path1/endpoint",
},
}
expected_result2 = {
"id": "85161",
"is_bot": False,
"left_at": None,
"joined_at": datetime(2023, 7, 2, 0, 0, 0, 0, tzinfo=timezone.utc),
"options": {},
"options": {
"name": "member2",
"username": "memberUser2",
"avatar": "https://" + self.endpoint + "/path1/endpoint2",
},
}

result = transformer.transform([raw_member1, raw_member2])
result = self.transformer.transform([raw_member1, raw_member2])
print(result)
self.assertEqual(result, [expected_result1, expected_result2])

0 comments on commit c94565e

Please sign in to comment.