Skip to content

Commit

Permalink
Merge pull request #164 from TogetherCrew/fix/hivemind-github-integra…
Browse files Browse the repository at this point in the history 
…tion

feat: getting org id from platform!
  • Loading branch information
@cyri113
cyri113 authored May 27, 2024
2 parents 686a4db + 6b9fd54 commit 33c5a91
Show file tree
Hide file tree
Showing 2 changed files with 186 additions and 52 deletions.
13 changes: 10 additions & 3 deletions dags/hivemind_etl_helpers/src/utils/modules/github.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ def get_learning_platforms(self):
"community_id": "community1",
"organization_ids": ["1111", "2222"],
"repo_ids": ["132", "45232"],
"from_date": datetime(2024, 1, 1)
# "from_date": datetime(2024, 1, 1)
"from_date": None
}]
```
"""
Expand All @@ -39,14 +40,20 @@ def get_learning_platforms(self):
if platform["name"] != self.platform_name:
continue

platform_id = platform["platform"]
organization_id = self.get_platform_metadata(
platform_id=platform_id,
metadata_name="installationId",
)
modules_options = platform["metadata"]

platforms_data.append(
{
"community_id": str(community),
"organization_ids": modules_options.get("organizationId", []),
"organization_ids": [organization_id],
"repo_ids": modules_options.get("repoIds", []),
"from_date": modules_options["fromDate"],
# "from_date": modules_options["fromDate"],
"from_date": None,
}
)

Expand Down
225 changes: 176 additions & 49 deletions dags/hivemind_etl_helpers/tests/integration/test_github_get_communities_org.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ class TestQueryGitHubModulesDB(unittest.TestCase):
def setUp(self):
client = MongoSingleton.get_instance().client
client["Core"].drop_collection("modules")
client["Core"].drop_collection("platforms")
self.modules_github = ModulesGitHub()

self.client = client
Expand All @@ -22,18 +23,37 @@ def test_get_github_communities_data_single_modules(self):
"""
single github platform for one community
"""
platform_id = ObjectId("6579c364f1120850414e0dc6")
community_id = ObjectId("6579c364f1120850414e0dc5")

self.client["Core"]["platforms"].insert_one(
{
"_id": platform_id,
"name": "github",
"metadata": {
"installationId": "345678",
"account": {},
},
"community": community_id,
"disconnectedAt": None,
"connectedAt": datetime.now(),
"createdAt": datetime.now(),
"updatedAt": datetime.now(),
}
)

self.client["Core"]["modules"].insert_one(
{
"name": "hivemind",
"community": ObjectId("6579c364f1120850414e0dc5"),
"community": community_id,
"options": {
"platforms": [
{
"platform": ObjectId("6579c364f1120850414e0dc6"),
"platform": platform_id,
"name": "github",
"metadata": {
"fromDate": datetime(2024, 1, 1),
"organizationId": ["1234"],
# "fromDate": datetime(2024, 1, 1),
# "organizationId": ["1234"],
"repoIds": ["111", "234"],
},
}
Expand All @@ -52,38 +72,70 @@ def test_get_github_communities_data_single_modules(self):
result[0],
{
"community_id": "6579c364f1120850414e0dc5",
"organization_ids": ["1234"],
"organization_ids": ["345678"],
"repo_ids": ["111", "234"],
"from_date": datetime(2024, 1, 1),
# "from_date": datetime(2024, 1, 1),
"from_date": None,
},
)

def test_get_github_communities_data_multiple_platforms(self):
"""
two github platform for one community
"""
platform_id = ObjectId("6579c364f1120850414e0dc6")
platform_id2 = ObjectId("6579c364f1120850414e0dc7")
community_id = ObjectId("6579c364f1120850414e0dc5")

self.client["Core"]["platforms"].insert_one(
{
"_id": platform_id,
"name": "github",
"metadata": {
"installationId": "11111",
"account": {},
},
"community": community_id,
"disconnectedAt": None,
"connectedAt": datetime.now(),
"createdAt": datetime.now(),
"updatedAt": datetime.now(),
}
)
self.client["Core"]["platforms"].insert_one(
{
"_id": platform_id2,
"name": "github",
"metadata": {
"installationId": "222222",
"account": {},
},
"community": community_id,
"disconnectedAt": None,
"connectedAt": datetime.now(),
"createdAt": datetime.now(),
"updatedAt": datetime.now(),
}
)
self.client["Core"]["modules"].insert_one(
{
"name": "hivemind",
"community": ObjectId("1009c364f1120850414e0dc5"),
"community": community_id,
"options": {
"platforms": [
{
"platform": ObjectId("6579c364f1120850414e0dc6"),
"platform": platform_id,
"name": "github",
"metadata": {
"fromDate": datetime(2024, 1, 1),
"organizationId": ["1234"],
"repoIds": ["111", "234"],
# "repoIds": ["111", "234"],
},
},
{
"platform": ObjectId("6579c364f1120850414e0dc7"),
"platform": platform_id2,
"name": "github",
"metadata": {
"fromDate": datetime(2024, 2, 2),
"organizationId": ["4321"],
"repoIds": ["2132", "8888"],
# "fromDate": datetime(2024, 2, 2),
# "repoIds": ["2132", "8888"],
},
},
]
Expand All @@ -96,22 +148,26 @@ def test_get_github_communities_data_multiple_platforms(self):
self.assertIsInstance(result, list)
self.assertEqual(len(result), 2)

print(result[0])

self.assertEqual(
result[0],
{
"community_id": "1009c364f1120850414e0dc5",
"organization_ids": ["1234"],
"repo_ids": ["111", "234"],
"from_date": datetime(2024, 1, 1),
"community_id": str(community_id),
"organization_ids": ["11111"],
"repo_ids": [],
# "from_date": datetime(2024, 1, 1),
"from_date": None,
},
)
self.assertEqual(
result[1],
{
"community_id": "1009c364f1120850414e0dc5",
"organization_ids": ["4321"],
"repo_ids": ["2132", "8888"],
"from_date": datetime(2024, 2, 2),
"community_id": str(community_id),
"organization_ids": ["222222"],
"repo_ids": [],
# "from_date": datetime(2024, 2, 2),
"from_date": None,
},
)

Expand All @@ -121,50 +177,117 @@ def test_get_github_communities_data_multiple_platforms_multiple_communities(
"""
two github platform for two separate communities
"""
platform_id = ObjectId("6579c364f1120850414e0dc6")
platform_id2 = ObjectId("6579c364f1120850414e0dc7")
platform_id3 = ObjectId("6579c364f1120850414e0dc8")
platform_id4 = ObjectId("6579c364f1120850414e0dc9")
community_id = ObjectId("6579c364f1120850414e0dc5")
community_id2 = ObjectId("2009c364f1120850414e0dc5")

self.client["Core"]["platforms"].insert_one(
{
"_id": platform_id,
"name": "github",
"metadata": {
"installationId": "11111",
"account": {},
},
"community": community_id,
"disconnectedAt": None,
"connectedAt": datetime.now(),
"createdAt": datetime.now(),
"updatedAt": datetime.now(),
}
)
self.client["Core"]["platforms"].insert_one(
{
"_id": platform_id2,
"name": "github",
"metadata": {
"installationId": "222222",
"account": {},
},
"community": community_id,
"disconnectedAt": None,
"connectedAt": datetime.now(),
"createdAt": datetime.now(),
"updatedAt": datetime.now(),
}
)
self.client["Core"]["platforms"].insert_one(
{
"_id": platform_id3,
"name": "github",
"metadata": {
"installationId": "333333",
"account": {},
},
"community": community_id2,
"disconnectedAt": None,
"connectedAt": datetime.now(),
"createdAt": datetime.now(),
"updatedAt": datetime.now(),
}
)
self.client["Core"]["platforms"].insert_one(
{
"_id": platform_id4,
"name": "discord",
"metadata": {
"learning": {},
"answering": {},
},
"community": community_id2,
"disconnectedAt": None,
"connectedAt": datetime.now(),
"createdAt": datetime.now(),
"updatedAt": datetime.now(),
}
)

self.client["Core"]["modules"].insert_many(
[
{
"name": "hivemind",
"community": ObjectId("1009c364f1120850414e0dc5"),
"community": community_id,
"options": {
"platforms": [
{
"platform": ObjectId("6579c364f1120850414e0dc6"),
"platform": platform_id,
"name": "github",
"metadata": {
"fromDate": datetime(2024, 1, 1),
"organizationId": ["1234"],
# "fromDate": datetime(2024, 1, 1),
},
},
{
"platform": ObjectId("6579c364f1120850414e0dc7"),
"platform": platform_id2,
"name": "github",
"metadata": {
"fromDate": datetime(2024, 2, 2),
"repoIds": ["1111"],
# "fromDate": datetime(2024, 2, 2),
"repoIds": ["AAAAA"],
},
},
]
},
},
{
"name": "hivemind",
"community": ObjectId("2009c364f1120850414e0dc5"),
"community": community_id2,
"options": {
"platforms": [
{
"platform": ObjectId("6579c364f1120850414e0db5"),
"platform": platform_id3,
"name": "github",
"metadata": {
"fromDate": datetime(2024, 3, 1),
"organizationId": ["111111"],
# "fromDate": datetime(2024, 3, 1),
# "organizationId": ["111111"],
},
},
{
"platform": ObjectId("6579c364f1120850414e0dc7"),
"platform": platform_id4,
"name": "discord",
"metadata": {
"fromDate": datetime(2024, 3, 1),
# "fromDate": datetime(2024, 3, 1),
"selectedChannels": ["666", "777"],
},
},
Expand All @@ -180,34 +303,38 @@ def test_get_github_communities_data_multiple_platforms_multiple_communities(
self.assertEqual(len(results), 3)

for res in results:
if res["organization_ids"] == ["1234"]:
print(res, "|||", str(community_id))
if res["organization_ids"] == ["11111"]:
self.assertEqual(
res,
{
"community_id": "1009c364f1120850414e0dc5",
"organization_ids": ["1234"],
"community_id": str(community_id),
"organization_ids": ["11111"],
"repo_ids": [],
"from_date": datetime(2024, 1, 1),
# "from_date": datetime(2024, 1, 1),
"from_date": None,
},
)
elif res["organization_ids"] == []:
elif res["organization_ids"] == ["222222"]:
self.assertEqual(
res,
{
"community_id": "1009c364f1120850414e0dc5",
"organization_ids": [],
"repo_ids": ["1111"],
"from_date": datetime(2024, 2, 2),
"community_id": str(community_id),
"organization_ids": ["222222"],
"repo_ids": ["AAAAA"],
# "from_date": datetime(2024, 2, 2),
"from_date": None,
},
)
elif res["organization_ids"] == ["111111"]:
elif res["organization_ids"] == ["333333"]:
self.assertEqual(
res,
{
"community_id": "2009c364f1120850414e0dc5",
"organization_ids": ["111111"],
"community_id": str(community_id2),
"organization_ids": ["333333"],
"repo_ids": [],
"from_date": datetime(2024, 3, 1),
# "from_date": datetime(2024, 3, 1),
"from_date": None,
},
)
else:
Expand Down

0 comments on commit 33c5a91

Please sign in to comment.