From 5c45403079423ba7cd2ee47dcf4147b61236b5fe Mon Sep 17 00:00:00 2001 From: primetheus Date: Wed, 24 Mar 2021 23:17:56 -0400 Subject: [PATCH 1/6] fixed bug where multi-installation fails to sync --- app.py | 60 ++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/app.py b/app.py index afcb6b3..10296d6 100644 --- a/app.py +++ b/app.py @@ -49,8 +49,12 @@ def sync_team(client=None, owner=None, team_id=None, slug=None): org = client.organization(owner) team = org.team(team_id) custom_map = load_custom_map() - directory_group = custom_map[slug] if slug in custom_map else slug - directory_members = directory_group_members(group=directory_group) + try: + directory_group = custom_map[slug] if slug in custom_map else slug + directory_members = directory_group_members(group=directory_group) + except Exception as e: + directory_members = [] + print(e) team_members = github_team_members( client=client, owner=owner, team_id=team_id, attribute="username" ) @@ -79,8 +83,12 @@ def directory_group_members(group=None): :return: group_members :rtype: list """ - members = directory.get_group_members(group_name=group) - group_members = [member for member in members] + try: + members = directory.get_group_members(group_name=group) + group_members = [member for member in members] + except Exception as e: + group_members = [] + print(e) return group_members @@ -220,6 +228,18 @@ def load_custom_map(file="syncmap.yml"): return syncmap +def get_app_installations(): + """ + Get a list of installations for this app + :return: + """ + with app.app_context() as ctx: + c = ctx.push() + gh = GitHubApp(c) + installations = gh.app_client.app_installations + return installations + + @scheduler.scheduled_job( trigger=CronTrigger.from_crontab(CRON_INTERVAL), id="sync_all_teams" ) @@ -228,26 +248,34 @@ def sync_all_teams(): Lookup teams in a GitHub org and synchronize all teams with your user directory :return: """ - pprint(f'Syncing all teams: {time.strftime("%A, %d. %B %Y %I:%M:%S %p")}') - with app.app_context() as ctx: - c = ctx.push() - gh = GitHubApp(c) - installations = gh.app_client.app_installations - for i in installations(): + print(f'Syncing all teams: {time.strftime("%A, %d. %B %Y %I:%M:%S %p")}') + installations = get_app_installations() + for i in installations(): + print("==============================================") + print(f"Processing Organization: {i.account['login']}") + try: + gh = GitHubApp(app.app_context().push()) client = gh.app_installation(installation_id=i.id) org = client.organization(i.account["login"]) for team in org.teams(): - sync_team( - client=client, - owner=i.account["login"], - team_id=team.id, - slug=team.slug, - ) + try: + sync_team( + client=client, owner=org.login, team_id=team.id, slug=team.slug, + ) + except Exception as e: + print(f"Organization: {org.login}") + print(f"Unable to sync team: {team.slug}") + print(f"DEBUG: {e}") + except Exception as e: + print(f"DEBUG: {e}") + +sync_all_teams() if __name__ == "__main__": sync_all_teams() app.run( host=os.environ.get("FLASK_RUN_HOST", "0.0.0.0"), port=os.environ.get("FLASK_RUN_PORT", "5000"), + use_reloader=False, ) From 510b2755d7f319ad226e00e86985b260a3b41f22 Mon Sep 17 00:00:00 2001 From: primetheus Date: Wed, 24 Mar 2021 23:20:16 -0400 Subject: [PATCH 2/6] remove unused use_reload function --- app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app.py b/app.py index 10296d6..da21f5f 100644 --- a/app.py +++ b/app.py @@ -277,5 +277,4 @@ def sync_all_teams(): app.run( host=os.environ.get("FLASK_RUN_HOST", "0.0.0.0"), port=os.environ.get("FLASK_RUN_PORT", "5000"), - use_reloader=False, ) From d80c78d9582843befa5e460aa2c57f3ffe3fc295 Mon Sep 17 00:00:00 2001 From: primetheus Date: Wed, 24 Mar 2021 23:36:24 -0400 Subject: [PATCH 3/6] updated exception handling --- app.py | 10 +++++----- githubapp/azuread.py | 13 ++++++++----- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/app.py b/app.py index da21f5f..e6bc366 100644 --- a/app.py +++ b/app.py @@ -1,8 +1,8 @@ import atexit import os import time +import json from distutils.util import strtobool -from pprint import pprint from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.cron import CronTrigger @@ -63,7 +63,7 @@ def sync_team(client=None, owner=None, team_id=None, slug=None): ) if TEST_MODE: print("Skipping execution due to TEST_MODE...") - pprint(compare) + print(json.dumps(compare, indent=2)) else: try: execute_sync(org=org, team=team, slug=slug, state=compare) @@ -179,13 +179,13 @@ def execute_sync(org, team, slug, state): for user in state["action"]["add"]: # Validate that user is in org if org.is_member(user): - pprint(f"Adding {user} to {slug}") + print(f"Adding {user} to {slug}") team.add_or_update_membership(user) else: - pprint(f"Skipping {user} as they are not part of the org") + print(f"Skipping {user} as they are not part of the org") for user in state["action"]["remove"]: - pprint(f"Removing {user} from {slug}") + print(f"Removing {user} from {slug}") team.revoke_membership(user) diff --git a/githubapp/azuread.py b/githubapp/azuread.py index 7f76cfe..3396d4c 100644 --- a/githubapp/azuread.py +++ b/githubapp/azuread.py @@ -75,11 +75,14 @@ def get_group_members(self, token=None, group_name=None): headers={"Authorization": f"Bearer {token}"}, ).json() # print("Graph API call result: %s" % json.dumps(graph_data, indent=2)) - group_info = json.loads(json.dumps(graph_data, indent=2))["value"][0] - members = requests.get( - f'{self.AZURE_API_ENDPOINT}/groups/{group_info["id"]}/members', - headers={"Authorization": f"Bearer {token}"}, - ).json()["value"] + try: + group_info = json.loads(json.dumps(graph_data, indent=2))["value"][0] + members = requests.get( + f'{self.AZURE_API_ENDPOINT}/groups/{group_info["id"]}/members', + headers={"Authorization": f"Bearer {token}"}, + ).json()["value"] + except IndexError as e: + members = [] for member in members: user_info = self.get_user_info(token=token, user=member["id"]) if self.AZURE_USER_IS_UPN: From 76359cb76200f900d2803278e2c7fa47ca0f9daa Mon Sep 17 00:00:00 2001 From: primetheus Date: Wed, 24 Mar 2021 23:40:43 -0400 Subject: [PATCH 4/6] pydoc var update --- githubapp/azuread.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/githubapp/azuread.py b/githubapp/azuread.py index 3396d4c..3d189a3 100644 --- a/githubapp/azuread.py +++ b/githubapp/azuread.py @@ -50,7 +50,7 @@ def get_access_token(self): result = app.acquire_token_for_client(scopes=self.AZURE_APP_SCOPE) if "access_token" in result: - print("Successfully authenticated!") + #print("Successfully authenticated!") return result["access_token"] else: @@ -64,7 +64,7 @@ def get_group_members(self, token=None, group_name=None): """ Get a list of members for a given group :param token: - :param group: + :param group_name: :return: """ token = self.get_access_token() if not token else token From 2657cb2c545b42dd86f43d5e65876c9e187525ce Mon Sep 17 00:00:00 2001 From: primetheus Date: Wed, 24 Mar 2021 23:45:04 -0400 Subject: [PATCH 5/6] improved debug output --- app.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index e6bc366..b5d4541 100644 --- a/app.py +++ b/app.py @@ -46,6 +46,8 @@ def sync_team(client=None, owner=None, team_id=None, slug=None): :param slug: :return: """ + print("-------------------------------") + print(f"Processing Team: {slug}") org = client.organization(owner) team = org.team(team_id) custom_map = load_custom_map() @@ -251,8 +253,9 @@ def sync_all_teams(): print(f'Syncing all teams: {time.strftime("%A, %d. %B %Y %I:%M:%S %p")}') installations = get_app_installations() for i in installations(): - print("==============================================") - print(f"Processing Organization: {i.account['login']}") + print("========================================================") + print(f"## Processing Organization: {i.account['login']}") + print("========================================================") try: gh = GitHubApp(app.app_context().push()) client = gh.app_installation(installation_id=i.id) From 227c4a5691d87fb0484e5e3acdbe2c1378202366 Mon Sep 17 00:00:00 2001 From: primetheus Date: Wed, 24 Mar 2021 23:46:14 -0400 Subject: [PATCH 6/6] reformatted with Black --- githubapp/azuread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/githubapp/azuread.py b/githubapp/azuread.py index 3d189a3..3326bc1 100644 --- a/githubapp/azuread.py +++ b/githubapp/azuread.py @@ -50,7 +50,7 @@ def get_access_token(self): result = app.acquire_token_for_client(scopes=self.AZURE_APP_SCOPE) if "access_token" in result: - #print("Successfully authenticated!") + # print("Successfully authenticated!") return result["access_token"] else: