From 7fdcbf72fe7e836139ac8cd3c1487f2b532b6fa4 Mon Sep 17 00:00:00 2001 From: nathan-roys <110457649+nathan-roys@users.noreply.github.com> Date: Wed, 9 Nov 2022 10:17:05 +0000 Subject: [PATCH] feat: add ability to delete or deactivate projects for archived status (#123) * feat: add ability to delete or deactivate projects when status in GH is archived * feat: update deactivate to delete call, ensure repo has archived populated * feat: add ability to reactivate projects should a GH repo become unarchived * fix: fix tests * fix: fix tests * fix: fix tests * fix: pylint * feat: add tests for delete/activation/reactivation * Change unarchive options to ignore * fix: fixes from PR comments --- README.md | 5 ++ app/app.py | 63 ++++++++++++--- app/gh_repo.py | 6 +- app/models.py | 5 ++ app/snyk_repo.py | 52 +++++++++++++ app/tests/test_snyk_scm_refresh.py | 120 +++++++++++++++++++++++++---- app/utils/snyk_helper.py | 29 ++++++- common.py | 15 ++++ 8 files changed, 266 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index e87d064..77c14df 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,11 @@ optional arguments: --container {on,off} scan for container projects, e.g. Dockerfile (on by default) --iac {on,off} scan for IAC manifests (experimental, off by default) --code {off} code analysis is deprecated with off only option + --on-archived {ignore,deactivate,delete} + Deletes or deactivates projects associated with archived repos (ignore by default) + --on-unarchived {ignore,reactivate} + If there is a deactivated project in Snyk, should the tool reactivate it if the repo is not + archived? (Warning: Use with caution, this will reactivate ALL projects associated with a repo) --dry-run Simulate processing of the script without making changes to Snyk --skip-scm-validation Skip validation of the TLS certificate used by the SCM diff --git a/app/app.py b/app/app.py index 80bb660..6e0f47b 100755 --- a/app/app.py +++ b/app/app.py @@ -21,6 +21,7 @@ log_audit_large_repo_result ) + def run(): """Begin application logic""" # pylint: disable=too-many-locals, too-many-branches, too-many-statements @@ -59,7 +60,7 @@ def run(): is_default_renamed = False app_print(snyk_repo.org_name, snyk_repo.full_name, - f"Processing {str(i+1)}/{str(len(snyk_repos))}") + f"Processing {str(i + 1)}/{str(len(snyk_repos))}") try: gh_repo_status = get_gh_repo_status(snyk_repo) @@ -70,13 +71,13 @@ def run(): app_print(snyk_repo.org_name, snyk_repo.full_name, f"Github Status {gh_repo_status.response_code}" \ - f"({gh_repo_status.response_message}) [{snyk_repo.origin}]") + f"({gh_repo_status.response_message}) [{snyk_repo.origin}]") - #if snyk_repo does not still exist (removed/404), then log and skip to next repo - if gh_repo_status.response_code == 404: # project no longer exists + # if snyk_repo does not still exist (removed/404), then log and skip to next repo + if gh_repo_status.response_code == 404: # project no longer exists log_potential_delete(snyk_repo.org_name, snyk_repo.full_name) - elif gh_repo_status.response_code == 200: # project exists and has not been renamed + elif gh_repo_status.response_code == 200: # project exists and has not been renamed # if --audit-large-repos is on if common.ARGS.audit_large_repos: is_truncated_str = \ @@ -90,12 +91,54 @@ def run(): ) # move to next repo without processing the rest of the code continue + + # If we've previously deactivated projects, we should activate them again + # if the repo becomes "unarchived" + if not gh_repo_status.archived and common.ARGS.on_unarchived == "reactivate": + for project in snyk_repo.snyk_projects: + if not project["is_monitored"]: + activated_projects = snyk_repo.activate_manifests(common.ARGS.dry_run) + for activated_project in activated_projects: + if not common.ARGS.dry_run: + app_print(snyk_repo.org_name, + snyk_repo.full_name, + f"Activated manifest: {activated_project['manifest']}") + else: + app_print(snyk_repo.org_name, + snyk_repo.full_name, + f"Would activate manifest: " + f"{activated_project['manifest']}") + break # We just needed to check if any one of the projects wasn't active + + if gh_repo_status.archived and common.ARGS.on_archived != "ignore": + app_print(snyk_repo.org_name, + snyk_repo.full_name, + f"Repo is archived") + + # Check what archival mode we're running in + on_archival_action = common.ARGS.on_archived + if on_archival_action == "deactivate": + deleted_projects = snyk_repo.deactivate_manifests(common.ARGS.dry_run) + elif on_archival_action == "delete": + deleted_projects = snyk_repo.delete_manifests(common.ARGS.dry_run) + + # And tell the user what has or would have happened + for project in deleted_projects: + if not common.ARGS.dry_run: + app_print(snyk_repo.org_name, + snyk_repo.full_name, + f"{on_archival_action.capitalize()}d manifest: " + f"{project['manifest']}") + else: + app_print(snyk_repo.org_name, + snyk_repo.full_name, + f"Would {on_archival_action} manifest: {project['manifest']}") # snyk has the wrong branch, re-import - if gh_repo_status.repo_default_branch != snyk_repo.branch: + elif gh_repo_status.repo_default_branch != snyk_repo.branch: app_print(snyk_repo.org_name, snyk_repo.full_name, - f"Default branch name changed from {snyk_repo.branch}" \ - f" -> {gh_repo_status.repo_default_branch}") + f"Default branch name changed from {snyk_repo.branch}" f" -> " + f"{gh_repo_status.repo_default_branch}") updated_projects = snyk_repo.update_branch( gh_repo_status.repo_default_branch, common.ARGS.dry_run) @@ -106,7 +149,7 @@ def run(): f"Monitored branch set to " \ f"{gh_repo_status.repo_default_branch} " \ f"for: {project['manifest']}") - else: #find deltas + else: # find deltas app_print(snyk_repo.org_name, snyk_repo.full_name, f"Checking {str(len(snyk_repo.snyk_projects))} " \ @@ -127,7 +170,7 @@ def run(): snyk_repo.full_name, "Checking for new manifests in source tree") - #if not common.ARGS.dry_run: + # if not common.ARGS.dry_run: projects_import = snyk_repo.add_new_manifests(common.ARGS.dry_run) if isinstance(projects_import, ImportStatus): diff --git a/app/gh_repo.py b/app/gh_repo.py index 6ac9860..9ebb006 100755 --- a/app/gh_repo.py +++ b/app/gh_repo.py @@ -183,6 +183,7 @@ def get_gh_repo_status(snyk_gh_repo): response_message = "" response_status_code = "" repo_default_branch = "" + archived = False # logging.debug(f"snyk_gh_repo origin: {snyk_gh_repo.origin}") @@ -214,6 +215,7 @@ def get_gh_repo_status(snyk_gh_repo): if response.status_code == 200: response_message = "Match" repo_default_branch = response.json()['default_branch'] + archived = response.json()['archived'] elif response.status_code == 404: response_message = "Not Found" @@ -231,6 +233,7 @@ def get_gh_repo_status(snyk_gh_repo): repo_new_full_name = follow_response.json()["full_name"] repo_owner = repo_new_full_name.split("/")[0] repo_name = repo_new_full_name.split("/")[1] + archived = follow_response.json()['archived'] else: repo_owner = "" repo_name = "" @@ -252,6 +255,7 @@ def get_gh_repo_status(snyk_gh_repo): snyk_gh_repo["org_id"], repo_owner, f"{repo_owner}/{repo_name}", - repo_default_branch + repo_default_branch, + archived ) return repo_status diff --git a/app/models.py b/app/models.py index 5daaa6e..d61f1cf 100644 --- a/app/models.py +++ b/app/models.py @@ -2,11 +2,13 @@ from dataclasses import dataclass from typing import List + @dataclass class ImportFile: """File being imported""" path: str + @dataclass class PendingDelete: """Projects needing deletion""" @@ -16,6 +18,7 @@ class PendingDelete: org_name: str pending_repo: str + @dataclass class ImportStatus: """Import job response""" @@ -29,6 +32,7 @@ class ImportStatus: files: List[ImportFile] pending_project_deletes: List[PendingDelete] +# pylint: disable=too-many-instance-attributes @dataclass class GithubRepoStatus: """Status of a Github repository""" @@ -39,4 +43,5 @@ class GithubRepoStatus: repo_owner: str repo_full_name: str repo_default_branch: str + archived: bool \ No newline at end of file diff --git a/app/snyk_repo.py b/app/snyk_repo.py index e42c432..112a40e 100755 --- a/app/snyk_repo.py +++ b/app/snyk_repo.py @@ -113,6 +113,58 @@ def delete_stale_manifests(self, dry_run): f" in org {snyk_project['org_id']}") return result + def delete_manifests(self, dry_run): + """ delete all snyk projects corresponding to a repo """ + result = [] + for snyk_project in self.snyk_projects: + # delete project, append on success + if not dry_run: + try: + app.utils.snyk_helper.delete_snyk_project(snyk_project["id"], + snyk_project["org_id"]) + result.append(snyk_project) + except snyk.errors.SnykNotFoundError: + print(f" - Project {snyk_project['id']} not found" \ + f" in org {snyk_project['org_id']}") + else: + result.append(snyk_project) + return result + + + def deactivate_manifests(self, dry_run): + """ deactivate all snyk projects corresponding to a repo """ + result = [] + for snyk_project in [x for x in self.snyk_projects if x["is_monitored"]]: + # delete project, append on success + if not dry_run: + try: + app.utils.snyk_helper.deactivate_snyk_project(snyk_project["id"], + snyk_project["org_id"]) + result.append(snyk_project) + except snyk.errors.SnykNotFoundError: + print(f" - Project {snyk_project['id']} not found" \ + f" in org {snyk_project['org_id']}") + else: + result.append(snyk_project) + return result + + def activate_manifests(self, dry_run): + """ deactivate all snyk projects corresponding to a repo """ + result = [] + for snyk_project in [x for x in self.snyk_projects if not x["is_monitored"]]: + # delete project, append on success + if not dry_run: + try: + app.utils.snyk_helper.activate_snyk_project(snyk_project["id"], + snyk_project["org_id"]) + result.append(snyk_project) + except snyk.errors.SnykNotFoundError: + print(f" - Project {snyk_project['id']} not found" \ + f" in org {snyk_project['org_id']}") + else: + result.append(snyk_project) + return result + def update_branch(self, new_branch_name, dry_run): """ update the branch for all snyk projects for this repo """ result = [] diff --git a/app/tests/test_snyk_scm_refresh.py b/app/tests/test_snyk_scm_refresh.py index 56ef1f3..514a0cf 100644 --- a/app/tests/test_snyk_scm_refresh.py +++ b/app/tests/test_snyk_scm_refresh.py @@ -27,18 +27,18 @@ def __init__(self, status_code): self.headers = {"Location": "test_location"} def json(self): - response = {"full_name": "new_owner/new_repo", "default_branch": "master"} + response = {"full_name": "new_owner/new_repo", "default_branch": "master", "archived": False} return response @pytest.mark.parametrize( - "status_code, response_message, repo, name, owner, default_branch", + "status_code, response_message, repo, name, owner, default_branch, archived", [ - (200, "Match", "test_org/test_repo", "test_repo", "test_owner", "master"), - (301, "Moved to new_repo", "new_owner/new_repo", "new_repo", "new_owner", ""), - (404, "Not Found", "test_org/test_repo", None, None, "") + (200, "Match", "test_org/test_repo", "test_repo", "test_owner", "master", False), + (301, "Moved to new_repo", "new_owner/new_repo", "new_repo", "new_owner", "", False), + (404, "Not Found", "test_org/test_repo", None, None, "", False) ], ) -def test_get_gh_repo_status_github(mocker, status_code, response_message, repo, name, owner, default_branch): +def test_get_gh_repo_status_github(mocker, status_code, response_message, repo, name, owner, default_branch, archived): # TODO: assumes a successful redirect for the 301 case mocker.patch( @@ -63,20 +63,21 @@ def test_get_gh_repo_status_github(mocker, status_code, response_message, repo, snyk_repo_github["org_id"], snyk_repo_github["full_name"].split("/")[0], snyk_repo_github["full_name"], - default_branch + default_branch, + archived ) assert get_gh_repo_status(snyk_repo_github) == repo_status @pytest.mark.parametrize( - "status_code, response_message, repo, name, owner, default_branch", + "status_code, response_message, repo, name, owner, default_branch, archived", [ - (200, "Match", "test_org/test_repo", "test_repo", "test_owner", "master"), - (301, "Moved to new_repo", "new_owner/new_repo", "new_repo", "new_owner", ""), - (404, "Not Found", "test_org/test_repo", None, None, "") + (200, "Match", "test_org/test_repo", "test_repo", "test_owner", "master", False), + (301, "Moved to new_repo", "new_owner/new_repo", "new_repo", "new_owner", "", False), + (404, "Not Found", "test_org/test_repo", None, None, "", False) ], ) -def test_get_gh_repo_status_github_enterprise_cloud(mocker, status_code, response_message, repo, name, owner, default_branch): +def test_get_gh_repo_status_github_enterprise_cloud(mocker, status_code, response_message, repo, name, owner, default_branch, archived): # TODO: assumes a successful redirect for the 301 case mocker.patch( @@ -101,7 +102,8 @@ def test_get_gh_repo_status_github_enterprise_cloud(mocker, status_code, respons snyk_repo_github_enterprise["org_id"], snyk_repo_github_enterprise["full_name"].split("/")[0], snyk_repo_github_enterprise["full_name"], - default_branch + default_branch, + archived ) assert get_gh_repo_status(snyk_repo_github_enterprise) == repo_status @@ -144,7 +146,8 @@ def test_get_snyk_repos_from_snyk_projects(): "type": "npm", "integration_id": "66d7ebef-9b36-464f-889c-b92c9ef5ce12", "branch_from_name": "", - "branch": "master" + "branch": "master", + "is_monitored": True }, { "id": "12345", @@ -159,7 +162,8 @@ def test_get_snyk_repos_from_snyk_projects(): "type": "npm", "integration_id": "66d7ebef-9b36-464f-889c-b92c9ef5ce12", "branch_from_name": "", - "branch": "master" + "branch": "master", + "is_monitored": True }, ] @@ -275,7 +279,93 @@ def test_passes_manifest_filter(): assert passes_manifest_filter(path_pass_2) == True assert passes_manifest_filter(path_fail_3) == False +@pytest.fixture +def snyk_projects_fixture(): + class TestModels(object): + # @pytest.fixture + def organization(self): + org = Organization( + name="My Other Org", id="a04d9cbd-ae6e-44af-b573-0556b0ad4bd2" + ) + org.client = SnykClient("token") + return org + + def base_url(self): + return "https://snyk.io/api/v1" + + def organization_url(self, base_url, organization): + return "%s/org/%s" % (base_url, organization.id) + + snyk_gh_projects = [ + { + "id": "12345", + "name": "scotte-snyk/test-project-1:package.json", + "repo_full_name": "scotte-snyk/test-project-1", + "repo_owner": "scotte-snyk", + "repo_name": "test-project-1", + "manifest": "package.json", + "org_id": "12345", + "org_name": "scotte-snyk", + "origin": "github", + "type": "npm", + "integration_id": "66d7ebef-9b36-464f-889c-b92c9ef5ce12", + "branch_from_name": "", + "branch": "master", + "is_monitored": True + }, + { + "id": "12345", + "name": "scotte-snyk/test-project-2:package.json", + "repo_full_name": "scotte-snyk/test-project-2", + "repo_owner": "scotte-snyk", + "repo_name": "test-project-2", + "manifest": "package.json", + "org_id": "12345", + "org_name": "scotte-snyk", + "origin": "github", + "type": "npm", + "integration_id": "66d7ebef-9b36-464f-889c-b92c9ef5ce12", + "branch_from_name": "", + "branch": "master", + "is_monitored": False + }, + ] + + snyk_repo_github_enterprise = SnykRepo( + 'new_owner/new_repo', + "1234-5678", + "new_owner", + "12345", + "github-enterprise", + "master", + snyk_gh_projects + ) + return snyk_repo_github_enterprise + +def test_archived_repo_delete(snyk_projects_fixture, mocker): + mock = mocker.patch( + "app.utils.snyk_helper.delete_snyk_project" + ) + snyk_projects_fixture.delete_manifests(dry_run=False) + assert mock.called_once + + +def test_archived_repo_deactivate(snyk_projects_fixture, mocker): + mock = mocker.patch( + "app.utils.snyk_helper.deactivate_snyk_project" + ) + snyk_projects_fixture.deactivate_manifests(dry_run=False) + assert mock.called_once + + +def test_unarchived_repo_reactivate(snyk_projects_fixture, mocker): + mock = mocker.patch( + "app.utils.snyk_helper.activate_snyk_project" + ) + snyk_projects_fixture.activate_manifests(dry_run=False) + assert mock.called + def test_import_manifest_exceeds_limit(mocker): """ Pytest snyk_helper.import_manifest exceeding limit of manifest projects diff --git a/app/utils/snyk_helper.py b/app/utils/snyk_helper.py index 255a2e7..31de850 100644 --- a/app/utils/snyk_helper.py +++ b/app/utils/snyk_helper.py @@ -71,8 +71,8 @@ def get_snyk_repos_from_snyk_projects(snyk_projects): snyk_projects[i]["integration_id"], snyk_projects[i]["origin"], snyk_projects[i]["branch"], - [x for x in snyk_projects \ - if x["repo_full_name"] == project["repo_full_name"]]) + [x for x in snyk_projects if x["repo_full_name"] == + project["repo_full_name"]]) ) repo_projects = [] @@ -153,7 +153,8 @@ def build_snyk_project_list(snyk_orgs, ARGS): "type": project.type, "integration_id": integration_id, "branch_from_name": branch_from_name, - "branch": project.branch + "branch": project.branch, + "is_monitored": project.isMonitored } ) @@ -238,6 +239,28 @@ def delete_snyk_project(project_id, org_id): print(f" - Project {project_id} not found in org {org_id} ...") return False +def deactivate_snyk_project(project_id, org_id): + """Deactivate a single Snyk project""" + org = common.snyk_client.organizations.get(org_id) + + try: + project = org.projects.get(project_id) + return project.deactivate() + except snyk.errors.SnykNotFoundError: + print(f" - Project {project_id} not found in org {org_id} ...") + return False + +def activate_snyk_project(project_id, org_id): + """Acitvate a single Syyk project""" + org = common.snyk_client.organizations.get(org_id) + + try: + project = org.projects.get(project_id) + return project.activate() + except snyk.errors.SnykNotFoundError: + print(f" - Project {project_id} not found in org {org_id} ...") + return False + def process_import_status_checks(import_status_checks): # pylint: disable=too-many-nested-blocks, too-many-branches # pylint: disable=too-many-locals diff --git a/common.py b/common.py index 1ced141..88808f4 100644 --- a/common.py +++ b/common.py @@ -112,6 +112,21 @@ def parse_command_line_args(): default=False, choices=['on', 'off'] ) + parser.add_argument( + "--on-archived", + help="Tells the tool what to do when a GitHub project is archived (Snyk projects ignored by default)", + required=False, + default="ignore", + choices=['ignore', 'deactivate', 'delete'] + ) + parser.add_argument( + "--on-unarchived", + help="If the tool detects a Snyk project deactivated whilst the GitHub repo is not archived, what should it do?" + " (By default the tool will ignore)", + required=False, + default="ignore", + choices=['ignore', 'reactivate'] + ) # show disabled argument help message and prevent invalidation of any existent "--code=off" verbose argument mode parser.add_argument( "--code",