diff --git a/.circleci/config.yml b/.circleci/config.yml index 0a8746c..ac3e52f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ version: 2.1 orbs: - snyk: snyk/snyk@0.0.10 + snyk: snyk/snyk@1.1.2 jobs: build-test: docker: diff --git a/README.md b/README.md index 2d5f0d4..3300aee 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,12 @@ # snyk-scm-refresh [![Known Vulnerabilities](https://snyk.io/test/github/snyk-tech-services/snyk-scm-refresh/badge.svg)](https://snyk.io/test/github/snyk-tech-services/snyk-scm-refresh) [![circleci](https://circleci.com/gh/snyk-tech-services/snyk-scm-refresh.svg?style=svg)](https://circleci.com/gh/snyk-tech-services/snyk-scm-refresh) +
+⚠️ WARNING: +Python 3.10 introduces breaking changes that are currently incompatible with this tool. You must use Python 3.7-3.9 +
+
+ Keeps Snyk projects in sync with their associated Github repos For repos with at least 1 project already in Snyk: @@ -11,6 +17,8 @@ For repos with at least 1 project already in Snyk: - Enable Snyk Code analysis for repos - Detect deleted repos and log for review + + **STOP NOW IF ANY OF THE FOLLOWING ARE TRUE** - Monitoring non-default branches - Using an SCM other than Github.com or Github Enterprise Server @@ -36,7 +44,7 @@ optional arguments: ``` ### Sync with defaults -`./snyk_scm_refresh.py --org-id=12345 +`./snyk_scm_refresh.py --org-id=12345` ### Sync SCA projects only `./snyk_scm_refresh.py --org-id=12345 --container=off` diff --git a/app/app.py b/app/app.py index 85936a7..d6671ec 100755 --- a/app/app.py +++ b/app/app.py @@ -60,10 +60,7 @@ def run(): f"Processing {str(i+1)}/{str(len(snyk_repos))}") try: - if snyk_repo.origin == "github": - gh_repo_status = get_gh_repo_status(snyk_repo, common.GITHUB_TOKEN) - elif snyk_repo.origin == "github-enterprise": - gh_repo_status = get_gh_repo_status(snyk_repo, common.GITHUB_ENTERPRISE_TOKEN, True) + gh_repo_status = get_gh_repo_status(snyk_repo) except RuntimeError as err: raise RuntimeError("Failed to query GitHub repository!") from err @@ -124,6 +121,7 @@ def run(): snyk_repo.full_name, f"Checking {str(len(snyk_repo.snyk_projects))} " \ f"projects for any stale manifests") + # print(f"snyk repo projects: {snyk_repo.snyk_projects}") deleted_projects = snyk_repo.delete_stale_manifests(common.ARGS.dry_run) for project in deleted_projects: if not common.ARGS.dry_run: diff --git a/app/gh_repo.py b/app/gh_repo.py index 26e6578..2830045 100755 --- a/app/gh_repo.py +++ b/app/gh_repo.py @@ -56,19 +56,28 @@ def passes_manifest_filter(path, skip_snyk_code=False): return passes_filter -def get_gh_repo_status(snyk_gh_repo, github_token, github_enterprise=False): +def get_gh_repo_status(snyk_gh_repo): """detect if repo still exists, has been removed, or renamed""" - repo_owner = snyk_gh_repo["owner"] - repo_name = snyk_gh_repo["name"] + repo_owner = snyk_gh_repo.full_name.split("/")[0] + repo_name = snyk_gh_repo.full_name.split("/")[1] response_message = "" repo_default_branch = "" + # print(f'snyk_gh_repo origin: {snyk_gh_repo.origin}') + + if snyk_gh_repo.origin == "github": + github_token = common.GITHUB_TOKEN + elif snyk_gh_repo.origin == "github-enterprise": + github_token = common.GITHUB_ENTERPRISE_TOKEN + headers = {"Authorization": "Bearer %s"} headers["Authorization"] = headers["Authorization"] % (github_token) - if not github_enterprise: + if snyk_gh_repo.origin == "github" or \ + (snyk_gh_repo.origin == "github-enterprise" and \ + common.USE_GHE_INTEGRATION_FOR_GH_CLOUD): request_url = f"https://api.github.com/repos/{snyk_gh_repo['full_name']}" # print("requestURL: " + requestURL) - else: + elif snyk_gh_repo.origin == "github-enterprise": request_url = f"https://{common.GITHUB_ENTERPRISE_HOST}" \ f"/api/v3/repos/{snyk_gh_repo['full_name']}" try: @@ -98,7 +107,7 @@ def get_gh_repo_status(snyk_gh_repo, github_token, github_enterprise=False): repo_owner = "" repo_name = "" - response_message = "Moved to %s" % repo_name + response_message = f"Moved to {repo_name}" repo_status = { "response_code": response.status_code, @@ -130,7 +139,7 @@ def is_default_branch_renamed(snyk_gh_repo, new_branch, github_token, github_ent try: response = requests.get(url=request_url, allow_redirects=False, headers=headers) - if response.status_code == 301 or response.status_code == 302: + if response.status_code in (301, 302): print('redirect response url: ' + response.headers["Location"]) if str(response.headers["Location"]).endswith(f"/{new_branch}"): # print('the redirect is pointing to the new branch') diff --git a/app/snyk_repo.py b/app/snyk_repo.py index 60b93f1..149e6ce 100755 --- a/app/snyk_repo.py +++ b/app/snyk_repo.py @@ -10,7 +10,7 @@ get_repo_manifests, passes_manifest_filter ) -import app.utils.snyk_helper +import app class SnykRepo(): """ SnykRepo object """ @@ -33,7 +33,7 @@ def __init__( self.branch = branch self.snyk_projects = snyk_projects def __getitem__(self, item): - return self.full_name + return self.__dict__[item] def get_projects(self): """ return list of projects for this repo """ @@ -107,7 +107,7 @@ def update_branch(self, new_branch_name, dry_run): for (i, snyk_project) in enumerate(self.snyk_projects): if snyk_project["branch"] != new_branch_name: if not dry_run: - sys.stdout.write("\r - %s/%s" % (i+1, len(self.snyk_projects))) + sys.stdout.write(f"\r - {i+1}/{len(self.snyk_projects)}") sys.stdout.flush() try: app.utils.snyk_helper.update_project_branch(snyk_project["id"], diff --git a/app/tests/test_snyk_scm_refresh.py b/app/tests/test_snyk_scm_refresh.py index b4a051d..e76c7b2 100644 --- a/app/tests/test_snyk_scm_refresh.py +++ b/app/tests/test_snyk_scm_refresh.py @@ -1,11 +1,13 @@ """test suite for snyk_scm_refresh.py""" +import os import pytest from snyk.models import Project +import common +from app.snyk_repo import SnykRepo from app.gh_repo import ( get_gh_repo_status, passes_manifest_filter, - ) from app.utils.snyk_helper import get_snyk_projects_for_repo @@ -27,34 +29,73 @@ def json(self): (404, "Not Found", "test_org/test_repo", None, None, "") ], ) -def test_get_gh_repo_status(mocker, status_code, response_message, repo, name, owner, default_branch): +def test_get_gh_repo_status_github(mocker, status_code, response_message, repo, name, owner, default_branch): # TODO: assumes a successful redirect for the 301 case mocker.patch( "requests.get", side_effect=[MockResponse(status_code), MockResponse(200)] ) + mocker.patch.dict(os.environ, {'GITHUB_ENTERPRISE_TOKEN': '1234', 'GITHUB_ENTERPRISE_HOST':common.GITHUB_CLOUD_API_HOST}) + + snyk_repo_github = SnykRepo( + 'new_owner/new_repo', + "1234-5678", + "new_owner", + "12345", + "github", + "master", + [] + ) - snyk_repo = { - "full_name": 'new_owner/new_repo', - "owner":'new_owner', - "name": 'new_repo', - "org_id": "1234-5678", - "gh_integration_id": "12345", - "branch_from_name": "", - "branch": "master" + repo_status = { + "response_code": status_code, + "response_message": response_message, + "repo_name": snyk_repo_github["full_name"].split("/")[1], + "snyk_org_id": snyk_repo_github["org_id"], + "repo_owner": snyk_repo_github["full_name"].split("/")[0], + "repo_full_name": snyk_repo_github["full_name"], + "repo_default_branch": default_branch } + assert get_gh_repo_status(snyk_repo_github) == repo_status + +@pytest.mark.parametrize( + "status_code, response_message, repo, name, owner, default_branch", + [ + (200, "Match", "test_org/test_repo", "test_repo", "test_owner", "master"), + (301, "Moved to new_repo", "new_owner/new_repo", "new_repo", "new_owner", ""), + (404, "Not Found", "test_org/test_repo", None, None, "") + ], +) +def test_get_gh_repo_status_github_enterprise_cloud(mocker, status_code, response_message, repo, name, owner, default_branch): + + # TODO: assumes a successful redirect for the 301 case + mocker.patch( + "requests.get", side_effect=[MockResponse(status_code), MockResponse(200)] + ) + mocker.patch.dict(os.environ, {'GITHUB_ENTERPRISE_TOKEN': '1234', 'GITHUB_ENTERPRISE_HOST':common.GITHUB_CLOUD_API_HOST}) + + snyk_repo_github_enterprise = SnykRepo( + 'new_owner/new_repo', + "1234-5678", + "new_owner", + "12345", + "github-enterprise", + "master", + [] + ) + repo_status = { "response_code": status_code, "response_message": response_message, - "repo_name": snyk_repo["name"], - "snyk_org_id": snyk_repo["org_id"], - "repo_owner": snyk_repo["owner"], - "repo_full_name": snyk_repo["full_name"], + "repo_name": snyk_repo_github_enterprise["full_name"].split("/")[1], + "snyk_org_id": snyk_repo_github_enterprise["org_id"], + "repo_owner": snyk_repo_github_enterprise["full_name"].split("/")[0], + "repo_full_name": snyk_repo_github_enterprise["full_name"], "repo_default_branch": default_branch } - assert get_gh_repo_status(snyk_repo, "test_token") == repo_status + assert get_gh_repo_status(snyk_repo_github_enterprise) == repo_status def test_get_gh_repo_status_unauthorized(mocker): """ test handling unauthorized token """ @@ -62,18 +103,20 @@ def test_get_gh_repo_status_unauthorized(mocker): "requests.get", side_effect=[MockResponse(401)] ) - snyk_repo = { - "full_name": 'test_org/test_repo', - "owner":'test_org', - "name": 'test_repo', - "org_id": "1234-5678", - "gh_integration_id": "12345", - "branch_from_name": "", - "branch": "master" - } + mocker.patch.dict(os.environ, {'GITHUB_TOKEN': 'test_token'}) + + snyk_repo = SnykRepo( + 'test_org/test_repo', + "1234-5678", + "new_owner", + "12345", + "github", + "master", + [] + ) with pytest.raises(RuntimeError): - get_gh_repo_status(snyk_repo, "test_token") + get_gh_repo_status(snyk_repo) def test_get_snyk_project_for_repo(): """ test collecting projects for a repo """ diff --git a/app/utils/snyk_helper.py b/app/utils/snyk_helper.py index 2ef4a65..2a33390 100644 --- a/app/utils/snyk_helper.py +++ b/app/utils/snyk_helper.py @@ -15,21 +15,22 @@ def app_print(org, repo, text): def log_potential_delete(org_name, repo_name): """ Log potential repo deletion """ app_print(org_name, repo_name, "Logging potential delete") - common.POTENTIAL_DELETES_FILE.write("%s,%s\n" % (org_name, repo_name)) + common.POTENTIAL_DELETES_FILE.write(f"{org_name},{repo_name}\n") def log_updated_project_branch(org_name, project_id, project_name, new_branch): """ Log project branch update """ - common.UPDATED_PROJECT_BRANCHES_FILE.write("%s,%s,%s,%s\n" % (org_name, - project_name, - project_id, - new_branch)) + common.UPDATED_PROJECT_BRANCHES_FILE.write(f"{org_name}," + f"{project_name}," + f"{project_id}," + f"{new_branch}\n") def log_update_project_branch_error(org_name, project_id, project_name, new_branch): """ Log project branch update """ - common.UPDATE_PROJECT_BRANCHES_ERRORS_FILE.write("%s,%s,%s,%s\n" % (org_name, - project_name, - project_id, - new_branch)) + common.UPDATE_PROJECT_BRANCHES_ERRORS_FILE.write( + f"{org_name}," + f"{project_name}," + f"{project_id}," + f"{new_branch}\n") def get_snyk_repos_from_snyk_orgs(snyk_orgs, ARGS): """Build list of repositories from a given list of Snyk orgs""" @@ -39,40 +40,60 @@ def get_snyk_repos_from_snyk_orgs(snyk_orgs, ARGS): repo_projects = [] # initialize to the first repo name - curr_repo_name = snyk_projects[0]["repo_full_name"] num_projects = len(snyk_projects) - for (i, project) in enumerate(snyk_projects): - if i == num_projects-1: - snyk_repos.append( - SnykRepo(snyk_projects[i]["repo_full_name"], - snyk_projects[i]["org_id"], - snyk_projects[i]["org_name"], - snyk_projects[i]["integration_id"], - snyk_projects[i]["origin"], - snyk_projects[i]["branch"], - repo_projects) - ) - - # we encountered a new repo, or reached the end of the project list - if project["repo_full_name"] != curr_repo_name: - # add repo to snyk_repos - snyk_repos.append( - SnykRepo(snyk_projects[i-1]["repo_full_name"], - snyk_projects[i-1]["org_id"], - snyk_projects[i-1]["org_name"], - snyk_projects[i-1]["integration_id"], - snyk_projects[i-1]["origin"], - snyk_projects[i-1]["branch"], - repo_projects) - ) - repo_projects = [project] + if num_projects > 0: + curr_repo_name = snyk_projects[0]["repo_full_name"] + print(f"curr repo name: {curr_repo_name}") + + for (i, project) in enumerate(snyk_projects): + #if i == num_projects-1: + # print(f"encountered base case") + # snyk_repos.append( + # SnykRepo(snyk_projects[i]["repo_full_name"], + # snyk_projects[i]["org_id"], + # snyk_projects[i]["org_name"], + # snyk_projects[i]["integration_id"], + # snyk_projects[i]["origin"], + # snyk_projects[i]["branch"], + # repo_projects) + # ) + + # we encountered a new repo, or reached the end of the project list + if project["repo_full_name"] != curr_repo_name: + # print(f"encountered a new repo name: {project['repo_full_name']}") + # add repo to snyk_repos + snyk_repos.append( + SnykRepo(snyk_projects[i-1]["repo_full_name"], + snyk_projects[i-1]["org_id"], + snyk_projects[i-1]["org_name"], + snyk_projects[i-1]["integration_id"], + snyk_projects[i-1]["origin"], + snyk_projects[i-1]["branch"], + repo_projects) + ) + repo_projects = [project] + # print(f"setting repo_projects to: {repo_projects}") + + if i == num_projects-1: + print("encountered last project") + snyk_repos.append( + SnykRepo(snyk_projects[i]["repo_full_name"], + snyk_projects[i]["org_id"], + snyk_projects[i]["org_name"], + snyk_projects[i]["integration_id"], + snyk_projects[i]["origin"], + snyk_projects[i]["branch"], + repo_projects) + ) - else: - # add to project list for this repo - repo_projects.append(project) + else: + # add to project list for this repo + repo_projects.append(project) + # print(f"adding project: {project['manifest']}") - curr_repo_name = project["repo_full_name"] + curr_repo_name = project["repo_full_name"] + # print(f"curr repo name set to: {curr_repo_name}") return snyk_repos def build_snyk_project_list(snyk_orgs, ARGS): @@ -172,7 +193,7 @@ def import_manifests(org_id, repo_full_name, integration_id, files=[]) -> Import repo_full_name = repo_full_name.split("/") org = common.snyk_client.organizations.get(org_id) - path = "org/%s/integrations/%s/import" % (org.id, integration_id) + path = f"org/{org.id}/integrations/{integration_id}/import" if len(files) > 0: payload = { @@ -231,8 +252,10 @@ def process_import_status_checks(import_status_checks): import_jobs_completed = [] import_logs_completed = [] - print("Checking import statuses, polling for up to %s minutes..." - % str((common.PENDING_REMOVAL_MAX_CHECKS * common.PENDING_REMOVAL_CHECK_INTERVAL)/60)) + polling_minutes = (common.PENDING_REMOVAL_MAX_CHECKS * common.PENDING_REMOVAL_CHECK_INTERVAL)/60 + + print(f"Checking import statuses, polling for up to " + f"{str(polling_minutes)} minutes...") # get unique import status checks with combined pending deletes (if present) seen_check_ids = [] @@ -248,9 +271,8 @@ def process_import_status_checks(import_status_checks): while check_count < common.PENDING_REMOVAL_MAX_CHECKS: if len(unique_import_status_checks) > len(import_jobs_completed): - sys.stdout.write("%s batch pending\n" % ( - len(unique_import_status_checks) - len(import_jobs_completed) - )) + sys.stdout.write(f"{len(unique_import_status_checks) - len(import_jobs_completed)} " + f"batch pending\n") sys.stdout.flush() # check each import job statuses for import_job in unique_import_status_checks: @@ -266,11 +288,9 @@ def process_import_status_checks(import_status_checks): uniq_import_log = import_status_log["name"] + \ '-' + import_status_log["created"] if uniq_import_log not in import_logs_completed: - print(" - [%s] Import Target status: %s (%s projects)" % ( - import_status_log["name"], - import_status_log["status"], - len(import_status_log["projects"]) - )) + print(f" - [{import_status_log['name']}] " + f"Import Target status: {import_status_log['status']} " + f"({len(import_status_log['projects'])} projects)") # if repo import status is complete, log # and delete any pending waiting on this repo import if import_status_log["status"] == "complete": @@ -283,12 +303,10 @@ def process_import_status_checks(import_status_checks): import_status_log["name"], f"Imported {imported_project}") # pylint: disable=line-too-long - common.COMPLETED_PROJECT_IMPORTS_FILE.write("%s,%s:%s,%s\n" % ( - import_job.org_name, - import_status_log["name"], - imported_project, - project["success"] - )) + common.COMPLETED_PROJECT_IMPORTS_FILE.write( + f"{import_job.org_name}," + f"{import_status_log['name']}:{imported_project}," + f"{project['success']}\n") if import_status["status"] != "pending": import_jobs_completed.append(import_job.import_job_id) @@ -297,20 +315,17 @@ def process_import_status_checks(import_status_checks): for pending_delete in import_job.pending_project_deletes: app_print(pending_delete['org_name'], pending_delete['repo_full_name'], - "delete stale project [%s]" % ( - pending_delete['id'])) + f"delete stale project [{pending_delete['id']}]") delete_snyk_project( pending_delete['id'], pending_delete['org_id'] ) - common.RENAMED_MANIFESTS_DELETED_FILE.write("%s,%s:%s\n" % ( - pending_delete['org_name'], - pending_delete['repo_full_name'], - pending_delete['manifest'] - )) - - print("Checking back in %d seconds..." % - common.PENDING_REMOVAL_CHECK_INTERVAL) + common.RENAMED_MANIFESTS_DELETED_FILE.write( + f"{pending_delete['org_name']}," + f"{pending_delete['repo_full_name']}:" + f"{pending_delete['manifest']}\n") + + print(f"Checking back in {common.PENDING_REMOVAL_CHECK_INTERVAL} seconds...") time.sleep(common.PENDING_REMOVAL_CHECK_INTERVAL) else: @@ -319,28 +334,21 @@ def process_import_status_checks(import_status_checks): check_count += 1 if check_count == common.PENDING_REMOVAL_MAX_CHECKS: - print( - "\nExiting with %d pending removals, logging...\n" % ( - len(unique_import_status_checks) - - len(import_jobs_completed) - ) - ) + print(f"\nExiting with {len(unique_import_status_checks) - len(import_jobs_completed)} " + f"pending removals, logging...\n") + for import_status_check in unique_import_status_checks: if import_status_check.import_job_id \ not in import_jobs_completed: common.RENAMED_MANIFESTS_PENDING_FILE.write( - "%s,%s/%s\n" % ( - import_status_check.org_name, - import_status_check.repo_owner, - import_status_check.repo_name - ) - ) + f"{import_status_check.org_name}," + f"{import_status_check.repo_owner}/{import_status_check.repo_name}\n") return def update_project_branch(project_id, project_name, org_id, new_branch_name): """ update snyk project monitored branch """ org = common.snyk_client.organizations.get(org_id) - path = "org/%s/project/%s" % (org.id, project_id) + path = "org/{org.id}/project/{project_id}" payload = { "branch": new_branch_name diff --git a/common.py b/common.py index 18c1791..8ef3b6e 100644 --- a/common.py +++ b/common.py @@ -16,9 +16,11 @@ MANIFEST_PATTERN_IAC = '.*[.](yaml|yml|tf)$' MANIFEST_PATTERN_CODE = '.*[.](js|cs|php|java|py)$' MANIFEST_PATTERN_EXCLUSIONS = '^.*(fixtures|tests\/|__tests__|test\/|__test__|[.].*ci\/|.*ci[.].yml|node_modules\/|bower_components\/|variables[.]tf|outputs[.]tf).*$' +GITHUB_CLOUD_API_HOST="api.github.com" GITHUB_ENABLED = False GITHUB_ENTERPRISE_ENABLED = False +USE_GHE_INTEGRATION_FOR_GH_CLOUD = False SNYK_TOKEN = getenv("SNYK_TOKEN") GITHUB_TOKEN = getenv("GITHUB_TOKEN") @@ -63,16 +65,25 @@ snyk_client = SnykClient(SNYK_TOKEN) +if (GITHUB_ENTERPRISE_HOST == GITHUB_CLOUD_API_HOST): + USE_GHE_INTEGRATION_FOR_GH_CLOUD = True + if (GITHUB_TOKEN): GITHUB_ENABLED = True gh_client = create_github_client(GITHUB_TOKEN) + print("created github.com client") if (GITHUB_ENTERPRISE_HOST): GITHUB_ENTERPRISE_ENABLED = True - gh_enterprise_client = create_github_enterprise_client(GITHUB_ENTERPRISE_TOKEN, GITHUB_ENTERPRISE_HOST) + if USE_GHE_INTEGRATION_FOR_GH_CLOUD: + gh_enterprise_client = create_github_client(GITHUB_ENTERPRISE_TOKEN) + print(f"created github client for enterprise host: {GITHUB_ENTERPRISE_HOST}") + else: + print(f"created GH enterprise client for host: {GITHUB_ENTERPRISE_HOST}") + gh_enterprise_client = create_github_enterprise_client(GITHUB_ENTERPRISE_TOKEN, GITHUB_ENTERPRISE_HOST) def parse_command_line_args(): - """Parse command-line arguments""" + """Parse command-line arguments""" parser = argparse.ArgumentParser() parser.add_argument( diff --git a/snyk_scm_refresh.py b/snyk_scm_refresh.py index 5d692e9..c54236e 100755 --- a/snyk_scm_refresh.py +++ b/snyk_scm_refresh.py @@ -11,12 +11,12 @@ if __name__ == "__main__": if common.ARGS.dry_run: - print("****** DRY-RUN MODE ******\n") + print("\n****** DRY-RUN MODE ******\n") for arg in vars(common.ARGS): if any(arg in x for x in ['sca', 'container', 'iac', 'code']): print(f"{arg}={common.toggle_to_bool(getattr(common.ARGS, arg))}") else: - print(f"{arg}={getattr(common.ARGS, arg)}") + print(f"{arg}={getattr(common.ARGS, arg)}") print("---") if getenv("SNYK_TOKEN") is None: