diff --git a/README.md b/README.md index eb24a0a..0129213 100644 --- a/README.md +++ b/README.md @@ -25,21 +25,24 @@ For repos with at least 1 project already in Snyk: ## Usage ``` -usage: snyk_scm_refresh.py [-h] [--org-id ORG_ID] [--repo-name REPO_NAME] [--sca {on,off}] - [--container {on,off}] [--iac {on,off}] [--code {on,off}] [--dry-run] [--debug] +usage: snyk_scm_refresh.py [-h] [--org-id ORG_ID] [--repo-name REPO_NAME] [--sca {on,off}] + [--container {on,off}] [--iac {on,off}] [--code {on,off}] [--dry-run] + [--skip-scm-validation] [--debug] optional arguments: -h, --help show this help message and exit - --org-id ORG_ID The Snyk Organisation Id found in Organization > Settings. - If omitted, process all orgs the Snyk user has access to. + --org-id ORG_ID The Snyk Organisation Id found in Organization > Settings. If omitted, + process all orgs the Snyk user has access to. --repo-name REPO_NAME - The full name of the repo to process (e.g. githubuser/githubrepo). - If omitted, process all repos in the Snyk org. + The full name of the repo to process (e.g. githubuser/githubrepo). If + omitted, process all repos in the Snyk org. --sca {on,off} scan for SCA manifests (on by default) --container {on,off} scan for container projects, e.g. Dockerfile (on by default) --iac {on,off} scan for IAC manifests (experimental, off by default) --code {on,off} create code analysis if not present (experimental, off by default) --dry-run Simulate processing of the script without making changes to Snyk + --skip-scm-validation + Skip validation of the TLS certificate used by the SCM --debug Write detailed debug data to snyk_scm_refresh.log for troubleshooting ``` @@ -87,13 +90,19 @@ If using the Snyk Github Enterprise Integration type for your Github.com reposit 1. In GitHub.com browse: https://github.com/settings/tokens/new. Or in GitHub Enterprise select your user icon (top-right), then 'Settings', then 'Developer settings', then 'Personal access tokens'. 2. Scopes - Public repos do not need a scope. If you want to scan private repos, then you'll need to enable this scope: `repo` (Full control of private repositories) +## Handling self-signed certificates +This tool uses the python requests library, therefore you can point [REQUESTS_CA_BUNDLE](https://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification) environment variable to the location of your cert bundle + +`export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt` + +If you are not able to validate the self-signed certificate, you may skip validation by providing the `--skip-scm-validation` option. + ## Instructions -Make sure to use a user *API Token* that has acess to the Snyk Orgs you need to process with the script. A service account will *not* work for GitHub, which is the only SCM currently supported at this time. +Make sure to use a user *API Token* that has access to the Snyk Orgs you need to process with the script. A service account will *not* work for GitHub, which is the only SCM currently supported at this time. Ensure that your GITHUB_TOKEN or GITHUB_ENTERPRISE_TOKEN has access to the repos contained in the Snyk Orgs in scope If unsure, try one org at a time with `--org-id` - **Recommended:** This tool will delete projects from Snyk that are detected as stale or have since been renamed diff --git a/app/gh_repo.py b/app/gh_repo.py index 5c53593..0b3ed5d 100755 --- a/app/gh_repo.py +++ b/app/gh_repo.py @@ -5,6 +5,12 @@ from app.models import GithubRepoStatus import common +# suppess InsecureRequestWarning when using --skip-scm-validation option +# due to pylint bug +# https://github.com/PyCQA/pylint/issues/4584) +# pylint: disable=no-member +requests.packages.urllib3.disable_warnings() + def get_repo_manifests(snyk_repo_name, origin, skip_snyk_code): """retrieve list of all supported manifests in a given github repo""" manifests = [] @@ -85,7 +91,10 @@ def get_gh_repo_status(snyk_gh_repo): request_url = f"https://{common.GITHUB_ENTERPRISE_HOST}" \ f"/api/v3/repos/{snyk_gh_repo['full_name']}" try: - response = requests.get(url=request_url, allow_redirects=False, headers=headers) + response = requests.get(url=request_url, + allow_redirects=False, + headers=headers, + verify=common.VERIFY_TLS) # logging.debug("response_code: %d" % response.status_code) # logging.debug(f"response default branch -> {response.json()['default_branch']}") @@ -103,7 +112,9 @@ def get_gh_repo_status(snyk_gh_repo): elif response.status_code == 301: follow_response = requests.get( - url=response.headers["Location"], headers=headers + url=response.headers["Location"], + headers=headers, + verify=common.VERIFY_TLS ) if follow_response.status_code == 200: repo_new_full_name = follow_response.json()["full_name"] diff --git a/app/utils/github_utils.py b/app/utils/github_utils.py index 64640bb..dc08a91 100644 --- a/app/utils/github_utils.py +++ b/app/utils/github_utils.py @@ -5,20 +5,20 @@ from github import Github # pylint: disable=invalid-name -def create_github_client(GITHUB_TOKEN): +def create_github_client(GITHUB_TOKEN, VERIFY_TLS): """ return a github client for given token """ try: - return Github(login_or_token=GITHUB_TOKEN) + return Github(login_or_token=GITHUB_TOKEN, verify=VERIFY_TLS) except KeyError as err: raise RuntimeError( "Failed to initialize GitHub client because GITHUB_TOKEN is not set!" ) from err -def create_github_enterprise_client(GITHUB_ENTERPRISE_TOKEN, GITHUB_ENTERPRISE_HOST): +def create_github_enterprise_client(GITHUB_ENTERPRISE_TOKEN, GITHUB_ENTERPRISE_HOST, VERIFY_TLS): """ return a github enterprise client for given token/host """ try: return Github(base_url=f"https://{GITHUB_ENTERPRISE_HOST}/api/v3", \ - login_or_token=GITHUB_ENTERPRISE_TOKEN) + login_or_token=GITHUB_ENTERPRISE_TOKEN, verify=VERIFY_TLS) except KeyError as err: raise RuntimeError( "Failed to initialize GitHub client because GITHUB_ENTERPRISE_TOKEN is not set!" diff --git a/common.py b/common.py index 8ef3b6e..025f71a 100644 --- a/common.py +++ b/common.py @@ -16,7 +16,7 @@ MANIFEST_PATTERN_IAC = '.*[.](yaml|yml|tf)$' MANIFEST_PATTERN_CODE = '.*[.](js|cs|php|java|py)$' MANIFEST_PATTERN_EXCLUSIONS = '^.*(fixtures|tests\/|__tests__|test\/|__test__|[.].*ci\/|.*ci[.].yml|node_modules\/|bower_components\/|variables[.]tf|outputs[.]tf).*$' -GITHUB_CLOUD_API_HOST="api.github.com" +GITHUB_CLOUD_API_HOST = "api.github.com" GITHUB_ENABLED = False GITHUB_ENTERPRISE_ENABLED = False @@ -63,25 +63,6 @@ PENDING_REMOVAL_MAX_CHECKS = 45 PENDING_REMOVAL_CHECK_INTERVAL = 20 -snyk_client = SnykClient(SNYK_TOKEN) - -if (GITHUB_ENTERPRISE_HOST == GITHUB_CLOUD_API_HOST): - USE_GHE_INTEGRATION_FOR_GH_CLOUD = True - -if (GITHUB_TOKEN): - GITHUB_ENABLED = True - gh_client = create_github_client(GITHUB_TOKEN) - print("created github.com client") - -if (GITHUB_ENTERPRISE_HOST): - GITHUB_ENTERPRISE_ENABLED = True - if USE_GHE_INTEGRATION_FOR_GH_CLOUD: - gh_enterprise_client = create_github_client(GITHUB_ENTERPRISE_TOKEN) - print(f"created github client for enterprise host: {GITHUB_ENTERPRISE_HOST}") - else: - print(f"created GH enterprise client for host: {GITHUB_ENTERPRISE_HOST}") - gh_enterprise_client = create_github_enterprise_client(GITHUB_ENTERPRISE_TOKEN, GITHUB_ENTERPRISE_HOST) - def parse_command_line_args(): """Parse command-line arguments""" @@ -134,6 +115,12 @@ def parse_command_line_args(): required=False, action="store_true", ) + parser.add_argument( + "--skip-scm-validation", + help="Skip validation of the TLS certificate used by the SCM", + required=False, + action="store_true", + ) parser.add_argument( "--debug", help="Write detailed debug data to snyk_scm_refresh.log for troubleshooting", @@ -152,6 +139,28 @@ def toggle_to_bool(toggle_value) -> bool: return False return toggle_value +snyk_client = SnykClient(SNYK_TOKEN) + +VERIFY_TLS = not ARGS.skip_scm_validation + +if (GITHUB_ENTERPRISE_HOST == GITHUB_CLOUD_API_HOST): + USE_GHE_INTEGRATION_FOR_GH_CLOUD = True + +if (GITHUB_TOKEN): + GITHUB_ENABLED = True + gh_client = create_github_client(GITHUB_TOKEN, VERIFY_TLS) + print("created github.com client") + +if (GITHUB_ENTERPRISE_HOST): + GITHUB_ENTERPRISE_ENABLED = True + if USE_GHE_INTEGRATION_FOR_GH_CLOUD: + gh_enterprise_client = create_github_client(GITHUB_ENTERPRISE_TOKEN, VERIFY_TLS) + print(f"created github client for enterprise host: {GITHUB_ENTERPRISE_HOST}") + else: + print(f"created GH enterprise client for host: {GITHUB_ENTERPRISE_HOST}") + gh_enterprise_client = create_github_enterprise_client(GITHUB_ENTERPRISE_TOKEN, \ + GITHUB_ENTERPRISE_HOST, VERIFY_TLS) + PROJECT_TYPE_ENABLED_SCA = toggle_to_bool(ARGS.sca) PROJECT_TYPE_ENABLED_CONTAINER = toggle_to_bool(ARGS.container) PROJECT_TYPE_ENABLED_IAC = toggle_to_bool(ARGS.iac)