Merge pull request #87 from snyk-tech-services/feat/skip-cert-validat…

…ion-option

feat: skip SCM certificate validation option

README.md

@@ -25,21 +25,24 @@ For repos with at least 1 project already in Snyk:
 
 ## Usage
 ```
-usage: snyk_scm_refresh.py [-h] [--org-id ORG_ID] [--repo-name REPO_NAME] [--sca {on,off}] 
-       [--container {on,off}] [--iac {on,off}] [--code {on,off}] [--dry-run] [--debug]
+usage: snyk_scm_refresh.py [-h] [--org-id ORG_ID] [--repo-name REPO_NAME] [--sca {on,off}]
+                           [--container {on,off}] [--iac {on,off}] [--code {on,off}] [--dry-run]
+                           [--skip-scm-validation] [--debug]
 
 optional arguments:
   -h, --help            show this help message and exit
-  --org-id ORG_ID       The Snyk Organisation Id found in Organization > Settings. 
-                        If omitted, process all orgs the Snyk user has access to.
+  --org-id ORG_ID       The Snyk Organisation Id found in Organization > Settings. If omitted,
+                        process all orgs the Snyk user has access to.
   --repo-name REPO_NAME
-                        The full name of the repo to process (e.g. githubuser/githubrepo). 
-                        If omitted, process all repos in the Snyk org.
+                        The full name of the repo to process (e.g. githubuser/githubrepo). If
+                        omitted, process all repos in the Snyk org.
   --sca {on,off}        scan for SCA manifests (on by default)
   --container {on,off}  scan for container projects, e.g. Dockerfile (on by default)
   --iac {on,off}        scan for IAC manifests (experimental, off by default)
   --code {on,off}       create code analysis if not present (experimental, off by default)
   --dry-run             Simulate processing of the script without making changes to Snyk
+  --skip-scm-validation
+                        Skip validation of the TLS certificate used by the SCM
   --debug               Write detailed debug data to snyk_scm_refresh.log for troubleshooting
 ```
 
@@ -87,13 +90,19 @@ If using the Snyk Github Enterprise Integration type for your Github.com reposit
 1. In GitHub.com browse: https://github.com/settings/tokens/new. Or in GitHub Enterprise select your user icon (top-right), then 'Settings', then 'Developer settings', then 'Personal access tokens'.
 2. Scopes - Public repos do not need a scope. If you want to scan private repos, then you'll need to enable this scope: `repo` (Full control of private repositories)
 
+## Handling self-signed certificates
+This tool uses the python requests library, therefore you can point [REQUESTS_CA_BUNDLE](https://docs.python-requests.org/en/master/user/advanced/#ssl-cert-verification) environment variable to the location of your cert bundle
+
+`export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt`
+
+If you are not able to validate the self-signed certificate, you may skip validation by providing the `--skip-scm-validation` option. 
+
 ## Instructions
-Make sure to use a user *API Token* that has acess to the Snyk Orgs you need to process with the script.  A service account will *not* work for GitHub, which is the only SCM currently supported at this time.
+Make sure to use a user *API Token* that has access to the Snyk Orgs you need to process with the script.  A service account will *not* work for GitHub, which is the only SCM currently supported at this time.
 
 Ensure that your GITHUB_TOKEN or GITHUB_ENTERPRISE_TOKEN has access to the repos contained in the Snyk Orgs in scope
 If unsure, try one org at a time with `--org-id`
 
-
 **Recommended:**
 This tool will delete projects from Snyk that are detected as stale or have since been renamed
 

app/gh_repo.py

@@ -5,6 +5,12 @@
 from app.models import GithubRepoStatus
 import common
 
+# suppess InsecureRequestWarning when using --skip-scm-validation option
+# due to pylint bug
+# https://github.com/PyCQA/pylint/issues/4584)
+# pylint: disable=no-member
+requests.packages.urllib3.disable_warnings()
+
 def get_repo_manifests(snyk_repo_name, origin, skip_snyk_code):
     """retrieve list of all supported manifests in a given github repo"""
     manifests = []
@@ -85,7 +91,10 @@ def get_gh_repo_status(snyk_gh_repo):
         request_url = f"https://{common.GITHUB_ENTERPRISE_HOST}" \
         f"/api/v3/repos/{snyk_gh_repo['full_name']}"
     try:
-        response = requests.get(url=request_url, allow_redirects=False, headers=headers)
+        response = requests.get(url=request_url,
+                                allow_redirects=False,
+                                headers=headers,
+                                verify=common.VERIFY_TLS)
         # logging.debug("response_code: %d" % response.status_code)
         # logging.debug(f"response default branch -> {response.json()['default_branch']}")
 
@@ -103,7 +112,9 @@ def get_gh_repo_status(snyk_gh_repo):
 
         elif response.status_code == 301:
             follow_response = requests.get(
-                url=response.headers["Location"], headers=headers
+                url=response.headers["Location"],
+                headers=headers,
+                verify=common.VERIFY_TLS
             )
             if follow_response.status_code == 200:
                 repo_new_full_name = follow_response.json()["full_name"]

app/utils/github_utils.py

@@ -5,20 +5,20 @@
 from github import Github
 
 # pylint: disable=invalid-name
-def create_github_client(GITHUB_TOKEN):
+def create_github_client(GITHUB_TOKEN, VERIFY_TLS):
     """ return a github client for given token """
     try:
-        return Github(login_or_token=GITHUB_TOKEN)
+        return Github(login_or_token=GITHUB_TOKEN, verify=VERIFY_TLS)
     except KeyError as err:
         raise RuntimeError(
             "Failed to initialize GitHub client because GITHUB_TOKEN is not set!"
         ) from err
 
-def create_github_enterprise_client(GITHUB_ENTERPRISE_TOKEN, GITHUB_ENTERPRISE_HOST):
+def create_github_enterprise_client(GITHUB_ENTERPRISE_TOKEN, GITHUB_ENTERPRISE_HOST, VERIFY_TLS):
     """ return a github enterprise client for given token/host """
     try:
         return Github(base_url=f"https://{GITHUB_ENTERPRISE_HOST}/api/v3", \
-            login_or_token=GITHUB_ENTERPRISE_TOKEN)
+            login_or_token=GITHUB_ENTERPRISE_TOKEN, verify=VERIFY_TLS)
     except KeyError as err:
         raise RuntimeError(
             "Failed to initialize GitHub client because GITHUB_ENTERPRISE_TOKEN is not set!"

common.py

@@ -16,7 +16,7 @@
 MANIFEST_PATTERN_IAC = '.*[.](yaml|yml|tf)$'
 MANIFEST_PATTERN_CODE = '.*[.](js|cs|php|java|py)$'
 MANIFEST_PATTERN_EXCLUSIONS = '^.*(fixtures|tests\/|__tests__|test\/|__test__|[.].*ci\/|.*ci[.].yml|node_modules\/|bower_components\/|variables[.]tf|outputs[.]tf).*$'
-GITHUB_CLOUD_API_HOST="api.github.com"
+GITHUB_CLOUD_API_HOST = "api.github.com"
 
 GITHUB_ENABLED = False
 GITHUB_ENTERPRISE_ENABLED = False
@@ -63,25 +63,6 @@
 PENDING_REMOVAL_MAX_CHECKS = 45
 PENDING_REMOVAL_CHECK_INTERVAL = 20
 
-snyk_client = SnykClient(SNYK_TOKEN)
-
-if (GITHUB_ENTERPRISE_HOST == GITHUB_CLOUD_API_HOST):
-   USE_GHE_INTEGRATION_FOR_GH_CLOUD = True
-
-if (GITHUB_TOKEN):
-    GITHUB_ENABLED = True
-    gh_client = create_github_client(GITHUB_TOKEN)
-    print("created github.com client")
-
-if (GITHUB_ENTERPRISE_HOST):
-    GITHUB_ENTERPRISE_ENABLED = True
-    if USE_GHE_INTEGRATION_FOR_GH_CLOUD:
-        gh_enterprise_client = create_github_client(GITHUB_ENTERPRISE_TOKEN)
-        print(f"created github client for enterprise host: {GITHUB_ENTERPRISE_HOST}")
-    else:
-        print(f"created GH enterprise client for host: {GITHUB_ENTERPRISE_HOST}")
-        gh_enterprise_client = create_github_enterprise_client(GITHUB_ENTERPRISE_TOKEN, GITHUB_ENTERPRISE_HOST)
-
 def parse_command_line_args():
     """Parse command-line arguments"""                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              
 
@@ -134,6 +115,12 @@ def parse_command_line_args():
         required=False,
         action="store_true",
     )
+    parser.add_argument(
+        "--skip-scm-validation",
+        help="Skip validation of the TLS certificate used by the SCM",
+        required=False,
+        action="store_true",
+    )
     parser.add_argument(
         "--debug",
         help="Write detailed debug data to snyk_scm_refresh.log for troubleshooting",
@@ -152,6 +139,28 @@ def toggle_to_bool(toggle_value) -> bool:
         return False
     return toggle_value
 
+snyk_client = SnykClient(SNYK_TOKEN)
+
+VERIFY_TLS = not ARGS.skip_scm_validation
+
+if (GITHUB_ENTERPRISE_HOST == GITHUB_CLOUD_API_HOST):
+   USE_GHE_INTEGRATION_FOR_GH_CLOUD = True
+
+if (GITHUB_TOKEN):
+    GITHUB_ENABLED = True
+    gh_client = create_github_client(GITHUB_TOKEN, VERIFY_TLS)
+    print("created github.com client")
+
+if (GITHUB_ENTERPRISE_HOST):
+    GITHUB_ENTERPRISE_ENABLED = True
+    if USE_GHE_INTEGRATION_FOR_GH_CLOUD:
+        gh_enterprise_client = create_github_client(GITHUB_ENTERPRISE_TOKEN, VERIFY_TLS)
+        print(f"created github client for enterprise host: {GITHUB_ENTERPRISE_HOST}")
+    else:
+        print(f"created GH enterprise client for host: {GITHUB_ENTERPRISE_HOST}")
+        gh_enterprise_client = create_github_enterprise_client(GITHUB_ENTERPRISE_TOKEN, \
+            GITHUB_ENTERPRISE_HOST, VERIFY_TLS)
+
 PROJECT_TYPE_ENABLED_SCA = toggle_to_bool(ARGS.sca)
 PROJECT_TYPE_ENABLED_CONTAINER = toggle_to_bool(ARGS.container)
 PROJECT_TYPE_ENABLED_IAC = toggle_to_bool(ARGS.iac)