Merge pull request #49 from snyk-tech-services/develop

release: v1.2

README.md

@@ -8,6 +8,7 @@ For repos with at least 1 project already in Snyk:
 - Remove projects for manifests that no longer exist
 - Update projects when a repo has been renamed 
 - Detect and update default branch change (not renaming)
+- Enable Snyk Code analysis for repos
 - Detect deleted repos and log for review
 
 **STOP NOW IF ANY OF THE FOLLOWING ARE TRUE**
@@ -16,27 +17,39 @@ For repos with at least 1 project already in Snyk:
 
 ## Usage
 ```
-usage: snyk_scm_refresh.py [-h] [--org-id=ORG_ID] [--repo-name=REPO_NAME]
-                           [--dry-run]
+usage: snyk_scm_refresh.py [-h] [--org-id ORG_ID] [--repo-name REPO_NAME] [--sca {on,off}] 
+       [--container {on,off}] [--iac {on,off}] [--code {on,off}] [--dry-run] [--debug]
 
 optional arguments:
   -h, --help            show this help message and exit
-  --org-id=ORG_ID       The Snyk Organisation Id found in Organization >
-                        Settings. If omitted, process all orgs the Snyk user
-                        has access to.
-  --repo-name=REPO_NAME
-                        The full name of the repo to process (e.g.
-                        githubuser/githubrepo). If omitted, process all repos
-                        in the Snyk org.
-  --dry-run             Simulate processing of the script without making
-                        changes to Snyk
+  --org-id ORG_ID       The Snyk Organisation Id found in Organization > Settings. 
+                        If omitted, process all orgs the Snyk user has access to.
+  --repo-name REPO_NAME
+                        The full name of the repo to process (e.g. githubuser/githubrepo). 
+                        If omitted, process all repos in the Snyk org.
+  --sca {on,off}        scan for SCA manifests (on by default)
+  --container {on,off}  scan for container projects, e.g. Dockerfile (on by default)
+  --iac {on,off}        scan for IAC manifests (experimental, off by default)
+  --code {on,off}       create code analysis if not present (experimental, off by default)
+  --dry-run             Simulate processing of the script without making changes to Snyk
+  --debug               Write detailed debug data to snyk_scm_refresh.log for troubleshooting
 ```
 
+### Sync with defaults
+`./snyk_scm_refresh.py --org-id=12345
 
+### Sync SCA projects only
+`./snyk_scm_refresh.py --org-id=12345 --container=off`
 
-## Dependencies
-pysnyk, PyGithub, requests
+### Sync Container projects only
+`./snyk_scm_refresh.py --org-id=12345 --sca=off --container=on`
+
+### Enable Snyk Code analysis for repos
+only: `./snyk_scm_refresh.py --org-id=12345 --sca=off --container=off --code=on` \
+defaults + snyk code enable: `./snyk_scm_refresh.py --org-id=12345 --code=on`
 
+
+## Dependencies
 ```
 pip install -r  requirements.txt
 ```

app/app.py

@@ -3,6 +3,7 @@
 """
 import sys
 import time
+import re
 import snyk.errors
 import common
 from app.models import ImportStatus
@@ -51,6 +52,7 @@ def run():
     import_status_checks = []
 
     for (i, snyk_repo) in enumerate(snyk_repos):
+        # snyk_repo.get_projects()
         deleted_projects = []
         is_default_renamed = False
         app_print(snyk_repo.org_name,
@@ -146,9 +148,15 @@ def run():
                               snyk_repo.full_name,
                               f"Found {len(projects_import.files)} to import")
                     for file in projects_import.files:
+                        import_message = ""
+                        if re.match(common.MANIFEST_PATTERN_CODE, file["path"]):
+                            import_message = "Triggering code analysis via"
+                        else:
+                            import_message = "Importing new manifest"
+
                         app_print(snyk_repo.org_name,
                                   snyk_repo.full_name,
-                                  f"Importing new manifest: {file['path']}")
+                                  f"{import_message}: {file['path']}")
 
         # if snyk_repo has been moved/renamed (301), then re-import the entire repo
         # with the new name and remove the old one (make optional)

app/gh_repo.py

@@ -3,7 +3,7 @@
 import requests
 import common
 
-def get_repo_manifests(snyk_repo_name, origin):
+def get_repo_manifests(snyk_repo_name, origin, skip_snyk_code):
     """retrieve list of all supported manifests in a given github repo"""
     manifests = []
     try:
@@ -19,19 +19,42 @@ def get_repo_manifests(snyk_repo_name, origin):
             gh_repo = common.gh_enterprise_client.get_user().get_repo(snyk_repo_name)
 
     contents = gh_repo.get_git_tree(gh_repo.default_branch, True).tree
-    #print(contents)
 
     while contents:
         file_content = contents.pop(0)
-        if passes_manifest_filter(file_content.path):
+        if passes_manifest_filter(file_content.path, skip_snyk_code):
             manifests.append(file_content.path)
+        if re.match(common.MANIFEST_PATTERN_CODE, file_content.path):
+            skip_snyk_code = True
+    #print(manifests)
     return manifests
 
-def passes_manifest_filter(path):
+def passes_manifest_filter(path, skip_snyk_code=False):
     """ check if given path should be imported based
         on configured search and exclusion filters """
-    return bool(re.match(common.MANIFEST_REGEX_PATTERN, path) and
-            not re.match(common.MANIFEST_EXCLUSION_REGEX_PATTERN, path))
+
+    passes_filter = False
+    if (common.PROJECT_TYPE_ENABLED_SCA and
+            re.match(common.MANIFEST_PATTERN_SCA, path)):
+        passes_filter = True
+        # print('passes SCA filter true')
+    if (common.PROJECT_TYPE_ENABLED_CONTAINER and
+            re.match(common.MANIFEST_PATTERN_CONTAINER, path)):
+        passes_filter = True
+        # print('passes CONTAINER filter true')
+    if (common.PROJECT_TYPE_ENABLED_IAC and
+            re.match(common.MANIFEST_PATTERN_IAC, path)):
+        passes_filter = True
+        # print('passes IAC filter true')
+    if (common.PROJECT_TYPE_ENABLED_CODE and
+            re.match(common.MANIFEST_PATTERN_CODE, path)):
+        if not skip_snyk_code:
+            passes_filter = True
+            # print('passes CODE filter true')
+    if re.match(common.MANIFEST_PATTERN_EXCLUSIONS, path):
+        passes_filter = False
+
+    return passes_filter
 
 def get_gh_repo_status(snyk_gh_repo, github_token, github_enterprise=False):
     """detect if repo still exists, has been removed, or renamed"""

app/snyk_repo.py

@@ -3,8 +3,13 @@
 snyk projects from the same repository
 """
 import sys
+import re
 import snyk
-from app.gh_repo import get_repo_manifests
+import common
+from app.gh_repo import (
+    get_repo_manifests,
+    passes_manifest_filter
+)
 import app.utils.snyk_helper
 
 class SnykRepo():
@@ -34,12 +39,22 @@ def get_projects(self):
         """ return list of projects for this repo """
         return self.snyk_projects
 
+    def has_snyk_code(self):
+        """ returns true if snyk already has a
+            snyk code project for this repo """
+        has_snyk_code = False
+        for snyk_project in self.snyk_projects:
+            if snyk_project["type"] == "sast":
+                has_snyk_code = True
+                break
+        return has_snyk_code
+
     def add_new_manifests(self, dry_run):
         """ find and import new projects """
         import_response = []
         files = []
 
-        gh_repo_manifests = get_repo_manifests(self.full_name, self.origin)
+        gh_repo_manifests = get_repo_manifests(self.full_name, self.origin, self.has_snyk_code())
 
         for gh_repo_manifest in gh_repo_manifests:
             if gh_repo_manifest not in {sp['manifest'] for sp in self.snyk_projects}:
@@ -55,18 +70,26 @@ def add_new_manifests(self, dry_run):
                     files)
         else:
             for file in files:
+                import_message = ""
+                if re.match(common.MANIFEST_PATTERN_CODE, file["path"]):
+                    import_message = "would trigger code analysis via"
+                else:
+                    import_message = "would import"
+
                 app.utils.snyk_helper.app_print(self.org_name,
                                                 self.full_name,
-                                                f"Would import: {file}")
+                                                f"{import_message}: {file['path']}")
         return import_response
 
     def delete_stale_manifests(self, dry_run):
         """ delete snyk projects for which the corresponding SCM file no longer exists """
         result = []
-        gh_repo_manifests = get_repo_manifests(self.full_name, self.origin)
+        gh_repo_manifests = get_repo_manifests(self.full_name, self.origin, True)
         for snyk_project in self.snyk_projects:
             # print(snyk_project["manifest"])
-            if snyk_project["manifest"] not in gh_repo_manifests:
+            if (snyk_project["type"] != "sast" and
+                    passes_manifest_filter(snyk_project["manifest"]) and
+                    snyk_project["manifest"] not in gh_repo_manifests):
                 # delete project, append on success
                 if not dry_run:
                     try:

app/tests/test_snyk_scm_refresh.py

@@ -4,7 +4,8 @@
 
 from app.gh_repo import (
     get_gh_repo_status,
-    passes_manifest_filter
+    passes_manifest_filter,
+
 )
 from app.utils.snyk_helper import get_snyk_projects_for_repo
 
@@ -159,4 +160,4 @@ def test_passes_manifest_filter():
     assert passes_manifest_filter(path_fail_1) == False
     assert passes_manifest_filter(path_pass_1) == True
     assert passes_manifest_filter(path_fail_2) == False
-    assert passes_manifest_filter(path_pass_2) == True
+    assert passes_manifest_filter(path_pass_2) == True

app/utils/snyk_helper.py

@@ -142,6 +142,7 @@ def build_snyk_project_list(snyk_orgs, ARGS):
                         "org_id": snyk_org.id,
                         "org_name": snyk_org.name,
                         "origin": project.origin,
+                        "type": project.type,
                         "integration_id": integration_id,
                         "branch_from_name": branch_from_name,
                         "branch": project.branch
@@ -218,6 +219,7 @@ def delete_snyk_project(project_id, org_id):
 
 def process_import_status_checks(import_status_checks):
     # pylint: disable=too-many-nested-blocks, too-many-branches
+    # pylint: disable=too-many-locals
     """
     Check status of pending import jobs
     up to PENDING_REMOVAL_MAX_CHECKS times,
@@ -275,15 +277,18 @@ def process_import_status_checks(import_status_checks):
                                 # print(import_status_log)
                                 import_logs_completed.append(uniq_import_log)
                                 for project in import_status_log["projects"]:
-                                    app_print(import_job.org_name,
-                                              import_status_log["name"],
-                                              f"Imported {project['targetFile']}")
-                                    common.COMPLETED_PROJECT_IMPORTS_FILE.write("%s,%s:%s,%s\n" % (
-                                        import_job.org_name,
-                                        import_status_log["name"],
-                                        project["targetFile"],
-                                        project["success"]
-                                    ))
+                                    if 'targetFile' in project:
+                                        imported_project = project['targetFile']
+                                        app_print(import_job.org_name,
+                                                  import_status_log["name"],
+                                                  f"Imported {imported_project}")
+                                        # pylint: disable=line-too-long
+                                        common.COMPLETED_PROJECT_IMPORTS_FILE.write("%s,%s:%s,%s\n" % (
+                                            import_job.org_name,
+                                            import_status_log["name"],
+                                            imported_project,
+                                            project["success"]
+                                        ))
 
                     if import_status["status"] != "pending":
                         import_jobs_completed.append(import_job.import_job_id)

common.py

@@ -1,13 +1,21 @@
-from os import getenv
+import sys
+from os import  (
+    getenv,
+    path
+)
 from snyk import SnykClient
 from app.utils.github_utils import (
     create_github_client,
     create_github_enterprise_client
 )
 import argparse
+import configparser
 
-MANIFEST_REGEX_PATTERN = '^(?![.]).*(package[.]json$|Gemfile[.]lock$|pom[.]xml$|build[.]gradle$|.*[.]lockfile$|build[.]sbt$|.*req.*[.]txt$|Gopkg[.]lock|go[.]mod|vendor[.]json|packages[.]config|.*[.]csproj|.*[.]fsproj|.*[.]vbproj|project[.]json|project[.]assets[.]json|composer[.]lock|Podfile|Podfile[.]lock|.*[.]yaml|.*[.]yml|Dockerfile)'
-MANIFEST_EXCLUSION_REGEX_PATTERN = '^.*(fixtures|\/tests\/|\/__tests__\/|\/test\/|__test__|[.].*ci\/|\/node_modules\/|\/bower_components\/).*$'
+MANIFEST_PATTERN_SCA = '^(?![.]).*(package[.]json|Gemfile[.]lock|pom[.]xml|build[.]gradle|.*[.]lockfile|build[.]sbt|.*req.*[.]txt|Gopkg[.]lock|go[.]mod|vendor[.]json|packages[.]config|.*[.]csproj|.*[.]fsproj|.*[.]vbproj|project[.]json|project[.]assets[.]json|composer[.]lock|Podfile|Podfile[.]lock)$'
+MANIFEST_PATTERN_CONTAINER = '^.*(Dockerfile)$'
+MANIFEST_PATTERN_IAC = '.*[.](yaml|yml|tf)$'
+MANIFEST_PATTERN_CODE = '.*[.](js|cs|php|java|py)$'
+MANIFEST_PATTERN_EXCLUSIONS = '^.*(fixtures|\/tests\/|\/__tests__\/|\/test\/|__test__|[.].*ci\/|.*ci[.].yml|\/node_modules\/|\/bower_components\/|variables[.]tf|outputs[.]tf).*$'
 
 GITHUB_ENABLED = False
 GITHUB_ENTERPRISE_ENABLED = False
@@ -81,13 +89,59 @@ def parse_command_line_args():
             If omitted, process all repos in the Snyk org.",
         required=False,
     )
+    parser.add_argument(
+        "--sca",
+        help="scan for SCA manifests (on by default)",
+        required=False,
+        default=True,
+        choices=['on', 'off']
+    )
+    parser.add_argument(
+        "--container",
+        help="scan for container projects, e.g. Dockerfile (on by default)",
+        required=False,
+        default=True,
+        choices=['on', 'off']
+    )
+    parser.add_argument(
+        "--iac",
+        help="scan for IAC manifests (experimental, off by default)",
+        required=False,
+        default=False,
+        choices=['on', 'off']
+    )
+    parser.add_argument(
+        "--code",
+        help="create code analysis if not present (experimental, off by default)",
+        required=False,
+        default=False,
+        choices=['on', 'off']
+    )
     parser.add_argument(
         "--dry-run",
         help="Simulate processing of the script without making changes to Snyk",
         required=False,
         action="store_true",
     )
+    parser.add_argument(
+        "--debug",
+        help="Write detailed debug data to snyk_scm_refresh.log for troubleshooting",
+        required=False,
+        action="store_true",
+    )
 
     return parser.parse_args()
 
-ARGS = parse_command_line_args()
+ARGS = parse_command_line_args()
+
+def toggle_to_bool(toggle_value) -> bool:
+    if toggle_value == "on":
+        return True
+    if toggle_value == "off":
+        return False
+    return toggle_value
+
+PROJECT_TYPE_ENABLED_SCA = toggle_to_bool(ARGS.sca)
+PROJECT_TYPE_ENABLED_CONTAINER = toggle_to_bool(ARGS.container)
+PROJECT_TYPE_ENABLED_IAC = toggle_to_bool(ARGS.iac)
+PROJECT_TYPE_ENABLED_CODE = toggle_to_bool(ARGS.code)

snyk_scm_refresh.py

@@ -13,7 +13,10 @@
     if common.ARGS.dry_run:
         print("****** DRY-RUN MODE ******\n")
     for arg in vars(common.ARGS):
-        print(f"{arg}={getattr(common.ARGS, arg)}")
+        if any(arg in x for x in ['sca', 'container', 'iac', 'code']):
+            print(f"{arg}={common.toggle_to_bool(getattr(common.ARGS, arg))}")
+        else:
+           print(f"{arg}={getattr(common.ARGS, arg)}")
     print("---")
 
     if getenv("SNYK_TOKEN") is None:
@@ -46,6 +49,9 @@
         print(f"{GITHUB_TOKEN_HIDDEN}")
 
     print("---")
-    logging.basicConfig(filename=common.LOG_FILENAME, level=logging.DEBUG, filemode="w")
+    if common.ARGS.debug:
+        logging.basicConfig(filename=common.LOG_FILENAME, level=logging.DEBUG, filemode="w")
+    else:
+        logging.basicConfig(filename=common.LOG_FILENAME, level=logging.INFO, filemode="w")
 
     run()