merge 2.8.0

.github/workflows/build-and-push.yml

@@ -0,0 +1,41 @@
+name: Build and push container image
+
+env:
+  IMAGE_NAME: "rapidast"
+  IMAGE_TAGS: "${{ github.sha }}"
+  IMAGE_REGISTRY: quay.io/redhatproductsecurity
+  IMAGE_REGISTRY_USER: ${{ secrets.IMAGE_REGISTRY_USER }}
+  IMAGE_REGISTRY_PASSWORD: ${{ secrets.IMAGE_REGISTRY_PASSWORD }}
+
+on:
+  push:
+    branches: ["development", "main"]
+
+jobs:
+
+  build-and-push:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    # https://github.com/redhat-actions/buildah-build#readme
+    - name: Build container image
+      id: build-image
+      uses: redhat-actions/buildah-build@v2
+      with:
+        image: ${{ env.IMAGE_NAME }}
+        tags: ${{ env.IMAGE_TAGS }}
+        dockerfiles: |
+            ./containerize/Containerfile
+
+    # https://github.com/redhat-actions/push-to-registry#readme
+    - name: Push to registry
+      id: push-image
+      uses: redhat-actions/push-to-registry@v2
+      with:
+        image: ${{ steps.build-image.outputs.image }}
+        tags: ${{ steps.build-image.outputs.tags }}
+        registry: ${{ env.IMAGE_REGISTRY }}
+        username: ${{ env.IMAGE_REGISTRY_USER }}
+        password: ${{ env.IMAGE_REGISTRY_PASSWORD }}

.github/workflows/build-image.yml

@@ -0,0 +1,26 @@
+name: Build container image
+
+env:
+  IMAGE_NAME: "rapidast"
+  IMAGE_TAGS: "${{ github.sha }}"
+
+on:
+  pull_request:
+    branches: ["development", "main"]
+
+jobs:
+
+  build-image:
+
+    runs-on: ubuntu-latest
+
+    # https://github.com/redhat-actions/buildah-build#readme
+    steps:
+    - uses: actions/checkout@v4
+    - name: Build container image
+      uses: redhat-actions/buildah-build@v2
+      with:
+        image: ${{ env.IMAGE_NAME }}
+        tags: ${{ env.IMAGE_TAGS }}
+        dockerfiles: |
+            ./containerize/Containerfile

.github/workflows/run-tests.yml

@@ -0,0 +1,32 @@
+name: Run tests
+
+on:
+  push:
+    branches: ["development", "main"]
+  pull_request:
+    branches: ["development", "main"]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.9"
+    - name: Install dependencies
+      run: |
+        python3 -m ensurepip --upgrade
+        pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt
+    - name: Test with pytest
+      run: |
+        pytest
+    - name: Lint with pre-commit hook
+      run: |
+        pre-commit run --all-files --show-diff-on-failure

.github/workflows/tag-image.yml

@@ -0,0 +1,33 @@
+name: Tag image on quay.io
+
+env:
+  IMAGE_NAME: "rapidast"
+  IMAGE_REGISTRY: quay.io/redhatproductsecurity
+  IMAGE_REGISTRY_USER: ${{ secrets.IMAGE_REGISTRY_USER }}
+  IMAGE_REGISTRY_PASSWORD: ${{ secrets.IMAGE_REGISTRY_PASSWORD }}
+
+on:
+  push:
+    tags: ["*"]
+
+jobs:
+
+  tag-image:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    # https://github.com/redhat-actions/podman-login
+    - name: Log in to quay.io
+      uses: redhat-actions/podman-login@v1
+      with:
+        registry: ${{ env.IMAGE_REGISTRY }}
+        username: ${{ env.IMAGE_REGISTRY_USER }}
+        password: ${{ env.IMAGE_REGISTRY_PASSWORD }}
+
+    - name: Tag image
+      run: |
+          # tag existing image on quay.io that has :<commit> tag with :<new> gh tag
+          SRC=${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
+          DST=${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}
+          skopeo copy docker://${SRC} docker://${DST}

.pre-commit-config.yaml

@@ -20,6 +20,8 @@ repos:
         args:
         - --safe
         - --quiet
+        - --line-length
+        - "120" # same as pylint below
         language_version: python3
         require_serial: true
 

README.md

@@ -619,6 +619,23 @@ Solutions:
 * Selenium, used to control Firefox, uses shared memory (`/dev/shm/`). When using the RapiDAST image or the ZAP image, the user needs to make sure that sufficient space is available in `/dev/shm/` (in podman, by default, its size is 64MB). A size of 2G is the recommended value by the Selenium community. In podman for example, the option would be `--shm-size=2g`.
 * Zap and Firefox can create a huge numbers of threads. Some container engines will default to 2048 concurrent pids, which is not sufficient for the Ajax Spider. Whenever possible, RapiDAST will check if that limit was reached, after the scan is finished, and prints a warning if this happened. In podman, increasing the maximum number of concurrent pids is done via the `--pids-limit=-1` option to prevent any limits.
 
+## Podman errors
+
+### subuid/subgid are not enabled
+
+If you see one of those errors:
+
+```
+Error: copying system image from manifest list: writing blob: adding layer with blob "sha256:82aabceedc2fbf89030cbb4ff98215b70d9ae35c780ade6c784d9b447b1109ed": processing tar file(potentially insufficient UIDs or GIDs available in user namespace (requested 0:42 for /etc/gshadow): Check /etc/subuid and /etc/subgid if configured locally and run "podman system migrate": lchown /etc/gshadow: invalid argument): exit status 1
+```
+ -or-
+```
+Error: parsing id map value "-1000": strconv.ParseUint: parsing "-1000": invalid syntax
+```
+
+Podman, in rootless mode (running as a regular user), needs subuid/subgit to be enabled: [rootless mode](https://docs.podman.io/en/latest/markdown/podman.1.html#rootless-mode)
+
+
 ## Caveats
 
 * Currently, RapiDAST does not clean up the temporary data when there is an error. The data may include:

config/config-template-trivy-k8s-scan.yaml

@@ -40,7 +40,7 @@ scanners:
     # 'inline' is used when container.type is not 'podman'
     # 'toolDir' specifies the default directory where inline scripts are located
     #toolDir: scanners/generic/tools
-    inline: "trivy k8s --kubeconfig=/home/rapidast/.kube/config -n default pod --scanners=misconfig --report all --format json -o /tmp/k8s_result.json && python3 convert_trivy_k8s_to_sarif.py -f /tmp/k8s_result.json"
+    inline: "trivy k8s --kubeconfig=/home/rapidast/.kube/config -n default pod --scanners=misconfig --report all --format json | convert_trivy_k8s_to_sarif.py"
 
     container:
       parameters:

config/config-template-zap-long.yaml

@@ -15,6 +15,10 @@ config:
   # all the results of all scanners will be stored under that location
   base_results_dir: "./results"
 
+  # In RapiDAST only: should RapiDAST verify certificates
+  # possible values: true [default], false, /path/to/a/PEM/file
+  tls_verify_for_rapidast_downloads: true
+
   # Import a particular environment, and inject it for each scanner
   environ:
     envFile: "path/to/env/file"
@@ -66,7 +70,7 @@ general:
     #
     # "browser" authentication will use firefox in the background to generate cookies
     #  - verifyUrl must return an error if the user is not logged in
-    #type: "browser"   
+    #type: "browser"
     #parameters:
     #  username: "user"
     #  password: "mypassw0rd"
@@ -135,6 +139,9 @@ scanners:
       url: ""        # url to start spidering from, default: application.url set above
 
     spiderAjax:
+      # The list of parameters: https://www.zaproxy.org/docs/desktop/addons/ajax-spider/automation/
+      #maxCrawlStates: 10   # this may be useful when running in a memory limited environment (default: 0 unlimited)
+      #maxCrawlDepth: 10    # default: unlimited
       maxDuration: 0 # in minutes, default: 0 unlimited
       url: ""        # url to start spidering from, default: application.url set above
       browserId: firefox-headless
@@ -145,6 +152,10 @@ scanners:
       disabledRules: "2,10015,10024,10027,10054,10096,10109,10112"
 
     activeScan:
+      # The list of parameters: https://www.zaproxy.org/docs/desktop/addons/ajax-spider/automation/
+      #maxRuleDurationInMins: max scan time for each Rule (default: unlimited)
+      #maxScanDurationInMins: max scan time for the entire scan. Useful for debugging automation
+      #
       # If no policy is chosen, a default ("API-scan-minimal") will be selected
       # The list of policies can be found in scanners/zap/policies/
       policy:  "API-scan-minimal"

configmodel/__init__.py

@@ -77,9 +77,7 @@ def delete(self, path):
         except AttributeError:
             pass
         # Failed to iterate until the end: the path does not exist
-        logging.warning(
-            f"RapidastConfigModel.delete(): Config path {path} was not found. No deletion"
-        )
+        logging.warning(f"RapidastConfigModel.delete(): Config path {path} was not found. No deletion")
         return False
 
     def exists(self, path):
@@ -122,9 +120,7 @@ def set(self, path, value, overwrite=True):
                 tmp = walk[key]
             # case 3: not a "dictionary" type: warn and overwrite (if True)
             if not isinstance(tmp, dict):
-                logging.warning(
-                    f"RapidastConfigModel.set: Incompatible {path} at {tmp}"
-                )
+                logging.warning(f"RapidastConfigModel.set: Incompatible {path} at {tmp}")
                 if not overwrite:
                     logging.info("RapidastConfigModel.set: no overwrite: early return")
                     return False
@@ -162,9 +158,7 @@ def merge(self, merge, preserve=False, root=None):
         if not merge:
             return
         if not isinstance(merge, dict):
-            raise TypeError(
-                f"RapidastConfigModel.merge: merge must be a dict (was: {type(merge)})"
-            )
+            raise TypeError(f"RapidastConfigModel.merge: merge must be a dict (was: {type(merge)})")
 
         root = path_to_list(root)
 
@@ -176,8 +170,54 @@ def merge(self, merge, preserve=False, root=None):
 
         deep_dict_merge(sub_conf, merge, preserve)
 
+    def subtree_to_dict(self, path):
+        """Given a path, returns its subtree as a dictionary.
+        This includes applying all the `*_from_var` transformation.
+        e.g.:
+        "{'a_from_var': 'A_VAR'}" would return "{'a': '<value of $A_VAR>'}"
+
+        Cases:
+        1- path does not exist: return None
+        2- path does not point to a dictionary: throw a KeyError instance
+        3- path exist and is a dictionary: copy it, walk the copy apply all _from_var, return the copy
+        """
+
+        # recursively descend the tree, and apply all the _from_var
+        def descend(root):
+            if isinstance(root, dict):
+                # Dictionary:
+                #  create a new dictionary, and apply the following logic:
+                #  if key matches `_from_var`, assume value is a string, and apply replacement
+                #  otherwise, copy key name and recursively descend on the value
+                new = {}
+                for key, val in root.items():
+                    if key.endswith("_from_var"):
+                        new[key.removesuffix("_from_var")] = os.environ[val]
+                        if not new[key.removesuffix("_from_var")]:
+                            logging.warning(f"configuration {key} points to environment variable {val}, which is empty")
+                    else:
+                        new[key] = descend(val)
+                return new
+            elif isinstance(root, list):
+                # List: apply on each entry, and return a new List
+                return [descend(val) for val in root]
+            else:
+                # root is just a value (integer, string), assuming it's immutable
+                return root
+
+        try:
+            subtree = self._get_from_conf(path_to_list(path))
+        except KeyError:
+            logging.debug(f"subtree_to_dict(): path '{path}' does not exist")
+            return None
+
+        if not isinstance(subtree, dict):
+            raise KeyError(f"subtree_to_dict(): '{path}' does not point to a dictionary in the config")
+
+        return descend(subtree)
+
     def get_official_app_name(self):
-        """ Shortcut: 
+        """Shortcut:
         Return a string corresponding to how the application should be called
         Based on the configuratoin.
         Prefer the full product name, but defer to short name if unavailable

configmodel/converter.py

@@ -45,9 +45,7 @@ def dispatch(version):
 def convert_configmodel(conf):
     """This is the base function, attached to error reporting"""
     version = conf.get("config.configVersion", 0)
-    raise RuntimeError(
-        f"There was an error in converting configuration. No convertion available for version {version}"
-    )
+    raise RuntimeError(f"There was an error in converting configuration. No convertion available for version {version}")
 
 
 @convert_configmodel.register(4)
@@ -60,9 +58,7 @@ def convert_from_version_4_to_5(old):
     new = copy.deepcopy(old)
 
     for key in old.conf["scanners"]:
-        if key.startswith("zap") and old.exists(
-            f"scanners.{key}.miscOptions.oauth2OpenapiManualDownload"
-        ):
+        if key.startswith("zap") and old.exists(f"scanners.{key}.miscOptions.oauth2OpenapiManualDownload"):
             new.move(
                 f"scanners.{key}.miscOptions.oauth2OpenapiManualDownload",
                 f"scanners.{key}.miscOptions.oauth2ManualDownload",
@@ -174,29 +170,22 @@ def convert_from_version_0_to_1(old):
     auth_method = old.get("scan.auth_method", default=None)
     if (
         auth_method == "scriptBasedAuthentication"
-        and old.get("scan.scriptAuth.authScriptFilePath", default="")
-        == "scripts/offline-token.js"
+        and old.get("scan.scriptAuth.authScriptFilePath", default="") == "scripts/offline-token.js"
     ):
         # probably OAuth2
         new.set(
             "general.authentication",
             {
                 "type": "oauth2_rtoken",
                 "parameters": {
-                    "client_id": old.get(
-                        "scan.scriptAuth.authClientID", default="cloud-services"
-                    ),
-                    "token_endpoint": old.get(
-                        "scan.scriptAuth.authTokenEndpoint", default=""
-                    ),
+                    "client_id": old.get("scan.scriptAuth.authClientID", default="cloud-services"),
+                    "token_endpoint": old.get("scan.scriptAuth.authTokenEndpoint", default=""),
                     "rtoken_var_name": "RTOKEN",
                 },
             },
         )
     else:
-        logging.warning(
-            "The config version translator does not support this particular authentication"
-        )
+        logging.warning("The config version translator does not support this particular authentication")
 
     # "Scanners.Zap" section
     new.set(
@@ -206,13 +195,9 @@ def convert_from_version_0_to_1(old):
 
     ### OpenAPI
     if old.get("openapi.importFromUrl", default=False):
-        new.set(
-            "scanners.zap.apiScan.apis.apiUrl", old.get("openapi.url", default=None)
-        )
+        new.set("scanners.zap.apiScan.apis.apiUrl", old.get("openapi.url", default=None))
     elif old.get("openapi.directory", default=""):
-        logging.warning(
-            "The config version translator does not support Directory based OpenAPI"
-        )
+        logging.warning("The config version translator does not support Directory based OpenAPI")
 
     ## Passive scan
     new.set("scanners.zap.passiveScan", {})
@@ -225,9 +210,7 @@ def convert_from_version_0_to_1(old):
     ## Active scan
     # Active scanner was always enabled, so we do the same:
     new.set("scanners.zap.activeScan", {})
-    new.set(
-        "scanners.zap.activeScan.policy", old.get("scan.policies.scanPolicyName", None)
-    )
+    new.set("scanners.zap.activeScan.policy", old.get("scan.policies.scanPolicyName", None))
 
     # Finally, set the correct version number
     new.set("config.configVersion", 1)