[ZAP] OAuth2: optionally dowload schemas (RedHatProductSecurity#117)

* [ZAP] OAuth2: optionally dowload schemas

ZAP can't use authenticated user when downloading schemas, such as
OpenAPI and GraphQL.

This workaround looks for places with URLs, downloads them and changes
the config to point at the file instead

Also prevent adding a NULL URL when application.url is missing
This causes a Null pointer dereference in ZAP.

* [ZAP] Prevents ZAP to run if application.url is missing

* [test] fixing pytest after application.url was made mandatory

config/config-template-generic-scan.yaml

@@ -8,7 +8,7 @@ config:
   # This value tells RapiDAST what schema should be used to read this configuration.
   # Therefore you should only change it if you update the configuration to a newer schema
   # It is intended to keep backward compatibility (newer RapiDAST running an older config)
-  configVersion: 4
+  configVersion: 5
 
 # `application` contains data related to the application, not to the scans.
 application:

config/config-template-multi-scan.yaml

@@ -10,7 +10,7 @@ config:
   # This value tells RapiDAST what schema should be used to read this configuration.
   # Therefore you should only change it if you update the configuration to a newer schema
   # It is intended to keep backward compatibility (newer RapiDAST running an older config)
-  configVersion: 4
+  configVersion: 5
 
 # `application` contains data related to the application, not to the scans.
 application:

config/config-template-zap-long.yaml

@@ -10,7 +10,7 @@ config:
   # This value tells RapiDAST what schema should be used to read this configuration.
   # Therefore you should only change it if you update the configuration to a newer schema
   # It is intended to keep backward compatibility (newer RapiDAST running an older config)
-  configVersion: 4
+  configVersion: 5
 
   # all the results of all scanners will be stored under that location
   base_results_dir: "./results"
@@ -150,8 +150,8 @@ scanners:
       enableUI: False
       # Defaults to True, set False to prevent auto update of ZAP plugins
       updateAddons: True
-      # If set to True and authentication is oauth2_rtoken and api.apiUrl is set, download the API outside of ZAP
-      oauth2OpenapiManualDownload: False
+      # If set to True and authentication is oauth2_rtoken: manually download schemas (e.g.: openAPI, GraphQL)
+      oauth2ManualDownload: False
 
       # overwrite the default port in case it is required. The default port was selected to avoid any collision with other services
       zapPort: 8080

config/config-template-zap-mac.yaml

@@ -10,7 +10,7 @@ config:
   # This value tells RapiDAST what schema should be used to read this configuration.
   # Therefore you should only change it if you update the configuration to a newer schema
   # It is intended to keep backward compatibility (newer RapiDAST running an older config)
-  configVersion: 4
+  configVersion: 5
 
 # `application` contains data related to the application, not to the scans.
 application:

config/config-template-zap-simple.yaml

@@ -10,7 +10,7 @@ config:
   # This value tells RapiDAST what schema should be used to read this configuration.
   # Therefore you should only change it if you update the configuration to a newer schema
   # It is intended to keep backward compatibility (newer RapiDAST running an older config)
-  configVersion: 4
+  configVersion: 5
 
 # `application` contains data related to the application, not to the scans.
 application:

configmodel/converter.py

@@ -5,7 +5,7 @@
 
 # WARNING: this needs to be incremented everytime a non-compatible change is made in the configuration.
 # A corresponding function also needs to be written
-CURR_CONFIG_VERSION = 4
+CURR_CONFIG_VERSION = 5
 
 
 def config_converter_dispatcher(func):
@@ -50,6 +50,30 @@ def convert_configmodel(conf):
     )
 
 
+@convert_configmodel.register(4)
+def convert_from_version_4_to_5(old):
+    """Returns a *copy* of the original rapidast config file, but updated to v5
+    scanner.zap.miscOptions.oauth2OpenapiManualDownload was renamed oauth2ManualDownload
+
+    Note: scanners can now have IDs (i.e.: scanner.zap_foo, not just scanner.zap)
+    """
+    new = copy.deepcopy(old)
+
+    for key in old.conf["scanners"]:
+        if key.startswith("zap") and old.exists(
+            f"scanners.{key}.miscOptions.oauth2OpenapiManualDownload"
+        ):
+            new.move(
+                f"scanners.{key}.miscOptions.oauth2OpenapiManualDownload",
+                f"scanners.{key}.miscOptions.oauth2ManualDownload",
+            )
+
+    # Finally, set the correct version number
+    new.set("config.configVersion", 5)
+
+    return new
+
+
 @convert_configmodel.register(3)
 def convert_from_version_3_to_4(old):
     """Returns a *copy* of the original rapidast config file, but updated to v4

scanners/zap/zap.py

@@ -270,15 +270,20 @@ def _setup_zap_automation(self):
         # Configure the basic environment target
         try:
             af_context = find_context(self.automation_config)
-            af_context["urls"].append(self.config.get("application.url"))
+            app_url = self.config.get("application.url")
+            if app_url:
+                af_context["urls"].append(app_url)
+            else:
+                logging.error("Configuration: ZAP requires an application.url entry")
+                raise KeyError("Missing `application.url` in configuration")
             af_context["includePaths"].extend(self.my_conf("urls.includes", default=[]))
             af_context["excludePaths"].extend(self.my_conf("urls.excludes", default=[]))
         except KeyError as exc:
             raise RuntimeError(
                 f"Something went wrong with the Zap scanner configuration, while creating the context':\n {str(exc)}"
             ) from exc
 
-        # authentication MUST happen first in case a user is created
+        # authentication MUST happen first in case a user is created, or authenticated manual download is needed
         self.authenticated = self.authentication_factory()
 
         # Create the AF configuration
@@ -680,35 +685,12 @@ def authentication_set_oauth2_rtoken(self):
         self.automation_config["jobs"].append(script)
         logging.info("ZAP configured with OAuth2 RTOKEN")
 
-        # quickhack: the openapi job currently does not run with user authentication.
-        # This is a problem when openapi requires an authenticated URL.
-        # => manually download the OAS, and change it to apiFile
-        # This can be deleted when https://github.com/zaproxy/zaproxy/issues/7739 is resolved
-        # Note: to avoid a temporary file, we download the file directly in its final destination in work_dir
-        #       This is not a problem: it will simply be ignored by _include_file()
-        oas_url = self.my_conf("apiScan.apis.apiUrl", default=None)
-        if oas_url and self.my_conf(
-            "miscOptions.oauth2OpenapiManualDownload", default=False
-        ):
-            logging.info("ZAP workaround: manually downloading the OpenAPI file")
-            if authenticated_download_with_rtoken(
-                url=oas_url,
-                dest=f"{self._host_work_dir()}/openapi.json",
+        if self.my_conf("miscOptions.oauth2ManualDownload"):
+            # See if manual authenticated downloads are required
+            self._manual_oauth2_download(
                 auth={"rtoken": rtoken, "client_id": client_id, "url": token_endpoint},
                 proxy=self.my_conf("proxy", default=None),
-            ):
-                logging.info(
-                    "Successful manual download of the OAS: replacing apiUrl by apiFile"
-                )
-                self.config.set(
-                    f"scanners.{self.ident}.apiScan.apis.apiFile",
-                    f"{self._host_work_dir()}/openapi.json",
-                )
-                self.config.delete(f"scanners.{self.ident}.apiScan.apis.apiUrl")
-            else:
-                logging.warning(
-                    "Failed to manually download the OAS. delegating to ZAP"
-                )
+            )
 
         return True
 
@@ -717,6 +699,54 @@ def authentication_set_oauth2_rtoken(self):
     # Special functions (other than __init__())                   #
     ###############################################################
 
+    def _manual_oauth2_download(self, auth, proxy):
+        """QUICKHACK: some ZAP requests can't be authenticated.
+        This is an issue for schema downloads behind a login (e.g.: openapi, graphQL)
+
+        IF a manual download is requested:
+        1) identify those URLs in the config
+        2) For each ones:
+          a) Download the said schema
+          b) Modify the config to use a file path instead of the URL
+
+        Example of issues: https://github.com/zaproxy/zaproxy/issues/7739 is resolved
+        Note: to avoid a temporary file, we download the files directly in its final destination in work_dir
+              This is not a problem: it will simply be ignored by _include_file()
+        """
+        logging.info("Looking for URLs to downloads manually")
+
+        # Preparation: list of all locations in the RapiDAST configuration that might need replacement
+        #  - config_url: URL's placement in the rapidast configuration, under the scanner
+        #  - path: destination for the download
+        #  - config_path: the RapiDAST config entry that will replace `config_url`
+        Change = namedtuple("Change", ["config_url", "path", "config_path"])
+        changes = [
+            Change(
+                "apiScan.apis.apiUrl",
+                f"{self._host_work_dir()}/openapi.json",
+                "apiScan.apis.apiFile",
+            ),
+            Change(
+                "graphql.schemaUrl",
+                f"{self._host_work_dir()}/schema.graphql",
+                "graphql.schemaFile",
+            ),
+        ]
+
+        for change in changes:
+            url = self.my_conf(change.config_url)
+            if url:
+                if authenticated_download_with_rtoken(url, change.path, auth, proxy):
+                    logging.info(
+                        f"Successful download of scanner's {change.config_url}"
+                    )
+                    self.config.set(
+                        f"scanners.{self.ident}.{change.config_path}", change.path
+                    )
+                    self.config.delete(f"scanners.{self.ident}.{change.config_url}")
+                else:
+                    logging.warning("Failed to download scanner's {change.config_url}")
+
 
 # Given an Automation Framework configuration, return its sub-dictionary corresponding to the context we're going to use
 def find_context(automation_config, context=Zap.DEFAULT_CONTEXT):

tests/configmodel/older-schemas/v4.yaml

@@ -0,0 +1,180 @@
+# This is a verbose configuration template. A lot of value do not need to be present, for most configuration.
+# See "config-template.yaml" for a simpler configuration file.
+# All the values are optional (except `config.configVersion`): if a key is missing, it will mean either "disabled" or a sensible default will be selected
+
+config:
+  # WARNING: `configVersion` indicates the schema version of the config file.
+  # This value tells RapiDAST what schema should be used to read this configuration.
+  # Therefore you should only change it if you update the configuration to a newer schema
+  # It is intended to keep backward compatibility (newer RapiDAST running an older config)
+  configVersion: 4
+
+  # all the results of all scanners will be stored under that location
+  base_results_dir: "./results"
+
+  # Import a particular environment, and inject it for each scanner
+  environ:
+    envFile: "path/to/env/file"
+
+  # (Optional) configure to export scan results to OWASP Defect Dojo
+  defectDojo:
+    url: "https://mydefectdojo.example.com/"
+    authorization:
+      username: "rapidast"
+      password: "password"
+        # or
+      token: "abc"
+
+# `application` contains data related to the application, not to the scans.
+application:
+  shortName: "MyApp-1.0"
+  url: "<Mandatory. root URL of the application>"
+
+# `general` is a section that will be applied to all scanners.
+# Any scanner can override a value by creating an entry of the same name in their own configuration
+general:
+
+
+  # remove `proxy` entirely for direct connection
+  proxy:
+    proxyHost: "<hostname>"
+    proxyPort: "<port>"
+
+  # remove `authentication` entirely for unauthenticated connection
+  authentication:
+    type: "oauth2_rtoken"
+    parameters:
+      client_id: "cloud-services"
+      token_endpoint: "<token retrieval URL>"
+      rtoken_from_var: "RTOKEN"     # referring to a env defined in general.environ.envFile
+    # Other types of authentication:
+    #type: "http_header"
+    #parameters:
+    #  name: "Authorization"
+    #  value: "MySecretHeader"
+    #type: "http_basic"
+    #parameters:
+    #  username: "user"
+    #  password: "mypassw0rd"
+    #type: "cookie"
+    #parameters:
+    #  name: "cookie name"
+    #  value: "cookie value"
+
+
+  container:
+    # This configures what technology is to be used for RapiDAST to run each scanner
+    # Currently supported: `podman`, `flatpak` and `none`
+    #   podman: RapiDAST runs each scanner using podman
+    #   flatpak: RapiDAST runs each scanner using flatpak
+    #   none: RapiDAST runs each scanner in the same host or container (where RapiDAST itself is running in a container)
+    #         container.
+    type: "podman"
+
+
+# `scanners' is a section that configures scanning options
+scanners:
+  zap:
+  # define a scan through the ZAP scanner
+    apiScan:
+      target: "<optional, if different from application.url>"
+      apis:
+        apiUrl: "<URL to openAPI>"
+        # alternative to apiURL: apiFile: "<local path to openAPI file>"
+
+    # A list of URLs can also be provided, from a text file (1 URL per line)
+    importUrlsFromFile: "<path to import URL>"
+
+    graphql:
+      endpoint: "<URL to GraphQL API endpoint>"
+      # schemaUrl: ""                       # String: URL pointing to a GraphQL Schema
+      # schemaFile: ""                      # String: Local file path of a GraphQL Schema
+      # maxQueryDepth: 5                    # The maximum query generation depth
+      # lenientMaxQueryDepthEnabled: true   # Whether or not Maximum Query Depth is enforced leniently
+      # maxAdditionalQueryDepth: 5          # The maximum additional query generation depth (used if enforced leniently)
+      # maxArgsDepth: 5                     # The maximum arguments generation depth
+      # optionalArgsEnabled: true           # Whether or not Optional Arguments should be specified
+      # argsType: both                      # Enum [inline, variables, both]: How arguments are specified
+      # querySplitType: leaf                # Enum [leaf, root_field, operation]: The level for which a single query is generated
+      # requestMethod: post_json            # Enum [post_json, post_graphql, get]: The request method
+
+    spider:
+      maxDuration: 0 # in minutes, default: 0 unlimited
+      url: ""        # url to start spidering from, default: application.url set above
+
+    spiderAjax:
+      maxDuration: 0 # in minutes, default: 0 unlimited
+      url: ""        # url to start spidering from, default: application.url set above
+      browserId: firefox-headless
+
+    passiveScan:
+      # optional comma-separated list of passive rules to disable
+      # Use https://www.zaproxy.org/docs/alerts/ to match rule with its ID
+      disabledRules: "2,10015,10027,10096,10024"
+
+    activeScan:
+      # If no policy is chosen, a default ("API-scan-minimal") will be selected
+      # The list of policies can be found in scanners/zap/policies/
+      policy:  "API-scan-minimal"
+
+    container:
+      parameters:
+        image: "docker.io/owasp/zap2docker-stable:latest" # for type such as podman
+        #podName: "mypod"  # optional: inject ZAP in an existing Pod
+
+        executable: "zap.sh"  # for Linux
+        # executable: "/Applications/OWASP ZAP.app/Contents/Java/zap.sh"    # for MacOS, when general.container.type is 'none' only
+
+
+    report:
+      format: ["json"]
+      # format: ["json","html","sarif","xml"]  # default: "json" only
+
+    urls:
+      # Optional, `includes` and `excludes` take a list of regexps.
+      # includes: A URL matching that regexp will be in the scope of scanning, in addition to application.url which is already in scope
+      # excludes: A URL matching that regexp will NOT be in the scope of scanning
+      # Note: The regular expressions MUST match the whole URL.
+      #       e.g.: 'http://example.com/do-not-descend-here/' will actually descend
+
+      #includes:
+      #  - "^https?://example.com:3000/.*$"
+      #excludes:
+      #  - "^https?://example.com:3000/do-not-descend-here/.*$"
+
+    miscOptions:
+      # enableUI (default: false), requires a compatible runtime (e.g.: flatpak or no containment)
+      enableUI: False
+      # Defaults to True, set False to prevent auto update of ZAP plugins
+      updateAddons: True
+      # If set to True and authentication is oauth2_rtoken and api.apiUrl is set, download the API outside of ZAP
+      oauth2OpenapiManualDownload: False
+
+      # overwrite the default port in case it is required. The default port was selected to avoid any collision with other services
+      zapPort: 8080
+
+    # (Optional) configure to export scan results to OWASP Defect Dojo.
+    # `config.defectDojo` must be configured first.
+    defectDojoExport:
+      type: "reimport" # choose between: import, reimport, False (disable export). Default (or other content): re-import if test is set
+      # Parameters contain data that will directly be sent as parameters to DefectDojo's import/reimport endpoints.
+      # For example: commit tag, version, push_to_jira, etc.
+      # See https://demo.defectdojo.org/api/v2/doc/ for a list of possibilities
+      # The minimum set of data is whatever is needed to identify which engagement/test needs to be chosen.
+      # If neither a test ID (`test` parameter), nor product_name and engagement_name were provided, sane default will be attempted:
+      #   - product_name chosen from either application.productName or application.shortName
+      #   - engagement_name:  "RapiDAST" [this way the same engagement will always be chosen, regardless of the scanner]
+      parameters:
+        product_name: "My Product"
+        engagement_name: "RapiDAST"
+        # - or -
+        #engagement: 3   # engagement ID
+        # - or -
+        #test_title: "ZAP"
+        # - or -
+        #test: 5       # test ID, that will force "reimport" mode
+        # additional options, see https://demo.defectdojo.org/api/v2/doc/ for list
+        auto_create_context: False  # Optional. set to True to auto-create engagement (requires product_name and engagement_name)
+
+
+# Other scanners to be defined(TBD)