Merge pull request #359 from camptocamp/fix

Audit: Improve messages and logs, add Python 3.12

.github/workflows/main.yaml

@@ -17,7 +17,7 @@ jobs:
   main:
     name: Continuous integration
     runs-on: ubuntu-22.04
-    timeout-minutes: 30
+    timeout-minutes: 45
     if: "!startsWith(github.event.head_commit.message, '[skip ci] ')"
 
     steps:

Dockerfile

@@ -115,7 +115,7 @@ RUN --mount=type=cache,target=/var/lib/apt/lists \
     && apt-get update \
     && apt-get install --assume-yes --no-install-recommends ${DEV_PACKAGES} \
     && git clone --depth=1 https://github.com/pyenv/pyenv.git /pyenv \
-    && pyenv install 3.7 3.8 3.9 3.10 3.11 \
+    && pyenv install 3.7 3.8 3.9 3.10 3.11 3.12 \
     && apt-get remove --purge --autoremove --yes ${DEV_PACKAGES}
 
 ENV PATH=${PATH}:/app/node_modules/.bin

github_app_geo_project/module/audit/__init__.py

@@ -194,14 +194,15 @@ async def _process_snyk_dpkg(
                     with open(".tool-versions", encoding="utf-8") as file:
                         for line in file:
                             if line.startswith("python "):
-                                python_version = ".".join(line.split(" ")[1].split(".")[0:2])
+                                python_version = ".".join(line.split(" ")[1].split(".")[0:2]).strip()
                                 break
                 try:
                     if python_version:
                         _use_python_version(python_version)
 
+                    logs_url = urllib.parse.urljoin(context.service_url, f"logs/{context.job_id}")
                     result, body, short_message, new_success = await audit_utils.snyk(
-                        branch, context.module_config.get("snyk", {}), local_config.get("snyk", {})
+                        branch, context.module_config.get("snyk", {}), local_config.get("snyk", {}), logs_url
                     )
                     success &= new_success
                     output_url = _process_error(
@@ -211,6 +212,11 @@ async def _process_snyk_dpkg(
                         [{"title": m.title, "children": [m.to_html("no-title")]} for m in result],
                         ", ".join(short_message),
                     )
+                    message: module_utils.Message = module_utils.HtmlMessage(
+                        "<a href='%s'>Output</a>" % output_url
+                    )
+                    message.title = "Output URL"
+                    _LOGGER.debug(message)
                     if output_url is not None:
                         short_message.append(f"[See also]({output_url})")
                 finally:
@@ -249,8 +255,11 @@ async def _process_snyk_dpkg(
                         if pull_request is not None:
                             issue_check.set_title(key, f"{key} ([Pull request]({pull_request.html_url}))")
 
-    except Exception:  # pylint: disable=broad-except
+    except Exception as exception:  # pylint: disable=broad-except
         _LOGGER.exception("Audit %s error", key)
+        return [f"Error while processing the audit {key}: {exception}"], False
+    finally:
+        os.chdir("/")
 
     return short_message, success
 

github_app_geo_project/module/audit/utils.py

@@ -23,7 +23,10 @@
 
 
 async def snyk(
-    branch: str, config: configuration.SnykConfiguration, local_config: configuration.SnykConfiguration
+    branch: str,
+    config: configuration.SnykConfiguration,
+    local_config: configuration.SnykConfiguration,
+    logs_url: str,
 ) -> tuple[list[module_utils.Message], module_utils.Message | None, list[str], bool]:
     """
     Audit the code with Snyk.
@@ -285,6 +288,7 @@ async def snyk(
 
     high_vulnerabilities: dict[str, int] = {}
     fixable_vulnerabilities: dict[str, int] = {}
+    fixable_vulnerabilities_summary: dict[str, str] = {}
     for row in test_json:
         message = module_utils.HtmlMessage(
             "\n".join(
@@ -329,6 +333,8 @@ async def snyk(
                 title += " [Patch available]."
             else:
                 title += "."
+            if vuln.get("fixedIn", []) or vuln.get("isUpgradable", False) or vuln.get("isPatchable", False):
+                fixable_vulnerabilities_summary[vuln["id"]] = title
             message = module_utils.HtmlMessage(
                 "<br>\n".join(
                     [
@@ -368,8 +374,19 @@ async def snyk(
         snyk_fix_success = snyk_fix_proc.returncode == 0
         if snyk_fix_proc.returncode != 0:
             message.title = "Error while fixing the project"
-            _LOGGER.error(message)
+            _LOGGER.warning(message)
             result.append(message)
+            message = module_utils.HtmlMessage(
+                "<br>\n".join(
+                    [
+                        *fixable_vulnerabilities_summary.values(),
+                        f"{os.path.basename(os.getcwd())}:{branch}",
+                        f"See logs: {logs_url}",
+                    ]
+                )
+            )
+            message.title = f"Unable to fix {len(fixable_vulnerabilities)} vulnerabilities"
+            _LOGGER.error(message)
         else:
             message.title = "Snyk fix applied"
             _LOGGER.debug(message)

github_app_geo_project/module/utils.py

@@ -308,6 +308,10 @@ def __str__(self) -> str:
         """Get the string representation."""
         return self.to_plain_text()
 
+    def __repr__(self) -> str:
+        """Get the string representation."""
+        return self.to_plain_text()
+
     def to_plain_text(self) -> str:
         """Get the ANSI message."""
         sanitizer = html_sanitizer.Sanitizer(