From 584a50f3d69ee9b101e4baab23fa2bbec11cbf6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Mon, 24 Jun 2024 10:16:57 +0200 Subject: [PATCH 1/5] Audit: Run git reset --hard event if there is no vulnerabilities --- github_app_geo_project/module/audit/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/github_app_geo_project/module/audit/utils.py b/github_app_geo_project/module/audit/utils.py index 592fde3ddc6..5b38ce84582 100644 --- a/github_app_geo_project/module/audit/utils.py +++ b/github_app_geo_project/module/audit/utils.py @@ -397,9 +397,8 @@ async def _snyk_fix( ) -> tuple[bool, module_utils.HtmlMessage | None]: snyk_fix_success = True snyk_fix_message = None + subprocess.run(["git", "reset", "--hard"], timeout=30) # nosec # pylint: disable=subprocess-run-check if fixable_vulnerabilities_summary: - subprocess.run(["git", "reset", "--hard"], timeout=30) # nosec # pylint: disable=subprocess-run-check - command = [ "snyk", "fix", From 6f2d468a5df260019b7f068688b55cfc25d1f7ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Mon, 24 Jun 2024 10:29:20 +0200 Subject: [PATCH 2/5] No raise timeout error --- github_app_geo_project/module/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/github_app_geo_project/module/utils.py b/github_app_geo_project/module/utils.py index c56fa0c5478..2fe8a1ca8de 100644 --- a/github_app_geo_project/module/utils.py +++ b/github_app_geo_project/module/utils.py @@ -552,7 +552,8 @@ async def run_timeout( _LOGGER.warning(message) return None, False, message else: - raise + _LOGGER.exception("TimeoutError: %s", exception) + return None, False, AnsiProcessMessage(command, None, "", "", str(exception)) def has_changes(include_un_followed: bool = False) -> bool: From dedf0f4c42d2eae6a5a27020ace706362667fdbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Mon, 24 Jun 2024 10:30:55 +0200 Subject: [PATCH 3/5] Versions: Add missing lock --- .../module/versions/__init__.py | 109 +++++++++--------- 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/github_app_geo_project/module/versions/__init__.py b/github_app_geo_project/module/versions/__init__.py index 8efb07faafc..7c459915d9b 100644 --- a/github_app_geo_project/module/versions/__init__.py +++ b/github_app_geo_project/module/versions/__init__.py @@ -231,64 +231,65 @@ async def process( assert context.module_event_data.version is not None version = context.module_event_data.version branch = context.module_config.get("version-mapping", {}).get(version, version) - with tempfile.TemporaryDirectory() as tmpdirname: - if os.environ.get("TEST") != "TRUE": - os.chdir(tmpdirname) - success = module_utils.git_clone(context.github_project, branch) - if not success: - raise VersionException("Failed to clone the repository") - - version_status = status.versions[version] - version_status.names_by_datasource.clear() - version_status.dependencies_by_datasource.clear() - transversal_status = context.transversal_status - - message = module_utils.HtmlMessage( - utils.format_json(json.loads(version_status.model_dump_json())["names_by_datasource"]) - ) - message.title = "Names cleaned:" + async with module_utils.WORKING_DIRECTORY_LOCK: + with tempfile.TemporaryDirectory() as tmpdirname: + if os.environ.get("TEST") != "TRUE": + os.chdir(tmpdirname) + success = module_utils.git_clone(context.github_project, branch) + if not success: + raise VersionException("Failed to clone the repository") + + version_status = status.versions[version] + version_status.names_by_datasource.clear() + version_status.dependencies_by_datasource.clear() + transversal_status = context.transversal_status + + message = module_utils.HtmlMessage( + utils.format_json(json.loads(version_status.model_dump_json())["names_by_datasource"]) + ) + message.title = "Names cleaned:" - _get_names( - context, - version_status.names_by_datasource, - version, - alternate_versions=context.module_event_data.alternate_versions, - ) - message = module_utils.HtmlMessage( - utils.format_json(json.loads(version_status.model_dump_json())["names_by_datasource"]) - ) - message.title = "Names:" - _LOGGER.debug(message) - _get_dependencies(context, version_status.dependencies_by_datasource) - message = module_utils.HtmlMessage( - utils.format_json( - json.loads(version_status.model_dump_json())["dependencies_by_datasource"] + _get_names( + context, + version_status.names_by_datasource, + version, + alternate_versions=context.module_event_data.alternate_versions, ) - ) - message.title = "Dependencies:" - _LOGGER.debug(message) - - message = module_utils.HtmlMessage( - utils.format_json_str( - transversal_status.repositories[ - f"{context.github_project.owner}/{context.github_project.repository}" - ] - .versions[version] - .model_dump_json(indent=2) + message = module_utils.HtmlMessage( + utils.format_json(json.loads(version_status.model_dump_json())["names_by_datasource"]) ) - ) - message.title = f"Version ({version}):" - _LOGGER.debug(message) - - message = module_utils.HtmlMessage( - utils.format_json_str( - transversal_status.repositories[ - f"{context.github_project.owner}/{context.github_project.repository}" - ].model_dump_json(indent=2) + message.title = "Names:" + _LOGGER.debug(message) + _get_dependencies(context, version_status.dependencies_by_datasource) + message = module_utils.HtmlMessage( + utils.format_json( + json.loads(version_status.model_dump_json())["dependencies_by_datasource"] + ) ) - ) - message.title = "Repo:" - _LOGGER.debug(message) + message.title = "Dependencies:" + _LOGGER.debug(message) + + message = module_utils.HtmlMessage( + utils.format_json_str( + transversal_status.repositories[ + f"{context.github_project.owner}/{context.github_project.repository}" + ] + .versions[version] + .model_dump_json(indent=2) + ) + ) + message.title = f"Version ({version}):" + _LOGGER.debug(message) + + message = module_utils.HtmlMessage( + utils.format_json_str( + transversal_status.repositories[ + f"{context.github_project.owner}/{context.github_project.repository}" + ].model_dump_json(indent=2) + ) + ) + message.title = "Repo:" + _LOGGER.debug(message) return ProcessOutput(transversal_status=context.transversal_status) raise VersionException("Invalid step") From 8e3f57bc903fc730c64d7274f6ce5bbb0e69aa78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Mon, 24 Jun 2024 13:02:42 +0200 Subject: [PATCH 4/5] Audit: Add snyk fix debug --- github_app_geo_project/module/audit/utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/github_app_geo_project/module/audit/utils.py b/github_app_geo_project/module/audit/utils.py index 5b38ce84582..92192c04557 100644 --- a/github_app_geo_project/module/audit/utils.py +++ b/github_app_geo_project/module/audit/utils.py @@ -65,6 +65,7 @@ async def snyk( logs_url, result, env_no_debug, + env, fixable_vulnerabilities_summary, ) npm_audit_fix_message, npm_audit_fix_success = await _npm_audit_fix(fixable_files_npm, result) @@ -393,6 +394,7 @@ async def _snyk_fix( logs_url: str, result: list[module_utils.Message], env_no_debug: dict[str, str], + env_debug: dict[str, str], fixable_vulnerabilities_summary: dict[str, str], ) -> tuple[bool, module_utils.HtmlMessage | None]: snyk_fix_success = True @@ -419,6 +421,15 @@ async def _snyk_fix( if fix_message: snyk_fix_message = module_utils.AnsiMessage(fix_message.strip()) if not snyk_fix_success: + await module_utils.run_timeout( + command, + env_debug, + int(os.environ.get("GHCI_SNYK_FIX_TIMEOUT", os.environ.get("GHCI_SNYK_TIMEOUT", "300"))), + "Snyk fix (debug)", + "Error while fixing the project (debug)", + "Timeout while fixing the project (debug)", + ) + cwd = module_utils.get_cwd() project = "-" if cwd is None else os.path.basename(cwd) message = module_utils.HtmlMessage( From c0251239f33d770df14112a199097a216d29c0d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Mon, 24 Jun 2024 13:20:32 +0200 Subject: [PATCH 5/5] Remove unworking info --- github_app_geo_project/scripts/process_queue.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/github_app_geo_project/scripts/process_queue.py b/github_app_geo_project/scripts/process_queue.py index 830a52c9d78..aec3a2a7c1d 100644 --- a/github_app_geo_project/scripts/process_queue.py +++ b/github_app_geo_project/scripts/process_queue.py @@ -34,7 +34,6 @@ _LOGGER_WSGI = logging.getLogger("prometheus_client.wsgi") _NB_JOBS = Gauge("ghci_jobs_number", "Number of jobs", ["status"]) -_JOBS = Info("ghci_jobs", "Running jobs") class _JobInfo(NamedTuple): @@ -810,25 +809,19 @@ def _watch(self) -> None: _NB_JOBS.labels(status.name).set( session.query(models.Queue).filter(models.Queue.status == status).count() ) - info = {} text = [] for id_, job in _RUNNING_JOBS.items(): text.append( f"{id_}: {job.module} {job.event_name} {job.repository} [{job.priority}] (Worker max priority {job.worker_max_priority})" ) - info[f"job-id-{id_}"] = ( - f"{job.module} {job.event_name} {job.repository} [{job.priority}] (Worker max priority {job.worker_max_priority})" - ) try: for task in asyncio.all_tasks(): txt = io.StringIO() task.print_stack(file=txt) text.append("-" * 30) text.append(txt.getvalue()) - info[f"task-{id(task)}"] = txt.getvalue() except RuntimeError as exception: text.append(str(exception)) - _JOBS.info(info) if time.time() - self.last_run > 300: error_message = ["Old Status"]