diff --git a/github_app_geo_project/module/audit/__init__.py b/github_app_geo_project/module/audit/__init__.py index 7680a3f876e..c42a1d1f989 100644 --- a/github_app_geo_project/module/audit/__init__.py +++ b/github_app_geo_project/module/audit/__init__.py @@ -245,7 +245,7 @@ async def _process_snyk_dpkg( else: repo = context.github_project.repo - new_success, pull_request = module_utils.create_commit_pull_request( + new_success, pull_request = await module_utils.create_commit_pull_request( branch, new_branch, f"Audit {key}", "" if body is None else body.to_markdown(), repo ) success &= new_success diff --git a/github_app_geo_project/module/audit/utils.py b/github_app_geo_project/module/audit/utils.py index 4865ae23708..f0ecf03e712 100644 --- a/github_app_geo_project/module/audit/utils.py +++ b/github_app_geo_project/module/audit/utils.py @@ -21,70 +21,6 @@ _LOGGER = logging.getLogger(__name__) -async def _run_timeout( - command: list[str], - env: dict[str, str] | None, - timeout: int, - success_message: str, - error_message: str, - timeout_message: str, - error_messages: list[module_utils.Message], - cwd: str | None = None, -) -> tuple[str | None, bool]: - async_proc = None - try: - async with asyncio.timeout(timeout): - async_proc = await asyncio.create_subprocess_exec( - *command, - cwd=cwd or os.getcwd(), - env=env, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - stdout, stderr = await async_proc.communicate() - assert async_proc.returncode is not None - message: module_utils.Message = module_utils.AnsiProcessMessage( - command, async_proc.returncode, stdout.decode(), stderr.decode() - ) - success = async_proc.returncode == 0 - if success: - message.title = success_message - _LOGGER.debug(message) - else: - message.title = error_message - _LOGGER.warning(message) - error_messages.append(message) - return stdout.decode(), success - except FileNotFoundError as exception: - _LOGGER.exception("%s not found: %s", command[0], exception) - proc = subprocess.run( # nosec # pylint: disable=subprocess-run-check - ["find", "/", "-name", command[0]], - capture_output=True, - encoding="utf-8", - timeout=30, - ) - message = module_utils.ansi_proc_message(proc) - message.title = f"Find {command[0]}" - _LOGGER.debug(message) - return None, False - except asyncio.TimeoutError as exception: - if async_proc: - async_proc.kill() - message = module_utils.AnsiProcessMessage( - command, - None, - "" if async_proc.stdout is None else (await async_proc.stdout.read()).decode(), - "" if async_proc.stderr is None else (await async_proc.stderr.read()).decode(), - error=str(exception), - ) - message.title = timeout_message - _LOGGER.warning(message) - error_messages.append(message) - return None, False - else: - raise - - async def snyk( branch: str, config: configuration.SnykConfiguration, @@ -181,7 +117,7 @@ async def _install_requirements_dependencies( if file in local_config.get("files-no-install", config.get("files-no-install", [])): continue - await _run_timeout( + _, _, proc_message = await module_utils.run_timeout( [ "python", "-m", @@ -195,8 +131,9 @@ async def _install_requirements_dependencies( f"Dependencies installed from {file}", f"Error while installing the dependencies from {file}", f"Timeout while installing the dependencies from {file}", - result, ) + if proc_message is not None: + result.append(proc_message) async def _install_pipenv_dependencies( @@ -221,7 +158,7 @@ async def _install_pipenv_dependencies( continue directory = os.path.dirname(os.path.abspath(file)) - await _run_timeout( + _, _, proc_message = await module_utils.run_timeout( [ "pipenv", "install", @@ -232,9 +169,10 @@ async def _install_pipenv_dependencies( f"Dependencies installed from {file}", f"Error while installing the dependencies from {file}", f"Timeout while installing the dependencies from {file}", - result, directory, ) + if proc_message is not None: + result.append(proc_message) async def _install_poetry_dependencies( @@ -261,7 +199,7 @@ async def _install_poetry_dependencies( if file in local_config.get("files-no-install", config.get("files-no-install", [])): continue - await _run_timeout( + _, _, proc_message = await module_utils.run_timeout( [ "poetry", "install", @@ -272,9 +210,10 @@ async def _install_poetry_dependencies( f"Dependencies installed from {file}", f"Error while installing the dependencies from {file}", f"Timeout while installing the dependencies from {file}", - result, os.path.dirname(os.path.abspath(file)), ) + if proc_message is not None: + result.append(proc_message) async def _snyk_monitor( @@ -314,15 +253,16 @@ async def _snyk_monitor( f"--project-tags={','.join(['='.join(tag) for tag in local_monitor_config.get('project-tags', monitor_config.get('project-tags', {}))])}" ) - await _run_timeout( + _, _, message = await module_utils.run_timeout( command, env, int(os.environ.get("GHCI_SNYK_TIMEOUT", "300")), "Project monitored", "Error while monitoring the project", "Timeout while monitoring the project", - result, ) + if message is not None: + result.append(message) async def _snyk_test( @@ -340,15 +280,16 @@ async def _snyk_test( "test-arguments", config.get("test-arguments", configuration.SNYK_TEST_ARGUMENTS_DEFAULT) ), ] - test_json_str, _ = await _run_timeout( + test_json_str, _, message = await module_utils.run_timeout( command, env_no_debug, int(os.environ.get("GHCI_SNYK_TIMEOUT", "300")), "Snyk test", "Error while testing the project", "Timeout while testing the project", - result, ) + if message is not None: + result.append(message) if test_json_str: message = module_utils.HtmlMessage(utils.format_json_str(test_json_str)) @@ -452,15 +393,16 @@ async def _snyk_fix( "fix-arguments", config.get("fix-arguments", configuration.SNYK_FIX_ARGUMENTS_DEFAULT) ), ] - fix_message, snyk_fix_success = await _run_timeout( + fix_message, snyk_fix_success, message = await module_utils.run_timeout( command, env_no_debug, int(os.environ.get("GHCI_SNYK_TIMEOUT", "300")), "Snyk fix", "Error while fixing the project", "Timeout while fixing the project", - result, ) + if message is not None: + result.append(message) if fix_message: snyk_fix_message = module_utils.AnsiMessage(fix_message.strip()) if not snyk_fix_success: @@ -488,16 +430,17 @@ async def _npm_audit_fix( messages.update(file_messages) _LOGGER.debug("Fixing vulnerabilities in %s with npm audit fix --force", package_lock_file_name) command = ["npm", "audit", "fix", "--force"] - _, success = await _run_timeout( + _, success, message = await module_utils.run_timeout( command, os.environ.copy(), int(os.environ.get("GHCI_SNYK_TIMEOUT", "300")), "Npm audit fix", "Error while fixing the project", "Timeout while fixing the project", - result, directory, ) + if message is not None: + result.append(message) _LOGGER.debug("Fixing version in %s", package_lock_file_name) # Remove the add '~' in the version in the package.json with open(os.path.join(directory, "package.json"), encoding="utf-8") as package_file: diff --git a/github_app_geo_project/module/standard/patch.py b/github_app_geo_project/module/standard/patch.py index 6cd11e07c09..aa2116fe2ff 100644 --- a/github_app_geo_project/module/standard/patch.py +++ b/github_app_geo_project/module/standard/patch.py @@ -156,7 +156,7 @@ async def process( _LOGGER.info(message) if utils.has_changes(include_un_followed=True): - success = utils.create_commit( + success = await utils.create_commit( f"{artifact.name[:-6]}\n\nFrom the artifact of the previous workflow run" ) if not success: diff --git a/github_app_geo_project/module/utils.py b/github_app_geo_project/module/utils.py index 9900d229932..74a5a1d62fc 100644 --- a/github_app_geo_project/module/utils.py +++ b/github_app_geo_project/module/utils.py @@ -1,5 +1,6 @@ """Module utility functions for the modules.""" +import asyncio import datetime import logging import os @@ -376,8 +377,8 @@ def __init__( self.args.append(arg) self.returncode = returncode - self.stdout = self._ansi_converter.convert(stdout, full=False) - self.stderr = self._ansi_converter.convert(stderr, full=False) + self.stdout = self._ansi_converter.convert(stdout or "", full=False) + self.stderr = self._ansi_converter.convert(stderr or "", full=False) message = [f"Command: {shlex.join(self.args)}"] if error: @@ -458,6 +459,84 @@ def ansi_proc_message( return AnsiProcessMessage.from_process(proc) +async def run_timeout( + command: list[str], + env: dict[str, str] | None, + timeout: int, + success_message: str, + error_message: str, + timeout_message: str, + cwd: str | None = None, +) -> tuple[str | None, bool, Message | None]: + """ + Run a command with a timeout. + + Arguments: + --------- + command: The command to run + env: The environment variables + timeout: The timeout + success_message: The message on success + error_message: The message on error + timeout_message: The message on timeout + cwd: The working directory + + Return: + ------ + The standard output, the success, the logged message + """ + async_proc = None + try: + async with asyncio.timeout(timeout): + async_proc = await asyncio.create_subprocess_exec( + *command, + cwd=cwd or os.getcwd(), + env=env, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await async_proc.communicate() + assert async_proc.returncode is not None + message: Message = AnsiProcessMessage( + command, async_proc.returncode, stdout.decode(), stderr.decode() + ) + success = async_proc.returncode == 0 + if success: + message.title = success_message + _LOGGER.debug(message) + else: + message.title = error_message + _LOGGER.warning(message) + return stdout.decode(), success, message + except FileNotFoundError as exception: + _LOGGER.exception("%s not found: %s", command[0], exception) + proc = subprocess.run( # nosec # pylint: disable=subprocess-run-check + ["find", "/", "-name", command[0]], + capture_output=True, + encoding="utf-8", + timeout=30, + ) + message = ansi_proc_message(proc) + message.title = f"Find {command[0]}" + _LOGGER.debug(message) + return None, False, message + except asyncio.TimeoutError as exception: + if async_proc: + async_proc.kill() + message = AnsiProcessMessage( + command, + None, + "" if async_proc.stdout is None else (await async_proc.stdout.read()).decode(), + "" if async_proc.stderr is None else (await async_proc.stderr.read()).decode(), + error=str(exception), + ) + message.title = timeout_message + _LOGGER.warning(message) + return None, False, message + else: + raise + + def has_changes(include_un_followed: bool = False) -> bool: """Check if there are changes.""" if include_un_followed: @@ -471,7 +550,7 @@ def has_changes(include_un_followed: bool = False) -> bool: return proc.returncode != 0 -def create_commit(message: str) -> bool: +async def create_commit(message: str) -> bool: """Do a commit.""" proc = subprocess.run( # nosec # pylint: disable=subprocess-run-check ["git", "add", "--all"], capture_output=True, encoding="utf-8", timeout=30 @@ -481,16 +560,15 @@ def create_commit(message: str) -> bool: proc_message.title = "Error adding files to commit" _LOGGER.warning(proc_message) return False - proc = subprocess.run( # nosec # pylint: disable=subprocess-run-check - ["git", "commit", f"--message={message}"], capture_output=True, encoding="utf-8", timeout=300 + _, success, _ = await run_timeout( + ["git", "commit", f"--message={message}"], + None, + 600, + "Commit", + "Error committing files", + "Timeout committing files", ) - if proc.returncode != 0: - proc_message = ansi_proc_message(proc) - proc_message.title = "Error committing files" - _LOGGER.warning(proc_message) - return False - - return True + return success def create_pull_request( @@ -540,7 +618,7 @@ def create_pull_request( return True, None -def create_commit_pull_request( +async def create_commit_pull_request( branch: str, new_branch: str, message: str, body: str, repo: github.Repository.Repository ) -> tuple[bool, github.PullRequest.PullRequest | None]: """Do a commit, then create a pull request.""" @@ -557,7 +635,7 @@ def create_commit_pull_request( _LOGGER.debug(proc_message) except FileNotFoundError: _LOGGER.debug("pre-commit not installed") - if not create_commit(message): + if not await create_commit(message): return False, None return create_pull_request(branch, new_branch, message, body, repo)