Skip to content

Commit

Permalink
Audit: Fix commit
Browse files Browse the repository at this point in the history
  • Loading branch information
@sbrunner
sbrunner committed Jun 16, 2024
1 parent ddbe281 commit 214e153
Show file tree
Hide file tree
Showing 4 changed files with 115 additions and 94 deletions.
2 changes: 1 addition & 1 deletion github_app_geo_project/module/audit/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ async def _process_snyk_dpkg(

else:
repo = context.github_project.repo
new_success, pull_request = module_utils.create_commit_pull_request(
new_success, pull_request = await module_utils.create_commit_pull_request(
branch, new_branch, f"Audit {key}", "" if body is None else body.to_markdown(), repo
)
success &= new_success
Expand Down
99 changes: 21 additions & 78 deletions github_app_geo_project/module/audit/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,70 +21,6 @@
_LOGGER = logging.getLogger(__name__)


async def _run_timeout(
command: list[str],
env: dict[str, str] | None,
timeout: int,
success_message: str,
error_message: str,
timeout_message: str,
error_messages: list[module_utils.Message],
cwd: str | None = None,
) -> tuple[str | None, bool]:
async_proc = None
try:
async with asyncio.timeout(timeout):
async_proc = await asyncio.create_subprocess_exec(
*command,
cwd=cwd or os.getcwd(),
env=env,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await async_proc.communicate()
assert async_proc.returncode is not None
message: module_utils.Message = module_utils.AnsiProcessMessage(
command, async_proc.returncode, stdout.decode(), stderr.decode()
)
success = async_proc.returncode == 0
if success:
message.title = success_message
_LOGGER.debug(message)
else:
message.title = error_message
_LOGGER.warning(message)
error_messages.append(message)
return stdout.decode(), success
except FileNotFoundError as exception:
_LOGGER.exception("%s not found: %s", command[0], exception)
proc = subprocess.run( # nosec # pylint: disable=subprocess-run-check
["find", "/", "-name", command[0]],
capture_output=True,
encoding="utf-8",
timeout=30,
)
message = module_utils.ansi_proc_message(proc)
message.title = f"Find {command[0]}"
_LOGGER.debug(message)
return None, False
except asyncio.TimeoutError as exception:
if async_proc:
async_proc.kill()
message = module_utils.AnsiProcessMessage(
command,
None,
"" if async_proc.stdout is None else (await async_proc.stdout.read()).decode(),
"" if async_proc.stderr is None else (await async_proc.stderr.read()).decode(),
error=str(exception),
)
message.title = timeout_message
_LOGGER.warning(message)
error_messages.append(message)
return None, False
else:
raise


async def snyk(
branch: str,
config: configuration.SnykConfiguration,
Expand Down Expand Up @@ -181,7 +117,7 @@ async def _install_requirements_dependencies(
if file in local_config.get("files-no-install", config.get("files-no-install", [])):
continue

await _run_timeout(
_, _, proc_message = await module_utils.run_timeout(
[
"python",
"-m",
Expand All @@ -195,8 +131,9 @@ async def _install_requirements_dependencies(
f"Dependencies installed from {file}",
f"Error while installing the dependencies from {file}",
f"Timeout while installing the dependencies from {file}",
result,
)
if proc_message is not None:
result.append(proc_message)


async def _install_pipenv_dependencies(
Expand All @@ -221,7 +158,7 @@ async def _install_pipenv_dependencies(
continue
directory = os.path.dirname(os.path.abspath(file))

await _run_timeout(
_, _, proc_message = await module_utils.run_timeout(
[
"pipenv",
"install",
Expand All @@ -232,9 +169,10 @@ async def _install_pipenv_dependencies(
f"Dependencies installed from {file}",
f"Error while installing the dependencies from {file}",
f"Timeout while installing the dependencies from {file}",
result,
directory,
)
if proc_message is not None:
result.append(proc_message)


async def _install_poetry_dependencies(
Expand All @@ -261,7 +199,7 @@ async def _install_poetry_dependencies(
if file in local_config.get("files-no-install", config.get("files-no-install", [])):
continue

await _run_timeout(
_, _, proc_message = await module_utils.run_timeout(
[
"poetry",
"install",
Expand All @@ -272,9 +210,10 @@ async def _install_poetry_dependencies(
f"Dependencies installed from {file}",
f"Error while installing the dependencies from {file}",
f"Timeout while installing the dependencies from {file}",
result,
os.path.dirname(os.path.abspath(file)),
)
if proc_message is not None:
result.append(proc_message)


async def _snyk_monitor(
Expand Down Expand Up @@ -314,15 +253,16 @@ async def _snyk_monitor(
f"--project-tags={','.join(['='.join(tag) for tag in local_monitor_config.get('project-tags', monitor_config.get('project-tags', {}))])}"
)

await _run_timeout(
_, _, message = await module_utils.run_timeout(
command,
env,
int(os.environ.get("GHCI_SNYK_TIMEOUT", "300")),
"Project monitored",
"Error while monitoring the project",
"Timeout while monitoring the project",
result,
)
if message is not None:
result.append(message)


async def _snyk_test(
Expand All @@ -340,15 +280,16 @@ async def _snyk_test(
"test-arguments", config.get("test-arguments", configuration.SNYK_TEST_ARGUMENTS_DEFAULT)
),
]
test_json_str, _ = await _run_timeout(
test_json_str, _, message = await module_utils.run_timeout(
command,
env_no_debug,
int(os.environ.get("GHCI_SNYK_TIMEOUT", "300")),
"Snyk test",
"Error while testing the project",
"Timeout while testing the project",
result,
)
if message is not None:
result.append(message)

if test_json_str:
message = module_utils.HtmlMessage(utils.format_json_str(test_json_str))
Expand Down Expand Up @@ -452,15 +393,16 @@ async def _snyk_fix(
"fix-arguments", config.get("fix-arguments", configuration.SNYK_FIX_ARGUMENTS_DEFAULT)
),
]
fix_message, snyk_fix_success = await _run_timeout(
fix_message, snyk_fix_success, message = await module_utils.run_timeout(
command,
env_no_debug,
int(os.environ.get("GHCI_SNYK_TIMEOUT", "300")),
"Snyk fix",
"Error while fixing the project",
"Timeout while fixing the project",
result,
)
if message is not None:
result.append(message)
if fix_message:
snyk_fix_message = module_utils.AnsiMessage(fix_message.strip())
if not snyk_fix_success:
Expand Down Expand Up @@ -488,16 +430,17 @@ async def _npm_audit_fix(
messages.update(file_messages)
_LOGGER.debug("Fixing vulnerabilities in %s with npm audit fix --force", package_lock_file_name)
command = ["npm", "audit", "fix", "--force"]
_, success = await _run_timeout(
_, success, message = await module_utils.run_timeout(
command,
os.environ.copy(),
int(os.environ.get("GHCI_SNYK_TIMEOUT", "300")),
"Npm audit fix",
"Error while fixing the project",
"Timeout while fixing the project",
result,
directory,
)
if message is not None:
result.append(message)
_LOGGER.debug("Fixing version in %s", package_lock_file_name)
# Remove the add '~' in the version in the package.json
with open(os.path.join(directory, "package.json"), encoding="utf-8") as package_file:
Expand Down
2 changes: 1 addition & 1 deletion github_app_geo_project/module/standard/patch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ async def process(
_LOGGER.info(message)

if utils.has_changes(include_un_followed=True):
success = utils.create_commit(
success = await utils.create_commit(
f"{artifact.name[:-6]}\n\nFrom the artifact of the previous workflow run"
)
if not success:
Expand Down
106 changes: 92 additions & 14 deletions github_app_geo_project/module/utils.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Module utility functions for the modules."""

import asyncio
import datetime
import logging
import os
Expand Down Expand Up @@ -376,8 +377,8 @@ def __init__(
self.args.append(arg)

self.returncode = returncode
self.stdout = self._ansi_converter.convert(stdout, full=False)
self.stderr = self._ansi_converter.convert(stderr, full=False)
self.stdout = self._ansi_converter.convert(stdout or "", full=False)
self.stderr = self._ansi_converter.convert(stderr or "", full=False)

message = [f"Command: {shlex.join(self.args)}"]
if error:
Expand Down Expand Up @@ -458,6 +459,84 @@ def ansi_proc_message(
return AnsiProcessMessage.from_process(proc)


async def run_timeout(
command: list[str],
env: dict[str, str] | None,
timeout: int,
success_message: str,
error_message: str,
timeout_message: str,
cwd: str | None = None,
) -> tuple[str | None, bool, Message | None]:
"""
Run a command with a timeout.
Arguments:
---------
command: The command to run
env: The environment variables
timeout: The timeout
success_message: The message on success
error_message: The message on error
timeout_message: The message on timeout
cwd: The working directory
Return:
------
The standard output, the success, the logged message
"""
async_proc = None
try:
async with asyncio.timeout(timeout):
async_proc = await asyncio.create_subprocess_exec(
*command,
cwd=cwd or os.getcwd(),
env=env,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await async_proc.communicate()
assert async_proc.returncode is not None
message: Message = AnsiProcessMessage(
command, async_proc.returncode, stdout.decode(), stderr.decode()
)
success = async_proc.returncode == 0
if success:
message.title = success_message
_LOGGER.debug(message)
else:
message.title = error_message
_LOGGER.warning(message)
return stdout.decode(), success, message
except FileNotFoundError as exception:
_LOGGER.exception("%s not found: %s", command[0], exception)
proc = subprocess.run( # nosec # pylint: disable=subprocess-run-check
["find", "/", "-name", command[0]],
capture_output=True,
encoding="utf-8",
timeout=30,
)
message = ansi_proc_message(proc)
message.title = f"Find {command[0]}"
_LOGGER.debug(message)
return None, False, message
except asyncio.TimeoutError as exception:
if async_proc:
async_proc.kill()
message = AnsiProcessMessage(
command,
None,
"" if async_proc.stdout is None else (await async_proc.stdout.read()).decode(),
"" if async_proc.stderr is None else (await async_proc.stderr.read()).decode(),
error=str(exception),
)
message.title = timeout_message
_LOGGER.warning(message)
return None, False, message
else:
raise


def has_changes(include_un_followed: bool = False) -> bool:
"""Check if there are changes."""
if include_un_followed:
Expand All @@ -471,7 +550,7 @@ def has_changes(include_un_followed: bool = False) -> bool:
return proc.returncode != 0


def create_commit(message: str) -> bool:
async def create_commit(message: str) -> bool:
"""Do a commit."""
proc = subprocess.run( # nosec # pylint: disable=subprocess-run-check
["git", "add", "--all"], capture_output=True, encoding="utf-8", timeout=30
Expand All @@ -481,16 +560,15 @@ def create_commit(message: str) -> bool:
proc_message.title = "Error adding files to commit"
_LOGGER.warning(proc_message)
return False
proc = subprocess.run( # nosec # pylint: disable=subprocess-run-check
["git", "commit", f"--message={message}"], capture_output=True, encoding="utf-8", timeout=300
_, success, _ = await run_timeout(
["git", "commit", f"--message={message}"],
None,
600,
"Commit",
"Error committing files",
"Timeout committing files",
)
if proc.returncode != 0:
proc_message = ansi_proc_message(proc)
proc_message.title = "Error committing files"
_LOGGER.warning(proc_message)
return False

return True
return success


def create_pull_request(
Expand Down Expand Up @@ -540,7 +618,7 @@ def create_pull_request(
return True, None


def create_commit_pull_request(
async def create_commit_pull_request(
branch: str, new_branch: str, message: str, body: str, repo: github.Repository.Repository
) -> tuple[bool, github.PullRequest.PullRequest | None]:
"""Do a commit, then create a pull request."""
Expand All @@ -557,7 +635,7 @@ def create_commit_pull_request(
_LOGGER.debug(proc_message)
except FileNotFoundError:
_LOGGER.debug("pre-commit not installed")
if not create_commit(message):
if not await create_commit(message):
return False, None
return create_pull_request(branch, new_branch, message, body, repo)

Expand Down

0 comments on commit 214e153

Please sign in to comment.