Skip to content

Commit

Permalink
Merge pull request #394 from camptocamp/fix
Browse files Browse the repository at this point in the history 
Some fix, see commits
  • Loading branch information
@sbrunner
sbrunner authored Jun 24, 2024
2 parents a6d6301 + c025123 commit 0136787
Show file tree
Hide file tree
Showing 4 changed files with 69 additions and 64 deletions.
14 changes: 12 additions & 2 deletions github_app_geo_project/module/audit/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ async def snyk(
logs_url,
result,
env_no_debug,
env,
fixable_vulnerabilities_summary,
)
npm_audit_fix_message, npm_audit_fix_success = await _npm_audit_fix(fixable_files_npm, result)
Expand Down Expand Up @@ -393,13 +394,13 @@ async def _snyk_fix(
logs_url: str,
result: list[module_utils.Message],
env_no_debug: dict[str, str],
env_debug: dict[str, str],
fixable_vulnerabilities_summary: dict[str, str],
) -> tuple[bool, module_utils.HtmlMessage | None]:
snyk_fix_success = True
snyk_fix_message = None
subprocess.run(["git", "reset", "--hard"], timeout=30) # nosec # pylint: disable=subprocess-run-check
if fixable_vulnerabilities_summary:
subprocess.run(["git", "reset", "--hard"], timeout=30) # nosec # pylint: disable=subprocess-run-check

command = [
"snyk",
"fix",
Expand All @@ -420,6 +421,15 @@ async def _snyk_fix(
if fix_message:
snyk_fix_message = module_utils.AnsiMessage(fix_message.strip())
if not snyk_fix_success:
await module_utils.run_timeout(
command,
env_debug,
int(os.environ.get("GHCI_SNYK_FIX_TIMEOUT", os.environ.get("GHCI_SNYK_TIMEOUT", "300"))),
"Snyk fix (debug)",
"Error while fixing the project (debug)",
"Timeout while fixing the project (debug)",
)

cwd = module_utils.get_cwd()
project = "-" if cwd is None else os.path.basename(cwd)
message = module_utils.HtmlMessage(
Expand Down
3 changes: 2 additions & 1 deletion github_app_geo_project/module/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -552,7 +552,8 @@ async def run_timeout(
_LOGGER.warning(message)
return None, False, message
else:
raise
_LOGGER.exception("TimeoutError: %s", exception)
return None, False, AnsiProcessMessage(command, None, "", "", str(exception))


def has_changes(include_un_followed: bool = False) -> bool:
Expand Down
109 changes: 55 additions & 54 deletions github_app_geo_project/module/versions/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,64 +231,65 @@ async def process(
assert context.module_event_data.version is not None
version = context.module_event_data.version
branch = context.module_config.get("version-mapping", {}).get(version, version)
with tempfile.TemporaryDirectory() as tmpdirname:
if os.environ.get("TEST") != "TRUE":
os.chdir(tmpdirname)
success = module_utils.git_clone(context.github_project, branch)
if not success:
raise VersionException("Failed to clone the repository")

version_status = status.versions[version]
version_status.names_by_datasource.clear()
version_status.dependencies_by_datasource.clear()
transversal_status = context.transversal_status

message = module_utils.HtmlMessage(
utils.format_json(json.loads(version_status.model_dump_json())["names_by_datasource"])
)
message.title = "Names cleaned:"
async with module_utils.WORKING_DIRECTORY_LOCK:
with tempfile.TemporaryDirectory() as tmpdirname:
if os.environ.get("TEST") != "TRUE":
os.chdir(tmpdirname)
success = module_utils.git_clone(context.github_project, branch)
if not success:
raise VersionException("Failed to clone the repository")

version_status = status.versions[version]
version_status.names_by_datasource.clear()
version_status.dependencies_by_datasource.clear()
transversal_status = context.transversal_status

message = module_utils.HtmlMessage(
utils.format_json(json.loads(version_status.model_dump_json())["names_by_datasource"])
)
message.title = "Names cleaned:"

_get_names(
context,
version_status.names_by_datasource,
version,
alternate_versions=context.module_event_data.alternate_versions,
)
message = module_utils.HtmlMessage(
utils.format_json(json.loads(version_status.model_dump_json())["names_by_datasource"])
)
message.title = "Names:"
_LOGGER.debug(message)
_get_dependencies(context, version_status.dependencies_by_datasource)
message = module_utils.HtmlMessage(
utils.format_json(
json.loads(version_status.model_dump_json())["dependencies_by_datasource"]
_get_names(
context,
version_status.names_by_datasource,
version,
alternate_versions=context.module_event_data.alternate_versions,
)
)
message.title = "Dependencies:"
_LOGGER.debug(message)

message = module_utils.HtmlMessage(
utils.format_json_str(
transversal_status.repositories[
f"{context.github_project.owner}/{context.github_project.repository}"
]
.versions[version]
.model_dump_json(indent=2)
message = module_utils.HtmlMessage(
utils.format_json(json.loads(version_status.model_dump_json())["names_by_datasource"])
)
)
message.title = f"Version ({version}):"
_LOGGER.debug(message)

message = module_utils.HtmlMessage(
utils.format_json_str(
transversal_status.repositories[
f"{context.github_project.owner}/{context.github_project.repository}"
].model_dump_json(indent=2)
message.title = "Names:"
_LOGGER.debug(message)
_get_dependencies(context, version_status.dependencies_by_datasource)
message = module_utils.HtmlMessage(
utils.format_json(
json.loads(version_status.model_dump_json())["dependencies_by_datasource"]
)
)
)
message.title = "Repo:"
_LOGGER.debug(message)
message.title = "Dependencies:"
_LOGGER.debug(message)

message = module_utils.HtmlMessage(
utils.format_json_str(
transversal_status.repositories[
f"{context.github_project.owner}/{context.github_project.repository}"
]
.versions[version]
.model_dump_json(indent=2)
)
)
message.title = f"Version ({version}):"
_LOGGER.debug(message)

message = module_utils.HtmlMessage(
utils.format_json_str(
transversal_status.repositories[
f"{context.github_project.owner}/{context.github_project.repository}"
].model_dump_json(indent=2)
)
)
message.title = "Repo:"
_LOGGER.debug(message)

return ProcessOutput(transversal_status=context.transversal_status)
raise VersionException("Invalid step")
Expand Down
7 changes: 0 additions & 7 deletions github_app_geo_project/scripts/process_queue.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
_LOGGER_WSGI = logging.getLogger("prometheus_client.wsgi")

_NB_JOBS = Gauge("ghci_jobs_number", "Number of jobs", ["status"])
_JOBS = Info("ghci_jobs", "Running jobs")


class _JobInfo(NamedTuple):
Expand Down Expand Up @@ -810,25 +809,19 @@ def _watch(self) -> None:
_NB_JOBS.labels(status.name).set(
session.query(models.Queue).filter(models.Queue.status == status).count()
)
info = {}
text = []
for id_, job in _RUNNING_JOBS.items():
text.append(
f"{id_}: {job.module} {job.event_name} {job.repository} [{job.priority}] (Worker max priority {job.worker_max_priority})"
)
info[f"job-id-{id_}"] = (
f"{job.module} {job.event_name} {job.repository} [{job.priority}] (Worker max priority {job.worker_max_priority})"
)
try:
for task in asyncio.all_tasks():
txt = io.StringIO()
task.print_stack(file=txt)
text.append("-" * 30)
text.append(txt.getvalue())
info[f"task-{id(task)}"] = txt.getvalue()
except RuntimeError as exception:
text.append(str(exception))
_JOBS.info(info)

if time.time() - self.last_run > 300:
error_message = ["Old Status"]
Expand Down

0 comments on commit 0136787

Please sign in to comment.