diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 00000000000..822658bf133 --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,3 @@ +ignored: + - DL3008 # Pin versions in apt get install. Instead of `apt-get install ` use `apt-get install =` + - DL3042 # Avoid cache directory with pip install --no-cache-dir diff --git a/Dockerfile b/Dockerfile index cad98611631..341113b143f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,9 @@ LABEL maintainer Camptocamp "info@camptocamp.com" # Print commands and their arguments as they are executed. SHELL ["/bin/bash", "-o", "pipefail", "-cux"] -RUN apt-get update \ +RUN --mount=type=cache,target=/var/lib/apt/lists \ + --mount=type=cache,target=/var/cache,sharing=locked \ + apt-get update \ && apt-get upgrade --assume-yes \ && apt-get install --assume-yes --no-install-recommends python3-pip postgresql-client docker.io libmagic1 git curl gnupg zlib1g libpq5 @@ -17,6 +19,11 @@ RUN rm -rf /usr/lib/python3.*/EXTERNALLY-MANAGED # We don't directly use `poetry install` because it force to use a virtual environment. FROM base-all as poetry +# Fail on error on pipe, see: https://github.com/hadolint/hadolint/wiki/DL4006. +# Treat unset variables as an error when substituting. +# Print commands and their arguments as they are executed. +SHELL ["/bin/bash", "-o", "pipefail", "-cux"] + # Install Poetry WORKDIR /tmp COPY requirements.txt ./ @@ -32,6 +39,11 @@ RUN poetry export --output=requirements.txt \ # Base, the biggest thing is to install the Python packages FROM base-all as base +# Fail on error on pipe, see: https://github.com/hadolint/hadolint/wiki/DL4006. +# Treat unset variables as an error when substituting. +# Print commands and their arguments as they are executed. +SHELL ["/bin/bash", "-o", "pipefail", "-cux"] + COPY .nvmrc /tmp RUN --mount=type=cache,target=/var/lib/apt/lists \ --mount=type=cache,target=/var/cache,sharing=locked \ @@ -41,17 +53,17 @@ RUN --mount=type=cache,target=/var/lib/apt/lists \ && apt-get update \ && apt-get install --assume-yes --no-install-recommends "nodejs=${NODE_MAJOR}.*" -# hadolint ignore=SC2086,DL3042,DL3008 +# Install some required dev packages RUN --mount=type=cache,target=/var/lib/apt/lists \ --mount=type=cache,target=/var/cache,sharing=locked \ + apt-get update \ + && apt-get install --assume-yes --no-install-recommends build-essential python3-dev libpq-dev libproj-dev + +RUN --mount=type=cache,target=/var/cache,sharing=locked \ --mount=type=cache,target=/root/.cache \ --mount=type=bind,from=poetry,source=/tmp,target=/poetry \ - DEV_PACKAGES="python3-dev libpq-dev build-essential" \ - && apt-get update \ - && apt-get install --assume-yes --no-install-recommends ${DEV_PACKAGES} \ - && python3 -m pip install --disable-pip-version-check --no-deps --requirement=/poetry/requirements.txt \ - && python3 -m compileall /usr/local/lib/python* /usr/lib/python* \ - && apt-get remove --purge --autoremove --yes ${DEV_PACKAGES} binutils + python3 -m pip install --disable-pip-version-check --no-deps --requirement=/poetry/requirements.txt \ + && python3 -m compileall /usr/local/lib/python* /usr/lib/python* # From c2cwsgiutils @@ -90,9 +102,10 @@ ENV PATH=/pyenv/shims:/pyenv/bin:${PATH} \ PYENV_ROOT=/pyenv # Install different Python version with pyenv +# hadolint ignore=SC2086 RUN --mount=type=cache,target=/var/lib/apt/lists \ --mount=type=cache,target=/var/cache,sharing=locked \ - DEV_PACKAGES="build-essential libffi-dev libssl-dev liblzma-dev libsqlite3-dev libcurses-ocaml-dev libreadline-dev libbz2-dev zlib1g-dev" \ + DEV_PACKAGES="libffi-dev libssl-dev liblzma-dev libsqlite3-dev libcurses-ocaml-dev libreadline-dev libbz2-dev zlib1g-dev" \ && apt-get update && apt-get install --assume-yes --no-install-recommends ${DEV_PACKAGES} \ && git clone --depth=1 https://github.com/pyenv/pyenv.git /pyenv \ && pyenv install 3.7 3.8 3.9 3.10 3.11 \ diff --git a/ci/dpkg-versions.yaml b/ci/dpkg-versions.yaml index 548dbf57761..7568c7948d2 100644 --- a/ci/dpkg-versions.yaml +++ b/ci/dpkg-versions.yaml @@ -3,10 +3,17 @@ camptocamp/github-app-geo-project:latest: ubuntu_23_10/apt: 2.7.3ubuntu0.1 ubuntu_23_10/base-passwd: 3.6.1 ubuntu_23_10/bash: 5.2.15-2ubuntu1 + ubuntu_23_10/binutils: 2.41-5ubuntu1 + ubuntu_23_10/binutils-common: 2.41-5ubuntu1 + ubuntu_23_10/binutils-x86-64-linux-gnu: 2.41-5ubuntu1 ubuntu_23_10/bsdutils: 1:2.39.1-4ubuntu2.2 + ubuntu_23_10/build-essential: 12.10ubuntu1 + ubuntu_23_10/bzip2: 1.0.8-5build1 ubuntu_23_10/ca-certificates: 20230311ubuntu1 ubuntu_23_10/containerd: 1.7.2-0ubuntu2 ubuntu_23_10/coreutils: 9.1-1ubuntu2.23.10.1 + ubuntu_23_10/cpp: 4:13.2.0-1ubuntu1 + ubuntu_23_10/cpp-13: 13.2.0-4ubuntu3 ubuntu_23_10/curl: 8.2.1-1ubuntu3.3 ubuntu_23_10/dash: 0.5.12-6ubuntu1 ubuntu_23_10/debconf: 1.5.82 @@ -15,11 +22,16 @@ camptocamp/github-app-geo-project:latest: ubuntu_23_10/dirmngr: 2.2.40-1.1ubuntu1 ubuntu_23_10/docker.io: 24.0.5-0ubuntu1 ubuntu_23_10/dpkg: 1.22.0ubuntu1.1 + ubuntu_23_10/dpkg-dev: 1.22.0ubuntu1.1 ubuntu_23_10/e2fsprogs: 1.47.0-2ubuntu1 ubuntu_23_10/findutils: 4.9.0-5 + ubuntu_23_10/g++: 4:13.2.0-1ubuntu1 + ubuntu_23_10/g++-13: 13.2.0-4ubuntu3 + ubuntu_23_10/gcc: 4:13.2.0-1ubuntu1 + ubuntu_23_10/gcc-13: 13.2.0-4ubuntu3 ubuntu_23_10/gcc-13-base: 13.2.0-4ubuntu3 - ubuntu_23_10/git: 1:2.40.1-1ubuntu1 - ubuntu_23_10/git-man: 1:2.40.1-1ubuntu1 + ubuntu_23_10/git: 1:2.40.1-1ubuntu1.1 + ubuntu_23_10/git-man: 1:2.40.1-1ubuntu1.1 ubuntu_23_10/gnupg: 2.2.40-1.1ubuntu1 ubuntu_23_10/gnupg-l10n: 2.2.40-1.1ubuntu1 ubuntu_23_10/gnupg-utils: 2.2.40-1.1ubuntu1 @@ -37,46 +49,64 @@ camptocamp/github-app-geo-project:latest: ubuntu_23_10/iptables: 1.8.9-2ubuntu2 ubuntu_23_10/libacl1: 2.3.1-3 ubuntu_23_10/libapt-pkg6.0: 2.7.3ubuntu0.1 + ubuntu_23_10/libasan8: 13.2.0-4ubuntu3 ubuntu_23_10/libassuan0: 2.5.6-1 + ubuntu_23_10/libatomic1: 13.2.0-4ubuntu3 ubuntu_23_10/libattr1: 1:2.5.1-4 ubuntu_23_10/libaudit-common: 1:3.1.1-1 ubuntu_23_10/libaudit1: 1:3.1.1-1 + ubuntu_23_10/libbinutils: 2.41-5ubuntu1 ubuntu_23_10/libblkid1: 2.39.1-4ubuntu2.2 ubuntu_23_10/libbrotli1: 1.0.9-2build8 ubuntu_23_10/libbz2-1.0: 1.0.8-5build1 - ubuntu_23_10/libc-bin: 2.38-1ubuntu6.2 - ubuntu_23_10/libc6: 2.38-1ubuntu6.2 + ubuntu_23_10/libc-bin: 2.38-1ubuntu6.3 + ubuntu_23_10/libc-dev-bin: 2.38-1ubuntu6.3 + ubuntu_23_10/libc6: 2.38-1ubuntu6.3 + ubuntu_23_10/libc6-dev: 2.38-1ubuntu6.3 ubuntu_23_10/libcap-ng0: 0.8.3-1build2 ubuntu_23_10/libcap2: 1:2.66-4ubuntu1 + ubuntu_23_10/libcc1-0: 13.2.0-4ubuntu3 ubuntu_23_10/libcom-err2: 1.47.0-2ubuntu1 + ubuntu_23_10/libcrypt-dev: 1:4.4.36-2 ubuntu_23_10/libcrypt1: 1:4.4.36-2 + ubuntu_23_10/libctf-nobfd0: 2.41-5ubuntu1 + ubuntu_23_10/libctf0: 2.41-5ubuntu1 ubuntu_23_10/libcurl3-gnutls: 8.2.1-1ubuntu3.3 ubuntu_23_10/libcurl4: 8.2.1-1ubuntu3.3 ubuntu_23_10/libdb5.3: 5.3.28+dfsg2-2 ubuntu_23_10/libdebconfclient0: 0.270ubuntu1 ubuntu_23_10/libdevmapper1.02.1: 2:1.02.185-2ubuntu1 + ubuntu_23_10/libdpkg-perl: 1.22.0ubuntu1.1 ubuntu_23_10/liberror-perl: 0.17029-2 ubuntu_23_10/libexpat1: 2.5.0-2ubuntu0.1 ubuntu_23_10/libext2fs2: 1.47.0-2ubuntu1 ubuntu_23_10/libffi8: 3.4.4-1 + ubuntu_23_10/libgcc-13-dev: 13.2.0-4ubuntu3 ubuntu_23_10/libgcc-s1: 13.2.0-4ubuntu3 ubuntu_23_10/libgcrypt20: 1.10.2-3ubuntu1 ubuntu_23_10/libgdbm-compat4: 1.23-3 ubuntu_23_10/libgdbm6: 1.23-3 ubuntu_23_10/libgmp10: 2:6.3.0+dfsg-2ubuntu4 ubuntu_23_10/libgnutls30: 3.8.1-4ubuntu1.3 + ubuntu_23_10/libgomp1: 13.2.0-4ubuntu3 ubuntu_23_10/libgpg-error0: 1.47-2 + ubuntu_23_10/libgprofng0: 2.41-5ubuntu1 ubuntu_23_10/libgssapi-krb5-2: 1.20.1-3ubuntu1 ubuntu_23_10/libhogweed6: 3.9.1-2 + ubuntu_23_10/libhwasan0: 13.2.0-4ubuntu3 ubuntu_23_10/libidn2-0: 2.3.4-1 ubuntu_23_10/libip4tc2: 1.8.9-2ubuntu2 ubuntu_23_10/libip6tc2: 1.8.9-2ubuntu2 + ubuntu_23_10/libisl23: 0.26-3 + ubuntu_23_10/libitm1: 13.2.0-4ubuntu3 + ubuntu_23_10/libjansson4: 2.14-2 ubuntu_23_10/libk5crypto3: 1.20.1-3ubuntu1 ubuntu_23_10/libkeyutils1: 1.6.3-2 ubuntu_23_10/libkrb5-3: 1.20.1-3ubuntu1 ubuntu_23_10/libkrb5support0: 1.20.1-3ubuntu1 ubuntu_23_10/libksba8: 1.6.4-2 ubuntu_23_10/libldap2: 2.6.6+dfsg-1~exp1ubuntu1 + ubuntu_23_10/liblsan0: 13.2.0-4ubuntu3 ubuntu_23_10/liblz4-1: 1.9.4-1 ubuntu_23_10/liblzma5: 5.4.1-0.2 ubuntu_23_10/libmagic-mgc: 1:5.44-3 @@ -84,6 +114,8 @@ camptocamp/github-app-geo-project:latest: ubuntu_23_10/libmd0: 1.1.0-1 ubuntu_23_10/libmnl0: 1.0.4-3ubuntu1 ubuntu_23_10/libmount1: 2.39.1-4ubuntu2.2 + ubuntu_23_10/libmpc3: 1.3.1-1 + ubuntu_23_10/libmpfr6: 4.2.1-1 ubuntu_23_10/libncursesw6: 6.4+20230625-2 ubuntu_23_10/libnetfilter-conntrack3: 1.0.9-5 ubuntu_23_10/libnettle8: 3.9.1-2 @@ -91,6 +123,7 @@ camptocamp/github-app-geo-project:latest: ubuntu_23_10/libnftnl11: 1.2.6-2 ubuntu_23_10/libnghttp2-14: 1.55.1-1ubuntu0.2 ubuntu_23_10/libnpth0: 1.6-3build2 + ubuntu_23_10/libnsl-dev: 1.3.0-2build2 ubuntu_23_10/libnsl2: 1.3.0-2build2 ubuntu_23_10/libp11-kit0: 0.25.0-4ubuntu1 ubuntu_23_10/libpam-modules: 1.5.2-6ubuntu1.1 @@ -99,12 +132,13 @@ camptocamp/github-app-geo-project:latest: ubuntu_23_10/libpam0g: 1.5.2-6ubuntu1.1 ubuntu_23_10/libpcre2-8-0: 10.42-4 ubuntu_23_10/libperl5.36: 5.36.0-9ubuntu1.1 - ubuntu_23_10/libpq5: 15.6-0ubuntu0.23.10.1 + ubuntu_23_10/libpq5: 15.7-0ubuntu0.23.10.1 ubuntu_23_10/libproc2-0: 2:4.0.3-1ubuntu1.23.10.1 ubuntu_23_10/libpsl5: 0.21.2-1 ubuntu_23_10/libpython3-stdlib: 3.11.4-5 ubuntu_23_10/libpython3.11-minimal: 3.11.6-3 ubuntu_23_10/libpython3.11-stdlib: 3.11.6-3 + ubuntu_23_10/libquadmath0: 13.2.0-4ubuntu3 ubuntu_23_10/libreadline8: 8.2-1.3 ubuntu_23_10/librtmp1: 2.4+20151223.gitfa8646d.1-2build4 ubuntu_23_10/libsasl2-2: 2.1.28+dfsg1-3 @@ -114,32 +148,40 @@ camptocamp/github-app-geo-project:latest: ubuntu_23_10/libsemanage-common: 3.5-1 ubuntu_23_10/libsemanage2: 3.5-1 ubuntu_23_10/libsepol2: 3.5-1 + ubuntu_23_10/libsframe1: 2.41-5ubuntu1 ubuntu_23_10/libsmartcols1: 2.39.1-4ubuntu2.2 ubuntu_23_10/libsqlite3-0: 3.42.0-1ubuntu0.1 ubuntu_23_10/libss2: 1.47.0-2ubuntu1 ubuntu_23_10/libssh-4: 0.10.5-3ubuntu1.2 ubuntu_23_10/libssl3: 3.0.10-1ubuntu2.3 + ubuntu_23_10/libstdc++-13-dev: 13.2.0-4ubuntu3 ubuntu_23_10/libstdc++6: 13.2.0-4ubuntu3 ubuntu_23_10/libsystemd0: 253.5-1ubuntu6.1 ubuntu_23_10/libtasn1-6: 4.19.0-3 ubuntu_23_10/libtinfo6: 6.4+20230625-2 ubuntu_23_10/libtirpc-common: 1.3.3+ds-1 + ubuntu_23_10/libtirpc-dev: 1.3.3+ds-1 ubuntu_23_10/libtirpc3: 1.3.3+ds-1 + ubuntu_23_10/libtsan2: 13.2.0-4ubuntu3 + ubuntu_23_10/libubsan1: 13.2.0-4ubuntu3 ubuntu_23_10/libudev1: 253.5-1ubuntu6.1 ubuntu_23_10/libunistring2: 1.0-2 ubuntu_23_10/libuuid1: 2.39.1-4ubuntu2.2 ubuntu_23_10/libxtables12: 1.8.9-2ubuntu2 ubuntu_23_10/libxxhash0: 0.8.1-1 ubuntu_23_10/libzstd1: 1.5.5+dfsg2-1ubuntu2 + ubuntu_23_10/linux-libc-dev: 6.5.0-35.35 ubuntu_23_10/login: 1:4.13+dfsg1-1ubuntu1.1 ubuntu_23_10/logsave: 1.47.0-2ubuntu1 + ubuntu_23_10/lto-disabled-list: '43' + ubuntu_23_10/make: 4.3-4.1build1 ubuntu_23_10/mawk: 1.3.4.20230730-1 ubuntu_23_10/media-types: 10.1.0 ubuntu_23_10/mount: 2.39.1-4ubuntu2.2 ubuntu_23_10/ncurses-base: 6.4+20230625-2 ubuntu_23_10/ncurses-bin: 6.4+20230625-2 ubuntu_23_10/netbase: '6.4' - ubuntu_23_10/nodejs: 18.13.0+dfsg1-1ubuntu2.2 + ubuntu_23_10/nodejs: 20.14.0-1nodesource1 ubuntu_23_10/openssl: 3.0.10-1ubuntu2.3 ubuntu_23_10/passwd: 1:4.13+dfsg1-1ubuntu1.1 ubuntu_23_10/patch: 2.7.6-7build2 @@ -148,7 +190,7 @@ camptocamp/github-app-geo-project:latest: ubuntu_23_10/perl-modules-5.36: 5.36.0-9ubuntu1.1 ubuntu_23_10/pinentry-curses: 1.2.1-1ubuntu1 ubuntu_23_10/postgresql-client: 15+253 - ubuntu_23_10/postgresql-client-15: 15.6-0ubuntu0.23.10.1 + ubuntu_23_10/postgresql-client-15: 15.7-0ubuntu0.23.10.1 ubuntu_23_10/postgresql-client-common: '253' ubuntu_23_10/procps: 2:4.0.3-1ubuntu1.23.10.1 ubuntu_23_10/python3: 3.11.4-5 @@ -162,6 +204,7 @@ camptocamp/github-app-geo-project:latest: ubuntu_23_10/python3.11: 3.11.6-3 ubuntu_23_10/python3.11-minimal: 3.11.6-3 ubuntu_23_10/readline-common: 8.2-1.3 + ubuntu_23_10/rpcsvc-proto: 1.4.2-0ubuntu6 ubuntu_23_10/runc: 1.1.7-0ubuntu2.2 ubuntu_23_10/sed: 4.9-1 ubuntu_23_10/sensible-utils: 0.0.20 diff --git a/github_app_geo_project/scripts/process_queue.py b/github_app_geo_project/scripts/process_queue.py index fc426bcc0d3..a69929c1811 100644 --- a/github_app_geo_project/scripts/process_queue.py +++ b/github_app_geo_project/scripts/process_queue.py @@ -12,7 +12,7 @@ import subprocess # nosec import sys import urllib.parse -from typing import Any, cast +from typing import Any, NamedTuple, cast import c2cwsgiutils.loader import c2cwsgiutils.setup_process @@ -33,6 +33,17 @@ _NB_JOBS = Gauge("ghci_jobs_number", "Number of jobs", ["status"]) +class _JobInfo(NamedTuple): + module: str + event_name: str + repository: str + priority: int + worker_max_priority: int + + +_RUNNING_JOBS: dict[int, _JobInfo] = {} + + class _Handler(logging.Handler): context_var: contextvars.ContextVar[int] = contextvars.ContextVar("job_id") @@ -656,6 +667,9 @@ async def _process_one_job( message.title = f"Start process job '{job.event_name}' id: {job.id}, on {job.owner}/{job.repository} on module: {job.module}, on application {job.application}" root_logger.addHandler(handler) _LOGGER.info(message) + _RUNNING_JOBS[job.id] = _JobInfo( + job.module or "-", job.event_name, job.repository, job.priority, max_priority + ) root_logger.removeHandler(handler) if make_pending: @@ -722,6 +736,7 @@ async def _process_one_job( job.status = models.JobStatus.ERROR job.finished_at = datetime.datetime.now(tz=datetime.timezone.utc) session.commit() + _RUNNING_JOBS.pop(job.id) _LOGGER.debug("Process one job (max priority: %i): Done", max_priority) return False @@ -784,6 +799,18 @@ async def __call__(self, *args: Any, **kwds: Any) -> Any: await asyncio.sleep(10) +async def _watch_dog() -> None: + while True: + _LOGGER.debug("Watch dog: alive") + with open("/watch_dog", "w", encoding="utf-8") as file_: + file_.write(datetime.datetime.now().isoformat()) + for id_, job in _RUNNING_JOBS.items(): + file_.write( + f"{id_}: {job.module} {job.event_name} {job.repository} [{job.priority}] (Worker max priority {job.worker_max_priority})\n" + ) + await asyncio.sleep(60) + + async def _async_main() -> None: """Process the jobs present in the database queue.""" parser = argparse.ArgumentParser(description=__doc__) @@ -816,6 +843,7 @@ async def _async_main() -> None: threads_call = [] if not args.exit_when_empty: threads_call.append(_UpdateCounter(Session)()) + threads_call.append(_watch_dog()) for priority in priority_groups: threads_call.append(_Run(config, Session, args.exit_when_empty, priority)()) @@ -825,8 +853,7 @@ async def _async_main() -> None: def main() -> None: """Process the jobs present in the database queue.""" socket.setdefaulttimeout(int(os.environ.get("GHCI_SOCKET_TIMEOUT", 120))) - with asyncio.Runner() as runner: - runner.run(_async_main()) + asyncio.run(_async_main()) if __name__ == "__main__":