Audit: Add dev packages, lint with hadolint

.hadolint.yaml

@@ -0,0 +1,3 @@
+ignored:
+  - DL3008 # Pin versions in apt get install. Instead of `apt-get install <package>` use `apt-get install <package>=<version>`
+  - DL3042 # Avoid cache directory with pip install --no-cache-dir

Dockerfile

@@ -7,7 +7,9 @@ LABEL maintainer Camptocamp "[email protected]"
 # Print commands and their arguments as they are executed.
 SHELL ["/bin/bash", "-o", "pipefail", "-cux"]
 
-RUN apt-get update \
+RUN --mount=type=cache,target=/var/lib/apt/lists \
+    --mount=type=cache,target=/var/cache,sharing=locked \
+    apt-get update \
     && apt-get upgrade --assume-yes \
     && apt-get install --assume-yes --no-install-recommends python3-pip postgresql-client docker.io libmagic1 git curl gnupg zlib1g libpq5
 
@@ -17,6 +19,11 @@ RUN rm -rf /usr/lib/python3.*/EXTERNALLY-MANAGED
 # We don't directly use `poetry install` because it force to use a virtual environment.
 FROM base-all as poetry
 
+# Fail on error on pipe, see: https://github.com/hadolint/hadolint/wiki/DL4006.
+# Treat unset variables as an error when substituting.
+# Print commands and their arguments as they are executed.
+SHELL ["/bin/bash", "-o", "pipefail", "-cux"]
+
 # Install Poetry
 WORKDIR /tmp
 COPY requirements.txt ./
@@ -32,6 +39,11 @@ RUN poetry export --output=requirements.txt \
 # Base, the biggest thing is to install the Python packages
 FROM base-all as base
 
+# Fail on error on pipe, see: https://github.com/hadolint/hadolint/wiki/DL4006.
+# Treat unset variables as an error when substituting.
+# Print commands and their arguments as they are executed.
+SHELL ["/bin/bash", "-o", "pipefail", "-cux"]
+
 COPY .nvmrc /tmp
 RUN --mount=type=cache,target=/var/lib/apt/lists \
     --mount=type=cache,target=/var/cache,sharing=locked \
@@ -41,17 +53,17 @@ RUN --mount=type=cache,target=/var/lib/apt/lists \
     && apt-get update \
     && apt-get install --assume-yes --no-install-recommends "nodejs=${NODE_MAJOR}.*"
 
-# hadolint ignore=SC2086,DL3042,DL3008
+# Install some required dev packages
 RUN --mount=type=cache,target=/var/lib/apt/lists \
     --mount=type=cache,target=/var/cache,sharing=locked \
+    apt-get update \
+    && apt-get install --assume-yes --no-install-recommends build-essential python3-dev libpq-dev libproj-dev
+
+RUN --mount=type=cache,target=/var/cache,sharing=locked \
     --mount=type=cache,target=/root/.cache \
     --mount=type=bind,from=poetry,source=/tmp,target=/poetry \
-    DEV_PACKAGES="python3-dev libpq-dev build-essential" \
-    && apt-get update \
-    && apt-get install --assume-yes --no-install-recommends ${DEV_PACKAGES} \
-    && python3 -m pip install --disable-pip-version-check --no-deps --requirement=/poetry/requirements.txt \
-    && python3 -m compileall /usr/local/lib/python* /usr/lib/python* \
-    && apt-get remove --purge --autoremove --yes ${DEV_PACKAGES} binutils
+    python3 -m pip install --disable-pip-version-check --no-deps --requirement=/poetry/requirements.txt \
+    && python3 -m compileall /usr/local/lib/python* /usr/lib/python*
 
 # From c2cwsgiutils
 
@@ -90,9 +102,10 @@ ENV PATH=/pyenv/shims:/pyenv/bin:${PATH} \
     PYENV_ROOT=/pyenv
 
 # Install different Python version with pyenv
+# hadolint ignore=SC2086
 RUN --mount=type=cache,target=/var/lib/apt/lists \
     --mount=type=cache,target=/var/cache,sharing=locked \
-    DEV_PACKAGES="build-essential libffi-dev libssl-dev liblzma-dev libsqlite3-dev libcurses-ocaml-dev libreadline-dev libbz2-dev zlib1g-dev" \
+    DEV_PACKAGES="libffi-dev libssl-dev liblzma-dev libsqlite3-dev libcurses-ocaml-dev libreadline-dev libbz2-dev zlib1g-dev" \
     && apt-get update && apt-get install --assume-yes --no-install-recommends ${DEV_PACKAGES} \
     && git clone --depth=1 https://github.com/pyenv/pyenv.git /pyenv \
     && pyenv install 3.7 3.8 3.9 3.10 3.11 \

ci/dpkg-versions.yaml

@@ -3,10 +3,17 @@ camptocamp/github-app-geo-project:latest:
   ubuntu_23_10/apt: 2.7.3ubuntu0.1
   ubuntu_23_10/base-passwd: 3.6.1
   ubuntu_23_10/bash: 5.2.15-2ubuntu1
+  ubuntu_23_10/binutils: 2.41-5ubuntu1
+  ubuntu_23_10/binutils-common: 2.41-5ubuntu1
+  ubuntu_23_10/binutils-x86-64-linux-gnu: 2.41-5ubuntu1
   ubuntu_23_10/bsdutils: 1:2.39.1-4ubuntu2.2
+  ubuntu_23_10/build-essential: 12.10ubuntu1
+  ubuntu_23_10/bzip2: 1.0.8-5build1
   ubuntu_23_10/ca-certificates: 20230311ubuntu1
   ubuntu_23_10/containerd: 1.7.2-0ubuntu2
   ubuntu_23_10/coreutils: 9.1-1ubuntu2.23.10.1
+  ubuntu_23_10/cpp: 4:13.2.0-1ubuntu1
+  ubuntu_23_10/cpp-13: 13.2.0-4ubuntu3
   ubuntu_23_10/curl: 8.2.1-1ubuntu3.3
   ubuntu_23_10/dash: 0.5.12-6ubuntu1
   ubuntu_23_10/debconf: 1.5.82
@@ -15,11 +22,16 @@ camptocamp/github-app-geo-project:latest:
   ubuntu_23_10/dirmngr: 2.2.40-1.1ubuntu1
   ubuntu_23_10/docker.io: 24.0.5-0ubuntu1
   ubuntu_23_10/dpkg: 1.22.0ubuntu1.1
+  ubuntu_23_10/dpkg-dev: 1.22.0ubuntu1.1
   ubuntu_23_10/e2fsprogs: 1.47.0-2ubuntu1
   ubuntu_23_10/findutils: 4.9.0-5
+  ubuntu_23_10/g++: 4:13.2.0-1ubuntu1
+  ubuntu_23_10/g++-13: 13.2.0-4ubuntu3
+  ubuntu_23_10/gcc: 4:13.2.0-1ubuntu1
+  ubuntu_23_10/gcc-13: 13.2.0-4ubuntu3
   ubuntu_23_10/gcc-13-base: 13.2.0-4ubuntu3
-  ubuntu_23_10/git: 1:2.40.1-1ubuntu1
-  ubuntu_23_10/git-man: 1:2.40.1-1ubuntu1
+  ubuntu_23_10/git: 1:2.40.1-1ubuntu1.1
+  ubuntu_23_10/git-man: 1:2.40.1-1ubuntu1.1
   ubuntu_23_10/gnupg: 2.2.40-1.1ubuntu1
   ubuntu_23_10/gnupg-l10n: 2.2.40-1.1ubuntu1
   ubuntu_23_10/gnupg-utils: 2.2.40-1.1ubuntu1
@@ -37,60 +49,81 @@ camptocamp/github-app-geo-project:latest:
   ubuntu_23_10/iptables: 1.8.9-2ubuntu2
   ubuntu_23_10/libacl1: 2.3.1-3
   ubuntu_23_10/libapt-pkg6.0: 2.7.3ubuntu0.1
+  ubuntu_23_10/libasan8: 13.2.0-4ubuntu3
   ubuntu_23_10/libassuan0: 2.5.6-1
+  ubuntu_23_10/libatomic1: 13.2.0-4ubuntu3
   ubuntu_23_10/libattr1: 1:2.5.1-4
   ubuntu_23_10/libaudit-common: 1:3.1.1-1
   ubuntu_23_10/libaudit1: 1:3.1.1-1
+  ubuntu_23_10/libbinutils: 2.41-5ubuntu1
   ubuntu_23_10/libblkid1: 2.39.1-4ubuntu2.2
   ubuntu_23_10/libbrotli1: 1.0.9-2build8
   ubuntu_23_10/libbz2-1.0: 1.0.8-5build1
-  ubuntu_23_10/libc-bin: 2.38-1ubuntu6.2
-  ubuntu_23_10/libc6: 2.38-1ubuntu6.2
+  ubuntu_23_10/libc-bin: 2.38-1ubuntu6.3
+  ubuntu_23_10/libc-dev-bin: 2.38-1ubuntu6.3
+  ubuntu_23_10/libc6: 2.38-1ubuntu6.3
+  ubuntu_23_10/libc6-dev: 2.38-1ubuntu6.3
   ubuntu_23_10/libcap-ng0: 0.8.3-1build2
   ubuntu_23_10/libcap2: 1:2.66-4ubuntu1
+  ubuntu_23_10/libcc1-0: 13.2.0-4ubuntu3
   ubuntu_23_10/libcom-err2: 1.47.0-2ubuntu1
+  ubuntu_23_10/libcrypt-dev: 1:4.4.36-2
   ubuntu_23_10/libcrypt1: 1:4.4.36-2
+  ubuntu_23_10/libctf-nobfd0: 2.41-5ubuntu1
+  ubuntu_23_10/libctf0: 2.41-5ubuntu1
   ubuntu_23_10/libcurl3-gnutls: 8.2.1-1ubuntu3.3
   ubuntu_23_10/libcurl4: 8.2.1-1ubuntu3.3
   ubuntu_23_10/libdb5.3: 5.3.28+dfsg2-2
   ubuntu_23_10/libdebconfclient0: 0.270ubuntu1
   ubuntu_23_10/libdevmapper1.02.1: 2:1.02.185-2ubuntu1
+  ubuntu_23_10/libdpkg-perl: 1.22.0ubuntu1.1
   ubuntu_23_10/liberror-perl: 0.17029-2
   ubuntu_23_10/libexpat1: 2.5.0-2ubuntu0.1
   ubuntu_23_10/libext2fs2: 1.47.0-2ubuntu1
   ubuntu_23_10/libffi8: 3.4.4-1
+  ubuntu_23_10/libgcc-13-dev: 13.2.0-4ubuntu3
   ubuntu_23_10/libgcc-s1: 13.2.0-4ubuntu3
   ubuntu_23_10/libgcrypt20: 1.10.2-3ubuntu1
   ubuntu_23_10/libgdbm-compat4: 1.23-3
   ubuntu_23_10/libgdbm6: 1.23-3
   ubuntu_23_10/libgmp10: 2:6.3.0+dfsg-2ubuntu4
   ubuntu_23_10/libgnutls30: 3.8.1-4ubuntu1.3
+  ubuntu_23_10/libgomp1: 13.2.0-4ubuntu3
   ubuntu_23_10/libgpg-error0: 1.47-2
+  ubuntu_23_10/libgprofng0: 2.41-5ubuntu1
   ubuntu_23_10/libgssapi-krb5-2: 1.20.1-3ubuntu1
   ubuntu_23_10/libhogweed6: 3.9.1-2
+  ubuntu_23_10/libhwasan0: 13.2.0-4ubuntu3
   ubuntu_23_10/libidn2-0: 2.3.4-1
   ubuntu_23_10/libip4tc2: 1.8.9-2ubuntu2
   ubuntu_23_10/libip6tc2: 1.8.9-2ubuntu2
+  ubuntu_23_10/libisl23: 0.26-3
+  ubuntu_23_10/libitm1: 13.2.0-4ubuntu3
+  ubuntu_23_10/libjansson4: 2.14-2
   ubuntu_23_10/libk5crypto3: 1.20.1-3ubuntu1
   ubuntu_23_10/libkeyutils1: 1.6.3-2
   ubuntu_23_10/libkrb5-3: 1.20.1-3ubuntu1
   ubuntu_23_10/libkrb5support0: 1.20.1-3ubuntu1
   ubuntu_23_10/libksba8: 1.6.4-2
   ubuntu_23_10/libldap2: 2.6.6+dfsg-1~exp1ubuntu1
+  ubuntu_23_10/liblsan0: 13.2.0-4ubuntu3
   ubuntu_23_10/liblz4-1: 1.9.4-1
   ubuntu_23_10/liblzma5: 5.4.1-0.2
   ubuntu_23_10/libmagic-mgc: 1:5.44-3
   ubuntu_23_10/libmagic1: 1:5.44-3
   ubuntu_23_10/libmd0: 1.1.0-1
   ubuntu_23_10/libmnl0: 1.0.4-3ubuntu1
   ubuntu_23_10/libmount1: 2.39.1-4ubuntu2.2
+  ubuntu_23_10/libmpc3: 1.3.1-1
+  ubuntu_23_10/libmpfr6: 4.2.1-1
   ubuntu_23_10/libncursesw6: 6.4+20230625-2
   ubuntu_23_10/libnetfilter-conntrack3: 1.0.9-5
   ubuntu_23_10/libnettle8: 3.9.1-2
   ubuntu_23_10/libnfnetlink0: 1.0.2-2
   ubuntu_23_10/libnftnl11: 1.2.6-2
   ubuntu_23_10/libnghttp2-14: 1.55.1-1ubuntu0.2
   ubuntu_23_10/libnpth0: 1.6-3build2
+  ubuntu_23_10/libnsl-dev: 1.3.0-2build2
   ubuntu_23_10/libnsl2: 1.3.0-2build2
   ubuntu_23_10/libp11-kit0: 0.25.0-4ubuntu1
   ubuntu_23_10/libpam-modules: 1.5.2-6ubuntu1.1
@@ -99,12 +132,13 @@ camptocamp/github-app-geo-project:latest:
   ubuntu_23_10/libpam0g: 1.5.2-6ubuntu1.1
   ubuntu_23_10/libpcre2-8-0: 10.42-4
   ubuntu_23_10/libperl5.36: 5.36.0-9ubuntu1.1
-  ubuntu_23_10/libpq5: 15.6-0ubuntu0.23.10.1
+  ubuntu_23_10/libpq5: 15.7-0ubuntu0.23.10.1
   ubuntu_23_10/libproc2-0: 2:4.0.3-1ubuntu1.23.10.1
   ubuntu_23_10/libpsl5: 0.21.2-1
   ubuntu_23_10/libpython3-stdlib: 3.11.4-5
   ubuntu_23_10/libpython3.11-minimal: 3.11.6-3
   ubuntu_23_10/libpython3.11-stdlib: 3.11.6-3
+  ubuntu_23_10/libquadmath0: 13.2.0-4ubuntu3
   ubuntu_23_10/libreadline8: 8.2-1.3
   ubuntu_23_10/librtmp1: 2.4+20151223.gitfa8646d.1-2build4
   ubuntu_23_10/libsasl2-2: 2.1.28+dfsg1-3
@@ -114,32 +148,40 @@ camptocamp/github-app-geo-project:latest:
   ubuntu_23_10/libsemanage-common: 3.5-1
   ubuntu_23_10/libsemanage2: 3.5-1
   ubuntu_23_10/libsepol2: 3.5-1
+  ubuntu_23_10/libsframe1: 2.41-5ubuntu1
   ubuntu_23_10/libsmartcols1: 2.39.1-4ubuntu2.2
   ubuntu_23_10/libsqlite3-0: 3.42.0-1ubuntu0.1
   ubuntu_23_10/libss2: 1.47.0-2ubuntu1
   ubuntu_23_10/libssh-4: 0.10.5-3ubuntu1.2
   ubuntu_23_10/libssl3: 3.0.10-1ubuntu2.3
+  ubuntu_23_10/libstdc++-13-dev: 13.2.0-4ubuntu3
   ubuntu_23_10/libstdc++6: 13.2.0-4ubuntu3
   ubuntu_23_10/libsystemd0: 253.5-1ubuntu6.1
   ubuntu_23_10/libtasn1-6: 4.19.0-3
   ubuntu_23_10/libtinfo6: 6.4+20230625-2
   ubuntu_23_10/libtirpc-common: 1.3.3+ds-1
+  ubuntu_23_10/libtirpc-dev: 1.3.3+ds-1
   ubuntu_23_10/libtirpc3: 1.3.3+ds-1
+  ubuntu_23_10/libtsan2: 13.2.0-4ubuntu3
+  ubuntu_23_10/libubsan1: 13.2.0-4ubuntu3
   ubuntu_23_10/libudev1: 253.5-1ubuntu6.1
   ubuntu_23_10/libunistring2: 1.0-2
   ubuntu_23_10/libuuid1: 2.39.1-4ubuntu2.2
   ubuntu_23_10/libxtables12: 1.8.9-2ubuntu2
   ubuntu_23_10/libxxhash0: 0.8.1-1
   ubuntu_23_10/libzstd1: 1.5.5+dfsg2-1ubuntu2
+  ubuntu_23_10/linux-libc-dev: 6.5.0-35.35
   ubuntu_23_10/login: 1:4.13+dfsg1-1ubuntu1.1
   ubuntu_23_10/logsave: 1.47.0-2ubuntu1
+  ubuntu_23_10/lto-disabled-list: '43'
+  ubuntu_23_10/make: 4.3-4.1build1
   ubuntu_23_10/mawk: 1.3.4.20230730-1
   ubuntu_23_10/media-types: 10.1.0
   ubuntu_23_10/mount: 2.39.1-4ubuntu2.2
   ubuntu_23_10/ncurses-base: 6.4+20230625-2
   ubuntu_23_10/ncurses-bin: 6.4+20230625-2
   ubuntu_23_10/netbase: '6.4'
-  ubuntu_23_10/nodejs: 18.13.0+dfsg1-1ubuntu2.2
+  ubuntu_23_10/nodejs: 20.14.0-1nodesource1
   ubuntu_23_10/openssl: 3.0.10-1ubuntu2.3
   ubuntu_23_10/passwd: 1:4.13+dfsg1-1ubuntu1.1
   ubuntu_23_10/patch: 2.7.6-7build2
@@ -148,7 +190,7 @@ camptocamp/github-app-geo-project:latest:
   ubuntu_23_10/perl-modules-5.36: 5.36.0-9ubuntu1.1
   ubuntu_23_10/pinentry-curses: 1.2.1-1ubuntu1
   ubuntu_23_10/postgresql-client: 15+253
-  ubuntu_23_10/postgresql-client-15: 15.6-0ubuntu0.23.10.1
+  ubuntu_23_10/postgresql-client-15: 15.7-0ubuntu0.23.10.1
   ubuntu_23_10/postgresql-client-common: '253'
   ubuntu_23_10/procps: 2:4.0.3-1ubuntu1.23.10.1
   ubuntu_23_10/python3: 3.11.4-5
@@ -162,6 +204,7 @@ camptocamp/github-app-geo-project:latest:
   ubuntu_23_10/python3.11: 3.11.6-3
   ubuntu_23_10/python3.11-minimal: 3.11.6-3
   ubuntu_23_10/readline-common: 8.2-1.3
+  ubuntu_23_10/rpcsvc-proto: 1.4.2-0ubuntu6
   ubuntu_23_10/runc: 1.1.7-0ubuntu2.2
   ubuntu_23_10/sed: 4.9-1
   ubuntu_23_10/sensible-utils: 0.0.20

github_app_geo_project/scripts/process_queue.py

@@ -12,7 +12,7 @@
 import subprocess  # nosec
 import sys
 import urllib.parse
-from typing import Any, cast
+from typing import Any, NamedTuple, cast
 
 import c2cwsgiutils.loader
 import c2cwsgiutils.setup_process
@@ -33,6 +33,17 @@
 _NB_JOBS = Gauge("ghci_jobs_number", "Number of jobs", ["status"])
 
 
+class _JobInfo(NamedTuple):
+    module: str
+    event_name: str
+    repository: str
+    priority: int
+    worker_max_priority: int
+
+
+_RUNNING_JOBS: dict[int, _JobInfo] = {}
+
+
 class _Handler(logging.Handler):
     context_var: contextvars.ContextVar[int] = contextvars.ContextVar("job_id")
 
@@ -656,6 +667,9 @@ async def _process_one_job(
         message.title = f"Start process job '{job.event_name}' id: {job.id}, on {job.owner}/{job.repository} on module: {job.module}, on application {job.application}"
         root_logger.addHandler(handler)
         _LOGGER.info(message)
+        _RUNNING_JOBS[job.id] = _JobInfo(
+            job.module or "-", job.event_name, job.repository, job.priority, max_priority
+        )
         root_logger.removeHandler(handler)
 
         if make_pending:
@@ -722,6 +736,7 @@ async def _process_one_job(
                 job.status = models.JobStatus.ERROR
             job.finished_at = datetime.datetime.now(tz=datetime.timezone.utc)
             session.commit()
+            _RUNNING_JOBS.pop(job.id)
 
         _LOGGER.debug("Process one job (max priority: %i): Done", max_priority)
         return False
@@ -784,6 +799,18 @@ async def __call__(self, *args: Any, **kwds: Any) -> Any:
             await asyncio.sleep(10)
 
 
+async def _watch_dog() -> None:
+    while True:
+        _LOGGER.debug("Watch dog: alive")
+        with open("/watch_dog", "w", encoding="utf-8") as file_:
+            file_.write(datetime.datetime.now().isoformat())
+            for id_, job in _RUNNING_JOBS.items():
+                file_.write(
+                    f"{id_}: {job.module} {job.event_name} {job.repository} [{job.priority}] (Worker max priority {job.worker_max_priority})\n"
+                )
+        await asyncio.sleep(60)
+
+
 async def _async_main() -> None:
     """Process the jobs present in the database queue."""
     parser = argparse.ArgumentParser(description=__doc__)
@@ -816,6 +843,7 @@ async def _async_main() -> None:
     threads_call = []
     if not args.exit_when_empty:
         threads_call.append(_UpdateCounter(Session)())
+        threads_call.append(_watch_dog())
 
     for priority in priority_groups:
         threads_call.append(_Run(config, Session, args.exit_when_empty, priority)())
@@ -825,8 +853,7 @@ async def _async_main() -> None:
 def main() -> None:
     """Process the jobs present in the database queue."""
     socket.setdefaulttimeout(int(os.environ.get("GHCI_SOCKET_TIMEOUT", 120)))
-    with asyncio.Runner() as runner:
-        runner.run(_async_main())
+    asyncio.run(_async_main())
 
 
 if __name__ == "__main__":