Add emerge jobs tmpdir blocks and files threshold options

--jobs-tmpdir-blocks-threshold[=RATIO]

  Specifies the maximum ratio of used blocks allowed (a
  floating-point number between 0.0 and 1.0) in PORTAGE_TMPDIR
  when starting a new job. With no argument, removes a previous
  blocks ratio threshold. For example, use a ratio of 0.50 to stop
  starting new jobs when the blocks usage in PORTAGE_TMPDIR
  exceeds 50%. This option conflicts with FEATURES="keepwork".
  WARNING: Since the job scheduler is unable to predict the future
  consumption of jobs that it has scheduled, users are advised to
  set a threshold that provides a significant amount of headroom,
  in order to decrease the probablility that jobs will fail due
  to ENOSPC errors.

--jobs-tmpdir-files-threshold[=RATIO]

  Specifies the maximum ratio of used files (inodes) allowed (a
  floating-point number between 0.0 and 1.0) in PORTAGE_TMPDIR
  when starting a new job. With no argument, removes a previous
  files ratio threshold. For example, use a ratio of 0.50 to stop
  starting new jobs when the files usage in PORTAGE_TMPDIR
  exceeds 50%. This option conflicts with FEATURES="keepwork".
  WARNING: Since the job scheduler is unable to predict the future
  consumption of jobs that it has scheduled, users are advised to
  set a threshold that provides a significant amount of headroom,
  in order to decrease the probablility that jobs will fail due
  to ENOSPC errors.

Bug: https://bugs.gentoo.org/934382
Signed-off-by: Zac Medico <[email protected]>

lib/_emerge/Scheduler.py

@@ -26,6 +26,7 @@
 from portage._sets.base import InternalPackageSet
 from portage.util import ensure_dirs, writemsg, writemsg_level
 from portage.util.futures import asyncio
+from portage.util.path import first_existing
 from portage.util.SlotObject import SlotObject
 from portage.util._async.SchedulerInterface import SchedulerInterface
 from portage.package.ebuild.digestcheck import digestcheck
@@ -64,7 +65,7 @@
 
 
 class Scheduler(PollScheduler):
-    # max time between loadavg checks (seconds)
+    # max time between loadavg and tmpdir statvfs checks (seconds)
     _loadavg_latency = 30
 
     # max time between display status updates (seconds)
@@ -229,6 +230,10 @@ def __init__(
         if max_jobs is None:
             max_jobs = 1
         self._set_max_jobs(max_jobs)
+        self._jobs_tmpdir_blocks_threshold = myopts.get(
+            "--jobs-tmpdir-blocks-threshold"
+        )
+        self._jobs_tmpdir_files_threshold = myopts.get("--jobs-tmpdir-files-threshold")
         self._running_root = trees[trees._running_eroot]["root_config"]
         self.edebug = 0
         if settings.get("PORTAGE_DEBUG", "") == "1":
@@ -1573,7 +1578,11 @@ def _main_loop(self):
         self._main_exit = self._event_loop.create_future()
 
         if (
-            self._max_load is not None
+            (
+                self._max_load is not None
+                or self._jobs_tmpdir_blocks_threshold is not None
+                or self._jobs_tmpdir_files_threshold is not None
+            )
             and self._loadavg_latency is not None
             and (self._max_jobs is True or self._max_jobs > 1)
         ):
@@ -1792,6 +1801,53 @@ def _is_work_scheduled(self):
     def _running_job_count(self):
         return self._jobs
 
+    def _can_add_job(self):
+        if not super()._can_add_job():
+            return False
+
+        running_job_count = self._running_job_count()
+        if running_job_count == 0 and not self._merge_wait_queue:
+            # Ensure there is forward progress if there are no running
+            # jobs and no jobs in the _merge_wait_queue.
+            return True
+
+        if (
+            self._jobs_tmpdir_blocks_threshold is not None
+            or self._jobs_tmpdir_files_threshold is not None
+        ) and hasattr(os, "statvfs"):
+            tmpdirs = set()
+            for root in self.trees:
+                settings = self.trees[root]["root_config"].settings
+                if settings["PORTAGE_TMPDIR"] in tmpdirs:
+                    continue
+                tmpdirs.add(settings["PORTAGE_TMPDIR"])
+                tmpdir = first_existing(
+                    os.path.join(settings["PORTAGE_TMPDIR"], "portage")
+                )
+                try:
+                    vfs_stat = os.statvfs(tmpdir)
+                except OSError as e:
+                    writemsg_level(
+                        f"!!! statvfs('{tmpdir}'): {e}\n",
+                        noiselevel=-1,
+                        level=logging.ERROR,
+                    )
+                else:
+                    if (
+                        self._jobs_tmpdir_blocks_threshold is not None
+                        and (vfs_stat.f_blocks - vfs_stat.f_bavail) / vfs_stat.f_blocks
+                    ) >= self._jobs_tmpdir_blocks_threshold:
+                        return False
+                    if (
+                        self._jobs_tmpdir_files_threshold is not None
+                        and vfs_stat.f_files > 0
+                        and ((vfs_stat.f_files - vfs_stat.f_favail) / vfs_stat.f_files)
+                        >= self._jobs_tmpdir_files_threshold
+                    ):
+                        return False
+
+        return True
+
     def _schedule_tasks(self):
         while True:
             state_change = 0

lib/_emerge/main.py

@@ -1,4 +1,4 @@
-# Copyright 1999-2023 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import argparse
@@ -165,6 +165,8 @@ def __contains__(self, s):
         "--getbinpkgonly": y_or_n,
         "--ignore-world": y_or_n,
         "--jobs": valid_integers,
+        "--jobs-tmpdir-blocks-threshold": valid_floats,
+        "--jobs-tmpdir-files-threshold": valid_floats,
         "--keep-going": y_or_n,
         "--load-average": valid_floats,
         "--onlydeps-with-ideps": y_or_n,
@@ -523,6 +525,14 @@ def parse_opts(tmpcmdline, silent=False):
             "help": "Specifies the number of packages to build " + "simultaneously.",
             "action": "store",
         },
+        "--jobs-tmpdir-blocks-threshold": {
+            "help": "Specifies maximum used blocks ratio when starting a new job.",
+            "action": "store",
+        },
+        "--jobs-tmpdir-files-threshold": {
+            "help": "Specifies maximum used files ratio when starting a new job.",
+            "action": "store",
+        },
         "--keep-going": {
             "help": "continue as much as possible after an error",
             "choices": true_y_or_n,
@@ -1033,6 +1043,42 @@ def parse_opts(tmpcmdline, silent=False):
 
         myoptions.jobs = jobs
 
+    if myoptions.jobs_tmpdir_blocks_threshold == "True":
+        myoptions.jobs_tmpdir_blocks_threshold = None
+
+    if myoptions.jobs_tmpdir_blocks_threshold:
+        try:
+            jobs_tmpdir_blocks_threshold = float(myoptions.jobs_tmpdir_blocks_threshold)
+        except ValueError:
+            jobs_tmpdir_blocks_threshold = 0.0
+
+        if jobs_tmpdir_blocks_threshold <= 0.0 or jobs_tmpdir_blocks_threshold > 1.0:
+            jobs_tmpdir_blocks_threshold = None
+            if not silent:
+                parser.error(
+                    f"Invalid --jobs-tmpdir-blocks-threshold: '{myoptions.jobs_tmpdir_blocks_threshold}'\n"
+                )
+
+        myoptions.jobs_tmpdir_blocks_threshold = jobs_tmpdir_blocks_threshold
+
+    if myoptions.jobs_tmpdir_files_threshold == "True":
+        myoptions.jobs_tmpdir_files_threshold = None
+
+    if myoptions.jobs_tmpdir_files_threshold:
+        try:
+            jobs_tmpdir_files_threshold = float(myoptions.jobs_tmpdir_files_threshold)
+        except ValueError:
+            jobs_tmpdir_files_threshold = 0.0
+
+        if jobs_tmpdir_files_threshold <= 0.0 or jobs_tmpdir_files_threshold > 1.0:
+            jobs_tmpdir_files_threshold = None
+            if not silent:
+                parser.error(
+                    f"Invalid --jobs-tmpdir-files-threshold: '{myoptions.jobs_tmpdir_files_threshold}'\n"
+                )
+
+        myoptions.jobs_tmpdir_files_threshold = jobs_tmpdir_files_threshold
+
     if myoptions.load_average == "True":
         myoptions.load_average = None
 
@@ -1304,6 +1350,27 @@ def emerge_main(args: Optional[list[str]] = None):
     emerge_config.action, emerge_config.opts, emerge_config.args = parse_opts(
         tmpcmdline
     )
+    if (
+        "--jobs-tmpdir-blocks-threshold" in emerge_config.opts
+        and "keepwork" in emerge_config.running_config.settings.features
+    ):
+        writemsg_level(
+            "--jobs-tmpdir-blocks-threshold conflicts with FEATURES=keepwork\n",
+            level=logging.ERROR,
+            noiselevel=-1,
+        )
+        return 1
+
+    if (
+        "--jobs-tmpdir-files-threshold" in emerge_config.opts
+        and "keepwork" in emerge_config.running_config.settings.features
+    ):
+        writemsg_level(
+            "--jobs-tmpdir-files-threshold conflicts with FEATURES=keepwork\n",
+            level=logging.ERROR,
+            noiselevel=-1,
+        )
+        return 1
 
     try:
         return run_action(emerge_config)

man/emerge.1

@@ -1,4 +1,4 @@
-.TH "EMERGE" "1" "May 2024" "Portage @VERSION@" "Portage"
+.TH "EMERGE" "1" "Jun 2024" "Portage @VERSION@" "Portage"
 .SH "NAME"
 emerge \- Command\-line interface to the Portage system
 .SH "SYNOPSIS"
@@ -693,6 +693,32 @@ Note that interactive packages currently force a setting
 of \fI\-\-jobs=1\fR. This issue can be temporarily avoided
 by specifying \fI\-\-accept\-properties=\-interactive\fR.
 .TP
+.BR \-\-jobs\-tmpdir\-blocks\-threshold[=RATIO]
+Specifies the maximum ratio of used blocks allowed (a floating\-point
+number between \fI0.0\fR and \fI1.0\fR) in \fBPORTAGE_TMPDIR\fR when
+starting a new job. With noargument, removes a previous blocks ratio
+threshold. For example, use a ratio of \fI0.50\fR to stop starting new
+jobs when the blocks usage in \fBPORTAGE_TMPDIR\fR exceeds \fI50%\fR.
+This option conflicts with \fBFEATURES="keepwork"\fR.
+\fBWARNING:\fR Since the job scheduler is unable to predict the future
+consumption of jobs that it has scheduled, users are advised to set a
+threshold that provides a significant amount of headroom, in order to
+decrease the probablility that jobs will fail due to \fIENOSPC\fR
+errors.
+.TP
+.BR \-\-jobs\-tmpdir\-files\-threshold[=RATIO]
+Specifies the maximum ratio of used files (inodes) allowed (a
+floating\-point number between \fI0.0\fR and \fI1.0\fR) in
+\fBPORTAGE_TMPDIR\fR when starting a new job. With no argument, removes
+a previous files ratio threshold. For example, use a ratio of \fI0.85\fR
+to stop starting new jobs when the files usage in \fBPORTAGE_TMPDIR\fR
+exceeds \fI85%\fR. This option conflicts with \fBFEATURES="keepwork"\fR.
+\fBWARNING:\fR Since the job scheduler is unable to predict the future
+consumption of jobs that it has scheduled, users are advised to set a
+threshold that provides a significant amount of headroom, in order to
+decrease the probablility that jobs will fail due to \fIENOSPC\fR
+errors.
+.TP
 .BR "\-\-keep\-going [ y | n ]"
 Continue as much as possible after an error. When an error occurs,
 dependencies are recalculated for remaining packages and any with