frontend: automatically EOL lifeless rolling chroots

Implements: https://github.com/fedora-copr/debate/blob/main/2024-04-23-rawhide-chroots-eol.md
Fixes: #2933

doc/how_to_manage_chroots.rst

@@ -20,6 +20,7 @@ Chroots can be easily managed with these few commands.
     copr-frontend branch-fedora <new-branched-version>
     copr-frontend rawhide-to-release <rawhide-chroot> <newly-created-chroot>
     copr-frontend chroots-template [--template PATH]
+    copr-frontend eol-lifeless-rolling-chroots
 
 However, `enablement process upon Fedora branching <#branching-process>`_ and also
 `chroot deactivation when Fedora reaches it's EOL phase <#eol-deactivation-process>`_, are not that simple.
@@ -110,6 +111,22 @@ When it is done, `send an information email to a mailing list <#mailing-lists>`_
 See the :ref:`the disable chroots template <disable_chroots_template>`.
 
 
+Rawhide (and other rolling) chroots EOL
+---------------------------------------
+
+Run ``copr-frontend eol-lifeless-rolling-chroots`` to mark existing rolling Copr
+chroots for the future removal/deactivation — if they appear lifeless.  You
+might want to run this daily in the ``copr-frontend-optional`` cron-job file.
+The logic in this command checks that no build happened in particular rolling
+chroot for a long time, so likely no work is being done there, and the old built
+packages are _likely_ non-installable anyway (as the rolling distro moves
+forward with dependencies, but no dependency resolution is being done with
+RPMs).  If such a chroot is marked EOL, this command applies the same
+notification policy/process as with the :ref:`eol_deactivation_process` so users
+can keep the chroot alive (either by prolonging the chroot, or triggering a new
+build).
+
+
 .. _managing_chroot_comments:
 
 Managing chroot comments

frontend/conf/cron.daily/copr-frontend-optional

@@ -2,6 +2,7 @@
 
 # Optional Copr frontend tasks to be executed daily.
 
+#runuser -c 'copr-frontend eol-lifeless-rolling-chroots' - copr-fe
 #runuser -c 'copr-frontend notify-outdated-chroots' - copr-fe
 #runuser -c 'copr-frontend delete-outdated-chroots' - copr-fe
 #/usr/libexec/copr_dump_db.sh /var/lib/copr/data/db_dumps/

frontend/coprs_frontend/alembic/versions/d23f84f87130_eol_rolling_builds.py

@@ -0,0 +1,30 @@
+"""
+record last copr_chroot build, allow marking Rawhide as rolling
+Create Date: 2024-05-13 08:56:31.557843
+"""
+
+import time
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.sql import text
+
+revision = 'd23f84f87130'
+down_revision = '41763f7a5185'
+
+def upgrade():
+    op.add_column('mock_chroot', sa.Column('rolling', sa.Boolean(), nullable=True))
+    op.add_column('copr_chroot', sa.Column('last_build_timestamp', sa.Integer(), nullable=True))
+    conn = op.get_bind()
+    conn.execute(
+        text("update copr_chroot set last_build_timestamp = :start_stamp;"),
+        {"start_stamp": int(time.time())},
+    )
+    op.create_index('copr_chroot_rolling_last_build_idx', 'copr_chroot',
+                    ['mock_chroot_id', 'last_build_timestamp', 'delete_after'],
+                    unique=False)
+
+
+def downgrade():
+    op.drop_index('copr_chroot_rolling_last_build_idx', table_name='copr_chroot')
+    op.drop_column('copr_chroot', 'last_build_timestamp')
+    op.drop_column('mock_chroot', 'rolling')

frontend/coprs_frontend/commands/eol_lifeless_rolling_chroots.py

@@ -0,0 +1,14 @@
+"""
+copr-frontend eol-lifeless-rolling-chroots command
+"""
+
+import click
+from coprs.logic.outdated_chroots_logic import OutdatedChrootsLogic
+
+@click.command()
+def eol_lifeless_rolling_chroots():
+    """
+    Go through all rolling CoprChroots and check whether they shouldn't be
+    scheduled for future removal.
+    """
+    OutdatedChrootsLogic.trigger_rolling_eol_policy()

frontend/coprs_frontend/config/copr.conf

@@ -207,6 +207,15 @@ HIDE_IMPORT_LOG_AFTER_DAYS = 14
 # failed builds.  Currently only integrated with the Fedora Copr instance.
 #LOG_DETECTIVE_BUTTON = False
 
+# After a given _INACTIVITY_WARNING period (DAYS), when no new build appeared
+# new build appeared in a rolling chroot, we start the EOL policy.  It means
+# that - after the next _INACTIVITY_REMOVAL DAYS - the chroot gets disabled, and
+# builds the corresponding builds removed to save storage.  This is only
+# applicable to MockChroots.rolling = True, and the ... TODO ... command must be
+# configured in cron.
+#ROLLING_CHROOTS_INACTIVITY_WARNING = 180
+#ROLLING_CHROOTS_INACTIVITY_REMOVAL = 180
+
 #############################
 ##### DEBUGGING Section #####
 

frontend/coprs_frontend/coprs/config.py

@@ -198,6 +198,10 @@ class Config(object):
 
     LOG_DETECTIVE_BUTTON = False
 
+    ROLLING_CHROOTS_INACTIVITY_WARNING = 180
+    ROLLING_CHROOTS_INACTIVITY_REMOVAL = 180
+
+
 class ProductionConfig(Config):
     DEBUG = False
     # SECRET_KEY = "put_some_secret_here"

frontend/coprs_frontend/coprs/logic/builds_logic.py

@@ -1469,6 +1469,7 @@ def new(cls, build, mock_chroot, **kwargs):
             copr_chroot = coprs_logic.CoprChrootsLogic.get_by_mock_chroot_id(
                 build.copr, mock_chroot.id
             ).one()
+        copr_chroot.build_done()
         return models.BuildChroot(
             mock_chroot=mock_chroot,
             copr_chroot=copr_chroot,

frontend/coprs_frontend/coprs/logic/outdated_chroots_logic.py

@@ -83,10 +83,43 @@ def expire(cls, copr_chroot):
         cls._update_copr_chroot(copr_chroot, delete_after_days)
 
     @classmethod
-    def _update_copr_chroot(cls, copr_chroot, delete_after_days):
+    def _update_copr_chroot(cls, copr_chroot, delete_after_days, actor=None):
         delete_after_timestamp = (
             datetime.now()
             + timedelta(days=delete_after_days)
         )
-        CoprChrootsLogic.update_chroot(flask.g.user, copr_chroot,
+
+        if actor is None:
+            actor = flask.g.user
+
+        CoprChrootsLogic.update_chroot(actor, copr_chroot,
                                        delete_after=delete_after_timestamp)
+
+
+    @classmethod
+    def trigger_rolling_eol_policy(cls):
+        """
+        Go through all the MockChroot.rolling -> CoprChroots, and check when the
+        last build has been done.  If it is more than
+        config.ROLLING_CHROOTS_INACTIVITY_WARNING days, mark the CoprChroot for
+        removal after ROLLING_CHROOTS_INACTIVITY_REMOVAL days.
+        """
+
+        period = app.config["ROLLING_CHROOTS_INACTIVITY_WARNING"] * 24 * 3600
+        warn_timestamp = int(datetime.now().timestamp()) - period
+
+        query = (
+            db.session.query(models.CoprChroot).join(models.MockChroot)
+            .filter(models.MockChroot.rolling.is_(True))
+            .filter(models.MockChroot.is_active.is_(True))
+            .filter(models.CoprChroot.delete_after.is_(None))
+            .filter(models.CoprChroot.last_build_timestamp.isnot(None))
+            .filter(models.CoprChroot.last_build_timestamp < warn_timestamp)
+        )
+
+        when = app.config["ROLLING_CHROOTS_INACTIVITY_REMOVAL"]
+        for chroot in query:
+            cls._update_copr_chroot(chroot, when,
+                                    models.AutomationUser("Rolling-Builds-Cleaner"))
+
+        db.session.commit()

frontend/coprs_frontend/coprs/models.py

@@ -104,13 +104,20 @@ def can_edit(self, copr, ignore_admin=False):
         """
         raise NotImplementedError
 
+    @property
+    def name(self):
+        """
+        Return the short username of the user, e.g. bkabrda
+        """
+        return self.username
+
 
 class AutomationUser(AbstractUser):
     """
     This user (instance of this class) can modify all projects; used for
     internal system operations like automatic build removals, etc.
     """
-    def can_edit(self, _copr, _ignore_admin=False):
+    def can_edit(self, _copr, ignore_admin=False):
         return True
 
     def __init__(self, name):
@@ -127,13 +134,6 @@ class User(db.Model, helpers.Serializer, AbstractUser):
     __table__ = outerjoin(_UserPublic.__table__, _UserPrivate.__table__)
     id = column_property(_UserPublic.__table__.c.id, _UserPrivate.__table__.c.user_id)
 
-    @property
-    def name(self):
-        """
-        Return the short username of the user, e.g. bkabrda
-        """
-
-        return self.username
 
     @property
     def copr_permissions(self):
@@ -1622,6 +1622,9 @@ class MockChroot(db.Model, TagMixin, helpers.Serializer):
 
     comment = db.Column(db.Text, nullable=True)
 
+    # rolling distribution, e.g. Fedora Rawhide
+    rolling = db.Column(db.Boolean, default=False)
+
     multilib_pairs = {
         'x86_64': 'i386',
     }
@@ -1695,6 +1698,8 @@ class CoprChroot(db.Model, helpers.Serializer):
         # slightly different configuration).
         db.UniqueConstraint("mock_chroot_id", "copr_id",
                             name="copr_chroot_mock_chroot_id_copr_id_uniq"),
+        db.Index("copr_chroot_rolling_last_build_idx",
+                 "mock_chroot_id", "last_build_timestamp", "delete_after"),
     )
 
     copr_id = db.Column(db.Integer, db.ForeignKey("copr.id"))
@@ -1734,6 +1739,8 @@ class CoprChroot(db.Model, helpers.Serializer):
     isolation = db.Column(db.Text, default="unchanged")
     deleted = db.Column(db.Boolean, default=False, index=True)
 
+    last_build_timestamp = db.Column(db.Integer)
+
     def update_comps(self, comps_xml):
         """
         save (compressed) the comps_xml file content (instance of bytes).
@@ -1914,6 +1921,13 @@ def isolation_setup(self):
             return {}
         return settings
 
+    def build_done(self):
+        """
+        Record that a build has been submitted into this CoprChroot.
+        """
+        self.delete_after = None
+        self.last_build_timestamp = int(time.time())
+
 
 class BuildChroot(db.Model, TagMixin, helpers.Serializer):
     """

frontend/coprs_frontend/manage.py

@@ -35,6 +35,7 @@
 import commands.vacuum_graphs
 import commands.notify_outdated_chroots
 import commands.delete_outdated_chroots
+import commands.eol_lifeless_rolling_chroots
 import commands.clean_expired_projects
 import commands.clean_old_builds
 import commands.delete_orphans
@@ -89,6 +90,7 @@
     "vacuum_graphs",
     "notify_outdated_chroots",
     "delete_outdated_chroots",
+    "eol_lifeless_rolling_chroots",
     "clean_expired_projects",
     "clean_old_builds",
     "delete_orphans",

frontend/coprs_frontend/tests/coprs_test_case.py

@@ -217,7 +217,8 @@ def f_mock_chroots(self):
         self.mc3.distgit_branch = self.mc2.distgit_branch
 
         self.mc4 = models.MockChroot(
-            os_release="fedora", os_version="rawhide", arch="i386", is_active=True)
+            os_release="fedora", os_version="rawhide", arch="i386",
+            is_active=True, rolling=True)
         self.mc4.distgit_branch = models.DistGitBranch(name='master')
 
         self.mc_basic_list = [self.mc1, self.mc2, self.mc3, self.mc4]

frontend/coprs_frontend/tests/test_logic/test_outdated_chroots_logic.py

@@ -377,3 +377,18 @@ def test_outdated_unclicked_repeat(self):
         CoprChrootsLogic.update_from_names(
             self.u2, self.c2, [chroot.name for chroot in self.c2.copr_chroots])
         assert not self.c2.copr_chroots[0].delete_after
+
+    @pytest.mark.usefixtures("f_users", "f_coprs", "f_mock_chroots", "f_db")
+    def test_rolling_warning(self):
+        for cc in self.models.CoprChroot.query.all():
+            if cc.copr.full_name == "user2/barcopr" and cc.name == "fedora-rawhide-i386":
+                # distant past!
+                cc.last_build_timestamp = 666
+        self.db.session.commit()
+        OutdatedChrootsLogic.trigger_rolling_eol_policy()
+        eols = self.db.session.query(self.models.CoprChroot)\
+                .filter(self.models.CoprChroot.delete_after.isnot(None)).all()
+
+        assert len(eols) == 1
+        assert eols[0].name == "fedora-rawhide-i386"
+        assert eols[0].delete_after_days == app.config["ROLLING_CHROOTS_INACTIVITY_REMOVAL"]-1