From e4c9e80c2d977930ecc5d179b2422a67ba757ee8 Mon Sep 17 00:00:00 2001 From: Jakub Kadlcik Date: Tue, 16 Apr 2024 16:32:04 +0200 Subject: [PATCH] frontend, backend: run rawhide-to-release for all CoprDirs Fix #2272 See #3016 --- backend/copr_backend/actions.py | 5 +- .../commands/rawhide_to_release.py | 157 ++++++++++-------- .../test_commands/test_rawhide_to_release.py | 6 +- 3 files changed, 91 insertions(+), 77 deletions(-) diff --git a/backend/copr_backend/actions.py b/backend/copr_backend/actions.py index cd4cb0077..356803f6f 100644 --- a/backend/copr_backend/actions.py +++ b/backend/copr_backend/actions.py @@ -423,14 +423,15 @@ def run(self): appstream = data["appstream"] result = BackendResultEnum("success") try: - chrootdir = os.path.join(self.opts.destdir, data["ownername"], data["projectname"], data["dest_chroot"]) + chrootdir = os.path.join(self.opts.destdir, data["ownername"], + data["copr_dir"], data["dest_chroot"]) if not os.path.exists(chrootdir): self.log.info("Create directory: %s", chrootdir) os.makedirs(chrootdir) for build in data["builds"]: srcdir = os.path.join(self.opts.destdir, data["ownername"], - data["projectname"], data["rawhide_chroot"], build) + data["copr_dir"], data["rawhide_chroot"], build) if os.path.exists(srcdir): destdir = os.path.join(chrootdir, build) diff --git a/frontend/coprs_frontend/commands/rawhide_to_release.py b/frontend/coprs_frontend/commands/rawhide_to_release.py index 01aaa6b14..0abb41568 100644 --- a/frontend/coprs_frontend/commands/rawhide_to_release.py +++ b/frontend/coprs_frontend/commands/rawhide_to_release.py @@ -70,80 +70,66 @@ def rawhide_to_release_function(rawhide_chroot, dest_chroot, retry_forked): copr_chroot = turn_on_the_chroot_for_copr(copr, rawhide_chroot, mock_chroot) - data = {"projectname": copr.name, - "ownername": copr.owner_name, - "rawhide_chroot": rawhide_chroot, - "appstream": copr.appstream, - "dest_chroot": dest_chroot, - "builds": []} - - latest_pkg_builds_in_rawhide = ( - db.session.query( - func.max(models.Build.id), - ) - .join(models.BuildChroot) - .filter(models.Build.copr_dir_id == copr.main_dir.id) - .filter(models.BuildChroot.mock_chroot_id == mock_rawhide_chroot.id) - .filter(models.BuildChroot.status == StatusEnum("succeeded")) - .group_by(models.Build.package_id) - ) - - fork_builds = ( - db.session.query(models.Build) - .options( - joinedload(models.Build.build_chroots) - .joinedload(models.BuildChroot.mock_chroot) - ) - .filter(models.Build.id.in_(latest_pkg_builds_in_rawhide)) - ).all() - - - # no builds to fork in this copr - if not len(fork_builds): - print("Createrepo for copr '{}', chroot '{}'".format(copr.full_name, mock_chroot.name)) - actions_logic.ActionsLogic.send_createrepo(copr, chroots=[mock_chroot.name]) - continue - - new_build_chroots = 0 - for build in fork_builds: - chroot_exists = mock_chroot in build.chroots - - if chroot_exists and not retry_forked: - # this build should already be forked + for copr_dir in copr.dirs: + data = {"projectname": copr.name, + "ownername": copr.owner_name, + "copr_dir": copr_dir.name, + "rawhide_chroot": rawhide_chroot, + "appstream": copr.appstream, + "dest_chroot": dest_chroot, + "builds": []} + + fork_builds = builds_to_fork(copr_dir, mock_rawhide_chroot) + + # No builds to fork in this copr, at least create an empty repo + if not len(fork_builds): + print("Createrepo for '{}', chroot '{}'".format( + copr_dir.full_name, mock_chroot.name)) + actions_logic.ActionsLogic.send_createrepo( + copr, dirnames=[copr_dir.name], chroots=[mock_chroot.name]) continue - # rbc means rawhide_build_chroot (we needed short variable) - rbc = None - for rbc in build.build_chroots: - if rbc.mock_chroot == mock_rawhide_chroot: - break - - if not chroot_exists: - # forked chroot may already exists, e.g. from prevoius - # 'rawhide-to-release-run' - new_build_chroots += 1 - dest_build_chroot = builds_logic.BuildChrootsLogic.new( - build=build, - mock_chroot=mock_chroot, - copr_chroot=copr_chroot, - **rbc.to_dict({ - "__columns_except__": ["id"], - }), - ) - dest_build_chroot.status = StatusEnum("forked") - db.session.add(dest_build_chroot) - - if rbc.result_dir: - data['builds'].append(rbc.result_dir) - - if data["builds"] or new_build_chroots: - print(" Fresh new build chroots: {}, regenerate {}".format( - new_build_chroots, - len(data["builds"]) - new_build_chroots, - )) - - if len(data["builds"]): - actions_logic.ActionsLogic.send_rawhide_to_release(copr, data) + new_build_chroots = 0 + + for build in fork_builds: + chroot_exists = mock_chroot in build.chroots + + if chroot_exists and not retry_forked: + # this build should already be forked + continue + + # rbc means rawhide_build_chroot (we needed short variable) + rbc = None + for rbc in build.build_chroots: + if rbc.mock_chroot == mock_rawhide_chroot: + break + + if not chroot_exists: + # forked chroot may already exists, e.g. from prevoius + # 'rawhide-to-release-run' + new_build_chroots += 1 + dest_build_chroot = builds_logic.BuildChrootsLogic.new( + build=build, + mock_chroot=mock_chroot, + copr_chroot=copr_chroot, + **rbc.to_dict({ + "__columns_except__": ["id"], + }), + ) + dest_build_chroot.status = StatusEnum("forked") + db.session.add(dest_build_chroot) + + if rbc.result_dir: + data['builds'].append(rbc.result_dir) + + if data["builds"] or new_build_chroots: + print(" Fresh new build chroots: {}, regenerate {}".format( + new_build_chroots, + len(data["builds"]) - new_build_chroots, + )) + + if len(data["builds"]): + actions_logic.ActionsLogic.send_rawhide_to_release(copr, data) db.session.commit() @@ -161,3 +147,30 @@ def turn_on_the_chroot_for_copr(copr, rawhide_name, mock_chroot): return chroot return coprs_logic.CoprChrootsLogic.create_chroot_from(rawhide_chroot, mock_chroot=mock_chroot) + + +def builds_to_fork(coprdir, mock_rawhide_chroot): + """ + Return a list of builds that needs to be forked for every CoprDir + """ + latest_pkg_builds_in_rawhide = ( + db.session.query( + func.max(models.Build.id), + ) + .join(models.BuildChroot) + .filter(models.Build.copr_dir_id == coprdir.id) + .filter(models.BuildChroot.mock_chroot_id == mock_rawhide_chroot.id) + .filter(models.BuildChroot.status == StatusEnum("succeeded")) + .group_by(models.Build.package_id) + ) + + fork_builds = ( + db.session.query(models.Build) + .options( + joinedload(models.Build.build_chroots) + .joinedload(models.BuildChroot.mock_chroot), + joinedload(models.Build.copr_dir), + ) + .filter(models.Build.id.in_(latest_pkg_builds_in_rawhide)) + ).all() + return fork_builds diff --git a/frontend/coprs_frontend/tests/test_commands/test_rawhide_to_release.py b/frontend/coprs_frontend/tests/test_commands/test_rawhide_to_release.py index c874052a2..93e483326 100644 --- a/frontend/coprs_frontend/tests/test_commands/test_rawhide_to_release.py +++ b/frontend/coprs_frontend/tests/test_commands/test_rawhide_to_release.py @@ -144,17 +144,17 @@ def test_branch_fedora(self, capsys): "Handling builds in copr 'user2/barcopr', chroot 'fedora-rawhide-i386'", " Fresh new build chroots: 1, regenerate 0", "Handling builds in copr 'user1/foocopr', chroot 'fedora-rawhide-x86_64'", - "Createrepo for copr 'user1/foocopr', chroot 'fedora-19-x86_64'", + "Createrepo for 'user1/foocopr', chroot 'fedora-19-x86_64'", "fedora-19-i386 - already exists.", "fedora-19-x86_64 - already exists.", "Handling builds in copr 'user2/barcopr', chroot 'fedora-rawhide-i386'", "Handling builds in copr 'user1/foocopr', chroot 'fedora-rawhide-x86_64'", - "Createrepo for copr 'user1/foocopr', chroot 'fedora-19-x86_64'", + "Createrepo for 'user1/foocopr', chroot 'fedora-19-x86_64'", "fedora-19-i386 - already exists.", "fedora-19-x86_64 - already exists.", "Handling builds in copr 'user2/barcopr', chroot 'fedora-rawhide-i386'", " Fresh new build chroots: 0, regenerate 1", "Handling builds in copr 'user1/foocopr', chroot 'fedora-rawhide-x86_64'", - "Createrepo for copr 'user1/foocopr', chroot 'fedora-19-x86_64'", + "Createrepo for 'user1/foocopr', chroot 'fedora-19-x86_64'", "" ])