backend, frontend: implement project and build deletion in Pulp

Fix #3318
Fix #3319

backend/copr_backend/actions.py

@@ -225,68 +225,22 @@ class Delete(Action):
     """
     Abstract class for all other Delete* classes.
     """
-    # pylint: disable=abstract-method
-    def _handle_delete_builds(self, ownername, projectname, project_dirname,
-                              chroot_builddirs, build_ids, appstream):
-        """ call /bin/copr-repo --delete """
-        devel = uses_devel_repo(self.front_url, ownername, projectname)
-        result = BackendResultEnum("success")
-        for chroot, subdirs in chroot_builddirs.items():
-            chroot_path = os.path.join(self.destdir, ownername, project_dirname,
-                                       chroot)
-            if not os.path.exists(chroot_path):
-                self.log.error("%s chroot path doesn't exist", chroot_path)
-                result = BackendResultEnum("failure")
-                continue
-
-            self.log.info("Deleting subdirs [%s] in %s",
-                          ", ".join(subdirs), chroot_path)
-
-            # Run createrepo first and then remove the files (to avoid old
-            # repodata temporarily pointing at non-existing files)!
-            if chroot != "srpm-builds":
-                # In srpm-builds we don't create repodata at all
-                if not call_copr_repo(chroot_path, delete=subdirs, devel=devel, appstream=appstream,
-                                      logger=self.log):
-                    result = BackendResultEnum("failure")
-
-            for build_id in build_ids or []:
-                log_paths = [
-                    os.path.join(chroot_path, build_chroot_log_name(build_id)),
-                    # we used to create those before
-                    os.path.join(chroot_path, 'build-{}.rsync.log'.format(build_id)),
-                    os.path.join(chroot_path, 'build-{}.log'.format(build_id))]
-                for log_path in log_paths:
-                    try:
-                        os.unlink(log_path)
-                    except OSError:
-                        self.log.debug("can't remove %s", log_path)
-        return result
-
 
 class DeleteProject(Delete):
     def run(self):
         self.log.debug("Action delete copr")
-        result = BackendResultEnum("success")
-
-        ext_data = json.loads(self.data["data"])
-        ownername = ext_data["ownername"]
-        project_dirnames = ext_data["project_dirnames"]
+        project_dirnames = self.ext_data["project_dirnames"]
 
-        if not ownername:
+        if not self.storage.owner:
             self.log.error("Received empty ownername!")
-            result = BackendResultEnum("failure")
-            return result
+            return BackendResultEnum("failure")
 
         for dirname in project_dirnames:
             if not dirname:
                 self.log.warning("Received empty dirname!")
                 continue
-            path = os.path.join(self.destdir, ownername, dirname)
-            if os.path.exists(path):
-                self.log.info("Removing copr dir %s", path)
-                shutil.rmtree(path)
-        return result
+            self.storage.delete_project(dirname)
+        return BackendResultEnum("success")
 
 
 class CompsUpdate(Action):
@@ -334,20 +288,14 @@ def run(self):
         #     srpm-builds: [00849545, 00849546]
         #     fedora-30-x86_64: [00849545-example, 00849545-foo]
         #   [...]
-        ext_data = json.loads(self.data["data"])
 
-        ownername = ext_data["ownername"]
-        projectname = ext_data["projectname"]
-        project_dirnames = ext_data["project_dirnames"]
-        build_ids = ext_data["build_ids"]
-        appstream = ext_data["appstream"]
+        project_dirnames = self.ext_data["project_dirnames"]
+        build_ids = self.ext_data["build_ids"]
 
         result = BackendResultEnum("success")
         for project_dirname, chroot_builddirs in project_dirnames.items():
-            if BackendResultEnum("failure") == \
-               self._handle_delete_builds(ownername, projectname,
-                                          project_dirname, chroot_builddirs,
-                                          build_ids, appstream):
+            success = self.storage.delete_builds(chroot_builddirs, build_ids)
+            if not success:
                 result = BackendResultEnum("failure")
         return result
 
@@ -363,22 +311,17 @@ def run(self):
         # chroot_builddirs:
         #   srpm-builds: [00849545]
         #   fedora-30-x86_64: [00849545-example]
-        ext_data = json.loads(self.data["data"])
 
-        try:
-            ownername = ext_data["ownername"]
-            build_ids = [self.data['object_id']]
-            projectname = ext_data["projectname"]
-            project_dirname = ext_data["project_dirname"]
-            chroot_builddirs = ext_data["chroot_builddirs"]
-            appstream = ext_data["appstream"]
-        except KeyError:
+        keys = {"ownername", "projectname", "project_dirname",
+                "chroot_builddirs", "appstream"}
+        if self.ext_data.keys() < keys or "object_id" not in self.data:
             self.log.exception("Invalid action data")
             return BackendResultEnum("failure")
 
-        return self._handle_delete_builds(ownername, projectname,
-                                          project_dirname, chroot_builddirs,
-                                          build_ids, appstream)
+        success = self.storage.delete_builds(
+            self.ext_data["chroot_builddirs"],
+            [self.data['object_id']])
+        return BackendResultEnum("success" if success else "failure")
 
 
 class DeleteChroot(Delete):

backend/copr_backend/pulp.py

@@ -189,3 +189,11 @@ def delete_distribution(self, distribution):
         """
         url = self.config["base_url"] + distribution
         return requests.delete(url, **self.request_params)
+
+    def list_packages(self, repository):
+        """
+        https://pulpproject.org/pulp_rpm/restapi/#tag/Content:-Packages/operation/content_rpm_packages_list
+        """
+        url = self.url("api/v3/content/rpm/packages/?")
+        url += urlencode({"repository_version": repository})
+        return requests.get(url, **self.request_params)

backend/copr_backend/storage.py

@@ -5,7 +5,7 @@
 import os
 import shutil
 from copr_common.enums import StorageEnum
-from copr_backend.helpers import call_copr_repo
+from copr_backend.helpers import call_copr_repo, build_chroot_log_name
 from copr_backend.pulp import PulpClient
 
 
@@ -71,6 +71,18 @@ def delete_repository(self, chroot):
         """
         raise NotImplementedError
 
+    def delete_project(self, dirname):
+        """
+        Delete the whole project and all of its repositories and builds
+        """
+        raise NotImplementedError
+
+    def delete_builds(self, chroot_builddirs, build_ids):
+        """
+        Delete multiple builds from the storage
+        """
+        raise NotImplementedError
+
 
 class BackendStorage(Storage):
     """
@@ -116,6 +128,48 @@ def delete_repository(self, chroot):
             return
         shutil.rmtree(chroot_path)
 
+    def delete_project(self, dirname):
+        path = os.path.join(self.opts.destdir, self.owner, dirname)
+        if os.path.exists(path):
+            self.log.info("Removing copr dir %s", path)
+            shutil.rmtree(path)
+
+    def delete_builds(self, chroot_builddirs, build_ids):
+        result = True
+        for chroot, subdirs in chroot_builddirs.items():
+            chroot_path = os.path.join(
+                self.opts.destdir, self.owner, self.project, chroot)
+            if not os.path.exists(chroot_path):
+                self.log.error("%s chroot path doesn't exist", chroot_path)
+                result = False
+                continue
+
+            self.log.info("Deleting subdirs [%s] in %s",
+                          ", ".join(subdirs), chroot_path)
+
+            # Run createrepo first and then remove the files (to avoid old
+            # repodata temporarily pointing at non-existing files)!
+            # In srpm-builds we don't create repodata at all
+            if chroot != "srpm-builds":
+                repo = call_copr_repo(
+                    chroot_path, delete=subdirs, devel=self.devel,
+                    appstream=self.appstream, logger=self.log)
+                if not repo:
+                    result = False
+
+            for build_id in build_ids or []:
+                log_paths = [
+                    os.path.join(chroot_path, build_chroot_log_name(build_id)),
+                    # we used to create those before
+                    os.path.join(chroot_path, 'build-{}.rsync.log'.format(build_id)),
+                    os.path.join(chroot_path, 'build-{}.log'.format(build_id))]
+                for log_path in log_paths:
+                    try:
+                        os.unlink(log_path)
+                    except OSError:
+                        self.log.debug("can't remove %s", log_path)
+        return result
+
 
 class PulpStorage(Storage):
     """
@@ -216,6 +270,28 @@ def delete_repository(self, chroot):
         self.client.delete_repository(repository)
         self.client.delete_distribution(distribution)
 
+    def delete_project(self, dirname):
+        # TODO Implement
+        raise NotImplementedError
+
+    def delete_builds(self, chroot_builddirs, build_ids):
+        result = True
+        for chroot, subdirs in chroot_builddirs.items():
+            if chroot == "srpm-builds":
+                continue
+
+            repository = self._get_latest_repository_version(chroot)
+            packages = self.client.list_packages(repository).json()["results"]
+            print(packages)
+            import ipdb; ipdb.set_trace()
+
+            # TODO Do some filtering based on subdirs
+            print(subdirs)
+
+            # TODO Remove the builds using
+            # pulp rpm repository content remove --repository frostyx/test-pulp-1/fedora-39-x86_64 --package-href TODO
+        return result
+
     def _repository_name(self, chroot, dirname=None):
         return "/".join([
             self.owner,
@@ -238,3 +314,15 @@ def _get_distribution(self, chroot):
         name = self._distribution_name(chroot)
         response = self.client.get_distribution(name)
         return response.json()["results"][0]["pulp_href"]
+
+    def _get_latest_repository_version(self, chroot):
+        name = self._repository_name(chroot)
+        response = self.client.get_repository(name)
+        href = response.json()["results"][0]["versions_href"]
+
+        # TODO Rewrite using limit, offset and sort by recent
+        # And probably put to pulp.py
+        import requests
+        url = self.client.config["base_url"] + href
+        response = requests.get(url, **self.client.request_params)
+        return response.json()["results"][-1]["pulp_href"]

frontend/coprs_frontend/coprs/logic/actions_logic.py

@@ -145,6 +145,7 @@ def send_delete_copr(cls, copr):
         data_dict = {
             "ownername": copr.owner_name,
             "project_dirnames": [copr_dir.name for copr_dir in copr.dirs],
+            "storage": copr.storage,
         }
         action = models.Action(action_type=ActionTypeEnum("delete"),
                                object_type="copr",
@@ -190,6 +191,8 @@ def get_build_delete_data(cls, build):
                 build.copr_dirname if build.copr_dir else build.copr_name,
             "chroot_builddirs": cls.get_chroot_builddirs(build),
             "appstream": build.appstream,
+            "devel": build.copr.devel_mode,
+            "storage": build.copr.storage,
         }
 
     @classmethod
@@ -216,7 +219,12 @@ def send_delete_multiple_builds(cls, builds):
         :type build: list of models.Build
         """
         project_dirnames = {}
-        data = {'project_dirnames': project_dirnames}
+        data = {
+            "project_dirnames": project_dirnames,
+            # We can pick any random build because the assumption is, they are
+            # all from the same project
+            "storage": builds[0].copr.storage if builds else None,
+        }
 
         build_ids = []
         for build in builds: