From dd90b97584b8b30f322ee1263fbea27ab1b10a52 Mon Sep 17 00:00:00 2001
From: jrcastro2 <jrcastro9515@gmail.com>
Date: Wed, 11 Sep 2024 13:37:31 +0200
Subject: [PATCH 01/76] fixtures: allow to load new vocab

---
 invenio_rdm_records/fixtures/vocabularies.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/invenio_rdm_records/fixtures/vocabularies.py b/invenio_rdm_records/fixtures/vocabularies.py
index cf7467baa..f82cc7e03 100644
--- a/invenio_rdm_records/fixtures/vocabularies.py
+++ b/invenio_rdm_records/fixtures/vocabularies.py
@@ -192,7 +192,8 @@ def load(self, reload=None):
             for v in VocabularyScheme.query.options(load_only("id", "parent_id")).all()
         ]
         self._loaded_vocabularies = set(v_type_ids + v_subtype_ids)
-        if reload:
+        # If it's not already loaded it means it's a new one
+        if reload and reload in self._loaded_vocabularies:
             self._loaded_vocabularies.remove(reload)
         # 1- Load from app_data_folder
         filepath = self._app_data_folder / self._filename

From db62df7e36671868b9ab4e7511cdb32d83222c71 Mon Sep 17 00:00:00 2001
From: jrcastro2 <jrcastro9515@gmail.com>
Date: Wed, 11 Sep 2024 13:37:52 +0200
Subject: [PATCH 02/76] release: v12.1.1

---
 CHANGES.rst                     | 5 +++++
 invenio_rdm_records/__init__.py | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index e086f87bc..b1f588f2d 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -11,6 +11,11 @@
 Changes
 =======
 
+Version v12.1.1 (released 2024-09-11)
+
+- resource: fix add record to community
+- controls: refactored isDisabled function
+
 Version v12.1.0 (released 2024-08-30)
 
 - config: added links for thumbnails (#1799)
diff --git a/invenio_rdm_records/__init__.py b/invenio_rdm_records/__init__.py
index fa764db52..62555abe9 100644
--- a/invenio_rdm_records/__init__.py
+++ b/invenio_rdm_records/__init__.py
@@ -10,6 +10,6 @@
 
 from .ext import InvenioRDMRecords
 
-__version__ = "12.1.0"
+__version__ = "12.1.1"
 
 __all__ = ("__version__", "InvenioRDMRecords")

From 9b69c9d9d2f32c9ff933425f9a5ccafb090e6371 Mon Sep 17 00:00:00 2001
From: David Eckhard <dabombeeer@gmail.com>
Date: Thu, 18 Apr 2024 12:02:55 +0200
Subject: [PATCH 03/76] tests: add serializer test for empty record

---
 tests/conftest.py                                   | 13 +++++++++++++
 tests/resources/serializers/test_csl_serializer.py  | 11 +++++++++++
 .../serializers/test_datacite_serializer.py         | 11 +++++++++++
 .../serializers/test_dublincore_serializer.py       | 11 +++++++++++
 .../serializers/test_schemaorg_serializer.py        | 13 +++++++++++++
 5 files changed, 59 insertions(+)

diff --git a/tests/conftest.py b/tests/conftest.py
index 29580de56..8a59f5b48 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -918,6 +918,19 @@ def minimal_record():
     }
 
 
+@pytest.fixture()
+def empty_record():
+    """Almost empty record data as dict coming from the external world."""
+    return {
+        "pids": {},
+        "access": {},
+        "files": {
+            "enabled": False,  # Most tests don't care about files
+        },
+        "metadata": {},
+    }
+
+
 @pytest.fixture()
 def minimal_restricted_record(minimal_record):
     """Data for restricted record."""
diff --git a/tests/resources/serializers/test_csl_serializer.py b/tests/resources/serializers/test_csl_serializer.py
index 97abe2378..319f63a3c 100644
--- a/tests/resources/serializers/test_csl_serializer.py
+++ b/tests/resources/serializers/test_csl_serializer.py
@@ -148,3 +148,14 @@ def test_citation_string_serializer_record(
             # in case of error, the response is JSON
             assert response.headers["content-type"] == "application/json"
             assert f"Citation string style not found." in body
+
+
+def test_citation_string_serializer_empty_record(running_app, empty_record):
+    """Test Citation String Serializer for an empty record."""
+
+    expected_data = {}
+
+    serializer = CSLJSONSchema()
+    serialized_record = serializer.dump(empty_record)
+
+    assert serialized_record == expected_data
diff --git a/tests/resources/serializers/test_datacite_serializer.py b/tests/resources/serializers/test_datacite_serializer.py
index 12a7432c9..967829aec 100644
--- a/tests/resources/serializers/test_datacite_serializer.py
+++ b/tests/resources/serializers/test_datacite_serializer.py
@@ -513,3 +513,14 @@ def test_datacite43_serializer_updated_date(running_app, full_modified_date_reco
 
     assert expected_dates == serialized_record["dates"]
     assert len(serialized_record["dates"]) == 3
+
+
+def test_datacite43_serializer_empty_record(running_app, empty_record):
+    """Test if the DataCite 4.3 JSON serializer handles an empty record."""
+
+    expected_data = {"schemaVersion": "http://datacite.org/schema/kernel-4"}
+
+    serializer = DataCite43JSONSerializer()
+    serialized_record = serializer.dump_obj(empty_record)
+
+    assert serialized_record == expected_data
diff --git a/tests/resources/serializers/test_dublincore_serializer.py b/tests/resources/serializers/test_dublincore_serializer.py
index 129914d4a..52c327d3d 100644
--- a/tests/resources/serializers/test_dublincore_serializer.py
+++ b/tests/resources/serializers/test_dublincore_serializer.py
@@ -90,6 +90,17 @@ def test_dublincorejson_serializer_minimal(running_app, updated_minimal_record):
     assert serialized_record == expected_data
 
 
+def test_dublincorejson_serializer_empty_record(running_app, empty_record):
+    """Test serializer to Dublin Core JSON with an empty record"""
+
+    expected_data = {}
+
+    serializer = DublinCoreJSONSerializer()
+    serialized_record = serializer.dump_obj(empty_record)
+
+    assert serialized_record == expected_data
+
+
 def test_vocabulary_type_error(running_app, updated_minimal_record):
     """Test error thrown on missing resource type."""
     updated_minimal_record["metadata"]["resource_type"]["id"] = "invalid"
diff --git a/tests/resources/serializers/test_schemaorg_serializer.py b/tests/resources/serializers/test_schemaorg_serializer.py
index b61122f64..6569a91ab 100644
--- a/tests/resources/serializers/test_schemaorg_serializer.py
+++ b/tests/resources/serializers/test_schemaorg_serializer.py
@@ -178,3 +178,16 @@ def test_schemaorg_serializer_minimal_record(running_app, minimal_record):
     serialized_record = serializer.dump_obj(minimal_record)
 
     assert serialized_record == expected_data
+
+
+def test_schemaorg_serializer_empty_record(running_app, empty_record):
+    """Test Schemaorg JSON-LD serializer with minimal record."""
+
+    expected_data = {
+        "@context": "http://schema.org",
+    }
+
+    serializer = SchemaorgJSONLDSerializer()
+    serialized_record = serializer.dump_obj(empty_record)
+
+    assert serialized_record == expected_data

From dc1e868ade727b9764fb119f9eb3d2e0dccb085d Mon Sep 17 00:00:00 2001
From: David Eckhard <dabombeeer@gmail.com>
Date: Thu, 18 Apr 2024 11:15:20 +0200
Subject: [PATCH 04/76] serializers: ensure values are not None before access

---
 .../resources/serializers/csl/schema.py       | 13 ++++-
 .../resources/serializers/datacite/schema.py  | 17 ++++--
 .../serializers/dublincore/schema.py          | 35 +++++++-----
 .../resources/serializers/marcxml/schema.py   | 54 ++++++++++---------
 .../resources/serializers/schemaorg/schema.py | 12 ++++-
 .../resources/serializers/schemas.py          |  8 ++-
 .../resources/serializers/ui/schema.py        | 20 ++++---
 7 files changed, 106 insertions(+), 53 deletions(-)

diff --git a/invenio_rdm_records/resources/serializers/csl/schema.py b/invenio_rdm_records/resources/serializers/csl/schema.py
index a0cce3056..e6d0f24ca 100644
--- a/invenio_rdm_records/resources/serializers/csl/schema.py
+++ b/invenio_rdm_records/resources/serializers/csl/schema.py
@@ -13,6 +13,7 @@
 from flask_resources.serializers import BaseSerializerSchema
 from marshmallow import Schema, fields, missing, pre_dump
 from marshmallow_utils.fields import SanitizedUnicode, StrippedHTML
+from pydash import py_
 
 from ..schemas import CommonFieldsMixin
 from ..utils import get_vocabulary_props
@@ -62,19 +63,27 @@ class CSLJSONSchema(BaseSerializerSchema, CommonFieldsMixin):
 
     def get_type(self, obj):
         """Get resource type."""
+        resource_type_id = py_.get(obj, "metadata.resource_type.id")
+        if not resource_type_id:
+            return missing
+
         props = get_vocabulary_props(
             "resourcetypes",
             [
                 "props.csl",
             ],
-            obj["metadata"]["resource_type"]["id"],
+            resource_type_id,
         )
         return props.get("csl", "article")  # article is CSL "Other"
 
     def get_issued(self, obj):
         """Get issued dates."""
+        publication_date = py_.get(obj, "metadata.publication_date")
+        if not publication_date:
+            return missing
+
         try:
-            parsed = parse_edtf(obj["metadata"].get("publication_date"))
+            parsed = parse_edtf(publication_date)
         except EDTFParseException:
             return missing
 
diff --git a/invenio_rdm_records/resources/serializers/datacite/schema.py b/invenio_rdm_records/resources/serializers/datacite/schema.py
index 17ed58d33..82d685e41 100644
--- a/invenio_rdm_records/resources/serializers/datacite/schema.py
+++ b/invenio_rdm_records/resources/serializers/datacite/schema.py
@@ -20,6 +20,7 @@
 from marshmallow import Schema, ValidationError, fields, missing, post_dump, validate
 from marshmallow_utils.fields import SanitizedUnicode
 from marshmallow_utils.html import strip_html
+from pydash import py_
 
 from ....proxies import current_rdm_records_service
 from ...serializers.ui.schema import current_default_locale
@@ -206,10 +207,14 @@ class DataCite43Schema(BaseSerializerSchema):
 
     def get_type(self, obj):
         """Get resource type."""
+        resource_type_id = py_.get(obj, "metadata.resource_type.id")
+        if not resource_type_id:
+            return missing
+
         props = get_vocabulary_props(
             "resourcetypes",
             ["props.datacite_general", "props.datacite_type"],
-            obj["metadata"]["resource_type"]["id"],
+            resource_type_id,
         )
         return {
             "resourceTypeGeneral": props.get("datacite_general", "Other"),
@@ -261,8 +266,11 @@ def get_descriptions(self, obj):
 
     def get_publication_year(self, obj):
         """Get publication year from edtf date."""
+        publication_date = py_.get(obj, "metadata.publication_date")
+        if not publication_date:
+            return missing
+
         try:
-            publication_date = obj["metadata"]["publication_date"]
             parsed_date = parse_edtf(publication_date)
             return str(parsed_date.lower_strict().tm_year)
         except ParseException:
@@ -274,7 +282,8 @@ def get_publication_year(self, obj):
 
     def get_dates(self, obj):
         """Get dates."""
-        dates = [{"date": obj["metadata"]["publication_date"], "dateType": "Issued"}]
+        pub_date = py_.get(obj, "metadata.publication_date")
+        dates = [{"date": pub_date, "dateType": "Issued"}] if pub_date else []
 
         updated = False
 
@@ -428,7 +437,7 @@ def get_related_identifiers(self, obj):
             if hasattr(obj, "parent"):
                 parent_record = obj.parent
             else:
-                parent_record = obj["parent"]
+                parent_record = obj.get("parent", {})
             parent_doi = parent_record.get("pids", {}).get("doi")
 
             if parent_doi:
diff --git a/invenio_rdm_records/resources/serializers/dublincore/schema.py b/invenio_rdm_records/resources/serializers/dublincore/schema.py
index 48cbc2745..541b762a7 100644
--- a/invenio_rdm_records/resources/serializers/dublincore/schema.py
+++ b/invenio_rdm_records/resources/serializers/dublincore/schema.py
@@ -12,6 +12,7 @@
 from flask import current_app
 from flask_resources.serializers import BaseSerializerSchema
 from marshmallow import fields, missing
+from pydash import py_
 
 from ..schemas import CommonFieldsMixin
 from ..ui.schema import current_default_locale
@@ -91,22 +92,22 @@ def get_relations(self, obj):
         # FIXME: Add after UI support is there
 
         # Alternate identifiers
-        for a in obj["metadata"].get("alternate_identifiers", []):
+        for a in obj.get("metadata", {}).get("alternate_identifiers", []):
             rels.append(self._transform_identifier(a["identifier"], a["scheme"]))
 
         # Related identifiers
-        for a in obj["metadata"].get("related_identifiers", []):
+        for a in obj.get("metadata", {}).get("related_identifiers", []):
             rels.append(self._transform_identifier(a["identifier"], a["scheme"]))
 
         # Communities
-        communities = obj["parent"].get("communities", {}).get("entries", [])
+        communities = obj.get("parent", {}).get("communities", {}).get("entries", [])
         for community in communities:
             slug = community["slug"]
             url = f"{current_app.config['SITE_UI_URL']}/communities/{slug}"
             rels.append(self._transform_identifier(url, "url"))
 
         # Parent doi
-        parent_pids = obj["parent"].get("pids", {})
+        parent_pids = obj.get("parent", {}).get("pids", {})
         for key, value in parent_pids.items():
             if key == "doi":
                 rels.append(self._transform_identifier(value["identifier"], key))
@@ -117,13 +118,14 @@ def get_rights(self, obj):
         """Get rights."""
         rights = []
 
-        access_right = obj["access"]["status"]
-        if access_right == "metadata-only":
-            access_right = "closed"
+        access_right = py_.get(obj, "access.status")
+        if access_right:
+            if access_right == "metadata-only":
+                access_right = "closed"
 
-        rights.append(f"info:eu-repo/semantics/{access_right}Access")
+            rights.append(f"info:eu-repo/semantics/{access_right}Access")
 
-        for right in obj["metadata"].get("rights", []):
+        for right in obj.get("metadata", {}).get("rights", []):
             rights.append(right.get("title").get(current_default_locale()))
             if right.get("id"):
                 license_url = right.get("props", {}).get("url")
@@ -138,9 +140,14 @@ def get_rights(self, obj):
 
     def get_dates(self, obj):
         """Get dates."""
-        dates = [obj["metadata"]["publication_date"]]
+        dates = []
 
-        if obj["access"]["status"] == "embargoed":
+        publication_date = py_.get(obj, "metadata.publication_date")
+        if publication_date:
+            dates.append(publication_date)
+
+        access_right = py_.get(obj, "access.status")
+        if access_right == "embargoed":
             date = obj["access"]["embargo"]["until"]
             dates.append(f"info:eu-repo/date/embargoEnd/{date}")
 
@@ -181,12 +188,16 @@ def get_subjects(self, obj):
 
     def get_types(self, obj):
         """Get resource type."""
+        resource_type_id = py_.get(obj, "metadata.resource_type.id")
+        if not resource_type_id:
+            return missing
+
         props = get_vocabulary_props(
             "resourcetypes",
             [
                 "props.eurepo",
             ],
-            obj["metadata"]["resource_type"]["id"],
+            resource_type_id,
         )
         t = props.get("eurepo")
         return [t] if t else missing
diff --git a/invenio_rdm_records/resources/serializers/marcxml/schema.py b/invenio_rdm_records/resources/serializers/marcxml/schema.py
index f2e70cebe..4af87ea17 100644
--- a/invenio_rdm_records/resources/serializers/marcxml/schema.py
+++ b/invenio_rdm_records/resources/serializers/marcxml/schema.py
@@ -14,6 +14,7 @@
 from flask_resources.serializers import BaseSerializerSchema
 from marshmallow import fields, missing
 from marshmallow_utils.html import sanitize_unicode
+from pydash import py_
 
 from ..schemas import CommonFieldsMixin
 from ..ui.schema import current_default_locale
@@ -491,30 +492,33 @@ def get_types_and_communities(self, obj):
         if communities:
             slugs = [community.get("slug") for community in communities]
             output += [{"a": f"user-{slug}"} for slug in slugs]
-        props = get_vocabulary_props(
-            "resourcetypes",
-            [
-                "props.eurepo",
-                "props.marc21_type",
-                "props.marc21_subtype",
-            ],
-            obj["metadata"]["resource_type"]["id"],
-        )
-        props_eurepo = props.get("eurepo")
-        if props_eurepo:
-            eurepo = {"a": props_eurepo}
-            output.append(eurepo)
-
-        resource_types = {}
-
-        resource_type = props.get("marc21_type")
-        if resource_type:
-            resource_types["a"] = resource_type
-        resource_subtype = props.get("marc21_subtype")
-        if resource_subtype:
-            resource_types["b"] = resource_subtype
-
-        if resource_types:
-            output.append(resource_types)
+
+        resource_type_id = py_.get(obj, "metadata.resource_type.id")
+        if resource_type_id:
+            props = get_vocabulary_props(
+                "resourcetypes",
+                [
+                    "props.eurepo",
+                    "props.marc21_type",
+                    "props.marc21_subtype",
+                ],
+                resource_type_id,
+            )
+            props_eurepo = props.get("eurepo")
+            if props_eurepo:
+                eurepo = {"a": props_eurepo}
+                output.append(eurepo)
+
+            resource_types = {}
+
+            resource_type = props.get("marc21_type")
+            if resource_type:
+                resource_types["a"] = resource_type
+            resource_subtype = props.get("marc21_subtype")
+            if resource_subtype:
+                resource_types["b"] = resource_subtype
+
+            if resource_types:
+                output.append(resource_types)
 
         return output or missing
diff --git a/invenio_rdm_records/resources/serializers/schemaorg/schema.py b/invenio_rdm_records/resources/serializers/schemaorg/schema.py
index 15d25f1f5..af1cdcad0 100644
--- a/invenio_rdm_records/resources/serializers/schemaorg/schema.py
+++ b/invenio_rdm_records/resources/serializers/schemaorg/schema.py
@@ -209,10 +209,14 @@ def get_id(self, obj):
 
     def get_type(self, obj):
         """Get type. Use the vocabulary service to get the schema.org type."""
+        resource_type_id = py_.get(obj, "metadata.resource_type.id")
+        if not resource_type_id:
+            return missing
+
         props = get_vocabulary_props(
             "resourcetypes",
             ["props.schema.org"],
-            py_.get(obj, "metadata.resource_type.id"),
+            resource_type_id,
         )
         ret = props.get("schema.org", "https://schema.org/CreativeWork")
         return ret
@@ -232,8 +236,12 @@ def get_format(self, obj):
 
     def get_publication_date(self, obj):
         """Get publication date."""
+        publication_date = py_.get(obj, "metadata.publication_date")
+        if not publication_date:
+            return missing
+
         try:
-            parsed_date = parse_edtf(py_.get(obj, "metadata.publication_date"))
+            parsed_date = parse_edtf(publication_date)
         except ParseException:
             return missing
 
diff --git a/invenio_rdm_records/resources/serializers/schemas.py b/invenio_rdm_records/resources/serializers/schemas.py
index 6d6b661b4..0a7034d56 100644
--- a/invenio_rdm_records/resources/serializers/schemas.py
+++ b/invenio_rdm_records/resources/serializers/schemas.py
@@ -8,6 +8,7 @@
 """Base parsing functions for the various serializers."""
 
 from marshmallow import missing
+from pydash import py_
 
 
 class CommonFieldsMixin:
@@ -55,7 +56,8 @@ def get_locations(self, obj):
 
     def get_titles(self, obj):
         """Get titles."""
-        return [obj["metadata"]["title"]]
+        title = py_.get(obj, "metadata.title")
+        return [title] if title else missing
 
     def get_identifiers(self, obj):
         """Get identifiers."""
@@ -67,7 +69,9 @@ def get_identifiers(self, obj):
 
     def get_creators(self, obj):
         """Get creators."""
-        return [c["person_or_org"]["name"] for c in obj["metadata"].get("creators", [])]
+        return [
+            c["person_or_org"]["name"] for c in obj["metadata"].get("creators", [])
+        ] or missing
 
     def get_publishers(self, obj):
         """Get publishers."""
diff --git a/invenio_rdm_records/resources/serializers/ui/schema.py b/invenio_rdm_records/resources/serializers/ui/schema.py
index 8f6fa6557..05395550a 100644
--- a/invenio_rdm_records/resources/serializers/ui/schema.py
+++ b/invenio_rdm_records/resources/serializers/ui/schema.py
@@ -14,6 +14,7 @@
 from functools import partial
 
 from babel_edtf import parse_edtf
+from edtf.parser.grammar import ParseException
 from flask import current_app, g
 from flask_resources import BaseObjectSchema
 from invenio_communities.communities.resources.ui_schema import (
@@ -30,6 +31,7 @@
 from marshmallow_utils.fields import FormatEDTF as FormatEDTF_
 from marshmallow_utils.fields import SanitizedHTML, SanitizedUnicode, StrippedHTML
 from marshmallow_utils.fields.babel import gettext_from_dict
+from pyparsing import ParseException
 
 from .fields import AccessStatusField
 
@@ -218,12 +220,18 @@ def _format_journal(journal, publication_date):
         journal_issue = journal.get("issue")
         journal_volume = journal.get("volume")
         journal_pages = journal.get("pages")
-        publication_date_edtf = (
-            parse_edtf(publication_date).lower_strict() if publication_date else None
-        )
-        publication_date_formatted = (
-            f"{publication_date_edtf.tm_year}" if publication_date_edtf else None
-        )
+
+        try:
+            publication_date_edtf = (
+                parse_edtf(publication_date).lower_strict()
+                if publication_date
+                else None
+            )
+            publication_date_formatted = (
+                f"{publication_date_edtf.tm_year}" if publication_date_edtf else None
+            )
+        except ParseException:
+            publication_date_formatted = None
 
         title = f"{journal_title}" if journal_title else None
         vol_issue = f"{journal_volume}" if journal_volume else None

From fa829a10002069b1d4cd1641e8413c59a48861d0 Mon Sep 17 00:00:00 2001
From: Sam Arbid <samaa2@kth.se>
Date: Sat, 14 Sep 2024 16:43:57 +0200
Subject: [PATCH 05/76] UI: Handle empty file uploads in FileUploader

* Display a warning for empty files, indicating they won't be included and listing file names.
* Feature controlled by `records-resources-allow-empty-files` config value.
* Continue uploading other files while showing the warning message.
---
 .../fields/FileUploader/FileUploader.js       | 45 +++++++++++++++++--
 1 file changed, 42 insertions(+), 3 deletions(-)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/FileUploader/FileUploader.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/FileUploader/FileUploader.js
index 1fd582bab..d829b63b8 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/FileUploader/FileUploader.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/FileUploader/FileUploader.js
@@ -3,6 +3,7 @@
 // Copyright (C) 2020-2022 Northwestern University.
 // Copyright (C)      2022 Graz University of Technology.
 // Copyright (C)      2022 TU Wien.
+// Copyright (C)      2024 KTH Royal Institute of Technology.
 //
 // Invenio-RDM-Records is free software; you can redistribute it and/or modify it
 // under the terms of the MIT License; see LICENSE file for more details.
@@ -40,6 +41,7 @@ export const FileUploaderComponent = ({
   isFileImportInProgress,
   decimalSizeDisplay,
   filesLocked,
+  allowEmptyFiles,
   ...uiProps
 }) => {
   // We extract the working copy of the draft stored as `values` in formik
@@ -80,10 +82,28 @@ export const FileUploaderComponent = ({
       const maxFileStorageReached = filesSize + acceptedFilesSize > quota.maxStorage;
 
       const filesNames = _map(filesList, "name");
-      const duplicateFiles = acceptedFiles.filter((acceptedFile) =>
-        filesNames.includes(acceptedFile.name)
+      const filesNamesSet = new Set(filesNames);
+
+      const { duplicateFiles, emptyFiles, nonEmptyFiles } = acceptedFiles.reduce(
+        (accumulators, file) => {
+          if (filesNamesSet.has(file.name)) {
+            accumulators.duplicateFiles.push(file);
+          }
+
+          if (file.size === 0) {
+            accumulators.emptyFiles.push(file);
+          } else {
+            accumulators.nonEmptyFiles.push(file);
+          }
+
+          return accumulators;
+        },
+        { duplicateFiles: [], emptyFiles: [], nonEmptyFiles: [] }
       );
 
+      const hasEmptyFiles = !_isEmpty(emptyFiles);
+      const hasNonEmptyFiles = !_isEmpty(nonEmptyFiles);
+
       if (maxFileNumberReached) {
         setWarningMsg(
           <div className="content">
@@ -130,7 +150,24 @@ export const FileUploaderComponent = ({
           </div>
         );
       } else {
-        uploadFiles(formikDraft, acceptedFiles);
+        if (!allowEmptyFiles && hasEmptyFiles) {
+          setWarningMsg(
+            <div className="content">
+              <Message
+                warning
+                icon="warning circle"
+                header={i18next.t("Could not upload all files.")}
+                content={i18next.t("Empty files were skipped.")}
+                list={_map(emptyFiles, "name")}
+              />
+            </div>
+          );
+        }
+
+        // Proceed with uploading the non-empty files or all files if empty files are allowed
+        if (allowEmptyFiles || hasNonEmptyFiles) {
+          uploadFiles(formikDraft, allowEmptyFiles ? acceptedFiles : nonEmptyFiles);
+        }
       }
     },
     multiple: true,
@@ -348,6 +385,7 @@ FileUploaderComponent.propTypes = {
   decimalSizeDisplay: PropTypes.bool,
   filesLocked: PropTypes.bool,
   permissions: PropTypes.object,
+  allowEmptyFiles: PropTypes.bool,
 };
 
 FileUploaderComponent.defaultProps = {
@@ -369,4 +407,5 @@ FileUploaderComponent.defaultProps = {
   importButtonText: i18next.t("Import files"),
   decimalSizeDisplay: true,
   filesLocked: false,
+  allowEmptyFiles: true,
 };

From 6d226ce7865e0deb0a3e23fe55a641dfb033567b Mon Sep 17 00:00:00 2001
From: Sam Arbid <samaa2@kth.se>
Date: Sat, 14 Sep 2024 22:43:36 +0200
Subject: [PATCH 06/76] Refactor: Enhance Warnings Handling

* Add the missing translation tags.
* Refactor the logic to display multiple warning messages, while allowing other files to upload, except for case where exceeding the file size limit.
---
 .../fields/FileUploader/FileUploader.js       | 66 +++++++++++--------
 1 file changed, 37 insertions(+), 29 deletions(-)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/FileUploader/FileUploader.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/FileUploader/FileUploader.js
index d829b63b8..eb787e306 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/FileUploader/FileUploader.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/FileUploader/FileUploader.js
@@ -88,9 +88,7 @@ export const FileUploaderComponent = ({
         (accumulators, file) => {
           if (filesNamesSet.has(file.name)) {
             accumulators.duplicateFiles.push(file);
-          }
-
-          if (file.size === 0) {
+          } else if (file.size === 0) {
             accumulators.emptyFiles.push(file);
           } else {
             accumulators.nonEmptyFiles.push(file);
@@ -102,7 +100,7 @@ export const FileUploaderComponent = ({
       );
 
       const hasEmptyFiles = !_isEmpty(emptyFiles);
-      const hasNonEmptyFiles = !_isEmpty(nonEmptyFiles);
+      const hasDuplicateFiles = !_isEmpty(duplicateFiles);
 
       if (maxFileNumberReached) {
         setWarningMsg(
@@ -110,10 +108,12 @@ export const FileUploaderComponent = ({
             <Message
               warning
               icon="warning circle"
-              header="Could not upload files."
-              content={`Uploading the selected files would result in ${
-                filesList.length + acceptedFiles.length
-              } files (max.${quota.maxFiles})`}
+              header={i18next.t("Could not upload files.")}
+              content={i18next.t(
+                `Uploading the selected files would result in ${
+                  filesList.length + acceptedFiles.length
+                } files (max.${quota.maxFiles})`
+              )}
             />
           </div>
         );
@@ -123,7 +123,7 @@ export const FileUploaderComponent = ({
             <Message
               warning
               icon="warning circle"
-              header="Could not upload files."
+              header={i18next.t("Could not upload files.")}
               content={
                 <>
                   {i18next.t("Uploading the selected files would result in")}{" "}
@@ -138,35 +138,43 @@ export const FileUploaderComponent = ({
             />
           </div>
         );
-      } else if (!_isEmpty(duplicateFiles)) {
-        setWarningMsg(
-          <div className="content">
+      } else {
+        let warnings = [];
+
+        if (hasDuplicateFiles) {
+          warnings.push(
             <Message
               warning
               icon="warning circle"
-              header={i18next.t(`The following files already exist`)}
+              header={i18next.t("The following files already exist")}
               list={_map(duplicateFiles, "name")}
             />
-          </div>
-        );
-      } else {
+          );
+        }
+
         if (!allowEmptyFiles && hasEmptyFiles) {
-          setWarningMsg(
-            <div className="content">
-              <Message
-                warning
-                icon="warning circle"
-                header={i18next.t("Could not upload all files.")}
-                content={i18next.t("Empty files were skipped.")}
-                list={_map(emptyFiles, "name")}
-              />
-            </div>
+          warnings.push(
+            <Message
+              warning
+              icon="warning circle"
+              header={i18next.t("Could not upload all files.")}
+              content={i18next.t("Empty files were skipped.")}
+              list={_map(emptyFiles, "name")}
+            />
           );
         }
 
-        // Proceed with uploading the non-empty files or all files if empty files are allowed
-        if (allowEmptyFiles || hasNonEmptyFiles) {
-          uploadFiles(formikDraft, allowEmptyFiles ? acceptedFiles : nonEmptyFiles);
+        if (!_isEmpty(warnings)) {
+          setWarningMsg(<div className="content">{warnings}</div>);
+        }
+
+        const filesToUpload = allowEmptyFiles
+          ? [...nonEmptyFiles, ...emptyFiles]
+          : nonEmptyFiles;
+
+        // Proceed with uploading files if there are any to upload
+        if (!_isEmpty(filesToUpload)) {
+          uploadFiles(formikDraft, filesToUpload);
         }
       }
     },

From ea9aceb7a7072c3edad9777eefc8ff692b1ffbe3 Mon Sep 17 00:00:00 2001
From: David Eckhard <dabombeeer@gmail.com>
Date: Wed, 10 Jul 2024 13:16:29 +0200
Subject: [PATCH 07/76] deposit: check permission and set disable tooltip for
 publish button

---
 .../controls/PublishButton/PublishButton.js   | 64 +++++++++++++++----
 .../SubmitReviewOrPublishButton.js            |  5 +-
 2 files changed, 54 insertions(+), 15 deletions(-)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/controls/PublishButton/PublishButton.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/controls/PublishButton/PublishButton.js
index 71dafb704..e92293e52 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/controls/PublishButton/PublishButton.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/controls/PublishButton/PublishButton.js
@@ -1,6 +1,7 @@
 // This file is part of Invenio-RDM-Records
 // Copyright (C) 2020-2023 CERN.
 // Copyright (C) 2020-2022 Northwestern University.
+// Copyright (C)      2024 Graz University of Technology.
 //
 // Invenio-RDM-Records is free software; you can redistribute it and/or modify it
 // under the terms of the MIT License; see LICENSE file for more details.
@@ -12,7 +13,7 @@ import _omit from "lodash/omit";
 import PropTypes from "prop-types";
 import React, { Component } from "react";
 import { connect } from "react-redux";
-import { Button, Icon, Message, Modal } from "semantic-ui-react";
+import { Button, Icon, Message, Modal, Popup } from "semantic-ui-react";
 import {
   DepositFormSubmitActions,
   DepositFormSubmitContext,
@@ -40,8 +41,8 @@ class PublishButtonComponent extends Component {
     this.closeConfirmModal();
   };
 
-  isDisabled = (values, isSubmitting, filesState) => {
-    if (isSubmitting) {
+  isDisabled = (values, isSubmitting, filesState, hasPublishPermission) => {
+    if (isSubmitting || !hasPublishPermission) {
       return true;
     }
 
@@ -59,6 +60,19 @@ class PublishButtonComponent extends Component {
     return !allCompleted;
   };
 
+  hasPermission = (permissions) => {
+    return permissions.can_publish;
+  };
+
+  getDisabledButtonPopupText = (hasPublishPermission) => {
+    let text = i18next.t("Required fields are missing");
+
+    if (!hasPublishPermission) {
+      text = i18next.t("You don't have permission to publish");
+    }
+    return text;
+  };
+
   render() {
     const {
       actionState,
@@ -67,6 +81,7 @@ class PublishButtonComponent extends Component {
       publishWithoutCommunity,
       formik,
       publishModalExtraContent,
+      permissions,
       ...ui
     } = this.props;
     const { isConfirmModalOpen } = this.state;
@@ -74,19 +89,38 @@ class PublishButtonComponent extends Component {
 
     const uiProps = _omit(ui, ["dispatch"]);
 
+    const hasPublishPermission = this.hasPermission(permissions);
+    const publishDisabled = this.isDisabled(
+      values,
+      isSubmitting,
+      filesState,
+      hasPublishPermission
+    );
+
+    // only used when button is disabled
+    const popupText = this.getDisabledButtonPopupText(hasPublishPermission);
+
     return (
       <>
-        <Button
-          disabled={this.isDisabled(values, isSubmitting, filesState)}
-          name="publish"
-          onClick={this.openConfirmModal}
-          positive
-          icon="upload"
-          loading={isSubmitting && actionState === DRAFT_PUBLISH_STARTED}
-          labelPosition="left"
-          content={buttonLabel}
-          {...uiProps}
-          type="button" // needed so the formik form doesn't handle it as submit button i.e enable HTML validation on required input fields
+        <Popup
+          disabled={!publishDisabled}
+          content={popupText}
+          trigger={
+            <span>
+              <Button
+                disabled={publishDisabled}
+                name="publish"
+                onClick={this.openConfirmModal}
+                positive
+                icon="upload"
+                loading={isSubmitting && actionState === DRAFT_PUBLISH_STARTED}
+                labelPosition="left"
+                content={buttonLabel}
+                {...uiProps}
+                type="button" // needed so the formik form doesn't handle it as submit button i.e enable HTML validation on required input fields
+              />
+            </span>
+          }
         />
         {isConfirmModalOpen && (
           <Modal
@@ -139,6 +173,7 @@ PublishButtonComponent.propTypes = {
   formik: PropTypes.object.isRequired,
   publishModalExtraContent: PropTypes.string,
   filesState: PropTypes.object,
+  permissions: PropTypes.object.isRequired,
 };
 
 PublishButtonComponent.defaultProps = {
@@ -153,6 +188,7 @@ const mapStateToProps = (state) => ({
   actionState: state.deposit.actionState,
   publishModalExtraContent: state.deposit.config.publish_modal_extra,
   filesState: state.files,
+  permissions: state.deposit.permissions,
 });
 
 export const PublishButton = connect(
diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/controls/PublishButton/SubmitReviewOrPublishButton.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/controls/PublishButton/SubmitReviewOrPublishButton.js
index 6d7a0bfdb..ed9612b24 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/controls/PublishButton/SubmitReviewOrPublishButton.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/controls/PublishButton/SubmitReviewOrPublishButton.js
@@ -29,6 +29,7 @@ class SubmitReviewOrPublishComponent extends Component {
       showDirectPublishButton,
       showSubmitForReviewButton,
       record,
+      permissions,
       ...ui
     } = this.props;
     const { modalOpen } = this.state;
@@ -61,13 +62,14 @@ class SubmitReviewOrPublishComponent extends Component {
           />
           <PublishButton
             buttonLabel={i18next.t("Publish without community")}
+            permissions={permissions}
             publishWithoutCommunity
             {...ui}
           />
         </>
       );
     } else {
-      result = <PublishButton {...ui} />;
+      result = <PublishButton permissions={permissions} {...ui} />;
     }
     return result;
   }
@@ -80,6 +82,7 @@ SubmitReviewOrPublishComponent.propTypes = {
   showDirectPublishButton: PropTypes.bool.isRequired,
   showSubmitForReviewButton: PropTypes.bool.isRequired,
   record: PropTypes.object.isRequired,
+  permissions: PropTypes.object.isRequired,
 };
 
 SubmitReviewOrPublishComponent.defaultProps = {

From c1a8ed36ad24e7287e266fef0df66173ed1781f0 Mon Sep 17 00:00:00 2001
From: Nicola Tarocco <ntarocco@gmail.com>
Date: Thu, 19 Sep 2024 11:29:36 +0200
Subject: [PATCH 08/76] release: v12.2.1

---
 CHANGES.rst                     | 7 +++++++
 invenio_rdm_records/__init__.py | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index b1f588f2d..18bcad75f 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -11,6 +11,13 @@
 Changes
 =======
 
+Version v12.2.1 (released 2024-09-19)
+
+- file upload: better handling of errors when uploading empty files
+- serializers: ensure that the vocab id is set before performing a look up
+- deposit: take into account the can_publish permission to control when the
+           Publish button should be enabled or disabled
+
 Version v12.1.1 (released 2024-09-11)
 
 - resource: fix add record to community
diff --git a/invenio_rdm_records/__init__.py b/invenio_rdm_records/__init__.py
index 62555abe9..402d03fbf 100644
--- a/invenio_rdm_records/__init__.py
+++ b/invenio_rdm_records/__init__.py
@@ -10,6 +10,6 @@
 
 from .ext import InvenioRDMRecords
 
-__version__ = "12.1.1"
+__version__ = "12.2.1"
 
 __all__ = ("__version__", "InvenioRDMRecords")

From 02edb6fbf2cae584ac09dca7dd02a016178788c7 Mon Sep 17 00:00:00 2001
From: Christoph Ladurner <christoph.ladurner@tugraz.at>
Date: Tue, 25 Jun 2024 13:20:36 +0200
Subject: [PATCH 09/76] fix: add compatibility layer to move to flask>=3

* flask-sqlalchemy moved pagination.

* this change has been added to have a smooth migration to flask>=3.0.0
  without creating a hard cut and major versions release.
---
 invenio_rdm_records/oaiserver/services/services.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/invenio_rdm_records/oaiserver/services/services.py b/invenio_rdm_records/oaiserver/services/services.py
index 392a63db9..516db9093 100644
--- a/invenio_rdm_records/oaiserver/services/services.py
+++ b/invenio_rdm_records/oaiserver/services/services.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2022-2023 Graz University of Technology.
+# Copyright (C) 2022-2024 Graz University of Technology.
 #
 # Invenio-RDM-Records is free software; you can redistribute it and/or modify
 # it under the terms of the MIT License; see LICENSE file for more details.
@@ -10,7 +10,6 @@
 import re
 
 from flask import current_app
-from flask_sqlalchemy import Pagination
 from invenio_db import db
 from invenio_i18n import lazy_gettext as _
 from invenio_oaiserver.models import OAISet
@@ -32,6 +31,13 @@
 )
 from .uow import OAISetCommitOp, OAISetDeleteOp
 
+try:
+    # flask_sqlalchemy<3.0.0
+    from flask_sqlalchemy import Pagination
+except ImportError:
+    # flask_sqlalchemy>=3.0.0
+    from flask_sqlalchemy.pagination import Pagination
+
 
 class OAIPMHServerService(Service):
     """OAI-PMH service."""

From 22f345506b35870ad29096a14b2760f3e02656ee Mon Sep 17 00:00:00 2001
From: Christoph Ladurner <christoph.ladurner@tugraz.at>
Date: Mon, 23 Sep 2024 22:27:04 +0200
Subject: [PATCH 10/76] fix: remove DeprecationWarning for sqlalchemy

* RemovedIn20Warning: Using strings to indicate column or relationship
  paths in loader options is deprecated and will be removed in SQLAlchemy
  2.0. Please use the class-bound attribute directly. (Background on
  SQLAlchemy 2.0

* version 1.2 of sqlalchemy introduced a new form of load_only handling.
  https://docs.sqlalchemy.org/en/14/orm/loading_columns.html#sqlalchemy.orm.load_only

* this change is forward compatible with version >= 2.0 of sqlalchemy
---
 invenio_rdm_records/fixtures/vocabularies.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/invenio_rdm_records/fixtures/vocabularies.py b/invenio_rdm_records/fixtures/vocabularies.py
index f82cc7e03..a802070fc 100644
--- a/invenio_rdm_records/fixtures/vocabularies.py
+++ b/invenio_rdm_records/fixtures/vocabularies.py
@@ -4,6 +4,7 @@
 # Copyright (C) 2021-2022 Northwestern University.
 # Copyright (C) 2024 TU Wien.
 # Copyright (C) 2024 KTH Royal Institute of Technology.
+# Copyright (C) 2024 Graz University of Technology.
 #
 # Invenio-RDM-Records is free software; you can redistribute it and/or modify
 # it under the terms of the MIT License; see LICENSE file for more details.
@@ -186,10 +187,15 @@ def load(self, reload=None):
         Fixtures found later are ignored.
         """
         # Prime with existing (sub)vocabularies
-        v_type_ids = [v.id for v in VocabularyType.query.options(load_only("id")).all()]
+        v_type_ids = [
+            v.id
+            for v in VocabularyType.query.options(load_only(VocabularyType.id)).all()
+        ]
         v_subtype_ids = [
             f"{v.parent_id}.{v.id}"
-            for v in VocabularyScheme.query.options(load_only("id", "parent_id")).all()
+            for v in VocabularyScheme.query.options(
+                load_only(VocabularyScheme.id, VocabularyScheme.parent_id)
+            ).all()
         ]
         self._loaded_vocabularies = set(v_type_ids + v_subtype_ids)
         # If it's not already loaded it means it's a new one

From be02ccd164b8156ea3157cdc747619b8d9d0349b Mon Sep 17 00:00:00 2001
From: Nicola Tarocco <ntarocco@gmail.com>
Date: Thu, 26 Sep 2024 14:05:25 +0200
Subject: [PATCH 11/76] Revert "deposit: check permission and set disable
 tooltip for publish button"

This reverts commit ea9aceb7a7072c3edad9777eefc8ff692b1ffbe3.
---
 .../controls/PublishButton/PublishButton.js   | 64 ++++---------------
 .../SubmitReviewOrPublishButton.js            |  5 +-
 2 files changed, 15 insertions(+), 54 deletions(-)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/controls/PublishButton/PublishButton.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/controls/PublishButton/PublishButton.js
index e92293e52..71dafb704 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/controls/PublishButton/PublishButton.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/controls/PublishButton/PublishButton.js
@@ -1,7 +1,6 @@
 // This file is part of Invenio-RDM-Records
 // Copyright (C) 2020-2023 CERN.
 // Copyright (C) 2020-2022 Northwestern University.
-// Copyright (C)      2024 Graz University of Technology.
 //
 // Invenio-RDM-Records is free software; you can redistribute it and/or modify it
 // under the terms of the MIT License; see LICENSE file for more details.
@@ -13,7 +12,7 @@ import _omit from "lodash/omit";
 import PropTypes from "prop-types";
 import React, { Component } from "react";
 import { connect } from "react-redux";
-import { Button, Icon, Message, Modal, Popup } from "semantic-ui-react";
+import { Button, Icon, Message, Modal } from "semantic-ui-react";
 import {
   DepositFormSubmitActions,
   DepositFormSubmitContext,
@@ -41,8 +40,8 @@ class PublishButtonComponent extends Component {
     this.closeConfirmModal();
   };
 
-  isDisabled = (values, isSubmitting, filesState, hasPublishPermission) => {
-    if (isSubmitting || !hasPublishPermission) {
+  isDisabled = (values, isSubmitting, filesState) => {
+    if (isSubmitting) {
       return true;
     }
 
@@ -60,19 +59,6 @@ class PublishButtonComponent extends Component {
     return !allCompleted;
   };
 
-  hasPermission = (permissions) => {
-    return permissions.can_publish;
-  };
-
-  getDisabledButtonPopupText = (hasPublishPermission) => {
-    let text = i18next.t("Required fields are missing");
-
-    if (!hasPublishPermission) {
-      text = i18next.t("You don't have permission to publish");
-    }
-    return text;
-  };
-
   render() {
     const {
       actionState,
@@ -81,7 +67,6 @@ class PublishButtonComponent extends Component {
       publishWithoutCommunity,
       formik,
       publishModalExtraContent,
-      permissions,
       ...ui
     } = this.props;
     const { isConfirmModalOpen } = this.state;
@@ -89,38 +74,19 @@ class PublishButtonComponent extends Component {
 
     const uiProps = _omit(ui, ["dispatch"]);
 
-    const hasPublishPermission = this.hasPermission(permissions);
-    const publishDisabled = this.isDisabled(
-      values,
-      isSubmitting,
-      filesState,
-      hasPublishPermission
-    );
-
-    // only used when button is disabled
-    const popupText = this.getDisabledButtonPopupText(hasPublishPermission);
-
     return (
       <>
-        <Popup
-          disabled={!publishDisabled}
-          content={popupText}
-          trigger={
-            <span>
-              <Button
-                disabled={publishDisabled}
-                name="publish"
-                onClick={this.openConfirmModal}
-                positive
-                icon="upload"
-                loading={isSubmitting && actionState === DRAFT_PUBLISH_STARTED}
-                labelPosition="left"
-                content={buttonLabel}
-                {...uiProps}
-                type="button" // needed so the formik form doesn't handle it as submit button i.e enable HTML validation on required input fields
-              />
-            </span>
-          }
+        <Button
+          disabled={this.isDisabled(values, isSubmitting, filesState)}
+          name="publish"
+          onClick={this.openConfirmModal}
+          positive
+          icon="upload"
+          loading={isSubmitting && actionState === DRAFT_PUBLISH_STARTED}
+          labelPosition="left"
+          content={buttonLabel}
+          {...uiProps}
+          type="button" // needed so the formik form doesn't handle it as submit button i.e enable HTML validation on required input fields
         />
         {isConfirmModalOpen && (
           <Modal
@@ -173,7 +139,6 @@ PublishButtonComponent.propTypes = {
   formik: PropTypes.object.isRequired,
   publishModalExtraContent: PropTypes.string,
   filesState: PropTypes.object,
-  permissions: PropTypes.object.isRequired,
 };
 
 PublishButtonComponent.defaultProps = {
@@ -188,7 +153,6 @@ const mapStateToProps = (state) => ({
   actionState: state.deposit.actionState,
   publishModalExtraContent: state.deposit.config.publish_modal_extra,
   filesState: state.files,
-  permissions: state.deposit.permissions,
 });
 
 export const PublishButton = connect(
diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/controls/PublishButton/SubmitReviewOrPublishButton.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/controls/PublishButton/SubmitReviewOrPublishButton.js
index ed9612b24..6d7a0bfdb 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/controls/PublishButton/SubmitReviewOrPublishButton.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/controls/PublishButton/SubmitReviewOrPublishButton.js
@@ -29,7 +29,6 @@ class SubmitReviewOrPublishComponent extends Component {
       showDirectPublishButton,
       showSubmitForReviewButton,
       record,
-      permissions,
       ...ui
     } = this.props;
     const { modalOpen } = this.state;
@@ -62,14 +61,13 @@ class SubmitReviewOrPublishComponent extends Component {
           />
           <PublishButton
             buttonLabel={i18next.t("Publish without community")}
-            permissions={permissions}
             publishWithoutCommunity
             {...ui}
           />
         </>
       );
     } else {
-      result = <PublishButton permissions={permissions} {...ui} />;
+      result = <PublishButton {...ui} />;
     }
     return result;
   }
@@ -82,7 +80,6 @@ SubmitReviewOrPublishComponent.propTypes = {
   showDirectPublishButton: PropTypes.bool.isRequired,
   showSubmitForReviewButton: PropTypes.bool.isRequired,
   record: PropTypes.object.isRequired,
-  permissions: PropTypes.object.isRequired,
 };
 
 SubmitReviewOrPublishComponent.defaultProps = {

From 34d67a41d132df44c7f827bcd838954fc60a3fdf Mon Sep 17 00:00:00 2001
From: Sam Arbid <36583694+Samk13@users.noreply.github.com>
Date: Mon, 30 Sep 2024 11:23:29 +0200
Subject: [PATCH 12/76] Record: Improve handling of draft PID in
 RecordCommunitiesService (#1822)

* Records: catch PIDUnregistered error
* Add draft class to RecordCommunitiesService
* Closes https://github.com/inveniosoftware/invenio-app-rdm/issues/2857
---
 .../services/communities/service.py               | 15 ++++++++++++---
 invenio_rdm_records/services/config.py            |  4 +++-
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/invenio_rdm_records/services/communities/service.py b/invenio_rdm_records/services/communities/service.py
index d8d4d7747..63c17cb22 100644
--- a/invenio_rdm_records/services/communities/service.py
+++ b/invenio_rdm_records/services/communities/service.py
@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2023-2024 CERN.
-# Copyright (C) 2024 Graz University of Technology.
+# Copyright (C) 2024      Graz University of Technology.
+# Copyright (C) 2024      KTH Royal Institute of Technology.
 #
 # Invenio-RDM-Records is free software; you can redistribute it and/or modify
 # it under the terms of the MIT License; see LICENSE file for more details.
@@ -17,7 +18,7 @@
 )
 from invenio_i18n import lazy_gettext as _
 from invenio_notifications.services.uow import NotificationOp
-from invenio_pidstore.errors import PIDDoesNotExistError
+from invenio_pidstore.errors import PIDDoesNotExistError, PIDUnregistered
 from invenio_records_resources.services import (
     RecordIndexerMixin,
     Service,
@@ -67,6 +68,11 @@ def record_cls(self):
         """Factory for creating a record class."""
         return self.config.record_cls
 
+    @property
+    def draft_cls(self):
+        """Factory for creating a draft class."""
+        return self.config.draft_cls
+
     def _exists(self, community_id, record):
         """Return the request id if an open request already exists, else None."""
         results = current_requests_service.search(
@@ -264,7 +270,10 @@ def search(
         **kwargs,
     ):
         """Search for record's communities."""
-        record = self.record_cls.pid.resolve(id_)
+        try:
+            record = self.record_cls.pid.resolve(id_)
+        except PIDUnregistered:
+            record = self.draft_cls.pid.resolve(id_, registered_only=False)
         self.require_permission(identity, "read", record=record)
 
         communities_ids = record.parent.communities.ids
diff --git a/invenio_rdm_records/services/config.py b/invenio_rdm_records/services/config.py
index c387b483d..7a9768c37 100644
--- a/invenio_rdm_records/services/config.py
+++ b/invenio_rdm_records/services/config.py
@@ -4,7 +4,8 @@
 # Copyright (C) 2020-2021 Northwestern University.
 # Copyright (C)      2021 TU Wien.
 # Copyright (C) 2021-2023 Graz University of Technology.
-# Copyright (C) 2022 Universität Hamburg
+# Copyright (C) 2022      Universität Hamburg
+# Copyright (C) 2024      KTH Royal Institute of Technology.
 #
 # Invenio-RDM-Records is free software; you can redistribute it and/or modify
 # it under the terms of the MIT License; see LICENSE file for more details.
@@ -260,6 +261,7 @@ class RDMRecordCommunitiesConfig(ServiceConfig, ConfiguratorMixin):
     service_id = "record-communities"
 
     record_cls = FromConfig("RDM_RECORD_CLS", default=RDMRecord)
+    draft_cls = FromConfig("RDM_DRAFT_CLS", default=RDMDraft)
     permission_policy_cls = FromConfig(
         "RDM_PERMISSION_POLICY", default=RDMRecordPermissionPolicy, import_string=True
     )

From 15831bf8b8d825b17eafbf8a361958ced3518551 Mon Sep 17 00:00:00 2001
From: Sam Arbid <samaa2@kth.se>
Date: Mon, 30 Sep 2024 14:03:02 +0200
Subject: [PATCH 13/76] release: v12.2.2

---
 CHANGES.rst                     | 8 ++++++++
 invenio_rdm_records/__init__.py | 3 ++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 18bcad75f..cc1a5e16f 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -2,6 +2,7 @@
 ..
     Copyright (C) 2019-2024 CERN.
     Copyright (C) 2019-2024 Northwestern University.
+    Copyright (C) 2024      KTH Royal Institute of Technology.
 
 
     Invenio-RDM-Records is free software; you can redistribute it and/or
@@ -11,6 +12,13 @@
 Changes
 =======
 
+Version v12.2.2 (released 2024-09-30)
+
+- Improve handling of draft PID in RecordCommunitiesService
+- Revert "deposit: check permission and set disable tooltip for publish button"
+- Remove DeprecationWarning for sqlalchemy
+- Add compatibility layer to move to flask>=3
+
 Version v12.2.1 (released 2024-09-19)
 
 - file upload: better handling of errors when uploading empty files
diff --git a/invenio_rdm_records/__init__.py b/invenio_rdm_records/__init__.py
index 402d03fbf..3806279db 100644
--- a/invenio_rdm_records/__init__.py
+++ b/invenio_rdm_records/__init__.py
@@ -2,6 +2,7 @@
 #
 # Copyright (C) 2019-2024 CERN.
 # Copyright (C) 2019-2024 Northwestern University.
+# Copyright (C) 2024      KTH Royal Institute of Technology.
 #
 # Invenio-RDM-Records is free software; you can redistribute it and/or modify
 # it under the terms of the MIT License; see LICENSE file for more details.
@@ -10,6 +11,6 @@
 
 from .ext import InvenioRDMRecords
 
-__version__ = "12.2.1"
+__version__ = "12.2.2"
 
 __all__ = ("__version__", "InvenioRDMRecords")

From a4fbec8dbde537244a58d905495fb6919029abaa Mon Sep 17 00:00:00 2001
From: Guillaume Viger <fenekku@fenekku.com>
Date: Fri, 27 Sep 2024 14:22:32 -0400
Subject: [PATCH 14/76] dependencies: bump flask-iiif

Did we forget to bump this dependency or was there a larger reason for
keeping it clamped so low? None of the flask-iiif improvements from the past year
have been adopted by InvenioRDM it means.

I will backport this to v12 maint branch once merged as it prevents our instance
from previewing specific medical images.
---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index e53590945..0901c7172 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -36,7 +36,7 @@ install_requires =
     datacite>=1.1.1,<2.0.0
     dcxml>=0.1.2,<1.0.0
     Faker>=2.0.3
-    flask-iiif>=0.6.2,<1.0.0
+    flask-iiif>=1.0.0,<2.0.0
     ftfy>=4.4.3,<5.0.0
     invenio-administration>=2.0.0,<3.0.0
     invenio-communities>=15.0.0,<16.0.0

From f3431763bf579b3b0790f6bd4a9e15286cded980 Mon Sep 17 00:00:00 2001
From: Christoph Ladurner <christoph.ladurner@tugraz.at>
Date: Tue, 1 Oct 2024 15:17:23 +0200
Subject: [PATCH 15/76] fix: propTypes warning

* the recordRestrictionGracePeriod property is a number. the object
  which it is in the beginning in python (timedelta object) is reduced to
  the days in https://github.com/inveniosoftware/invenio-app-rdm/blob/master/invenio_app_rdm/records_ui/templates/semantic-ui/invenio_app_rdm/records/deposit.html#L45
---
 .../src/deposit/fields/AccessField/AccessRightField.js        | 4 ++--
 .../deposit/fields/AccessField/components/MetadataAccess.js   | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AccessField/AccessRightField.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AccessField/AccessRightField.js
index c77e505c7..47b801192 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AccessField/AccessRightField.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AccessField/AccessRightField.js
@@ -104,7 +104,7 @@ AccessRightFieldCmp.propTypes = {
   showMetadataAccess: PropTypes.bool,
   community: PropTypes.object,
   record: PropTypes.object.isRequired,
-  recordRestrictionGracePeriod: PropTypes.object.isRequired,
+  recordRestrictionGracePeriod: PropTypes.number.isRequired,
   allowRecordRestriction: PropTypes.bool.isRequired,
 };
 
@@ -140,7 +140,7 @@ AccessRightField.propTypes = {
   labelIcon: PropTypes.string,
   isMetadataOnly: PropTypes.bool,
   record: PropTypes.object.isRequired,
-  recordRestrictionGracePeriod: PropTypes.object.isRequired,
+  recordRestrictionGracePeriod: PropTypes.number.isRequired,
   allowRecordRestriction: PropTypes.bool.isRequired,
 };
 
diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AccessField/components/MetadataAccess.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AccessField/components/MetadataAccess.js
index fc3a904b0..c146f8762 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AccessField/components/MetadataAccess.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AccessField/components/MetadataAccess.js
@@ -62,6 +62,6 @@ MetadataAccess.propTypes = {
   recordAccess: PropTypes.string.isRequired,
   communityAccess: PropTypes.string.isRequired,
   record: PropTypes.object.isRequired,
-  recordRestrictionGracePeriod: PropTypes.object.isRequired,
+  recordRestrictionGracePeriod: PropTypes.number.isRequired,
   allowRecordRestriction: PropTypes.bool.isRequired,
 };

From 40d69993e35eeebe76eec79aa0e626921002eaf6 Mon Sep 17 00:00:00 2001
From: Christoph Ladurner <christoph.ladurner@tugraz.at>
Date: Tue, 1 Oct 2024 15:15:07 +0200
Subject: [PATCH 16/76] fix: propTypes warning

* the canRestrictRecord property is optional in
  ProtectionButtonsComponent but required in ProtectionButtons.

* Since it is working without a problem, because of the default value in
  ProtectionButtonsComponent, the required has been removed from
  ProtectionButtons and replaced by a defaultProps definition to be
  consistent with the ProtectionButtonsComponent definition.
---
 .../fields/AccessField/components/ProtectionButtons.js    | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AccessField/components/ProtectionButtons.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AccessField/components/ProtectionButtons.js
index 6cfe988d4..b3ec39d77 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AccessField/components/ProtectionButtons.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AccessField/components/ProtectionButtons.js
@@ -1,7 +1,7 @@
 // This file is part of Invenio-RDM-Records
 // Copyright (C) 2020-2023 CERN.
 // Copyright (C) 2020-2022 Northwestern University.
-// Copyright (C) 2021 Graz University of Technology.
+// Copyright (C) 2021-2024 Graz University of Technology.
 //
 // Invenio-RDM-Records is free software; you can redistribute it and/or modify it
 // under the terms of the MIT License; see LICENSE file for more details.
@@ -109,7 +109,11 @@ export class ProtectionButtons extends Component {
   }
 }
 
+ProtectionButtons.defaultProps = {
+  canRestrictRecord: true,
+};
+
 ProtectionButtons.propTypes = {
-  canRestrictRecord: PropTypes.bool.isRequired,
+  canRestrictRecord: PropTypes.bool,
   fieldPath: PropTypes.string.isRequired,
 };

From 0c0ed320710c22f338268ef55a8aa427f334c760 Mon Sep 17 00:00:00 2001
From: alejandromumo <manuel.alejandro.de.oliveira.da.costa@cern.ch>
Date: Wed, 25 Sep 2024 17:12:06 +0200
Subject: [PATCH 17/76] collections: added core backend

---
 invenio_rdm_records/collections/__init__.py  |  16 +
 invenio_rdm_records/collections/api.py       | 343 +++++++++++++++++++
 invenio_rdm_records/collections/models.py    | 136 ++++++++
 invenio_rdm_records/collections/searchapp.py |  37 ++
 invenio_rdm_records/collections/service.py   | 107 ++++++
 invenio_rdm_records/ext.py                   |   3 +
 setup.cfg                                    |   1 +
 7 files changed, 643 insertions(+)
 create mode 100644 invenio_rdm_records/collections/__init__.py
 create mode 100644 invenio_rdm_records/collections/api.py
 create mode 100644 invenio_rdm_records/collections/models.py
 create mode 100644 invenio_rdm_records/collections/searchapp.py
 create mode 100644 invenio_rdm_records/collections/service.py

diff --git a/invenio_rdm_records/collections/__init__.py b/invenio_rdm_records/collections/__init__.py
new file mode 100644
index 000000000..abd0193da
--- /dev/null
+++ b/invenio_rdm_records/collections/__init__.py
@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""Collections entrypoint."""
+
+from .models import Collection, CollectionTree
+from .searchapp import search_app_context
+
+__all__ = (
+    "Collection",
+    "CollectionTree",
+    "search_app_context",
+)
diff --git a/invenio_rdm_records/collections/api.py b/invenio_rdm_records/collections/api.py
new file mode 100644
index 000000000..803cadfe2
--- /dev/null
+++ b/invenio_rdm_records/collections/api.py
@@ -0,0 +1,343 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""Collections programmatic API."""
+
+from types import ClassMethodDescriptorType
+
+from invenio_db import db
+from sqlalchemy import select
+from werkzeug.utils import cached_property
+
+from .models import Collection as CollectionModel
+from .models import CollectionTree as CollectionTreeModel
+
+
+class Collection:
+    """Collection Object."""
+
+    model_cls = CollectionModel
+
+    def __init__(self, model=None):
+        """Instantiate a Collection object."""
+        self.model = model
+
+    @classmethod
+    def create(cls, slug, title, query, ctree=None, parent=None, order=None, logo=None):
+        """Create a new collection."""
+        _ctree = None
+        if parent:
+            path = f"{parent.path}{parent.id},"
+            _ctree = parent.collection_tree
+        elif ctree:
+            path = ","
+            _ctree = ctree
+        else:
+            raise ValueError("Either parent or ctree must be set.")
+
+        return cls(
+            cls.model_cls.create(
+                slug=slug,
+                path=path,
+                title=title,
+                query_text=query,
+                order=order,
+                logo=logo,
+                ctree_or_id=_ctree,
+            )
+        )
+
+    @classmethod
+    def resolve(cls, id_, ctree_id=None, use_slug=False):
+        """Resolve a collection by ID or slug.
+
+        To resolve by slug, the collection tree ID must be provided.
+        """
+        if not use_slug:
+            return cls.get(id_)
+        if not ctree_id:
+            raise ValueError(
+                "Collection tree ID is required to resolve a collection by slug."
+            )
+        return cls.get_by_slug(id_, ctree_id)
+
+    @classmethod
+    def get(cls, id_):
+        """Get a collection by ID."""
+        model = cls.model_cls.get(id_)
+        if not model:
+            return None
+        return cls(model)
+
+    @classmethod
+    def get_by_slug(cls, slug, ctree_id):
+        """Get a collection by slug."""
+        model = cls.model_cls.get_by_slug(slug, ctree_id)
+        if not model:
+            return None
+        return cls(model)
+
+    def add(self, slug, title, query, order=None, logo=None):
+        """Add a subcollection to the collection."""
+        return self.create(
+            slug=slug,
+            title=title,
+            query=query,
+            parent=self,
+            order=order,
+            logo=logo,
+        )
+
+    @property
+    def id(self):
+        """Get the collection ID."""
+        return self.model.id
+
+    @property
+    def path(self):
+        """Get the collection path."""
+        return self.model.path
+
+    @property
+    def ctree_id(self):
+        """Get the collection tree ID."""
+        return self.model.tree_id
+
+    @property
+    def order(self):
+        """Get the collection order."""
+        return self.model.order
+
+    @property
+    def title(self):
+        """Get the collection title."""
+        return self.model.title
+
+    @property
+    def ctree_title(self):
+        """Get the collection tree title."""
+        return self.model.collection_tree.title
+
+    @property
+    def collection_tree(self):
+        """Get the collection tree object.
+
+        Note: this will execute a query to the collection tree table.
+        """
+        return self.model.collection_tree
+
+    @property
+    def depth(self):
+        """Get the collection depth in its tree."""
+        return self.model.depth
+
+    @property
+    def slug(self):
+        """Get the collection slug."""
+        return self.model.slug
+
+    @cached_property
+    def community(self):
+        """Get the community object."""
+        return self.collection_tree.community
+
+    @property
+    def logo(self):
+        """Get the collection logo."""
+        return self.model.logo
+
+    @property
+    def query(self):
+        """Get the collection query."""
+        q = ""
+        for _a in self.ancestors:
+            q += f"({_a.model.query_text}) AND "
+        q += f"({self.model.query_text})"
+        return q
+
+    @cached_property
+    def ancestors(self):
+        """Get the collection ancestors."""
+        if not self.model:
+            return None
+
+        cps = self.path.split(",")
+        ret = []
+        for cid in cps:
+            if not cid:
+                continue
+            cl = Collection.get(cid)
+            ret.append(cl)
+        return list(sorted(ret, key=lambda x: (x.path, x.order)))
+
+    @cached_property
+    def sub_collections(self):
+        """Fetch all the descendants."""
+        return self.get_subcollections()
+
+    @cached_property
+    def direct_subcollections(self):
+        """Fetch only direct descendants."""
+        return self.get_direct_subcollections()
+
+    def get_direct_subcollections(self):
+        """Get the collection first level (direct) children.
+
+        More preformant query to retrieve descendants, executes an exact match query.
+        """
+        if not self.model:
+            return None
+        stmt = (
+            select(self.model_cls)
+            .filter(
+                self.model_cls.path == f"{self.path}{self.id},",
+                self.model_cls.tree_id == self.ctree_id,
+            )
+            .order_by(self.model_cls.path, self.model_cls.order)
+        )
+        ret = db.session.execute(stmt).scalars().all()
+        return [type(self)(r) for r in ret]
+
+    def get_subcollections(self, max_depth=3):
+        """Get the collection subcollections.
+
+        This query executes a LIKE query on the path column.
+        """
+        if not self.model:
+            return None
+
+        stmt = (
+            select(self.model_cls)
+            .filter(
+                self.model_cls.path.like(f"{self.path}{self.id},%"),
+                self.model_cls.depth < self.model.depth + max_depth,
+            )
+            .order_by(self.model_cls.path, self.model_cls.order)
+        )
+        ret = db.session.execute(stmt).scalars().all()
+        return [type(self)(r) for r in ret]
+
+    @classmethod
+    def dump(cls, collection):
+        """Transform the collection into a dictionary."""
+        res = {
+            "title": collection.title,
+            "slug": collection.slug,
+            "depth": collection.depth,
+            "logo": collection.logo,
+            "order": collection.order,
+            "id": collection.id,
+            "query": collection.query,
+        }
+        return res
+
+    def to_dict(self) -> dict:
+        """Return a dictionary representation of the collection.
+
+        Uses an adjacency list.
+        """
+        ret = {
+            "root": self.id,
+            self.id: {**Collection.dump(self), "children": set()},
+        }
+
+        for _c in self.sub_collections:
+            # Add the collection itself to the dictionary
+            if _c.id not in ret:
+                ret[_c.id] = {**Collection.dump(_c), "children": set()}
+
+            # Find the parent ID from the collection's path (last valid ID in the path)
+            path_parts = [int(part) for part in _c.path.split(",") if part.strip()]
+            if path_parts:
+                parent_id = path_parts[-1]
+                # Add the collection as a child of its parent
+                ret[parent_id]["children"].add(_c.id)
+        for k, v in ret.items():
+            if isinstance(v, dict):
+                v["children"] = list(v["children"])
+        return ret
+
+    def __repr__(self) -> str:
+        """Return a string representation of the collection."""
+        if self.model:
+            return f"Collection {self.id} ({self.path})"
+        else:
+            return "Collection (None)"
+
+
+class CollectionTree:
+    """Collection Tree Object."""
+
+    model_cls = CollectionTreeModel
+
+    def __init__(self, model):
+        """Instantiate a CollectionTree object."""
+        self.model = model
+
+    @classmethod
+    def create(cls, title, slug, community_id=None, order=None):
+        """Create a new collection tree."""
+        return cls(
+            cls.model_cls.create(
+                title=title, slug=slug, community_id=community_id, order=order
+            )
+        )
+
+    @classmethod
+    def resolve(cls, id_, community_id=None, use_slug=False):
+        """Resolve a CollectionTree."""
+        if not use_slug:
+            return cls.get(id_)
+
+        if not community_id:
+            raise ValueError(
+                "Community ID is required to resolve a collection tree by slug."
+            )
+        return cls.get_by_slug(id_, community_id)
+
+    @classmethod
+    def get(cls, id_):
+        """Get a collection tree by ID."""
+        model = cls.model_cls.get(id_)
+        if not model:
+            return None
+        return cls(model)
+
+    @classmethod
+    def get_by_slug(cls, slug, community_id):
+        """Get a collection tree by slug.
+
+        Community ID is required to avoid ambiguity.
+        """
+        model = cls.model_cls.get_by_slug(slug, community_id)
+        if not model:
+            return None
+        return cls(model)
+
+    @property
+    def id(self):
+        """Get the collection tree ID."""
+        return self.model.id
+
+    @property
+    def title(self):
+        """Get the collection tree title."""
+        return self.model.title
+
+    @property
+    def slug(self):
+        """Get the collection tree slug."""
+        return self.model.slug
+
+    @property
+    def community_id(self):
+        """Get the community ID."""
+        return self.model.community_id
+
+    @property
+    def order(self):
+        """Get the collection tree order."""
+        return self.model.order
diff --git a/invenio_rdm_records/collections/models.py b/invenio_rdm_records/collections/models.py
new file mode 100644
index 000000000..151cdec36
--- /dev/null
+++ b/invenio_rdm_records/collections/models.py
@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM-records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""Collections models."""
+
+from invenio_communities.communities.records.models import CommunityMetadata
+from invenio_db import db
+from invenio_records.models import Timestamp
+from sqlalchemy import UniqueConstraint
+from sqlalchemy_utils.types import UUIDType
+from traitlets import ClassBasedTraitType
+
+
+# CollectionTree Table
+class CollectionTree(db.Model, Timestamp):
+    """Collection tree model."""
+
+    __tablename__ = "collection_tree"
+
+    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
+    community_id = db.Column(
+        UUIDType,
+        db.ForeignKey(CommunityMetadata.id, ondelete="SET NULL"),
+        nullable=True,
+        index=True,
+    )
+    title = db.Column(db.String(255), nullable=False)
+    order = db.Column(db.Integer, nullable=True)
+    slug = db.Column(db.String(255), nullable=False, index=True)
+
+    # Relationship to Collection
+    collections = db.relationship("Collection", back_populates="collection_tree")
+    community = db.relationship(CommunityMetadata, backref="collection_trees")
+
+    # Unique constraint on slug and community_id. Slugs should be unique within a community.
+    UniqueConstraint(
+        "slug", "community_id", name="uq_collection_tree_slug_community_id"
+    )
+
+    @classmethod
+    def create(cls, title, slug, community_id=None, order=None):
+        """Create a new collection tree."""
+        with db.session.begin_nested():
+            collection_tree = cls(
+                title=title, slug=slug, community_id=community_id, order=order
+            )
+            db.session.add(collection_tree)
+        return collection_tree
+
+    @classmethod
+    def get(cls, id_):
+        """Get a collection tree by ID."""
+        return cls.query.get(id_)
+
+    @classmethod
+    def get_by_slug(cls, slug, community_id):
+        """Get a collection tree by slug."""
+        return cls.query.filter(
+            cls.slug == slug, cls.community_id == community_id
+        ).one_or_none()
+
+
+# Collection Table
+class Collection(db.Model, Timestamp):
+    """Collection model.
+
+    Indices:
+    - id
+    - collection_tree_id
+    - path
+    - slug
+    """
+
+    __tablename__ = "collections"
+
+    id = db.Column(db.Integer, primary_key=True, autoincrement=True)
+    slug = db.Column(db.String(255), unique=False, nullable=False, index=True)
+    path = db.Column(db.Text, nullable=False, index=True)
+    tree_id = db.Column(db.Integer, db.ForeignKey("collection_tree.id"), nullable=False)
+    logo = db.Column(db.Text, nullable=True)
+    title = db.Column(db.String(255), nullable=False)
+    query_text = db.Column("query", db.Text, nullable=False)
+    order = db.Column(db.Integer, nullable=True)
+    # TODO index depth
+    depth = db.Column(
+        db.Integer, db.Computed("array_length(string_to_array(path, ','), 1) - 2")
+    )  # TODO client side
+    num_records = db.Column(db.Integer, nullable=True)
+
+    # Relationship to CollectionTree
+    collection_tree = db.relationship("CollectionTree", back_populates="collections")
+
+    # Unique constraint on slug and tree_id. Slugs should be unique within a tree.
+    UniqueConstraint("slug", "tree_id", name="uq_collections_slug_tree_id")
+
+    @classmethod
+    def create(cls, slug, path, title, query_text, ctree_or_id, **kwargs):
+        """Create a new collection."""
+        with db.session.begin_nested():
+            if isinstance(ctree_or_id, CollectionTree):
+                collection = cls(
+                    slug=slug,
+                    path=path,
+                    title=title,
+                    query_text=query_text,
+                    collection_tree=ctree_or_id,
+                    **kwargs,
+                )
+            elif isinstance(ctree_or_id, int):
+                collection = cls(
+                    slug=slug,
+                    path=path,
+                    title=title,
+                    query_text=query_text,
+                    tree_id=ctree_or_id,
+                    **kwargs,
+                )
+            else:
+                raise ValueError(
+                    "Either `collection_tree` or `collection_tree_id` must be provided."
+                )
+            db.session.add(collection)
+        return collection
+
+    @classmethod
+    def get(cls, id_):
+        """Get a collection by ID."""
+        return cls.query.get(id_)
+
+    @classmethod
+    def get_by_slug(cls, slug, tree_id):
+        """Get a collection by slug."""
+        return cls.query.filter(cls.slug == slug, cls.tree_id == tree_id).one_or_none()
diff --git a/invenio_rdm_records/collections/searchapp.py b/invenio_rdm_records/collections/searchapp.py
new file mode 100644
index 000000000..ea22306e0
--- /dev/null
+++ b/invenio_rdm_records/collections/searchapp.py
@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""Collection search app helpers for React-SearchKit."""
+
+
+from functools import partial
+
+from flask import current_app
+from invenio_search_ui.searchconfig import search_app_config
+
+
+def search_app_context():
+    """Search app context."""
+    # endpoint = None
+    # if collection.community is None:
+    #     endpoint = "/api/records/"
+    # else:
+    #     endpoint = f"/api/communities/{collection.community.id}/records/"
+    return {
+        "search_app_collection_config": partial(
+            search_app_config,
+            config_name="RDM_SEARCH",
+            available_facets=current_app.config["RDM_FACETS"],
+            sort_options=current_app.config["RDM_SORT_OPTIONS"],
+            headers={"Accept": "application/vnd.inveniordm.v1+json"},
+            pagination_options=(10, 25, 50, 100),
+            # endpoint=/communities/eu/records
+            # endpoint=/api/records
+            # hidden_params=[
+            #     ["q", collection.query]
+            # ]
+        )
+    }
diff --git a/invenio_rdm_records/collections/service.py b/invenio_rdm_records/collections/service.py
new file mode 100644
index 000000000..b503f79dd
--- /dev/null
+++ b/invenio_rdm_records/collections/service.py
@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""Collections service."""
+
+from invenio_communities.proxies import current_communities
+
+from .api import Collection, CollectionTree
+
+
+class CollectionItem:
+    """Collection item."""
+
+    def __init__(self, collection: Collection):
+        """Instantiate a Collection object.
+
+        Optionally pass a community to cache its information in the collection's instance.
+        """
+        self._collection = collection
+
+    def to_dict(self):
+        """Serialize the collection to a dictionary and add links."""
+        res = {**self._collection.to_dict()}
+        res[self._collection.id]["links"] = self.links
+
+        return res
+
+    @property
+    def links(self):
+        """Return the links of the collection."""
+        self_html = None
+        search = None
+        tree_slug = self._collection.collection_tree.slug
+        if self._collection.community:
+            self_html = f"/communities/{self._collection.community.slug}/{tree_slug}/{self._collection.slug}"
+            search = f"/api/communities/{self._collection.community.slug}/records"
+        else:
+            self_html = f"/collections/{tree_slug}/{self._collection.slug}"
+            search = "/api/records"
+        return {
+            "search": search,
+            "self_html": self_html,
+        }
+
+    def __getattr__(self, name):
+        """Delegate attribute access to the collection."""
+        return getattr(self._collection, name)
+
+
+class CollectionsService:
+    """Collections service."""
+
+    collection_cls = Collection
+
+    def create(self, identity, community_id, tree_slug, slug, title, query, **kwargs):
+        """Create a new collection."""
+        current_communities.service.require_permission(
+            identity, "update", community_id=community_id
+        )
+        ctree = CollectionTree.get_by_slug(tree_slug, community_id)
+        if not ctree:
+            raise ValueError(f"Collection tree {tree_slug} not found.")
+        collection = self.collection_cls.create(
+            slug=slug, title=title, query=query, ctree=ctree, **kwargs
+        )
+        return CollectionItem(collection)
+
+    def read(self, identity, id_):
+        """Get a collection by ID or slug."""
+        collection = self.collection_cls.resolve(id_)
+        if not collection:
+            raise ValueError(f"Collection {id_} not found.")
+        if collection.community:
+            current_communities.service.require_permission(
+                identity, "read", community_id=collection.community.id
+            )
+
+        return CollectionItem(collection)
+
+    def read_slug(self, identity, community_id, tree_slug, slug):
+        """Get a collection by slug."""
+        current_communities.service.require_permission(
+            identity, "read", community_id=community_id
+        )
+
+        ctree = CollectionTree.get_by_slug(tree_slug, community_id)
+        if not ctree:
+            raise ValueError(f"Collection tree {tree_slug} not found.")
+
+        collection = self.collection_cls.resolve(slug, ctree.id, use_slug=True)
+        if not collection:
+            raise ValueError(f"Collection {slug} not found.")
+
+        return CollectionItem(collection)
+
+    def add(self, identity, collection, slug, title, query, **kwargs):
+        """Add a subcollection to a collection."""
+        current_communities.service.require_permission(
+            identity, "update", community_id=collection.community.id
+        )
+        collection = self.collection_cls.create(
+            parent=collection, slug=slug, title=title, query=query, **kwargs
+        )
+        return CollectionItem(collection)
diff --git a/invenio_rdm_records/ext.py b/invenio_rdm_records/ext.py
index b9802d80f..470c9ef42 100644
--- a/invenio_rdm_records/ext.py
+++ b/invenio_rdm_records/ext.py
@@ -19,6 +19,7 @@
 from invenio_records_resources.resources.files import FileResource
 
 from . import config
+from .collections.service import CollectionsService
 from .oaiserver.resources.config import OAIPMHServerResourceConfig
 from .oaiserver.resources.resources import OAIPMHServerResource
 from .oaiserver.services.config import OAIPMHServerServiceConfig
@@ -203,6 +204,8 @@ def init_services(self, app):
             config=service_configs.oaipmh_server,
         )
 
+        self.collections_service = CollectionsService()
+
     def init_resource(self, app):
         """Initialize resources."""
         self.records_resource = RDMRecordResource(
diff --git a/setup.cfg b/setup.cfg
index 0901c7172..f9ede4779 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -111,6 +111,7 @@ invenio_celery.tasks =
     invenio_rdm_records_user_moderation = invenio_rdm_records.requests.user_moderation.tasks
 invenio_db.models =
     invenio_rdm_records = invenio_rdm_records.records.models
+    invenio_rdm_records_collections = invenio_rdm_records.collections.models
 invenio_db.alembic =
     invenio_rdm_records = invenio_rdm_records:alembic
 invenio_jsonschemas.schemas =

From 11a05cc67e5013fef9bc3fb4455d288d9b9aef98 Mon Sep 17 00:00:00 2001
From: alejandromumo <manuel.alejandro.de.oliveira.da.costa@cern.ch>
Date: Wed, 25 Sep 2024 17:12:21 +0200
Subject: [PATCH 18/76] tests: added collection tests

---
 invenio_rdm_records/collections/searchapp.py |  5 --
 tests/collections/__init__.py                |  7 ++
 tests/collections/test_collections_api.py    | 71 +++++++++++++++++++
 tests/services/test_collections_service.py   | 73 ++++++++++++++++++++
 4 files changed, 151 insertions(+), 5 deletions(-)
 create mode 100644 tests/collections/__init__.py
 create mode 100644 tests/collections/test_collections_api.py
 create mode 100644 tests/services/test_collections_service.py

diff --git a/invenio_rdm_records/collections/searchapp.py b/invenio_rdm_records/collections/searchapp.py
index ea22306e0..a48e2699a 100644
--- a/invenio_rdm_records/collections/searchapp.py
+++ b/invenio_rdm_records/collections/searchapp.py
@@ -15,11 +15,6 @@
 
 def search_app_context():
     """Search app context."""
-    # endpoint = None
-    # if collection.community is None:
-    #     endpoint = "/api/records/"
-    # else:
-    #     endpoint = f"/api/communities/{collection.community.id}/records/"
     return {
         "search_app_collection_config": partial(
             search_app_config,
diff --git a/tests/collections/__init__.py b/tests/collections/__init__.py
new file mode 100644
index 000000000..03664b765
--- /dev/null
+++ b/tests/collections/__init__.py
@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""Tests for collections."""
diff --git a/tests/collections/test_collections_api.py b/tests/collections/test_collections_api.py
new file mode 100644
index 000000000..056cc0c13
--- /dev/null
+++ b/tests/collections/test_collections_api.py
@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""Test suite for the collections programmatic API."""
+
+from invenio_rdm_records.collections.api import Collection
+from invenio_rdm_records.collections.models import CollectionTree
+
+
+def test_create(running_app, db, community, community_owner):
+    """Test collection creation via API."""
+    tree = CollectionTree(
+        title="Tree 1",
+        order=10,
+        community_id=community.id,
+    )
+
+    # Use ORM object (collection tree)
+    collection = Collection.create(
+        title="My Collection",
+        query="*:*",
+        slug="my-collection",
+        ctree=tree,
+    )
+
+    assert collection.id
+    assert collection.title == "My Collection"
+    assert collection.collection_tree == tree
+
+    # Use collection tree id
+    collection = Collection.create(
+        title="My Collection 2",
+        query="*:*",
+        slug="my-collection-2",
+        ctree=tree.id,
+    )
+    assert collection.id
+    assert collection.title == "My Collection 2"
+    assert collection.collection_tree == tree
+
+
+def test_as_dict(running_app, db):
+    """Test collection as dict."""
+    tree = CollectionTree(
+        title="Tree 1",
+        order=10,
+    )
+    c1 = Collection.create(
+        title="My Collection",
+        query="*:*",
+        slug="my-collection",
+        ctree=tree,
+    )
+
+    c2 = Collection.create(
+        title="My Collection 2",
+        query="*:*",
+        slug="my-collection-2",
+        parent=c1,
+    )
+
+    c3 = Collection.create(title="3", query="*", slug="my-collection-3", parent=c2)
+    res = c1.to_dict()
+    assert all(k in res for k in (c1.id, c2.id, c3.id))
+    assert res[1]["title"] == "My Collection"
+    assert res[1]["children"] == [c2.id]
+    assert res[2]["children"] == [c3.id]
+    assert res[3]["children"] == []
diff --git a/tests/services/test_collections_service.py b/tests/services/test_collections_service.py
new file mode 100644
index 000000000..e4b3c4324
--- /dev/null
+++ b/tests/services/test_collections_service.py
@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+""" TODO """
+
+import pytest
+
+from invenio_rdm_records.collections.api import Collection
+from invenio_rdm_records.collections.models import CollectionTree
+from invenio_rdm_records.proxies import current_rdm_records
+
+
+@pytest.fixture()
+def collections_service():
+    """Get collections service fixture."""
+    return current_rdm_records.collections_service
+
+
+@pytest.fixture(autouse=True)
+def add_collections(running_app, db, community):
+    """Create collections on demand."""
+
+    def _inner():
+        """Add collections to the app."""
+        tree = CollectionTree(
+            title="Tree 1",
+            order=10,
+            community_id=community.id,
+        )
+        db.session.add(tree)
+        db.session.flush()
+        db.session.refresh(tree)
+        c1 = Collection.create(
+            title="Collection 1", query="*:*", slug="collection-1", ctree=tree
+        )
+        db.session.add(c1.model)
+        db.session.flush()
+        db.session.refresh(tree)
+        c2 = Collection.create(
+            title="Collection 2",
+            query="*:*",
+            slug="collection-2",
+            ctree=tree,
+            parent=c1,
+        )
+        db.session.add(c2.model)
+        db.session.commit()
+        return [c1, c2]
+
+    return _inner
+
+
+def test_collections_service_read(
+    running_app, db, add_collections, collections_service, community_owner
+):
+    """Test collections service."""
+    collections = add_collections()
+    c0 = collections[0]
+    c1 = collections[1]
+    res = collections_service.read(community_owner, c0.id)
+    assert res._collection.id == c0.id
+
+
+def test_collections_service_create(
+    running_app, db, collections_service, community_owner
+):
+    """Test collections service create."""
+    collections_service.create(
+        community_owner, 1, title="My Collection", query="*:*", slug="my-collection"
+    )

From 371a3fd26ba295d2e46abcfe54522666dd58786c Mon Sep 17 00:00:00 2001
From: alejandromumo <manuel.alejandro.de.oliveira.da.costa@cern.ch>
Date: Thu, 26 Sep 2024 10:45:40 +0200
Subject: [PATCH 19/76] alembic: added collections tables

* collections: fix create api
---
 .../425b691f768b_create_collections_tables.py | 103 ++++++++++++++++++
 invenio_rdm_records/collections/api.py        |  35 +++---
 invenio_rdm_records/collections/models.py     |  44 +++++---
 invenio_rdm_records/collections/service.py    |  17 ++-
 tests/collections/test_collections_api.py     |  21 ++--
 tests/services/test_collections_service.py    |  25 +----
 6 files changed, 177 insertions(+), 68 deletions(-)
 create mode 100644 invenio_rdm_records/alembic/425b691f768b_create_collections_tables.py

diff --git a/invenio_rdm_records/alembic/425b691f768b_create_collections_tables.py b/invenio_rdm_records/alembic/425b691f768b_create_collections_tables.py
new file mode 100644
index 000000000..a66e0d06d
--- /dev/null
+++ b/invenio_rdm_records/alembic/425b691f768b_create_collections_tables.py
@@ -0,0 +1,103 @@
+#
+# This file is part of Invenio.
+# Copyright (C) 2016-2018 CERN.
+#
+# Invenio is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""Create collections tables."""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy_utils import UUIDType
+
+# revision identifiers, used by Alembic.
+revision = "425b691f768b"
+down_revision = "ff9bec971d30"
+branch_labels = ()
+depends_on = None
+
+
+def upgrade():
+    """Upgrade database."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "collections_collection_tree",
+        sa.Column("created", sa.DateTime(), nullable=False),
+        sa.Column("updated", sa.DateTime(), nullable=False),
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("community_id", UUIDType(), nullable=True),
+        sa.Column("title", sa.String(length=255), nullable=False),
+        sa.Column("order", sa.Integer(), nullable=True),
+        sa.Column("slug", sa.String(length=255), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["community_id"],
+            ["communities_metadata.id"],
+            name=op.f(
+                "fk_collections_collection_tree_community_id_communities_metadata"
+            ),
+            ondelete="SET NULL",
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_collections_collection_tree")),
+        sa.UniqueConstraint(
+            "slug",
+            "community_id",
+            name="uq_collections_collection_tree_slug_community_id",
+        ),
+    )
+    op.create_index(
+        op.f("ix_collections_collection_tree_community_id"),
+        "collections_collection_tree",
+        ["community_id"],
+        unique=False,
+    )
+
+    op.create_table(
+        "collections_collection",
+        sa.Column("created", sa.DateTime(), nullable=False),
+        sa.Column("updated", sa.DateTime(), nullable=False),
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("slug", sa.String(length=255), nullable=False),
+        sa.Column("path", sa.Text(), nullable=False),
+        sa.Column("tree_id", sa.Integer(), nullable=False),
+        sa.Column("title", sa.String(length=255), nullable=False),
+        sa.Column("query", sa.Text(), nullable=False),
+        sa.Column("order", sa.Integer(), nullable=True),
+        sa.Column(
+            "depth",
+            sa.Integer(),
+            sa.Computed(
+                "array_length(string_to_array(path, ','), 1) - 2",
+            ),
+            nullable=True,
+        ),
+        sa.Column("num_records", sa.Integer(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["tree_id"],
+            ["collections_collection_tree.id"],
+            name=op.f("fk_collections_collection_tree_id_collections_collection_tree"),
+        ),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_collections_collection")),
+        sa.UniqueConstraint(
+            "slug", "tree_id", name="uq_collections_collection_slug_tree_id"
+        ),
+    )
+    op.create_index(
+        op.f("ix_collections_collection_path"),
+        "collections_collection",
+        ["path"],
+        unique=False,
+    )
+
+
+def downgrade():
+    """Downgrade database."""
+    op.drop_index(
+        op.f("ix_collections_collection_path"), table_name="collections_collection"
+    )
+    op.drop_table("collections_collection")
+    op.drop_index(
+        op.f("ix_collections_collection_tree_community_id"),
+        table_name="collections_collection_tree",
+    )
+    op.drop_table("collections_collection_tree")
diff --git a/invenio_rdm_records/collections/api.py b/invenio_rdm_records/collections/api.py
index 803cadfe2..68200f60a 100644
--- a/invenio_rdm_records/collections/api.py
+++ b/invenio_rdm_records/collections/api.py
@@ -26,15 +26,15 @@ def __init__(self, model=None):
         self.model = model
 
     @classmethod
-    def create(cls, slug, title, query, ctree=None, parent=None, order=None, logo=None):
+    def create(cls, slug, title, query, ctree=None, parent=None, order=None):
         """Create a new collection."""
         _ctree = None
         if parent:
             path = f"{parent.path}{parent.id},"
-            _ctree = parent.collection_tree
+            _ctree = parent.collection_tree.model
         elif ctree:
             path = ","
-            _ctree = ctree
+            _ctree = ctree if isinstance(ctree, int) else ctree.model
         else:
             raise ValueError("Either parent or ctree must be set.")
 
@@ -43,9 +43,8 @@ def create(cls, slug, title, query, ctree=None, parent=None, order=None, logo=No
                 slug=slug,
                 path=path,
                 title=title,
-                query_text=query,
+                search_query=query,
                 order=order,
-                logo=logo,
                 ctree_or_id=_ctree,
             )
         )
@@ -80,7 +79,13 @@ def get_by_slug(cls, slug, ctree_id):
             return None
         return cls(model)
 
-    def add(self, slug, title, query, order=None, logo=None):
+    def add(
+        self,
+        slug,
+        title,
+        query,
+        order=None,
+    ):
         """Add a subcollection to the collection."""
         return self.create(
             slug=slug,
@@ -88,7 +93,6 @@ def add(self, slug, title, query, order=None, logo=None):
             query=query,
             parent=self,
             order=order,
-            logo=logo,
         )
 
     @property
@@ -127,7 +131,7 @@ def collection_tree(self):
 
         Note: this will execute a query to the collection tree table.
         """
-        return self.model.collection_tree
+        return CollectionTree(self.model.collection_tree)
 
     @property
     def depth(self):
@@ -144,18 +148,13 @@ def community(self):
         """Get the community object."""
         return self.collection_tree.community
 
-    @property
-    def logo(self):
-        """Get the collection logo."""
-        return self.model.logo
-
     @property
     def query(self):
         """Get the collection query."""
         q = ""
         for _a in self.ancestors:
-            q += f"({_a.model.query_text}) AND "
-        q += f"({self.model.query_text})"
+            q += f"({_a.model.search_query}) AND "
+        q += f"({self.model.search_query})"
         return q
 
     @cached_property
@@ -227,7 +226,6 @@ def dump(cls, collection):
             "title": collection.title,
             "slug": collection.slug,
             "depth": collection.depth,
-            "logo": collection.logo,
             "order": collection.order,
             "id": collection.id,
             "query": collection.query,
@@ -341,3 +339,8 @@ def community_id(self):
     def order(self):
         """Get the collection tree order."""
         return self.model.order
+
+    @property
+    def community(self):
+        """Get the community object."""
+        return self.model.community
diff --git a/invenio_rdm_records/collections/models.py b/invenio_rdm_records/collections/models.py
index 151cdec36..ad3da0d27 100644
--- a/invenio_rdm_records/collections/models.py
+++ b/invenio_rdm_records/collections/models.py
@@ -18,7 +18,16 @@
 class CollectionTree(db.Model, Timestamp):
     """Collection tree model."""
 
-    __tablename__ = "collection_tree"
+    __tablename__ = "collections_collection_tree"
+
+    __table_args__ = (
+        # Unique constraint on slug and community_id. Slugs should be unique within a community.
+        UniqueConstraint(
+            "slug",
+            "community_id",
+            name="uq_collections_collection_tree_slug_community_id",
+        ),
+    )
 
     id = db.Column(db.Integer, primary_key=True, autoincrement=True)
     community_id = db.Column(
@@ -29,17 +38,12 @@ class CollectionTree(db.Model, Timestamp):
     )
     title = db.Column(db.String(255), nullable=False)
     order = db.Column(db.Integer, nullable=True)
-    slug = db.Column(db.String(255), nullable=False, index=True)
+    slug = db.Column(db.String(255), nullable=False)
 
     # Relationship to Collection
     collections = db.relationship("Collection", back_populates="collection_tree")
     community = db.relationship(CommunityMetadata, backref="collection_trees")
 
-    # Unique constraint on slug and community_id. Slugs should be unique within a community.
-    UniqueConstraint(
-        "slug", "community_id", name="uq_collection_tree_slug_community_id"
-    )
-
     @classmethod
     def create(cls, title, slug, community_id=None, order=None):
         """Create a new collection tree."""
@@ -74,15 +78,22 @@ class Collection(db.Model, Timestamp):
     - slug
     """
 
-    __tablename__ = "collections"
+    __tablename__ = "collections_collection"
+    __table_args__ = (
+        # Unique constraint on slug and tree_id. Slugs should be unique within a tree.
+        UniqueConstraint(
+            "slug", "tree_id", name="uq_collections_collection_slug_tree_id"
+        ),
+    )
 
     id = db.Column(db.Integer, primary_key=True, autoincrement=True)
-    slug = db.Column(db.String(255), unique=False, nullable=False, index=True)
+    slug = db.Column(db.String(255), nullable=False)
     path = db.Column(db.Text, nullable=False, index=True)
-    tree_id = db.Column(db.Integer, db.ForeignKey("collection_tree.id"), nullable=False)
-    logo = db.Column(db.Text, nullable=True)
+    tree_id = db.Column(
+        db.Integer, db.ForeignKey("collections_collection_tree.id"), nullable=False
+    )
     title = db.Column(db.String(255), nullable=False)
-    query_text = db.Column("query", db.Text, nullable=False)
+    search_query = db.Column("query", db.Text, nullable=False)
     order = db.Column(db.Integer, nullable=True)
     # TODO index depth
     depth = db.Column(
@@ -93,11 +104,8 @@ class Collection(db.Model, Timestamp):
     # Relationship to CollectionTree
     collection_tree = db.relationship("CollectionTree", back_populates="collections")
 
-    # Unique constraint on slug and tree_id. Slugs should be unique within a tree.
-    UniqueConstraint("slug", "tree_id", name="uq_collections_slug_tree_id")
-
     @classmethod
-    def create(cls, slug, path, title, query_text, ctree_or_id, **kwargs):
+    def create(cls, slug, path, title, search_query, ctree_or_id, **kwargs):
         """Create a new collection."""
         with db.session.begin_nested():
             if isinstance(ctree_or_id, CollectionTree):
@@ -105,7 +113,7 @@ def create(cls, slug, path, title, query_text, ctree_or_id, **kwargs):
                     slug=slug,
                     path=path,
                     title=title,
-                    query_text=query_text,
+                    search_query=search_query,
                     collection_tree=ctree_or_id,
                     **kwargs,
                 )
@@ -114,7 +122,7 @@ def create(cls, slug, path, title, query_text, ctree_or_id, **kwargs):
                     slug=slug,
                     path=path,
                     title=title,
-                    query_text=query_text,
+                    search_query=search_query,
                     tree_id=ctree_or_id,
                     **kwargs,
                 )
diff --git a/invenio_rdm_records/collections/service.py b/invenio_rdm_records/collections/service.py
index b503f79dd..bad4ef23c 100644
--- a/invenio_rdm_records/collections/service.py
+++ b/invenio_rdm_records/collections/service.py
@@ -45,9 +45,20 @@ def links(self):
             "self_html": self_html,
         }
 
-    def __getattr__(self, name):
-        """Delegate attribute access to the collection."""
-        return getattr(self._collection, name)
+    @property
+    def community(self):
+        """Get the collection community."""
+        return self._collection.community
+
+    @property
+    def query(self):
+        """Get the collection query."""
+        return self._collection.query
+
+    @property
+    def title(self):
+        """Get the collection title."""
+        return self._collection.title
 
 
 class CollectionsService:
diff --git a/tests/collections/test_collections_api.py b/tests/collections/test_collections_api.py
index 056cc0c13..d6b715a80 100644
--- a/tests/collections/test_collections_api.py
+++ b/tests/collections/test_collections_api.py
@@ -6,16 +6,16 @@
 # it under the terms of the MIT License; see LICENSE file for more details.
 """Test suite for the collections programmatic API."""
 
-from invenio_rdm_records.collections.api import Collection
-from invenio_rdm_records.collections.models import CollectionTree
+from invenio_rdm_records.collections.api import Collection, CollectionTree
 
 
 def test_create(running_app, db, community, community_owner):
     """Test collection creation via API."""
-    tree = CollectionTree(
+    tree = CollectionTree.create(
         title="Tree 1",
         order=10,
         community_id=community.id,
+        slug="tree-1",
     )
 
     # Use ORM object (collection tree)
@@ -28,7 +28,7 @@ def test_create(running_app, db, community, community_owner):
 
     assert collection.id
     assert collection.title == "My Collection"
-    assert collection.collection_tree == tree
+    assert collection.collection_tree.id == tree.id
 
     # Use collection tree id
     collection = Collection.create(
@@ -39,14 +39,15 @@ def test_create(running_app, db, community, community_owner):
     )
     assert collection.id
     assert collection.title == "My Collection 2"
-    assert collection.collection_tree == tree
+    assert collection.collection_tree.id == tree.id
 
 
 def test_as_dict(running_app, db):
     """Test collection as dict."""
-    tree = CollectionTree(
+    tree = CollectionTree.create(
         title="Tree 1",
         order=10,
+        slug="tree-1",
     )
     c1 = Collection.create(
         title="My Collection",
@@ -65,7 +66,7 @@ def test_as_dict(running_app, db):
     c3 = Collection.create(title="3", query="*", slug="my-collection-3", parent=c2)
     res = c1.to_dict()
     assert all(k in res for k in (c1.id, c2.id, c3.id))
-    assert res[1]["title"] == "My Collection"
-    assert res[1]["children"] == [c2.id]
-    assert res[2]["children"] == [c3.id]
-    assert res[3]["children"] == []
+    assert res[c1.id]["title"] == "My Collection"
+    assert res[c1.id]["children"] == [c2.id]
+    assert res[c2.id]["children"] == [c3.id]
+    assert res[c3.id]["children"] == []
diff --git a/tests/services/test_collections_service.py b/tests/services/test_collections_service.py
index e4b3c4324..cf97becdb 100644
--- a/tests/services/test_collections_service.py
+++ b/tests/services/test_collections_service.py
@@ -8,8 +8,7 @@
 
 import pytest
 
-from invenio_rdm_records.collections.api import Collection
-from invenio_rdm_records.collections.models import CollectionTree
+from invenio_rdm_records.collections.api import Collection, CollectionTree
 from invenio_rdm_records.proxies import current_rdm_records
 
 
@@ -25,20 +24,15 @@ def add_collections(running_app, db, community):
 
     def _inner():
         """Add collections to the app."""
-        tree = CollectionTree(
+        tree = CollectionTree.create(
             title="Tree 1",
             order=10,
             community_id=community.id,
+            slug="tree-1",
         )
-        db.session.add(tree)
-        db.session.flush()
-        db.session.refresh(tree)
         c1 = Collection.create(
             title="Collection 1", query="*:*", slug="collection-1", ctree=tree
         )
-        db.session.add(c1.model)
-        db.session.flush()
-        db.session.refresh(tree)
         c2 = Collection.create(
             title="Collection 2",
             query="*:*",
@@ -46,8 +40,6 @@ def _inner():
             ctree=tree,
             parent=c1,
         )
-        db.session.add(c2.model)
-        db.session.commit()
         return [c1, c2]
 
     return _inner
@@ -60,14 +52,5 @@ def test_collections_service_read(
     collections = add_collections()
     c0 = collections[0]
     c1 = collections[1]
-    res = collections_service.read(community_owner, c0.id)
+    res = collections_service.read(community_owner.identity, c0.id)
     assert res._collection.id == c0.id
-
-
-def test_collections_service_create(
-    running_app, db, collections_service, community_owner
-):
-    """Test collections service create."""
-    collections_service.create(
-        community_owner, 1, title="My Collection", query="*:*", slug="my-collection"
-    )

From c5ef37549890a6a7694ecad77a2d9dd3ef5bfc65 Mon Sep 17 00:00:00 2001
From: alejandromumo <manuel.alejandro.de.oliveira.da.costa@cern.ch>
Date: Tue, 1 Oct 2024 14:16:17 +0200
Subject: [PATCH 20/76] collections: added uow and refactor depth calculation

---
 invenio_rdm_records/collections/models.py  |  7 ++++---
 invenio_rdm_records/collections/service.py | 22 +++++++++++++---------
 2 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/invenio_rdm_records/collections/models.py b/invenio_rdm_records/collections/models.py
index ad3da0d27..d3f510dba 100644
--- a/invenio_rdm_records/collections/models.py
+++ b/invenio_rdm_records/collections/models.py
@@ -96,9 +96,7 @@ class Collection(db.Model, Timestamp):
     search_query = db.Column("query", db.Text, nullable=False)
     order = db.Column(db.Integer, nullable=True)
     # TODO index depth
-    depth = db.Column(
-        db.Integer, db.Computed("array_length(string_to_array(path, ','), 1) - 2")
-    )  # TODO client side
+    depth = db.Column(db.Integer, nullable=False)
     num_records = db.Column(db.Integer, nullable=True)
 
     # Relationship to CollectionTree
@@ -107,6 +105,7 @@ class Collection(db.Model, Timestamp):
     @classmethod
     def create(cls, slug, path, title, search_query, ctree_or_id, **kwargs):
         """Create a new collection."""
+        depth = len([int(part) for part in path.split(",") if part.strip()])
         with db.session.begin_nested():
             if isinstance(ctree_or_id, CollectionTree):
                 collection = cls(
@@ -115,6 +114,7 @@ def create(cls, slug, path, title, search_query, ctree_or_id, **kwargs):
                     title=title,
                     search_query=search_query,
                     collection_tree=ctree_or_id,
+                    depth=depth,
                     **kwargs,
                 )
             elif isinstance(ctree_or_id, int):
@@ -124,6 +124,7 @@ def create(cls, slug, path, title, search_query, ctree_or_id, **kwargs):
                     title=title,
                     search_query=search_query,
                     tree_id=ctree_or_id,
+                    depth=depth,
                     **kwargs,
                 )
             else:
diff --git a/invenio_rdm_records/collections/service.py b/invenio_rdm_records/collections/service.py
index bad4ef23c..98b7f9e8f 100644
--- a/invenio_rdm_records/collections/service.py
+++ b/invenio_rdm_records/collections/service.py
@@ -7,6 +7,7 @@
 """Collections service."""
 
 from invenio_communities.proxies import current_communities
+from invenio_records_resources.services.uow import ModelCommitOp, unit_of_work
 
 from .api import Collection, CollectionTree
 
@@ -14,11 +15,8 @@
 class CollectionItem:
     """Collection item."""
 
-    def __init__(self, collection: Collection):
-        """Instantiate a Collection object.
-
-        Optionally pass a community to cache its information in the collection's instance.
-        """
+    def __init__(self, collection):
+        """Instantiate a Collection object."""
         self._collection = collection
 
     def to_dict(self):
@@ -66,7 +64,10 @@ class CollectionsService:
 
     collection_cls = Collection
 
-    def create(self, identity, community_id, tree_slug, slug, title, query, **kwargs):
+    @unit_of_work()
+    def create(
+        self, identity, community_id, tree_slug, slug, title, query, uow=None, **kwargs
+    ):
         """Create a new collection."""
         current_communities.service.require_permission(
             identity, "update", community_id=community_id
@@ -77,6 +78,7 @@ def create(self, identity, community_id, tree_slug, slug, title, query, **kwargs
         collection = self.collection_cls.create(
             slug=slug, title=title, query=query, ctree=ctree, **kwargs
         )
+        uow.register(ModelCommitOp(collection.model))
         return CollectionItem(collection)
 
     def read(self, identity, id_):
@@ -107,12 +109,14 @@ def read_slug(self, identity, community_id, tree_slug, slug):
 
         return CollectionItem(collection)
 
-    def add(self, identity, collection, slug, title, query, **kwargs):
+    @unit_of_work()
+    def add(self, identity, collection, slug, title, query, uow=None, **kwargs):
         """Add a subcollection to a collection."""
         current_communities.service.require_permission(
             identity, "update", community_id=collection.community.id
         )
-        collection = self.collection_cls.create(
+        new_collection = self.collection_cls.create(
             parent=collection, slug=slug, title=title, query=query, **kwargs
         )
-        return CollectionItem(collection)
+        uow.register(ModelCommitOp(new_collection.model))
+        return CollectionItem(new_collection)

From ac37cf8089e86c316e2fb96d298295a417471e87 Mon Sep 17 00:00:00 2001
From: alejandromumo <manuel.alejandro.de.oliveira.da.costa@cern.ch>
Date: Tue, 1 Oct 2024 14:33:57 +0200
Subject: [PATCH 21/76] collections: updated depth column

---
 .../alembic/425b691f768b_create_collections_tables.py    | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/invenio_rdm_records/alembic/425b691f768b_create_collections_tables.py b/invenio_rdm_records/alembic/425b691f768b_create_collections_tables.py
index a66e0d06d..75068011e 100644
--- a/invenio_rdm_records/alembic/425b691f768b_create_collections_tables.py
+++ b/invenio_rdm_records/alembic/425b691f768b_create_collections_tables.py
@@ -63,14 +63,7 @@ def upgrade():
         sa.Column("title", sa.String(length=255), nullable=False),
         sa.Column("query", sa.Text(), nullable=False),
         sa.Column("order", sa.Integer(), nullable=True),
-        sa.Column(
-            "depth",
-            sa.Integer(),
-            sa.Computed(
-                "array_length(string_to_array(path, ','), 1) - 2",
-            ),
-            nullable=True,
-        ),
+        sa.Column("depth", sa.Integer(), nullable=True),
         sa.Column("num_records", sa.Integer(), nullable=True),
         sa.ForeignKeyConstraint(
             ["tree_id"],

From 44e093e6ffea929641ea77fb869a2e990c12b8c3 Mon Sep 17 00:00:00 2001
From: alejandromumo <manuel.alejandro.de.oliveira.da.costa@cern.ch>
Date: Thu, 3 Oct 2024 14:55:57 +0200
Subject: [PATCH 22/76] release: v13.0.0

---
 CHANGES.rst                     | 6 ++++++
 invenio_rdm_records/__init__.py | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index cc1a5e16f..92ef99d00 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -12,6 +12,12 @@
 Changes
 =======
 
+Version v13.0.0 (released 2024-10-03)
+
+- collections: added feature, containing core functionalities and DB models
+- ui: fixed propTypes warnings
+- dependencies: bump flask-iiif to >1.0.0
+
 Version v12.2.2 (released 2024-09-30)
 
 - Improve handling of draft PID in RecordCommunitiesService
diff --git a/invenio_rdm_records/__init__.py b/invenio_rdm_records/__init__.py
index 3806279db..55d0945b2 100644
--- a/invenio_rdm_records/__init__.py
+++ b/invenio_rdm_records/__init__.py
@@ -11,6 +11,6 @@
 
 from .ext import InvenioRDMRecords
 
-__version__ = "12.2.2"
+__version__ = "13.0.0"
 
 __all__ = ("__version__", "InvenioRDMRecords")

From a7f35c0ecd4cc51e894dc8d91425a7fe7fa056b7 Mon Sep 17 00:00:00 2001
From: Fatimah Zulfiqar <fatimah0zulfiqar@gmail.com>
Date: Fri, 4 Oct 2024 16:10:28 +0200
Subject: [PATCH 23/76] tests: fixed names vocablularies test

---
 tests/resources/vocabularies/test_names_vocabulary.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/resources/vocabularies/test_names_vocabulary.py b/tests/resources/vocabularies/test_names_vocabulary.py
index 6ff6df70b..0bb7c32bc 100644
--- a/tests/resources/vocabularies/test_names_vocabulary.py
+++ b/tests/resources/vocabularies/test_names_vocabulary.py
@@ -39,20 +39,20 @@ def example_name(app, db, search_clear, superuser_identity, names_service):
     names_service.delete(superuser_identity, name.id)
 
 
-def test_names_get(client, example_name, headers):
+def test_names_get(client_with_login, example_name, headers):
     """Test the endpoint to retrieve a single item."""
     id_ = example_name.id
 
-    res = client.get(f"/names/{id_}", headers=headers)
+    res = client_with_login.get(f"/names/{id_}", headers=headers)
     assert res.status_code == 200
     assert res.json["id"] == id_
     # Test links
     assert res.json["links"] == {"self": f"https://127.0.0.1:5000/api/names/{id_}"}
 
 
-def test_names_search(client, example_name, headers):
+def test_names_search(client_with_login, example_name, headers):
     """Test a successful search."""
-    res = client.get("/names", headers=headers)
+    res = client_with_login.get("/names", headers=headers)
 
     assert res.status_code == 200
     assert res.json["hits"]["total"] == 1

From 584cd62cb39c313ed137d6c422d86d8d500442f0 Mon Sep 17 00:00:00 2001
From: Pablo Tamarit <pablo.tamarit@cern.ch>
Date: Thu, 3 Oct 2024 15:43:41 +0200
Subject: [PATCH 24/76] installation: bump invenio-vocabularies &
 invenio-communities

---
 setup.cfg | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index f9ede4779..0eb9f9ec0 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -39,7 +39,7 @@ install_requires =
     flask-iiif>=1.0.0,<2.0.0
     ftfy>=4.4.3,<5.0.0
     invenio-administration>=2.0.0,<3.0.0
-    invenio-communities>=15.0.0,<16.0.0
+    invenio-communities>=16.0.0,<17.0.0
     invenio-drafts-resources>=5.0.0,<6.0.0
     invenio-records-resources>=6.0.0,<7.0.0
     invenio-github>=1.0.0,<2.0.0
@@ -48,7 +48,7 @@ install_requires =
     invenio-oaiserver>=2.0.0,<3.0.0
     invenio-oauth2server>=2.0.0
     invenio-stats>=4.0.0,<5.0.0
-    invenio-vocabularies>=5.0.0,<6.0.0
+    invenio-vocabularies>=6.0.0,<7.0.0
     nameparser>=1.1.1
     pycountry>=22.3.5
     pydash>=6.0.0,<7.0.0

From 9519c1bd765b47b618e7820dd4fd47bfc9571c4d Mon Sep 17 00:00:00 2001
From: Pablo Tamarit <pablo.tamarit@cern.ch>
Date: Fri, 4 Oct 2024 14:59:57 +0200
Subject: [PATCH 25/76] =?UTF-8?q?=F0=9F=93=A6=20release:=20v14.0.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 CHANGES.rst                     | 4 ++++
 invenio_rdm_records/__init__.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 92ef99d00..3586e66bf 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -12,6 +12,10 @@
 Changes
 =======
 
+Version v14.0.0 (released 2024-10-04)
+
+- installation: bump invenio-vocabularies & invenio-communities
+
 Version v13.0.0 (released 2024-10-03)
 
 - collections: added feature, containing core functionalities and DB models
diff --git a/invenio_rdm_records/__init__.py b/invenio_rdm_records/__init__.py
index 55d0945b2..c5c7c1b8d 100644
--- a/invenio_rdm_records/__init__.py
+++ b/invenio_rdm_records/__init__.py
@@ -11,6 +11,6 @@
 
 from .ext import InvenioRDMRecords
 
-__version__ = "13.0.0"
+__version__ = "14.0.0"
 
 __all__ = ("__version__", "InvenioRDMRecords")

From 2c24e666dce83678edd1db04ad8704754a099742 Mon Sep 17 00:00:00 2001
From: Fatimah Zulfiqar <fatimah0zulfiqar@gmail.com>
Date: Mon, 23 Sep 2024 09:44:15 +0200
Subject: [PATCH 26/76] relations: added subject relation in awards

---
 invenio_rdm_records/records/api.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/invenio_rdm_records/records/api.py b/invenio_rdm_records/records/api.py
index e8f91b4e6..d89952eb7 100644
--- a/invenio_rdm_records/records/api.py
+++ b/invenio_rdm_records/records/api.py
@@ -150,10 +150,17 @@ class CommonFieldsMixin:
         funding_award=PIDListRelation(
             "metadata.funding",
             relation_field="award",
-            keys=["title", "number", "identifiers", "acronym", "program"],
+            keys=["title", "number", "identifiers", "acronym", "program", "subjects", "organizations"],
             pid_field=Award.pid,
             cache_key="awards",
         ),
+        funding_award_subjects=PIDListRelation(
+            "metadata.funding",
+            relation_field="award.subjects",
+            keys=["subject", "scheme", "identifiers", "props"],
+            pid_field=Subject.pid,
+            cache_key="subjects",
+        ),
         languages=PIDListRelation(
             "metadata.languages",
             keys=["title"],

From 69b19ca58f53d5d21935a56e70594869836bdb17 Mon Sep 17 00:00:00 2001
From: Fatimah Zulfiqar <fatimah0zulfiqar@gmail.com>
Date: Sat, 28 Sep 2024 11:06:14 +0200
Subject: [PATCH 27/76] awards: added subjects and orgs, updated mappings

---
 invenio_rdm_records/records/api.py            |  18 ++-
 .../records/dumpers/__init__.py               |   2 +
 .../records/dumpers/subject_hierarchy.py      |  68 +++++++++++
 .../records/definitions-v2.0.0.json           | 106 ++++++++++--------
 .../jsonschemas/records/record-v6.0.0.json    |  31 ++---
 .../os-v1/rdmrecords/drafts/draft-v6.0.0.json | 103 +++++++++++++++++
 .../rdmrecords/records/record-v7.0.0.json     | 105 +++++++++++++++++
 .../os-v2/rdmrecords/drafts/draft-v6.0.0.json | 103 +++++++++++++++++
 .../rdmrecords/records/record-v7.0.0.json     | 105 +++++++++++++++++
 .../v7/rdmrecords/drafts/draft-v6.0.0.json    | 103 +++++++++++++++++
 .../v7/rdmrecords/records/record-v6.0.0.json  | 105 +++++++++++++++++
 11 files changed, 777 insertions(+), 72 deletions(-)
 create mode 100644 invenio_rdm_records/records/dumpers/subject_hierarchy.py

diff --git a/invenio_rdm_records/records/api.py b/invenio_rdm_records/records/api.py
index d89952eb7..0348f53bb 100644
--- a/invenio_rdm_records/records/api.py
+++ b/invenio_rdm_records/records/api.py
@@ -52,6 +52,7 @@
     EDTFListDumperExt,
     GrantTokensDumperExt,
     StatisticsDumperExt,
+    SubjectHierarchyDumperExt,
 )
 from .systemfields import (
     HasDraftCheckField,
@@ -122,6 +123,7 @@ class CommonFieldsMixin:
             CombinedSubjectsDumperExt(),
             CustomFieldsDumperExt(fields_var="RDM_CUSTOM_FIELDS"),
             StatisticsDumperExt("stats"),
+            SubjectHierarchyDumperExt(),
         ]
     )
 
@@ -150,14 +152,22 @@ class CommonFieldsMixin:
         funding_award=PIDListRelation(
             "metadata.funding",
             relation_field="award",
-            keys=["title", "number", "identifiers", "acronym", "program", "subjects", "organizations"],
+            keys=[
+                "title",
+                "number",
+                "identifiers",
+                "acronym",
+                "program",
+                "subjects",
+                "organizations",
+            ],
             pid_field=Award.pid,
             cache_key="awards",
         ),
-        funding_award_subjects=PIDListRelation(
+        funding_award_subjects=PIDNestedListRelation(
             "metadata.funding",
             relation_field="award.subjects",
-            keys=["subject", "scheme", "identifiers", "props"],
+            keys=["subject", "scheme", "props"],
             pid_field=Subject.pid,
             cache_key="subjects",
         ),
@@ -176,7 +186,7 @@ class CommonFieldsMixin:
         ),
         subjects=PIDListRelation(
             "metadata.subjects",
-            keys=["subject", "scheme"],
+            keys=["subject", "scheme", "props"],
             pid_field=Subject.pid,
             cache_key="subjects",
         ),
diff --git a/invenio_rdm_records/records/dumpers/__init__.py b/invenio_rdm_records/records/dumpers/__init__.py
index 85515d095..d2b25b92e 100644
--- a/invenio_rdm_records/records/dumpers/__init__.py
+++ b/invenio_rdm_records/records/dumpers/__init__.py
@@ -13,6 +13,7 @@
 from .locations import LocationsDumper
 from .pids import PIDsDumperExt
 from .statistics import StatisticsDumperExt
+from .subject_hierarchy import SubjectHierarchyDumperExt
 
 __all__ = (
     "CombinedSubjectsDumperExt",
@@ -22,4 +23,5 @@
     "GrantTokensDumperExt",
     "LocationsDumper",
     "StatisticsDumperExt",
+    "SubjectHierarchyDumperExt",
 )
diff --git a/invenio_rdm_records/records/dumpers/subject_hierarchy.py b/invenio_rdm_records/records/dumpers/subject_hierarchy.py
new file mode 100644
index 000000000..e95f7cac2
--- /dev/null
+++ b/invenio_rdm_records/records/dumpers/subject_hierarchy.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Search dumpers for subject hierarchy support."""
+
+from invenio_records.dumpers import SearchDumperExt
+
+
+class SubjectHierarchyDumperExt(SearchDumperExt):
+    """Search dumper extension for subject hierarchy support.
+
+    It parses the values of the `subjects` field in the document, builds hierarchical
+    parent notations, and adds entries to the `hierarchy` field for each subject in award.
+
+    This dumper needs to be placed after the RelationDumper for subjects as it relies
+    on dereferenced subjects with scheme, subject, and props.
+    """
+
+    def __init__(self, splitchar=","):
+        """Constructor.
+
+        :param splitchar: string to use to combine subject ids in hierarchy
+        """
+        super().__init__()
+        self._splitchar = splitchar
+
+    def dump(self, record, data):
+        """Dump the data to secondary storage (OpenSearch-like)."""
+        awards = data.get("metadata", {}).get("funding", [])
+
+        def build_hierarchy(parents_str, current_subject_id):
+            """Build the hierarchy by progressively combining parent notations."""
+            parents = parents_str.split(self._splitchar)  # Split the parent notations
+            hierarchy = []
+            current_hierarchy = parents[0]  # Start with the top-level parent
+
+            hierarchy.append(current_hierarchy)
+            for parent in parents[1:]:
+                current_hierarchy = f"{current_hierarchy}{self._splitchar}{parent}"
+                hierarchy.append(current_hierarchy)
+
+            hierarchy.append(
+                f"{current_hierarchy}{self._splitchar}{current_subject_id}"
+            )
+            return hierarchy
+
+        for award in awards:
+            subjects = award.get("award", {}).get("subjects", [])
+            for subject in subjects:
+                parents = subject.get("props", {}).get("parents", "")
+                current_subject_id = subject.get("id", "")
+                if parents and current_subject_id:
+                    subject_hierarchy = build_hierarchy(parents, current_subject_id)
+                    subject.setdefault("props", {})["hierarchy"] = subject_hierarchy
+
+        data["metadata"] = data.get("metadata", {})
+
+    def load(self, data, record_cls):
+        """Load the data from secondary storage (OpenSearch-like).
+
+        This is run against the parent too (for some reason), so presence of any
+        field cannot be assumed.
+        """
+        pass
diff --git a/invenio_rdm_records/records/jsonschemas/records/definitions-v2.0.0.json b/invenio_rdm_records/records/jsonschemas/records/definitions-v2.0.0.json
index beb527ae5..866cc7285 100644
--- a/invenio_rdm_records/records/jsonschemas/records/definitions-v2.0.0.json
+++ b/invenio_rdm_records/records/jsonschemas/records/definitions-v2.0.0.json
@@ -13,7 +13,7 @@
   },
   "affiliations": {
     "type": "array",
-    "items": {"$ref": "#/affiliation"}
+    "items": { "$ref": "#/affiliation" }
   },
   "agent": {
     "description": "An agent (user, software process, community, ...).",
@@ -72,16 +72,11 @@
       {
         "title": "GeoJSON Point",
         "type": "object",
-        "required": [
-          "type",
-          "coordinates"
-        ],
+        "required": ["type", "coordinates"],
         "properties": {
           "type": {
             "type": "string",
-            "enum": [
-              "Point"
-            ]
+            "enum": ["Point"]
           },
           "coordinates": {
             "type": "array",
@@ -102,16 +97,11 @@
       {
         "title": "GeoJSON LineString",
         "type": "object",
-        "required": [
-          "type",
-          "coordinates"
-        ],
+        "required": ["type", "coordinates"],
         "properties": {
           "type": {
             "type": "string",
-            "enum": [
-              "LineString"
-            ]
+            "enum": ["LineString"]
           },
           "coordinates": {
             "type": "array",
@@ -136,16 +126,11 @@
       {
         "title": "GeoJSON Polygon",
         "type": "object",
-        "required": [
-          "type",
-          "coordinates"
-        ],
+        "required": ["type", "coordinates"],
         "properties": {
           "type": {
             "type": "string",
-            "enum": [
-              "Polygon"
-            ]
+            "enum": ["Polygon"]
           },
           "coordinates": {
             "type": "array",
@@ -173,16 +158,11 @@
       {
         "title": "GeoJSON MultiPoint",
         "type": "object",
-        "required": [
-          "type",
-          "coordinates"
-        ],
+        "required": ["type", "coordinates"],
         "properties": {
           "type": {
             "type": "string",
-            "enum": [
-              "MultiPoint"
-            ]
+            "enum": ["MultiPoint"]
           },
           "coordinates": {
             "type": "array",
@@ -206,16 +186,11 @@
       {
         "title": "GeoJSON MultiLineString",
         "type": "object",
-        "required": [
-          "type",
-          "coordinates"
-        ],
+        "required": ["type", "coordinates"],
         "properties": {
           "type": {
             "type": "string",
-            "enum": [
-              "MultiLineString"
-            ]
+            "enum": ["MultiLineString"]
           },
           "coordinates": {
             "type": "array",
@@ -243,16 +218,11 @@
       {
         "title": "GeoJSON MultiPolygon",
         "type": "object",
-        "required": [
-          "type",
-          "coordinates"
-        ],
+        "required": ["type", "coordinates"],
         "properties": {
           "type": {
             "type": "string",
-            "enum": [
-              "MultiPolygon"
-            ]
+            "enum": ["MultiPolygon"]
           },
           "coordinates": {
             "type": "array",
@@ -311,10 +281,7 @@
   "nameType": {
     "description": "Type of name.",
     "type": "string",
-    "enum": [
-      "personal",
-      "organizational"
-    ]
+    "enum": ["personal", "organizational"]
   },
   "person_or_org": {
     "type": "object",
@@ -381,12 +348,55 @@
       },
       "subject": {
         "type": "string"
+      },
+      "scheme": {
+        "description": "Identifier of the subject scheme.",
+        "$ref": "local://definitions-v1.0.0.json#/identifier"
+      },
+      "props": {
+        "type": "object",
+        "patternProperties": {
+          "^.*$": {
+            "type": "string"
+          }
+        }
+      },
+      "identifiers": {
+        "description": "Alternate identifiers for the subject.",
+        "type": "array",
+        "items": {
+          "$ref": "local://definitions-v2.0.0.json#/identifiers_with_scheme"
+        },
+        "uniqueItems": true
       }
     }
   },
   "subjects": {
     "type": "array",
-    "items": {"$ref": "#/subject"}
+    "items": { "$ref": "#/subject" }
+  },
+  "organization": {
+    "description": "Award's organizations.",
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {
+      "scheme": {
+        "description": "Identifier of the organization scheme.",
+        "$ref": "local://definitions-v1.0.0.json#/identifier"
+      },
+      "id": {
+        "description": "Identifier of the organization for the given scheme.",
+        "$ref": "local://definitions-v1.0.0.json#/identifier"
+      },
+      "organization": {
+        "description": "Human readable label.",
+        "type": "string"
+      }
+    }
+  },
+  "organizations": {
+    "type": "array",
+    "items": { "$ref": "#/organization" }
   },
   "title_type": {
     "description": "Type of title.",
diff --git a/invenio_rdm_records/records/jsonschemas/records/record-v6.0.0.json b/invenio_rdm_records/records/jsonschemas/records/record-v6.0.0.json
index 84d02b795..a51b22ff5 100644
--- a/invenio_rdm_records/records/jsonschemas/records/record-v6.0.0.json
+++ b/invenio_rdm_records/records/jsonschemas/records/record-v6.0.0.json
@@ -316,6 +316,12 @@
                     "items": {
                       "$ref": "local://definitions-v1.0.0.json#/identifiers_with_scheme"
                     }
+                  },
+                  "subjects": {
+                    "$ref": "local://records/definitions-v2.0.0.json#/subjects"
+                  },
+                  "organizations": {
+                    "$ref": "local://records/definitions-v2.0.0.json#/organizations"
                   }
                 }
               }
@@ -400,18 +406,12 @@
         "record": {
           "description": "Record visibility (public or restricted)",
           "type": "string",
-          "enum": [
-            "public",
-            "restricted"
-          ]
+          "enum": ["public", "restricted"]
         },
         "files": {
           "description": "Files visibility (public or restricted)",
           "type": "string",
-          "enum": [
-            "public",
-            "restricted"
-          ]
+          "enum": ["public", "restricted"]
         },
         "embargo": {
           "description": "Description of the embargo on the record.",
@@ -420,25 +420,16 @@
           "properties": {
             "active": {
               "description": "Whether or not the embargo is (still) active.",
-              "type": [
-                "boolean",
-                "null"
-              ]
+              "type": ["boolean", "null"]
             },
             "until": {
               "description": "Embargo date of record (ISO8601 formatted date time in UTC). At this time both metadata and files will be made public.",
-              "type": [
-                "string",
-                "null"
-              ],
+              "type": ["string", "null"],
               "format": "date"
             },
             "reason": {
               "description": "The reason why the record is under embargo.",
-              "type": [
-                "string",
-                "null"
-              ]
+              "type": ["string", "null"]
             }
           }
         }
diff --git a/invenio_rdm_records/records/mappings/os-v1/rdmrecords/drafts/draft-v6.0.0.json b/invenio_rdm_records/records/mappings/os-v1/rdmrecords/drafts/draft-v6.0.0.json
index 7e7b4cddc..3c532097e 100644
--- a/invenio_rdm_records/records/mappings/os-v1/rdmrecords/drafts/draft-v6.0.0.json
+++ b/invenio_rdm_records/records/mappings/os-v1/rdmrecords/drafts/draft-v6.0.0.json
@@ -353,6 +353,39 @@
                                     "type": "keyword"
                                   }
                                 }
+                              },
+                              "subjects": {
+                                "properties": {
+                                  "@v": {
+                                    "type": "keyword"
+                                  },
+                                  "id": {
+                                    "type": "keyword"
+                                  },
+                                  "subject": {
+                                    "type": "keyword"
+                                  },
+                                  "scheme": {
+                                    "type": "keyword"
+                                  },
+                                  "props": {
+                                    "type": "object",
+                                    "dynamic": "true"
+                                  }
+                                }
+                              },
+                              "organizations": {
+                                "properties": {
+                                  "scheme": {
+                                    "type": "keyword"
+                                  },
+                                  "id": {
+                                    "type": "keyword"
+                                  },
+                                  "organization": {
+                                    "type": "keyword"
+                                  }
+                                }
                               }
                             }
                           },
@@ -512,6 +545,39 @@
                                         "type": "keyword"
                                       }
                                     }
+                                  },
+                                  "subjects": {
+                                    "properties": {
+                                      "@v": {
+                                        "type": "keyword"
+                                      },
+                                      "id": {
+                                        "type": "keyword"
+                                      },
+                                      "subject": {
+                                        "type": "keyword"
+                                      },
+                                      "scheme": {
+                                        "type": "keyword"
+                                      },
+                                      "props": {
+                                        "type": "object",
+                                        "dynamic": "true"
+                                      }
+                                    }
+                                  },
+                                  "organizations": {
+                                    "properties": {
+                                      "scheme": {
+                                        "type": "keyword"
+                                      },
+                                      "id": {
+                                        "type": "keyword"
+                                      },
+                                      "organization": {
+                                        "type": "keyword"
+                                      }
+                                    }
                                   }
                                 }
                               },
@@ -896,6 +962,39 @@
                         "type": "keyword"
                       }
                     }
+                  },
+                  "subjects": {
+                    "properties": {
+                      "@v": {
+                        "type": "keyword"
+                      },
+                      "id": {
+                        "type": "keyword"
+                      },
+                      "subject": {
+                        "type": "keyword"
+                      },
+                      "scheme": {
+                        "type": "keyword"
+                      },
+                      "props": {
+                        "type": "object",
+                        "dynamic": "true"
+                      }
+                    }
+                  },
+                  "organizations": {
+                    "properties": {
+                      "scheme": {
+                        "type": "keyword"
+                      },
+                      "id": {
+                        "type": "keyword"
+                      },
+                      "organization": {
+                        "type": "keyword"
+                      }
+                    }
                   }
                 }
               },
@@ -1135,6 +1234,10 @@
               },
               "scheme": {
                 "type": "keyword"
+              },
+              "props": {
+                "type": "object",
+                "dynamic": "true"
               }
             }
           },
diff --git a/invenio_rdm_records/records/mappings/os-v1/rdmrecords/records/record-v7.0.0.json b/invenio_rdm_records/records/mappings/os-v1/rdmrecords/records/record-v7.0.0.json
index 3e29366cd..3fd7e98d4 100644
--- a/invenio_rdm_records/records/mappings/os-v1/rdmrecords/records/record-v7.0.0.json
+++ b/invenio_rdm_records/records/mappings/os-v1/rdmrecords/records/record-v7.0.0.json
@@ -18,6 +18,8 @@
       "metadata.funding.award.identifiers.identifier",
       "metadata.funding.award.acronym.text",
       "metadata.funding.award.number",
+      "metadata.funding.award.subjects",
+      "metadata.funding.award.organizations",
       "metadata.funding.funder.name",
       "metadata.identifiers.identifier",
       "metadata.locations.features.place",
@@ -409,6 +411,39 @@
                               }
                             }
                           },
+                          "subjects": {
+                            "properties": {
+                              "@v": {
+                                "type": "keyword"
+                              },
+                              "id": {
+                                "type": "keyword"
+                              },
+                              "subject": {
+                                "type": "keyword"
+                              },
+                              "scheme": {
+                                "type": "keyword"
+                              },
+                              "props": {
+                                "type": "object",
+                                "dynamic": "true"
+                              }
+                            }
+                          },
+                          "organizations": {
+                            "properties": {
+                              "scheme": {
+                                "type": "keyword"
+                              },
+                              "id": {
+                                "type": "keyword"
+                              },
+                              "organization": {
+                                "type": "keyword"
+                              }
+                            }
+                          },
                           "funder": {
                             "type": "object",
                             "properties": {
@@ -565,6 +600,39 @@
                                         "type": "keyword"
                                       }
                                     }
+                                  },
+                                  "subjects": {
+                                    "properties": {
+                                      "@v": {
+                                        "type": "keyword"
+                                      },
+                                      "id": {
+                                        "type": "keyword"
+                                      },
+                                      "subject": {
+                                        "type": "keyword"
+                                      },
+                                      "scheme": {
+                                        "type": "keyword"
+                                      },
+                                      "props": {
+                                        "type": "object",
+                                        "dynamic": "true"
+                                      }
+                                    }
+                                  },
+                                  "organizations": {
+                                    "properties": {
+                                      "scheme": {
+                                        "type": "keyword"
+                                      },
+                                      "id": {
+                                        "type": "keyword"
+                                      },
+                                      "organization": {
+                                        "type": "keyword"
+                                      }
+                                    }
                                   }
                                 }
                               },
@@ -911,6 +979,39 @@
                         "type": "keyword"
                       }
                     }
+                  },
+                  "subjects": {
+                    "properties": {
+                      "@v": {
+                        "type": "keyword"
+                      },
+                      "id": {
+                        "type": "keyword"
+                      },
+                      "subject": {
+                        "type": "keyword"
+                      },
+                      "scheme": {
+                        "type": "keyword"
+                      },
+                      "props": {
+                        "type": "object",
+                        "dynamic": "true"
+                      }
+                    }
+                  },
+                  "organizations": {
+                    "properties": {
+                      "scheme": {
+                        "type": "keyword"
+                      },
+                      "id": {
+                        "type": "keyword"
+                      },
+                      "organization": {
+                        "type": "keyword"
+                      }
+                    }
                   }
                 }
               },
@@ -1150,6 +1251,10 @@
               },
               "scheme": {
                 "type": "keyword"
+              },
+              "props": {
+                "type": "object",
+                "dynamic": "true"
               }
             }
           },
diff --git a/invenio_rdm_records/records/mappings/os-v2/rdmrecords/drafts/draft-v6.0.0.json b/invenio_rdm_records/records/mappings/os-v2/rdmrecords/drafts/draft-v6.0.0.json
index 7e7b4cddc..3c532097e 100644
--- a/invenio_rdm_records/records/mappings/os-v2/rdmrecords/drafts/draft-v6.0.0.json
+++ b/invenio_rdm_records/records/mappings/os-v2/rdmrecords/drafts/draft-v6.0.0.json
@@ -353,6 +353,39 @@
                                     "type": "keyword"
                                   }
                                 }
+                              },
+                              "subjects": {
+                                "properties": {
+                                  "@v": {
+                                    "type": "keyword"
+                                  },
+                                  "id": {
+                                    "type": "keyword"
+                                  },
+                                  "subject": {
+                                    "type": "keyword"
+                                  },
+                                  "scheme": {
+                                    "type": "keyword"
+                                  },
+                                  "props": {
+                                    "type": "object",
+                                    "dynamic": "true"
+                                  }
+                                }
+                              },
+                              "organizations": {
+                                "properties": {
+                                  "scheme": {
+                                    "type": "keyword"
+                                  },
+                                  "id": {
+                                    "type": "keyword"
+                                  },
+                                  "organization": {
+                                    "type": "keyword"
+                                  }
+                                }
                               }
                             }
                           },
@@ -512,6 +545,39 @@
                                         "type": "keyword"
                                       }
                                     }
+                                  },
+                                  "subjects": {
+                                    "properties": {
+                                      "@v": {
+                                        "type": "keyword"
+                                      },
+                                      "id": {
+                                        "type": "keyword"
+                                      },
+                                      "subject": {
+                                        "type": "keyword"
+                                      },
+                                      "scheme": {
+                                        "type": "keyword"
+                                      },
+                                      "props": {
+                                        "type": "object",
+                                        "dynamic": "true"
+                                      }
+                                    }
+                                  },
+                                  "organizations": {
+                                    "properties": {
+                                      "scheme": {
+                                        "type": "keyword"
+                                      },
+                                      "id": {
+                                        "type": "keyword"
+                                      },
+                                      "organization": {
+                                        "type": "keyword"
+                                      }
+                                    }
                                   }
                                 }
                               },
@@ -896,6 +962,39 @@
                         "type": "keyword"
                       }
                     }
+                  },
+                  "subjects": {
+                    "properties": {
+                      "@v": {
+                        "type": "keyword"
+                      },
+                      "id": {
+                        "type": "keyword"
+                      },
+                      "subject": {
+                        "type": "keyword"
+                      },
+                      "scheme": {
+                        "type": "keyword"
+                      },
+                      "props": {
+                        "type": "object",
+                        "dynamic": "true"
+                      }
+                    }
+                  },
+                  "organizations": {
+                    "properties": {
+                      "scheme": {
+                        "type": "keyword"
+                      },
+                      "id": {
+                        "type": "keyword"
+                      },
+                      "organization": {
+                        "type": "keyword"
+                      }
+                    }
                   }
                 }
               },
@@ -1135,6 +1234,10 @@
               },
               "scheme": {
                 "type": "keyword"
+              },
+              "props": {
+                "type": "object",
+                "dynamic": "true"
               }
             }
           },
diff --git a/invenio_rdm_records/records/mappings/os-v2/rdmrecords/records/record-v7.0.0.json b/invenio_rdm_records/records/mappings/os-v2/rdmrecords/records/record-v7.0.0.json
index 3e29366cd..0719293e9 100644
--- a/invenio_rdm_records/records/mappings/os-v2/rdmrecords/records/record-v7.0.0.json
+++ b/invenio_rdm_records/records/mappings/os-v2/rdmrecords/records/record-v7.0.0.json
@@ -18,6 +18,8 @@
       "metadata.funding.award.identifiers.identifier",
       "metadata.funding.award.acronym.text",
       "metadata.funding.award.number",
+      "metadata.funding.award.subjects",
+      "metadata.funding.award.organizations",
       "metadata.funding.funder.name",
       "metadata.identifiers.identifier",
       "metadata.locations.features.place",
@@ -406,6 +408,39 @@
                                     "type": "keyword"
                                   }
                                 }
+                              },
+                              "subjects": {
+                                "properties": {
+                                  "@v": {
+                                    "type": "keyword"
+                                  },
+                                  "id": {
+                                    "type": "keyword"
+                                  },
+                                  "subject": {
+                                    "type": "keyword"
+                                  },
+                                  "scheme": {
+                                    "type": "keyword"
+                                  },
+                                  "props": {
+                                    "type": "object",
+                                    "dynamic": "true"
+                                  }
+                                }
+                              },
+                              "organizations": {
+                                "properties": {
+                                  "scheme": {
+                                    "type": "keyword"
+                                  },
+                                  "id": {
+                                    "type": "keyword"
+                                  },
+                                  "organization": {
+                                    "type": "keyword"
+                                  }
+                                }
                               }
                             }
                           },
@@ -565,6 +600,39 @@
                                         "type": "keyword"
                                       }
                                     }
+                                  },
+                                  "subjects": {
+                                    "properties": {
+                                      "@v": {
+                                        "type": "keyword"
+                                      },
+                                      "id": {
+                                        "type": "keyword"
+                                      },
+                                      "subject": {
+                                        "type": "keyword"
+                                      },
+                                      "scheme": {
+                                        "type": "keyword"
+                                      },
+                                      "props": {
+                                        "type": "object",
+                                        "dynamic": "true"
+                                      }
+                                    }
+                                  },
+                                  "organizations": {
+                                    "properties": {
+                                      "scheme": {
+                                        "type": "keyword"
+                                      },
+                                      "id": {
+                                        "type": "keyword"
+                                      },
+                                      "organization": {
+                                        "type": "keyword"
+                                      }
+                                    }
                                   }
                                 }
                               },
@@ -911,6 +979,39 @@
                         "type": "keyword"
                       }
                     }
+                  },
+                  "subjects": {
+                    "properties": {
+                      "@v": {
+                        "type": "keyword"
+                      },
+                      "id": {
+                        "type": "keyword"
+                      },
+                      "subject": {
+                        "type": "keyword"
+                      },
+                      "scheme": {
+                        "type": "keyword"
+                      },
+                      "props": {
+                        "type": "object",
+                        "dynamic": "true"
+                      }
+                    }
+                  },
+                  "organizations": {
+                    "properties": {
+                      "scheme": {
+                        "type": "keyword"
+                      },
+                      "id": {
+                        "type": "keyword"
+                      },
+                      "organization": {
+                        "type": "keyword"
+                      }
+                    }
                   }
                 }
               },
@@ -1150,6 +1251,10 @@
               },
               "scheme": {
                 "type": "keyword"
+              },
+              "props": {
+                "type": "object",
+                "dynamic": "true"
               }
             }
           },
diff --git a/invenio_rdm_records/records/mappings/v7/rdmrecords/drafts/draft-v6.0.0.json b/invenio_rdm_records/records/mappings/v7/rdmrecords/drafts/draft-v6.0.0.json
index 4a8feb346..1a901cbfe 100644
--- a/invenio_rdm_records/records/mappings/v7/rdmrecords/drafts/draft-v6.0.0.json
+++ b/invenio_rdm_records/records/mappings/v7/rdmrecords/drafts/draft-v6.0.0.json
@@ -356,6 +356,39 @@
                                     "type": "keyword"
                                   }
                                 }
+                              },
+                              "subjects": {
+                                "properties": {
+                                  "@v": {
+                                    "type": "keyword"
+                                  },
+                                  "id": {
+                                    "type": "keyword"
+                                  },
+                                  "subject": {
+                                    "type": "keyword"
+                                  },
+                                  "scheme": {
+                                    "type": "keyword"
+                                  },
+                                  "props": {
+                                    "type": "object",
+                                    "dynamic": "true"
+                                  }
+                                }
+                              },
+                              "organizations": {
+                                "properties": {
+                                  "scheme": {
+                                    "type": "keyword"
+                                  },
+                                  "id": {
+                                    "type": "keyword"
+                                  },
+                                  "organization": {
+                                    "type": "keyword"
+                                  }
+                                }
                               }
                             }
                           },
@@ -515,6 +548,39 @@
                                         "type": "keyword"
                                       }
                                     }
+                                  },
+                                  "subjects": {
+                                    "properties": {
+                                      "@v": {
+                                        "type": "keyword"
+                                      },
+                                      "id": {
+                                        "type": "keyword"
+                                      },
+                                      "subject": {
+                                        "type": "keyword"
+                                      },
+                                      "scheme": {
+                                        "type": "keyword"
+                                      },
+                                      "props": {
+                                        "type": "object",
+                                        "dynamic": "true"
+                                      }
+                                    }
+                                  },
+                                  "organizations": {
+                                    "properties": {
+                                      "scheme": {
+                                        "type": "keyword"
+                                      },
+                                      "id": {
+                                        "type": "keyword"
+                                      },
+                                      "organization": {
+                                        "type": "keyword"
+                                      }
+                                    }
                                   }
                                 }
                               },
@@ -896,6 +962,39 @@
                         "type": "keyword"
                       }
                     }
+                  },
+                  "subjects": {
+                    "properties": {
+                      "@v": {
+                        "type": "keyword"
+                      },
+                      "id": {
+                        "type": "keyword"
+                      },
+                      "subject": {
+                        "type": "keyword"
+                      },
+                      "scheme": {
+                        "type": "keyword"
+                      },
+                      "props": {
+                        "type": "object",
+                        "dynamic": "true"
+                      }
+                    }
+                  },
+                  "organizations": {
+                    "properties": {
+                      "scheme": {
+                        "type": "keyword"
+                      },
+                      "id": {
+                        "type": "keyword"
+                      },
+                      "organization": {
+                        "type": "keyword"
+                      }
+                    }
                   }
                 }
               },
@@ -1135,6 +1234,10 @@
               },
               "scheme": {
                 "type": "keyword"
+              },
+              "props": {
+                "type": "object",
+                "dynamic": "true"
               }
             }
           },
diff --git a/invenio_rdm_records/records/mappings/v7/rdmrecords/records/record-v6.0.0.json b/invenio_rdm_records/records/mappings/v7/rdmrecords/records/record-v6.0.0.json
index 3145c3bb6..941fca4e4 100644
--- a/invenio_rdm_records/records/mappings/v7/rdmrecords/records/record-v6.0.0.json
+++ b/invenio_rdm_records/records/mappings/v7/rdmrecords/records/record-v6.0.0.json
@@ -356,6 +356,39 @@
                                     "type": "keyword"
                                   }
                                 }
+                              },
+                              "subjects": {
+                                "properties": {
+                                  "@v": {
+                                    "type": "keyword"
+                                  },
+                                  "id": {
+                                    "type": "keyword"
+                                  },
+                                  "subject": {
+                                    "type": "keyword"
+                                  },
+                                  "scheme": {
+                                    "type": "keyword"
+                                  },
+                                  "props": {
+                                    "type": "object",
+                                    "dynamic": "true"
+                                  }
+                                }
+                              },
+                              "organizations": {
+                                "properties": {
+                                  "scheme": {
+                                    "type": "keyword"
+                                  },
+                                  "id": {
+                                    "type": "keyword"
+                                  },
+                                  "organization": {
+                                    "type": "keyword"
+                                  }
+                                }
                               }
                             }
                           },
@@ -515,6 +548,39 @@
                                         "type": "keyword"
                                       }
                                     }
+                                  },
+                                  "subjects": {
+                                    "properties": {
+                                      "@v": {
+                                        "type": "keyword"
+                                      },
+                                      "id": {
+                                        "type": "keyword"
+                                      },
+                                      "subject": {
+                                        "type": "keyword"
+                                      },
+                                      "scheme": {
+                                        "type": "keyword"
+                                      },
+                                      "props": {
+                                        "type": "object",
+                                        "dynamic": "true"
+                                      }
+                                    }
+                                  },
+                                  "organizations": {
+                                    "properties": {
+                                      "scheme": {
+                                        "type": "keyword"
+                                      },
+                                      "id": {
+                                        "type": "keyword"
+                                      },
+                                      "organization": {
+                                        "type": "keyword"
+                                      }
+                                    }
                                   }
                                 }
                               },
@@ -853,6 +919,39 @@
                         "type": "keyword"
                       }
                     }
+                  },
+                  "subjects": {
+                    "properties": {
+                      "@v": {
+                        "type": "keyword"
+                      },
+                      "id": {
+                        "type": "keyword"
+                      },
+                      "subject": {
+                        "type": "keyword"
+                      },
+                      "scheme": {
+                        "type": "keyword"
+                      },
+                      "props": {
+                        "type": "object",
+                        "dynamic": "true"
+                      }
+                    }
+                  },
+                  "organizations": {
+                    "properties": {
+                      "scheme": {
+                        "type": "keyword"
+                      },
+                      "id": {
+                        "type": "keyword"
+                      },
+                      "organization": {
+                        "type": "keyword"
+                      }
+                    }
                   }
                 }
               },
@@ -1092,6 +1191,10 @@
               },
               "scheme": {
                 "type": "keyword"
+              },
+              "props": {
+                "type": "object",
+                "dynamic": "true"
               }
             }
           },
@@ -1440,6 +1543,8 @@
       "metadata.funding.award.identifiers.identifier",
       "metadata.funding.award.acronym.text",
       "metadata.funding.award.number",
+      "metadata.funding.award.subjects",
+      "metadata.funding.award.organizations",
       "metadata.funding.funder.name",
       "metadata.identifiers.identifier",
       "metadata.locations.features.place",

From ec1305fd1951f99ccc97c6537696a122d51c2a50 Mon Sep 17 00:00:00 2001
From: Fatimah Zulfiqar <fatimah0zulfiqar@gmail.com>
Date: Tue, 1 Oct 2024 10:49:17 +0200
Subject: [PATCH 28/76] dumper: refactor and updated docstring

---
 .../records/dumpers/subject_hierarchy.py      | 37 ++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)

diff --git a/invenio_rdm_records/records/dumpers/subject_hierarchy.py b/invenio_rdm_records/records/dumpers/subject_hierarchy.py
index e95f7cac2..48dfa309f 100644
--- a/invenio_rdm_records/records/dumpers/subject_hierarchy.py
+++ b/invenio_rdm_records/records/dumpers/subject_hierarchy.py
@@ -18,6 +18,40 @@ class SubjectHierarchyDumperExt(SearchDumperExt):
 
     This dumper needs to be placed after the RelationDumper for subjects as it relies
     on dereferenced subjects with scheme, subject, and props.
+
+    Example
+    "subjects" :
+        [
+            {
+            "id" : "euroscivoc:425",
+            "subject" : "Energy and fuels",
+            "scheme" : "EuroSciVoc",
+            "props" : {
+                "parents" : "euroscivoc:25,euroscivoc:67",
+            },
+            "@v" : "ef9f1c4c-b469-4645-a4b2-0db9b1c42096::1"
+            }
+        ]
+
+    The above subject is dumped with hierarchy field and is transformed to
+
+    "subjects" :
+        [
+            {
+            "id" : "euroscivoc:425",
+            "subject" : "Energy and fuels",
+            "scheme" : "EuroSciVoc",
+            "props" : {
+                "parents" : "euroscivoc:25,euroscivoc:67",
+                "hierarchy" : [
+                    "euroscivoc:25",
+                    "euroscivoc:25,euroscivoc:67",
+                    "euroscivoc:25,euroscivoc:67,euroscivoc:425"
+                ]
+            },
+            "@v" : "ef9f1c4c-b469-4645-a4b2-0db9b1c42096::1"
+            }
+        ]
     """
 
     def __init__(self, splitchar=","):
@@ -57,7 +91,8 @@ def build_hierarchy(parents_str, current_subject_id):
                     subject_hierarchy = build_hierarchy(parents, current_subject_id)
                     subject.setdefault("props", {})["hierarchy"] = subject_hierarchy
 
-        data["metadata"] = data.get("metadata", {})
+        if awards:
+            data["metadata"]["funding"] = awards
 
     def load(self, data, record_cls):
         """Load the data from secondary storage (OpenSearch-like).

From d3a2750d5b497ea774728d70bd57c483e4665310 Mon Sep 17 00:00:00 2001
From: Fatimah Zulfiqar <fatimah0zulfiqar@gmail.com>
Date: Tue, 8 Oct 2024 12:26:14 +0200
Subject: [PATCH 29/76] jsonschema: removed subjects and orgs from awards

---
 .../records/definitions-v2.0.0.json           | 43 -------------------
 .../jsonschemas/records/record-v6.0.0.json    |  6 ---
 2 files changed, 49 deletions(-)

diff --git a/invenio_rdm_records/records/jsonschemas/records/definitions-v2.0.0.json b/invenio_rdm_records/records/jsonschemas/records/definitions-v2.0.0.json
index 866cc7285..56a857ec5 100644
--- a/invenio_rdm_records/records/jsonschemas/records/definitions-v2.0.0.json
+++ b/invenio_rdm_records/records/jsonschemas/records/definitions-v2.0.0.json
@@ -348,26 +348,6 @@
       },
       "subject": {
         "type": "string"
-      },
-      "scheme": {
-        "description": "Identifier of the subject scheme.",
-        "$ref": "local://definitions-v1.0.0.json#/identifier"
-      },
-      "props": {
-        "type": "object",
-        "patternProperties": {
-          "^.*$": {
-            "type": "string"
-          }
-        }
-      },
-      "identifiers": {
-        "description": "Alternate identifiers for the subject.",
-        "type": "array",
-        "items": {
-          "$ref": "local://definitions-v2.0.0.json#/identifiers_with_scheme"
-        },
-        "uniqueItems": true
       }
     }
   },
@@ -375,29 +355,6 @@
     "type": "array",
     "items": { "$ref": "#/subject" }
   },
-  "organization": {
-    "description": "Award's organizations.",
-    "type": "object",
-    "additionalProperties": false,
-    "properties": {
-      "scheme": {
-        "description": "Identifier of the organization scheme.",
-        "$ref": "local://definitions-v1.0.0.json#/identifier"
-      },
-      "id": {
-        "description": "Identifier of the organization for the given scheme.",
-        "$ref": "local://definitions-v1.0.0.json#/identifier"
-      },
-      "organization": {
-        "description": "Human readable label.",
-        "type": "string"
-      }
-    }
-  },
-  "organizations": {
-    "type": "array",
-    "items": { "$ref": "#/organization" }
-  },
   "title_type": {
     "description": "Type of title.",
     "type": "object",
diff --git a/invenio_rdm_records/records/jsonschemas/records/record-v6.0.0.json b/invenio_rdm_records/records/jsonschemas/records/record-v6.0.0.json
index a51b22ff5..d426401cc 100644
--- a/invenio_rdm_records/records/jsonschemas/records/record-v6.0.0.json
+++ b/invenio_rdm_records/records/jsonschemas/records/record-v6.0.0.json
@@ -316,12 +316,6 @@
                     "items": {
                       "$ref": "local://definitions-v1.0.0.json#/identifiers_with_scheme"
                     }
-                  },
-                  "subjects": {
-                    "$ref": "local://records/definitions-v2.0.0.json#/subjects"
-                  },
-                  "organizations": {
-                    "$ref": "local://records/definitions-v2.0.0.json#/organizations"
                   }
                 }
               }

From 89989056c2e50c065e6f9f01c9dd766e37c26131 Mon Sep 17 00:00:00 2001
From: Fatimah Zulfiqar <fatimah0zulfiqar@gmail.com>
Date: Tue, 8 Oct 2024 15:15:32 +0200
Subject: [PATCH 30/76] dumper: added condition for no parent

---
 invenio_rdm_records/records/dumpers/subject_hierarchy.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/invenio_rdm_records/records/dumpers/subject_hierarchy.py b/invenio_rdm_records/records/dumpers/subject_hierarchy.py
index 48dfa309f..d10e91c49 100644
--- a/invenio_rdm_records/records/dumpers/subject_hierarchy.py
+++ b/invenio_rdm_records/records/dumpers/subject_hierarchy.py
@@ -68,6 +68,11 @@ def dump(self, record, data):
 
         def build_hierarchy(parents_str, current_subject_id):
             """Build the hierarchy by progressively combining parent notations."""
+            if not parents_str:
+                return [
+                    current_subject_id
+                ]  # No parents, so the hierarchy is just the current ID.
+
             parents = parents_str.split(self._splitchar)  # Split the parent notations
             hierarchy = []
             current_hierarchy = parents[0]  # Start with the top-level parent
@@ -87,7 +92,7 @@ def build_hierarchy(parents_str, current_subject_id):
             for subject in subjects:
                 parents = subject.get("props", {}).get("parents", "")
                 current_subject_id = subject.get("id", "")
-                if parents and current_subject_id:
+                if current_subject_id:
                     subject_hierarchy = build_hierarchy(parents, current_subject_id)
                     subject.setdefault("props", {})["hierarchy"] = subject_hierarchy
 

From 03d01440d6ec58a3b6348341cd0569aa382ad253 Mon Sep 17 00:00:00 2001
From: Pablo Tamarit <pablo.tamarit@cern.ch>
Date: Tue, 8 Oct 2024 14:34:12 +0200
Subject: [PATCH 31/76] installation: bump invenio-communities

---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 0eb9f9ec0..e3a126f86 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -39,7 +39,7 @@ install_requires =
     flask-iiif>=1.0.0,<2.0.0
     ftfy>=4.4.3,<5.0.0
     invenio-administration>=2.0.0,<3.0.0
-    invenio-communities>=16.0.0,<17.0.0
+    invenio-communities>=17.0.0,<18.0.0
     invenio-drafts-resources>=5.0.0,<6.0.0
     invenio-records-resources>=6.0.0,<7.0.0
     invenio-github>=1.0.0,<2.0.0

From 39d81175ddbd09b6b273e09423956498c0e9f87d Mon Sep 17 00:00:00 2001
From: Pablo Tamarit <pablo.tamarit@cern.ch>
Date: Tue, 8 Oct 2024 14:50:47 +0200
Subject: [PATCH 32/76] =?UTF-8?q?=F0=9F=93=A6=20release:=20v15.0.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 CHANGES.rst                     | 7 +++++++
 invenio_rdm_records/__init__.py | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 3586e66bf..eb556657a 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -12,6 +12,13 @@
 Changes
 =======
 
+Version v15.0.0 (released 2024-10-08)
+
+- installation: bump invenio-communities
+- dumper: refactor and updated docstring
+- awards: added subjects and orgs, updated mappings
+- relations: added subject relation in awards
+
 Version v14.0.0 (released 2024-10-04)
 
 - installation: bump invenio-vocabularies & invenio-communities
diff --git a/invenio_rdm_records/__init__.py b/invenio_rdm_records/__init__.py
index c5c7c1b8d..5e424643c 100644
--- a/invenio_rdm_records/__init__.py
+++ b/invenio_rdm_records/__init__.py
@@ -11,6 +11,6 @@
 
 from .ext import InvenioRDMRecords
 
-__version__ = "14.0.0"
+__version__ = "15.0.0"
 
 __all__ = ("__version__", "InvenioRDMRecords")

From e39c1eb4e990332566274878b704e5e591d19de6 Mon Sep 17 00:00:00 2001
From: Karolina Przerwa <karolina.m.przerwa@gmail.com>
Date: Fri, 23 Aug 2024 17:51:21 +0200
Subject: [PATCH 33/76] jobs: register embargos update

---
 invenio_rdm_records/jobs/__init__.py |  0
 invenio_rdm_records/jobs/jobs.py     | 12 ++++++++++++
 setup.cfg                            |  2 ++
 3 files changed, 14 insertions(+)
 create mode 100644 invenio_rdm_records/jobs/__init__.py
 create mode 100644 invenio_rdm_records/jobs/jobs.py

diff --git a/invenio_rdm_records/jobs/__init__.py b/invenio_rdm_records/jobs/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/invenio_rdm_records/jobs/jobs.py b/invenio_rdm_records/jobs/jobs.py
new file mode 100644
index 000000000..11a2874bb
--- /dev/null
+++ b/invenio_rdm_records/jobs/jobs.py
@@ -0,0 +1,12 @@
+from invenio_jobs.jobs import RegisteredTask
+
+from invenio_rdm_records.services.tasks import update_expired_embargos
+
+update_expired_embargos_cls = RegisteredTask.factory(
+    arguments_schema=None,
+    job_cls_name="UpdateEmbargoesJob",
+    id_="update_expired_embargos",
+    task=update_expired_embargos,
+    description="Updates expired embargos",
+    title="Update expired embargoes"
+)
diff --git a/setup.cfg b/setup.cfg
index e3a126f86..28bf42bfe 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -141,6 +141,8 @@ invenio_users_resources.moderation.actions =
     block = invenio_rdm_records.requests.user_moderation.actions:on_block
     restore = invenio_rdm_records.requests.user_moderation.actions:on_restore
     approve = invenio_rdm_records.requests.user_moderation.actions:on_approve
+invenio_jobs.jobs =
+    update_expired_embargos = invenio_rdm_records.jobs.jobs:update_expired_embargos_cls
 
 [build_sphinx]
 source-dir = docs/

From 92dc8037d40b0e9c6734b89bfe820a8db9cbbe13 Mon Sep 17 00:00:00 2001
From: Karolina Przerwa <karolina.m.przerwa@gmail.com>
Date: Tue, 10 Sep 2024 18:00:01 +0200
Subject: [PATCH 34/76] jobs: rename to job type

---
 invenio_rdm_records/jobs/jobs.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/invenio_rdm_records/jobs/jobs.py b/invenio_rdm_records/jobs/jobs.py
index 11a2874bb..e4d7a26c8 100644
--- a/invenio_rdm_records/jobs/jobs.py
+++ b/invenio_rdm_records/jobs/jobs.py
@@ -1,8 +1,8 @@
-from invenio_jobs.jobs import RegisteredTask
+from invenio_jobs.jobs import JobType
 
 from invenio_rdm_records.services.tasks import update_expired_embargos
 
-update_expired_embargos_cls = RegisteredTask.factory(
+update_expired_embargos_cls = JobType.create(
     arguments_schema=None,
     job_cls_name="UpdateEmbargoesJob",
     id_="update_expired_embargos",

From 35fb73957b307ed078cace9576a0215d3c76c8b4 Mon Sep 17 00:00:00 2001
From: Karolina Przerwa <karolina.m.przerwa@gmail.com>
Date: Wed, 2 Oct 2024 09:19:14 +0200
Subject: [PATCH 35/76] installation: bump invenio-jobs

---
 invenio_rdm_records/jobs/__init__.py |  8 ++++++++
 invenio_rdm_records/jobs/jobs.py     | 11 ++++++++++-
 setup.cfg                            |  2 +-
 3 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/invenio_rdm_records/jobs/__init__.py b/invenio_rdm_records/jobs/__init__.py
index e69de29bb..b394efcad 100644
--- a/invenio_rdm_records/jobs/__init__.py
+++ b/invenio_rdm_records/jobs/__init__.py
@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Jobs module."""
diff --git a/invenio_rdm_records/jobs/jobs.py b/invenio_rdm_records/jobs/jobs.py
index e4d7a26c8..7f0d56a5f 100644
--- a/invenio_rdm_records/jobs/jobs.py
+++ b/invenio_rdm_records/jobs/jobs.py
@@ -1,3 +1,12 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Jobs definition module."""
+
 from invenio_jobs.jobs import JobType
 
 from invenio_rdm_records.services.tasks import update_expired_embargos
@@ -8,5 +17,5 @@
     id_="update_expired_embargos",
     task=update_expired_embargos,
     description="Updates expired embargos",
-    title="Update expired embargoes"
+    title="Update expired embargoes",
 )
diff --git a/setup.cfg b/setup.cfg
index 28bf42bfe..450fe0bf4 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -44,7 +44,7 @@ install_requires =
     invenio-records-resources>=6.0.0,<7.0.0
     invenio-github>=1.0.0,<2.0.0
     invenio-i18n>=2.0.0,<3.0.0
-    invenio-jobs>=0.1.0,<1.0.0
+    invenio-jobs>=1.0.0,<2.0.0
     invenio-oaiserver>=2.0.0,<3.0.0
     invenio-oauth2server>=2.0.0
     invenio-stats>=4.0.0,<5.0.0

From 0363ef683a8b5e98f927beca1c244e3f2848b47d Mon Sep 17 00:00:00 2001
From: Karolina Przerwa <karolina.m.przerwa@gmail.com>
Date: Thu, 10 Oct 2024 09:53:51 +0200
Subject: [PATCH 36/76] release: v15.1.0

---
 CHANGES.rst                     | 5 +++++
 invenio_rdm_records/__init__.py | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index eb556657a..8888b0ba6 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -12,6 +12,11 @@
 Changes
 =======
 
+Version v15.1.0 (released 2024-10-10)
+
+- jobs: register embargo update job type
+- installation: upgrade invenio-jbs
+
 Version v15.0.0 (released 2024-10-08)
 
 - installation: bump invenio-communities
diff --git a/invenio_rdm_records/__init__.py b/invenio_rdm_records/__init__.py
index 5e424643c..b0e5d554e 100644
--- a/invenio_rdm_records/__init__.py
+++ b/invenio_rdm_records/__init__.py
@@ -11,6 +11,6 @@
 
 from .ext import InvenioRDMRecords
 
-__version__ = "15.0.0"
+__version__ = "15.1.0"
 
 __all__ = ("__version__", "InvenioRDMRecords")

From 32fd5f1b174e30b810a4f0ef22014bcd503a21c9 Mon Sep 17 00:00:00 2001
From: Anika Churilova <nyuta.churilova@gmail.com>
Date: Mon, 7 Oct 2024 10:50:13 +0200
Subject: [PATCH 37/76] dependencies: update axios

* closes https://github.com/inveniosoftware/react-searchkit/issues/278
---
 .../assets/semantic-ui/js/invenio_rdm_records/package.json  | 6 +++---
 invenio_rdm_records/webpack.py                              | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/package.json b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/package.json
index 0b9c95a14..899a63671 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/package.json
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/package.json
@@ -20,7 +20,7 @@
     "@translations/invenio_rdm_records": "../../translations/invenio_rdm_records",
     "ajv": "^8.0.0",
     "ajv-keywords": "^5.0.0",
-    "axios": "^0.21.0",
+    "axios": "^1.7.7",
     "coveralls": "^3.0.0",
     "enzyme": "^3.10.0",
     "enzyme-adapter-react-16": "^1.15.0",
@@ -40,11 +40,11 @@
     "react-dom": "^16.13.0",
     "react-dropzone": "^11.0.0",
     "react-i18next": "^11.11.0",
-    "react-invenio-forms": "^3.0.0",
+    "react-invenio-forms": "^4.0.0",
     "react-overridable": "^0.0.3",
     "react-redux": "^7.2.0",
     "react-scripts": "^5.0.1",
-    "react-searchkit": "^2.0.0",
+    "react-searchkit": "^3.0.0",
     "redux": "^4.0.0",
     "redux-thunk": "^2.3.0",
     "rimraf": "^3.0.0",
diff --git a/invenio_rdm_records/webpack.py b/invenio_rdm_records/webpack.py
index 31e9f2cc7..4d06b72e7 100644
--- a/invenio_rdm_records/webpack.py
+++ b/invenio_rdm_records/webpack.py
@@ -37,11 +37,11 @@
                 "react-dropzone": "^11.0.0",
                 "react-i18next": "^11.11.0",
                 "react-invenio-forms": "^4.0.0",
-                "react-searchkit": "^2.0.0",
+                "react-searchkit": "^3.0.0",
                 "tinymce": "^6.7.2",
                 "yup": "^0.32.0",
                 "@semantic-ui-react/css-patch": "^1.0.0",
-                "axios": "^0.21.0",
+                "axios": "^1.7.7",
                 "react": "^16.13.0",
                 "react-dom": "^16.13.0",
                 "react-redux": "^7.2.0",

From cdd272cd72d07e874e0e5fe978cdad938cae126e Mon Sep 17 00:00:00 2001
From: Zacharias Zacharodimos <zacharias.zacharodimos@cern.ch>
Date: Thu, 10 Oct 2024 16:15:57 +0200
Subject: [PATCH 38/76] tests: fix js tests

---
 .../assets/semantic-ui/js/invenio_rdm_records/package.json    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/package.json b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/package.json
index 899a63671..593cca25c 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/package.json
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/package.json
@@ -3,7 +3,7 @@
     "This package.json is needed to run the JS tests, locally and CI."
   ],
   "scripts": {
-    "test": "react-scripts test"
+    "test": "react-scripts test --transformIgnorePatterns /\"node_modules/(?!axios)/\""
   },
   "devDependencies": {
     "@babel/cli": "^7.5.0",
@@ -59,4 +59,4 @@
     "typescript": "^4.9.5",
     "yup": "^0.32.11"
   }
-}
+}
\ No newline at end of file

From 0d313c61bda1c509c8518c1c30f8abc337e96967 Mon Sep 17 00:00:00 2001
From: Zacharias Zacharodimos <zacharias.zacharodimos@cern.ch>
Date: Thu, 10 Oct 2024 16:44:27 +0200
Subject: [PATCH 39/76] release: v15.2.0

---
 CHANGES.rst                     | 4 ++++
 invenio_rdm_records/__init__.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 8888b0ba6..dedb03d1f 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -12,6 +12,10 @@
 Changes
 =======
 
+Version v15.2.0 (released 2024-10-10)
+
+- webpack: update axios and react-searchkit(due to axios) major versions
+
 Version v15.1.0 (released 2024-10-10)
 
 - jobs: register embargo update job type
diff --git a/invenio_rdm_records/__init__.py b/invenio_rdm_records/__init__.py
index b0e5d554e..5cafefe69 100644
--- a/invenio_rdm_records/__init__.py
+++ b/invenio_rdm_records/__init__.py
@@ -11,6 +11,6 @@
 
 from .ext import InvenioRDMRecords
 
-__version__ = "15.1.0"
+__version__ = "15.2.0"
 
 __all__ = ("__version__", "InvenioRDMRecords")

From 2e4f5aa55310497a53db1187f1ce0de5d8e4f373 Mon Sep 17 00:00:00 2001
From: Saksham <sakshamarora1001@gmail.com>
Date: Mon, 16 Sep 2024 10:26:29 +0200
Subject: [PATCH 40/76] Services: Communities: Add feature to require community
 based on config

---
 invenio_rdm_records/config.py                 |   6 +
 invenio_rdm_records/resources/config.py       |   1 +
 .../services/communities/service.py           |  16 +-
 invenio_rdm_records/services/generators.py    |  11 ++
 invenio_rdm_records/services/permissions.py   |  27 ++-
 tests/resources/test_resources_communities.py | 177 ++++++++++++++++++
 6 files changed, 234 insertions(+), 4 deletions(-)

diff --git a/invenio_rdm_records/config.py b/invenio_rdm_records/config.py
index ddc1d1f11..6e8710e8d 100644
--- a/invenio_rdm_records/config.py
+++ b/invenio_rdm_records/config.py
@@ -131,6 +131,12 @@ def always_valid(identifier):
 RDM_ALLOW_RESTRICTED_RECORDS = True
 """Allow users to set restricted/private records."""
 
+#
+# Record communities
+#
+RDM_RECORD_ALWAYS_IN_COMMUNITY = True
+"""Enforces at least one community per record on remove community function."""
+
 #
 # Search configuration
 #
diff --git a/invenio_rdm_records/resources/config.py b/invenio_rdm_records/resources/config.py
index 3ea7673d3..80ab769bf 100644
--- a/invenio_rdm_records/resources/config.py
+++ b/invenio_rdm_records/resources/config.py
@@ -187,6 +187,7 @@ class RDMRecordResourceConfig(RecordResourceConfig, ConfiguratorMixin):
     )
 
     error_handlers = {
+        **ErrorHandlersMixin.error_handlers,
         DeserializerError: create_error_handler(
             lambda exc: HTTPJSONException(
                 code=400,
diff --git a/invenio_rdm_records/services/communities/service.py b/invenio_rdm_records/services/communities/service.py
index 63c17cb22..8817bdf3c 100644
--- a/invenio_rdm_records/services/communities/service.py
+++ b/invenio_rdm_records/services/communities/service.py
@@ -14,7 +14,6 @@
 from invenio_communities.proxies import current_communities
 from invenio_drafts_resources.services.records.uow import (
     ParentRecordCommitOp,
-    RecordCommitOp,
 )
 from invenio_i18n import lazy_gettext as _
 from invenio_notifications.services.uow import NotificationOp
@@ -24,7 +23,10 @@
     Service,
     ServiceSchemaWrapper,
 )
-from invenio_records_resources.services.errors import PermissionDeniedError
+from invenio_records_resources.services.errors import (
+    PermissionDeniedError,
+    CannotRemoveCommunityError,
+)
 from invenio_records_resources.services.uow import (
     IndexRefreshOp,
     RecordIndexOp,
@@ -211,6 +213,16 @@ def _remove(self, identity, community_id, record):
         if community_id not in record.parent.communities.ids:
             raise RecordCommunityMissing(record.id, community_id)
 
+        # If config is True and there is only 1 communities left to remove
+        # Then, check for permissions to remove last community
+        if (
+            current_app.config["RDM_RECORD_ALWAYS_IN_COMMUNITY"]
+            and len(record.parent.communities.ids) == 1
+        ):
+            if not self.check_permission(
+                identity, "remove_community", record=record, community_id=community_id
+            ):
+                raise CannotRemoveCommunityError()
         # check permission here, per community: curator cannot remove another community
         self.require_permission(
             identity, "remove_community", record=record, community_id=community_id
diff --git a/invenio_rdm_records/services/generators.py b/invenio_rdm_records/services/generators.py
index 270476b27..b24959ba1 100644
--- a/invenio_rdm_records/services/generators.py
+++ b/invenio_rdm_records/services/generators.py
@@ -414,3 +414,14 @@ def needs(self, request=None, **kwargs):
             return [AccessRequestTokenNeed(request["payload"]["token"])]
 
         return []
+
+
+class IfOneCommunity(ConditionalGenerator):
+    """Conditional generator for records always in communities case."""
+
+    def _condition(self, record=None, **kwargs):
+        """Check if the record is associated with zero or one community."""
+        if record is None:
+            return True
+        rec_communities = record.parent.communities.ids
+        return len(rec_communities) == 1
diff --git a/invenio_rdm_records/services/permissions.py b/invenio_rdm_records/services/permissions.py
index 52b7f6417..486be998c 100644
--- a/invenio_rdm_records/services/permissions.py
+++ b/invenio_rdm_records/services/permissions.py
@@ -30,6 +30,7 @@
     AccessGrant,
     CommunityInclusionReviewers,
     GuestAccessRequestToken,
+    IfOneCommunity,
     IfCreate,
     IfDeleted,
     IfExternalDOIRecord,
@@ -199,7 +200,17 @@ class RDMRecordPermissionPolicy(RecordPermissionPolicy):
         ),
     ]
     # Allow publishing a new record or changes to an existing record.
-    can_publish = can_review
+    can_publish = [
+        IfConfig(
+            "RDM_RECORD_ALWAYS_IN_COMMUNITY",
+            then_=[
+                IfOneCommunity(
+                    then_=can_review, else_=[Administration(), SystemProcess()]
+                )
+            ],
+            else_=can_review,
+        )
+    ]
     # Allow lifting a record or draft.
     can_lift_embargo = can_manage
 
@@ -209,11 +220,23 @@ class RDMRecordPermissionPolicy(RecordPermissionPolicy):
     # Who can add record to a community
     can_add_community = can_manage
     # Who can remove a community from a record
-    can_remove_community = [
+    can_remove_community_ = [
         RecordOwners(),
         CommunityCurators(),
         SystemProcess(),
     ]
+    can_remove_community = [
+        IfConfig(
+            "RDM_RECORD_ALWAYS_IN_COMMUNITY",
+            then_=[
+                IfOneCommunity(
+                    then_=[Administration(), SystemProcess()],
+                    else_=can_remove_community_,
+                )
+            ],
+            else_=can_remove_community_,
+        )
+    ]
     # Who can remove records from a community
     can_remove_record = [CommunityCurators()]
     # Who can add records to a community in bulk
diff --git a/tests/resources/test_resources_communities.py b/tests/resources/test_resources_communities.py
index f4ad9c4dd..b3f63343f 100644
--- a/tests/resources/test_resources_communities.py
+++ b/tests/resources/test_resources_communities.py
@@ -7,6 +7,7 @@
 
 """Tests record's communities resources."""
 
+from contextlib import contextmanager
 from copy import deepcopy
 
 import pytest
@@ -889,3 +890,179 @@ def test_add_record_to_restricted_community_submission_open_member(
     assert not response.json.get("errors")
     processed = response.json["processed"]
     assert len(processed) == 1
+
+# Assure Records community exists tests
+# -------------------------------------
+from invenio_records_resources.services.errors import CommunityNotSelectedError, CannotRemoveCommunityError
+
+
+@contextmanager
+def ensure_record_community_exists_config(app):
+    """
+        Context manager to ensure record community exists config
+        is set to True during a specific code block.
+        Parameters:
+        app: app fixture
+        Usage:
+        with ensure_record_community_exists_config(app):
+        # code block that requires the flag to be set
+    """
+    try:
+        app.config["RDM_RECORD_ALWAYS_IN_COMMUNITY"] = True
+        yield
+    finally:
+        app.config["RDM_RECORD_ALWAYS_IN_COMMUNITY"] = False
+
+def test_restricted_record_creation(
+    app, record_community, uploader, curator, community_owner, test_user, superuser
+):
+    """Verify CommunityNotSelectedError is raised when direct publish a record"""
+    # You can directly publish a record when the config is disabled
+    rec = record_community.create_record()
+    assert rec.id
+    with ensure_record_community_exists_config(app):
+        # You can't directly publish
+        users = [
+            curator,
+            test_user,
+            uploader,
+            community_owner,
+        ]
+        for user in users:
+            with pytest.raises(CommunityNotSelectedError):
+                record_community.create_record(uploader=user)
+
+        # Super user can!
+        super_user_rec = record_community.create_record(uploader=superuser)
+        assert super_user_rec.id
+
+
+def test_remove_last_existing_non_existing_community(
+    app, client, uploader, record_community, headers, community
+):
+    """Test removal of an existing and non-existing community from the record,
+    while ensuring at least one community exists."""
+    data = {
+        "communities": [
+            {"id": "wrong-id"},
+            {"id": community.id},
+            {"id": "wrong-id2"},
+        ]
+    }
+
+    client = uploader.login(client)
+    record = record_community.create_record()
+    with ensure_record_community_exists_config(app):
+        with pytest.raises(CannotRemoveCommunityError):
+            response = client.delete(
+                f"/records/{record.pid.pid_value}/communities",
+                headers=headers,
+                json=data,
+            )
+            assert response.status_code == 400
+            # Should get 3 errors: Can't remove community, 2 bad IDs
+            assert len(response.json["errors"]) == 3
+            record_saved = client.get(f"/records/{record.pid.pid_value}", headers=headers)
+            assert record_saved.json["parent"]["communities"]
+
+
+def test_remove_last_community_api_error_handling(
+    record_community,
+    community,
+    uploader,
+    headers,
+    curator,
+    client,
+    app,
+):
+    """Testing error message when trying to remove last community."""
+    record = record_community.create_record()
+    data = {"communities": [{"id": community.id}]}
+    for user in [uploader, curator]:
+        client = user.login(client)
+        response = client.get(
+            f"/communities/{community.id}/records",
+            headers=headers,
+            json=data,
+        )
+        assert (
+            len(response.json["hits"]["hits"][0]["parent"]["communities"]["ids"]) == 1
+        )
+        with ensure_record_community_exists_config(app):
+            response = client.delete(
+                f"/records/{record.pid.pid_value}/communities",
+                headers=headers,
+                json=data,
+            )
+            assert response.is_json
+            assert response.status_code == 400
+
+            record_saved = client.get(
+                f"/records/{record.pid.pid_value}", headers=headers
+            )
+            assert record_saved.json["parent"]["communities"]
+
+            client = user.logout(client)
+            # check communities number
+            response = client.get(
+                f"/communities/{community.id}/records",
+                headers=headers,
+                json=data,
+            )
+            assert (
+                len(response.json["hits"]["hits"][0]["parent"]["communities"]["ids"])
+                == 1
+            )
+
+
+def test_remove_record_last_community_with_multiple_communities(
+    closed_review_community,
+    open_review_community,
+    record_community,
+    community2,
+    uploader,
+    headers,
+    client,
+    app,
+    db,
+):
+    """Testing correct removal of multiple communities"""
+    client = uploader.login(client)
+
+    record = record_community.create_record()
+    comm = [
+        community2,
+        open_review_community,
+        closed_review_community,
+    ]  # one more in the rec fixture so it's 4
+    for com in comm:
+        _add_to_community(db, record, com)
+    assert len(record.parent.communities.ids) == 4
+
+    with ensure_record_community_exists_config(app):
+        data = {"communities": [{"id": x} for x in record.parent.communities.ids]}
+
+        response = client.delete(
+            f"/records/{record.pid.pid_value}/communities",
+            headers=headers,
+            json=data,
+        )
+        # You get res 200 with error msg if all communities you are deleting
+        assert response.status_code == 200
+        assert "error" in str(response.data)
+
+        rec_com_left = client.get(f"/records/{record.pid.pid_value}", headers=headers)
+        assert len(rec_com_left.json["parent"]["communities"]["ids"]) == 1
+
+        # You get res 400 with error msg if you Delete the last one only.
+        response = client.delete(
+            f"/records/{record.pid.pid_value}/communities",
+            headers=headers,
+            json={"communities": [{"id": str(record.parent.communities.ids[0])}]},
+        )
+        assert response.status_code == 400
+        assert "error" in str(response.data)
+
+        record_saved = client.get(f"/records/{record.pid.pid_value}", headers=headers)
+        # check that only one community ID is associated with the record
+        assert len(record_saved.json["parent"]["communities"]["ids"]) == 1

From 9b74cd0897a398da35bdc8dd47544efe655bec81 Mon Sep 17 00:00:00 2001
From: Saksham <sakshamarora1001@gmail.com>
Date: Fri, 20 Sep 2024 17:28:51 +0200
Subject: [PATCH 41/76] services: Override publish and remove APIs to check for
 new config

---
 invenio_rdm_records/resources/config.py       |  7 +++
 .../services/communities/service.py           | 38 ++++++++-------
 invenio_rdm_records/services/errors.py        | 12 +++++
 invenio_rdm_records/services/permissions.py   |  2 +-
 invenio_rdm_records/services/services.py      | 24 ++++++++++
 tests/resources/test_resources_communities.py | 47 ++++++++++---------
 6 files changed, 90 insertions(+), 40 deletions(-)

diff --git a/invenio_rdm_records/resources/config.py b/invenio_rdm_records/resources/config.py
index 80ab769bf..6675de01c 100644
--- a/invenio_rdm_records/resources/config.py
+++ b/invenio_rdm_records/resources/config.py
@@ -36,6 +36,7 @@
 
 from ..services.errors import (
     AccessRequestExistsError,
+    CommunityNotSelectedError,
     GrantExistsError,
     InvalidAccessRestrictions,
     RecordDeletedException,
@@ -256,6 +257,12 @@ class RDMRecordResourceConfig(RecordResourceConfig, ConfiguratorMixin):
                 description=e.description,
             )
         ),
+        CommunityNotSelectedError: create_error_handler(
+            HTTPJSONException(
+                code=400,
+                description="Cannot publish without selecting a community.",
+            )
+        ),
     }
 
 
diff --git a/invenio_rdm_records/services/communities/service.py b/invenio_rdm_records/services/communities/service.py
index 8817bdf3c..7e7d44cf6 100644
--- a/invenio_rdm_records/services/communities/service.py
+++ b/invenio_rdm_records/services/communities/service.py
@@ -12,9 +12,7 @@
 from flask import current_app
 from invenio_access.permissions import system_identity
 from invenio_communities.proxies import current_communities
-from invenio_drafts_resources.services.records.uow import (
-    ParentRecordCommitOp,
-)
+from invenio_drafts_resources.services.records.uow import ParentRecordCommitOp
 from invenio_i18n import lazy_gettext as _
 from invenio_notifications.services.uow import NotificationOp
 from invenio_pidstore.errors import PIDDoesNotExistError, PIDUnregistered
@@ -23,10 +21,7 @@
     Service,
     ServiceSchemaWrapper,
 )
-from invenio_records_resources.services.errors import (
-    PermissionDeniedError,
-    CannotRemoveCommunityError,
-)
+from invenio_records_resources.services.errors import PermissionDeniedError
 from invenio_records_resources.services.uow import (
     IndexRefreshOp,
     RecordIndexOp,
@@ -41,6 +36,7 @@
 from ...proxies import current_rdm_records, current_rdm_records_service
 from ...requests import CommunityInclusion
 from ..errors import (
+    CannotRemoveCommunityError,
     CommunityAlreadyExists,
     InvalidAccessRestrictions,
     OpenRequestAlreadyExists,
@@ -213,20 +209,22 @@ def _remove(self, identity, community_id, record):
         if community_id not in record.parent.communities.ids:
             raise RecordCommunityMissing(record.id, community_id)
 
-        # If config is True and there is only 1 communities left to remove
+        # If config is true and there is only 1 communities left to remove
+        is_community_required = current_app.config["RDM_RECORD_ALWAYS_IN_COMMUNITY"]
+        is_last_community = len(record.parent.communities.ids) == 1
         # Then, check for permissions to remove last community
+        can_remove_last_community = self.check_permission(
+            identity, "remove_community", record=record, community_id=community_id
+        )
         if (
-            current_app.config["RDM_RECORD_ALWAYS_IN_COMMUNITY"]
-            and len(record.parent.communities.ids) == 1
+            is_community_required
+            and is_last_community
+            and not can_remove_last_community
         ):
-            if not self.check_permission(
-                identity, "remove_community", record=record, community_id=community_id
-            ):
-                raise CannotRemoveCommunityError()
+            raise CannotRemoveCommunityError()
         # check permission here, per community: curator cannot remove another community
-        self.require_permission(
-            identity, "remove_community", record=record, community_id=community_id
-        )
+        elif not can_remove_last_community:
+            raise PermissionDeniedError("remove_community")
 
         # Default community is deleted when the exact same community is removed from the record
         record.parent.communities.remove(community_id)
@@ -251,7 +249,11 @@ def remove(self, identity, id_, data, uow):
             try:
                 self._remove(identity, community_id, record)
                 processed.append({"community": community_id})
-            except (RecordCommunityMissing, PermissionDeniedError) as ex:
+            except (
+                RecordCommunityMissing,
+                PermissionDeniedError,
+                CannotRemoveCommunityError,
+            ) as ex:
                 errors.append(
                     {
                         "community": community_id,
diff --git a/invenio_rdm_records/services/errors.py b/invenio_rdm_records/services/errors.py
index 024ab39ee..5757c5d55 100644
--- a/invenio_rdm_records/services/errors.py
+++ b/invenio_rdm_records/services/errors.py
@@ -206,3 +206,15 @@ class RecordSubmissionClosedCommunityError(PermissionDenied):
     """Record submission policy forbids non-members from submitting records to community."""
 
     description = "Submission to this community is only allowed to community members."
+
+
+class CommunityNotSelectedError(Exception):
+    """Error thrown when a record is being created/updated with less than 1 community."""
+
+    description = "Cannot publish without selecting a community."
+
+
+class CannotRemoveCommunityError(PermissionDenied):
+    """Error thrown when the last community is being removed from the record."""
+
+    description = "Cannot remove. A record should be part of atleast 1 community."
diff --git a/invenio_rdm_records/services/permissions.py b/invenio_rdm_records/services/permissions.py
index 486be998c..ea5efac14 100644
--- a/invenio_rdm_records/services/permissions.py
+++ b/invenio_rdm_records/services/permissions.py
@@ -30,12 +30,12 @@
     AccessGrant,
     CommunityInclusionReviewers,
     GuestAccessRequestToken,
-    IfOneCommunity,
     IfCreate,
     IfDeleted,
     IfExternalDOIRecord,
     IfFileIsLocal,
     IfNewRecord,
+    IfOneCommunity,
     IfRecordDeleted,
     IfRequestType,
     IfRestricted,
diff --git a/invenio_rdm_records/services/services.py b/invenio_rdm_records/services/services.py
index 3841fdb29..25c10b159 100644
--- a/invenio_rdm_records/services/services.py
+++ b/invenio_rdm_records/services/services.py
@@ -20,6 +20,7 @@
 from invenio_drafts_resources.services.records import RecordService
 from invenio_drafts_resources.services.records.uow import ParentRecordCommitOp
 from invenio_records_resources.services import LinksTemplate, ServiceSchemaWrapper
+from invenio_records_resources.services.errors import PermissionDeniedError
 from invenio_records_resources.services.uow import (
     RecordCommitOp,
     RecordIndexDeleteOp,
@@ -37,6 +38,7 @@
 
 from ..records.systemfields.deletion_status import RecordDeletionStatusEnum
 from .errors import (
+    CommunityNotSelectedError,
     DeletionStatusException,
     EmbargoNotLiftedError,
     RecordDeletedException,
@@ -404,6 +406,28 @@ def purge_record(self, identity, id_, uow=None):
 
         raise NotImplementedError()
 
+    def publish(self, identity, id_, uow=None, expand=False):
+        """Publish a draft.
+
+        Check for permissions to publish a draft and then call invenio_drafts_resourcs.services.records.services.publish()
+        """
+        # Get the draft
+        draft = self.draft_cls.pid.resolve(id_, registered_only=False)
+
+        # If config is true and there are no communities selected
+        is_community_required = current_app.config["RDM_RECORD_ALWAYS_IN_COMMUNITY"]
+        is_community_missing = len(draft.parent.communities.ids) == 0
+        # Then, check for permissions to upload without community
+        can_publish_draft = self.check_permission(identity, "publish", record=draft)
+
+        if is_community_required and is_community_missing and not can_publish_draft:
+            raise CommunityNotSelectedError()
+        elif not can_publish_draft:
+            # If community is not missing or the community is not required, raise error if user doesn't have permissions
+            raise PermissionDeniedError("publish")
+
+        return super().publish(identity, id_, uow=uow, expand=expand)
+
     #
     # Search functions
     #
diff --git a/tests/resources/test_resources_communities.py b/tests/resources/test_resources_communities.py
index b3f63343f..2aff90fee 100644
--- a/tests/resources/test_resources_communities.py
+++ b/tests/resources/test_resources_communities.py
@@ -19,7 +19,10 @@
 )
 from invenio_rdm_records.records.api import RDMDraft, RDMRecord
 from invenio_rdm_records.requests.community_inclusion import CommunityInclusion
-from invenio_rdm_records.services.errors import InvalidAccessRestrictions
+from invenio_rdm_records.services.errors import (
+    CommunityNotSelectedError,
+    InvalidAccessRestrictions,
+)
 
 
 def _add_to_community(db, record, community):
@@ -891,21 +894,21 @@ def test_add_record_to_restricted_community_submission_open_member(
     processed = response.json["processed"]
     assert len(processed) == 1
 
+
 # Assure Records community exists tests
 # -------------------------------------
-from invenio_records_resources.services.errors import CommunityNotSelectedError, CannotRemoveCommunityError
 
 
 @contextmanager
 def ensure_record_community_exists_config(app):
     """
-        Context manager to ensure record community exists config
-        is set to True during a specific code block.
-        Parameters:
-        app: app fixture
-        Usage:
-        with ensure_record_community_exists_config(app):
-        # code block that requires the flag to be set
+    Context manager to ensure record community exists config
+    is set to True during a specific code block.
+    Parameters:
+    app: app fixture
+    Usage:
+    with ensure_record_community_exists_config(app):
+    # code block that requires the flag to be set
     """
     try:
         app.config["RDM_RECORD_ALWAYS_IN_COMMUNITY"] = True
@@ -913,6 +916,7 @@ def ensure_record_community_exists_config(app):
     finally:
         app.config["RDM_RECORD_ALWAYS_IN_COMMUNITY"] = False
 
+
 def test_restricted_record_creation(
     app, record_community, uploader, curator, community_owner, test_user, superuser
 ):
@@ -953,17 +957,17 @@ def test_remove_last_existing_non_existing_community(
     client = uploader.login(client)
     record = record_community.create_record()
     with ensure_record_community_exists_config(app):
-        with pytest.raises(CannotRemoveCommunityError):
-            response = client.delete(
-                f"/records/{record.pid.pid_value}/communities",
-                headers=headers,
-                json=data,
-            )
-            assert response.status_code == 400
-            # Should get 3 errors: Can't remove community, 2 bad IDs
-            assert len(response.json["errors"]) == 3
-            record_saved = client.get(f"/records/{record.pid.pid_value}", headers=headers)
-            assert record_saved.json["parent"]["communities"]
+        response = client.delete(
+            f"/records/{record.pid.pid_value}/communities",
+            headers=headers,
+            json=data,
+        )
+        assert response.is_json
+        assert response.status_code == 200
+        # Should get 3 errors: Can't remove community, 2 bad IDs
+        assert len(response.json["errors"]) == 3
+        record_saved = client.get(f"/records/{record.pid.pid_value}", headers=headers)
+        assert record_saved.json["parent"]["communities"]
 
 
 def test_remove_last_community_api_error_handling(
@@ -995,12 +999,13 @@ def test_remove_last_community_api_error_handling(
                 json=data,
             )
             assert response.is_json
-            assert response.status_code == 400
+            assert response.status_code == 200
 
             record_saved = client.get(
                 f"/records/{record.pid.pid_value}", headers=headers
             )
             assert record_saved.json["parent"]["communities"]
+            assert len(response.json["errors"]) == 1
 
             client = user.logout(client)
             # check communities number

From 2b0637e04538287fc5b82a9bf0126af8ba6cd624 Mon Sep 17 00:00:00 2001
From: Saksham <sakshamarora1001@gmail.com>
Date: Mon, 23 Sep 2024 15:25:55 +0200
Subject: [PATCH 42/76] tests: Select community before publishing

---
 invenio_rdm_records/config.py                 |  4 +-
 invenio_rdm_records/services/errors.py        |  2 +-
 invenio_rdm_records/services/services.py      |  1 +
 tests/conftest.py                             | 36 ++++++++++++
 tests/resources/test_resources_communities.py | 57 +++++++++++--------
 5 files changed, 74 insertions(+), 26 deletions(-)

diff --git a/invenio_rdm_records/config.py b/invenio_rdm_records/config.py
index 6e8710e8d..e7ebfc32c 100644
--- a/invenio_rdm_records/config.py
+++ b/invenio_rdm_records/config.py
@@ -134,8 +134,8 @@ def always_valid(identifier):
 #
 # Record communities
 #
-RDM_RECORD_ALWAYS_IN_COMMUNITY = True
-"""Enforces at least one community per record on remove community function."""
+RDM_RECORD_ALWAYS_IN_COMMUNITY = False
+"""Enforces at least one community per record."""
 
 #
 # Search configuration
diff --git a/invenio_rdm_records/services/errors.py b/invenio_rdm_records/services/errors.py
index 5757c5d55..011460fcc 100644
--- a/invenio_rdm_records/services/errors.py
+++ b/invenio_rdm_records/services/errors.py
@@ -214,7 +214,7 @@ class CommunityNotSelectedError(Exception):
     description = "Cannot publish without selecting a community."
 
 
-class CannotRemoveCommunityError(PermissionDenied):
+class CannotRemoveCommunityError(Exception):
     """Error thrown when the last community is being removed from the record."""
 
     description = "Cannot remove. A record should be part of atleast 1 community."
diff --git a/invenio_rdm_records/services/services.py b/invenio_rdm_records/services/services.py
index 25c10b159..66ea2697b 100644
--- a/invenio_rdm_records/services/services.py
+++ b/invenio_rdm_records/services/services.py
@@ -406,6 +406,7 @@ def purge_record(self, identity, id_, uow=None):
 
         raise NotImplementedError()
 
+    @unit_of_work()
     def publish(self, identity, id_, uow=None, expand=False):
         """Publish a draft.
 
diff --git a/tests/conftest.py b/tests/conftest.py
index 8a59f5b48..6a89872c3 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -67,6 +67,7 @@
 from invenio_records_resources.proxies import current_service_registry
 from invenio_records_resources.references.entity_resolvers import ServiceResultResolver
 from invenio_records_resources.services.custom_fields import TextCF
+from invenio_records_resources.services.uow import UnitOfWork
 from invenio_requests.notifications.builders import (
     CommentRequestEventCreateNotificationBuilder,
 )
@@ -2093,6 +2094,41 @@ def create_record(
     return RecordFactory()
 
 
+@pytest.fixture()
+def record_required_community(db, uploader, minimal_record, community):
+    """Creates a record that belongs to a community before publishing."""
+
+    class Record:
+        """Test record class."""
+
+        def create_record(
+            self,
+            record_dict=minimal_record,
+            uploader=uploader,
+            community=community,
+        ):
+            """Creates new record that belongs to the same community."""
+            # create draft
+            draft = current_rdm_records_service.create(uploader.identity, record_dict)
+            record = draft._record
+            # add the record to the community
+            community_record = community._record
+            record.parent.communities.add(community_record, default=False)
+            record.parent.commit()
+            db.session.commit()
+            current_rdm_records_service.indexer.index(
+                record, arguments={"refresh": True}
+            )
+
+            # publish and get record
+            community_record = current_rdm_records_service.publish(
+                uploader.identity, draft.id
+            )
+            return community_record
+
+    return Record()
+
+
 @pytest.fixture(scope="session")
 def headers():
     """Default headers for making requests."""
diff --git a/tests/resources/test_resources_communities.py b/tests/resources/test_resources_communities.py
index 2aff90fee..cb6ca4ee9 100644
--- a/tests/resources/test_resources_communities.py
+++ b/tests/resources/test_resources_communities.py
@@ -918,7 +918,13 @@ def ensure_record_community_exists_config(app):
 
 
 def test_restricted_record_creation(
-    app, record_community, uploader, curator, community_owner, test_user, superuser
+    app,
+    record_community,
+    uploader,
+    curator,
+    community_owner,
+    test_user,
+    superuser,
 ):
     """Verify CommunityNotSelectedError is raised when direct publish a record"""
     # You can directly publish a record when the config is disabled
@@ -942,7 +948,7 @@ def test_restricted_record_creation(
 
 
 def test_remove_last_existing_non_existing_community(
-    app, client, uploader, record_community, headers, community
+    app, client, uploader, record_required_community, headers, community
 ):
     """Test removal of an existing and non-existing community from the record,
     while ensuring at least one community exists."""
@@ -955,23 +961,24 @@ def test_remove_last_existing_non_existing_community(
     }
 
     client = uploader.login(client)
-    record = record_community.create_record()
+    record = record_required_community.create_record()
+    record_pid = record._record.pid.pid_value
     with ensure_record_community_exists_config(app):
         response = client.delete(
-            f"/records/{record.pid.pid_value}/communities",
+            f"/records/{record_pid}/communities",
             headers=headers,
             json=data,
         )
         assert response.is_json
-        assert response.status_code == 200
+        assert response.status_code == 400
         # Should get 3 errors: Can't remove community, 2 bad IDs
         assert len(response.json["errors"]) == 3
-        record_saved = client.get(f"/records/{record.pid.pid_value}", headers=headers)
+        record_saved = client.get(f"/records/{record_pid}", headers=headers)
         assert record_saved.json["parent"]["communities"]
 
 
 def test_remove_last_community_api_error_handling(
-    record_community,
+    record_required_community,
     community,
     uploader,
     headers,
@@ -980,7 +987,8 @@ def test_remove_last_community_api_error_handling(
     app,
 ):
     """Testing error message when trying to remove last community."""
-    record = record_community.create_record()
+    record = record_required_community.create_record()
+    record_pid = record._record.pid.pid_value
     data = {"communities": [{"id": community.id}]}
     for user in [uploader, curator]:
         client = user.login(client)
@@ -994,16 +1002,14 @@ def test_remove_last_community_api_error_handling(
         )
         with ensure_record_community_exists_config(app):
             response = client.delete(
-                f"/records/{record.pid.pid_value}/communities",
+                f"/records/{record_pid}/communities",
                 headers=headers,
                 json=data,
             )
             assert response.is_json
-            assert response.status_code == 200
+            assert response.status_code == 400
 
-            record_saved = client.get(
-                f"/records/{record.pid.pid_value}", headers=headers
-            )
+            record_saved = client.get(f"/records/{record_pid}", headers=headers)
             assert record_saved.json["parent"]["communities"]
             assert len(response.json["errors"]) == 1
 
@@ -1023,7 +1029,7 @@ def test_remove_last_community_api_error_handling(
 def test_remove_record_last_community_with_multiple_communities(
     closed_review_community,
     open_review_community,
-    record_community,
+    record_required_community,
     community2,
     uploader,
     headers,
@@ -1034,21 +1040,24 @@ def test_remove_record_last_community_with_multiple_communities(
     """Testing correct removal of multiple communities"""
     client = uploader.login(client)
 
-    record = record_community.create_record()
+    record = record_required_community.create_record()
+    record_pid = record._record.pid.pid_value
     comm = [
         community2,
         open_review_community,
         closed_review_community,
     ]  # one more in the rec fixture so it's 4
     for com in comm:
-        _add_to_community(db, record, com)
-    assert len(record.parent.communities.ids) == 4
+        _add_to_community(db, record._record, com)
+    assert len(record._record.parent.communities.ids) == 4
 
     with ensure_record_community_exists_config(app):
-        data = {"communities": [{"id": x} for x in record.parent.communities.ids]}
+        data = {
+            "communities": [{"id": x} for x in record._record.parent.communities.ids]
+        }
 
         response = client.delete(
-            f"/records/{record.pid.pid_value}/communities",
+            f"/records/{record_pid}/communities",
             headers=headers,
             json=data,
         )
@@ -1056,18 +1065,20 @@ def test_remove_record_last_community_with_multiple_communities(
         assert response.status_code == 200
         assert "error" in str(response.data)
 
-        rec_com_left = client.get(f"/records/{record.pid.pid_value}", headers=headers)
+        rec_com_left = client.get(f"/records/{record_pid}", headers=headers)
         assert len(rec_com_left.json["parent"]["communities"]["ids"]) == 1
 
         # You get res 400 with error msg if you Delete the last one only.
         response = client.delete(
-            f"/records/{record.pid.pid_value}/communities",
+            f"/records/{record_pid}/communities",
             headers=headers,
-            json={"communities": [{"id": str(record.parent.communities.ids[0])}]},
+            json={
+                "communities": [{"id": str(record._record.parent.communities.ids[0])}]
+            },
         )
         assert response.status_code == 400
         assert "error" in str(response.data)
 
-        record_saved = client.get(f"/records/{record.pid.pid_value}", headers=headers)
+        record_saved = client.get(f"/records/{record_pid}", headers=headers)
         # check that only one community ID is associated with the record
         assert len(record_saved.json["parent"]["communities"]["ids"]) == 1

From fef9cff3e86f5c656f86e01896fafd5d6a79546e Mon Sep 17 00:00:00 2001
From: Saksham <sakshamarora1001@gmail.com>
Date: Thu, 26 Sep 2024 11:25:28 +0200
Subject: [PATCH 43/76] services: Handle case for draft republish with multiple
 communities

---
 invenio_rdm_records/services/generators.py    | 11 +++++
 invenio_rdm_records/services/permissions.py   |  3 +-
 invenio_rdm_records/services/services.py      | 11 +++--
 tests/resources/test_resources_communities.py | 41 +++++++++++++++++++
 4 files changed, 59 insertions(+), 7 deletions(-)

diff --git a/invenio_rdm_records/services/generators.py b/invenio_rdm_records/services/generators.py
index b24959ba1..3754254a1 100644
--- a/invenio_rdm_records/services/generators.py
+++ b/invenio_rdm_records/services/generators.py
@@ -425,3 +425,14 @@ def _condition(self, record=None, **kwargs):
             return True
         rec_communities = record.parent.communities.ids
         return len(rec_communities) == 1
+
+
+class IfAtleastOneCommunity(ConditionalGenerator):
+    """Conditional generator for records always in communities case."""
+
+    def _condition(self, record=None, **kwargs):
+        """Check if the record is associated with zero or one community."""
+        if record is None:
+            return True
+        rec_communities = record.parent.communities.ids
+        return len(rec_communities) > 0
diff --git a/invenio_rdm_records/services/permissions.py b/invenio_rdm_records/services/permissions.py
index ea5efac14..7abc37db3 100644
--- a/invenio_rdm_records/services/permissions.py
+++ b/invenio_rdm_records/services/permissions.py
@@ -30,6 +30,7 @@
     AccessGrant,
     CommunityInclusionReviewers,
     GuestAccessRequestToken,
+    IfAtleastOneCommunity,
     IfCreate,
     IfDeleted,
     IfExternalDOIRecord,
@@ -204,7 +205,7 @@ class RDMRecordPermissionPolicy(RecordPermissionPolicy):
         IfConfig(
             "RDM_RECORD_ALWAYS_IN_COMMUNITY",
             then_=[
-                IfOneCommunity(
+                IfAtleastOneCommunity(
                     then_=can_review, else_=[Administration(), SystemProcess()]
                 )
             ],
diff --git a/invenio_rdm_records/services/services.py b/invenio_rdm_records/services/services.py
index 66ea2697b..616cb9493 100644
--- a/invenio_rdm_records/services/services.py
+++ b/invenio_rdm_records/services/services.py
@@ -419,13 +419,12 @@ def publish(self, identity, id_, uow=None, expand=False):
         is_community_required = current_app.config["RDM_RECORD_ALWAYS_IN_COMMUNITY"]
         is_community_missing = len(draft.parent.communities.ids) == 0
         # Then, check for permissions to upload without community
-        can_publish_draft = self.check_permission(identity, "publish", record=draft)
-
-        if is_community_required and is_community_missing and not can_publish_draft:
+        if (
+            is_community_required
+            and is_community_missing
+            and not self.check_permission(identity, "publish", record=draft)
+        ):
             raise CommunityNotSelectedError()
-        elif not can_publish_draft:
-            # If community is not missing or the community is not required, raise error if user doesn't have permissions
-            raise PermissionDeniedError("publish")
 
         return super().publish(identity, id_, uow=uow, expand=expand)
 
diff --git a/tests/resources/test_resources_communities.py b/tests/resources/test_resources_communities.py
index cb6ca4ee9..895479e15 100644
--- a/tests/resources/test_resources_communities.py
+++ b/tests/resources/test_resources_communities.py
@@ -947,6 +947,47 @@ def test_restricted_record_creation(
         assert super_user_rec.id
 
 
+def test_republish_with_mulitple_communities(
+    app,
+    db,
+    headers,
+    client,
+    minimal_record,
+    open_review_community,
+    record_required_community,
+    community2,
+    uploader,
+):
+    """Verify new version of record with multiple communities can be re-published"""
+    client = uploader.login(client)
+    comm = [
+        community2,
+        open_review_community,
+    ]
+    with ensure_record_community_exists_config(app):
+        record = record_required_community.create_record()
+        record_pid = record._record.pid.pid_value
+        for com in comm:
+            _add_to_community(db, record._record, com)
+        assert len(record._record.parent.communities.ids) == 3
+        response = client.post(
+            f"/records/{record_pid}/versions",
+            headers=headers,
+        )
+        assert response.is_json
+        assert response.status_code == 201
+        current_rdm_records_service.update_draft(
+            uploader.identity, response.json["id"], minimal_record
+        )
+        result_item = current_rdm_records_service.publish(
+            uploader.identity, response.json["id"]
+        )
+        new_record_pid = result_item._record.pid.pid_value
+
+        new_record = client.get(f"/records/{new_record_pid}", headers=headers)
+        assert len(new_record.json["parent"]["communities"]["ids"]) == 3
+
+
 def test_remove_last_existing_non_existing_community(
     app, client, uploader, record_required_community, headers, community
 ):

From 5daabf63159e512266a48c53728fae059cf76105 Mon Sep 17 00:00:00 2001
From: Saksham <sakshamarora1001@gmail.com>
Date: Wed, 2 Oct 2024 14:16:48 +0200
Subject: [PATCH 44/76] services: Add elevated permissions check for publish
 and community removal

---
 .../services/communities/service.py           | 16 +++++----
 invenio_rdm_records/services/errors.py        |  8 +++--
 invenio_rdm_records/services/generators.py    | 22 ------------
 invenio_rdm_records/services/permissions.py   | 36 +++++--------------
 invenio_rdm_records/services/services.py      |  9 +++--
 5 files changed, 30 insertions(+), 61 deletions(-)

diff --git a/invenio_rdm_records/services/communities/service.py b/invenio_rdm_records/services/communities/service.py
index 7e7d44cf6..c2c57eaa8 100644
--- a/invenio_rdm_records/services/communities/service.py
+++ b/invenio_rdm_records/services/communities/service.py
@@ -209,12 +209,19 @@ def _remove(self, identity, community_id, record):
         if community_id not in record.parent.communities.ids:
             raise RecordCommunityMissing(record.id, community_id)
 
-        # If config is true and there is only 1 communities left to remove
+        # Check for permission to remove a community from a record
+        self.require_permission(
+            identity, "remove_community", record=record, community_id=community_id
+        )
         is_community_required = current_app.config["RDM_RECORD_ALWAYS_IN_COMMUNITY"]
         is_last_community = len(record.parent.communities.ids) == 1
-        # Then, check for permissions to remove last community
+        # If community is required for a record and if it is the last community to remove
+        # Then, only users with special permissions can remove
         can_remove_last_community = self.check_permission(
-            identity, "remove_community", record=record, community_id=community_id
+            identity,
+            "remove_community_elevated",
+            record=record,
+            community_id=community_id,
         )
         if (
             is_community_required
@@ -222,9 +229,6 @@ def _remove(self, identity, community_id, record):
             and not can_remove_last_community
         ):
             raise CannotRemoveCommunityError()
-        # check permission here, per community: curator cannot remove another community
-        elif not can_remove_last_community:
-            raise PermissionDeniedError("remove_community")
 
         # Default community is deleted when the exact same community is removed from the record
         record.parent.communities.remove(community_id)
diff --git a/invenio_rdm_records/services/errors.py b/invenio_rdm_records/services/errors.py
index 011460fcc..33c6e5d27 100644
--- a/invenio_rdm_records/services/errors.py
+++ b/invenio_rdm_records/services/errors.py
@@ -205,16 +205,18 @@ def description(self):
 class RecordSubmissionClosedCommunityError(PermissionDenied):
     """Record submission policy forbids non-members from submitting records to community."""
 
-    description = "Submission to this community is only allowed to community members."
+    description = _(
+        "Submission to this community is only allowed to community members."
+    )
 
 
 class CommunityNotSelectedError(Exception):
     """Error thrown when a record is being created/updated with less than 1 community."""
 
-    description = "Cannot publish without selecting a community."
+    description = _("Cannot publish without selecting a community.")
 
 
 class CannotRemoveCommunityError(Exception):
     """Error thrown when the last community is being removed from the record."""
 
-    description = "Cannot remove. A record should be part of atleast 1 community."
+    description = _("Cannot remove. A record should be part of at least 1 community.")
diff --git a/invenio_rdm_records/services/generators.py b/invenio_rdm_records/services/generators.py
index 3754254a1..270476b27 100644
--- a/invenio_rdm_records/services/generators.py
+++ b/invenio_rdm_records/services/generators.py
@@ -414,25 +414,3 @@ def needs(self, request=None, **kwargs):
             return [AccessRequestTokenNeed(request["payload"]["token"])]
 
         return []
-
-
-class IfOneCommunity(ConditionalGenerator):
-    """Conditional generator for records always in communities case."""
-
-    def _condition(self, record=None, **kwargs):
-        """Check if the record is associated with zero or one community."""
-        if record is None:
-            return True
-        rec_communities = record.parent.communities.ids
-        return len(rec_communities) == 1
-
-
-class IfAtleastOneCommunity(ConditionalGenerator):
-    """Conditional generator for records always in communities case."""
-
-    def _condition(self, record=None, **kwargs):
-        """Check if the record is associated with zero or one community."""
-        if record is None:
-            return True
-        rec_communities = record.parent.communities.ids
-        return len(rec_communities) > 0
diff --git a/invenio_rdm_records/services/permissions.py b/invenio_rdm_records/services/permissions.py
index 7abc37db3..de9681569 100644
--- a/invenio_rdm_records/services/permissions.py
+++ b/invenio_rdm_records/services/permissions.py
@@ -30,13 +30,11 @@
     AccessGrant,
     CommunityInclusionReviewers,
     GuestAccessRequestToken,
-    IfAtleastOneCommunity,
     IfCreate,
     IfDeleted,
     IfExternalDOIRecord,
     IfFileIsLocal,
     IfNewRecord,
-    IfOneCommunity,
     IfRecordDeleted,
     IfRequestType,
     IfRestricted,
@@ -68,6 +66,7 @@ class RDMRecordPermissionPolicy(RecordPermissionPolicy):
         RecordOwners(),
         RecordCommunitiesAction("curate"),
         AccessGrant("manage"),
+        Administration(),
         SystemProcess(),
     ]
     can_curate = can_manage + [AccessGrant("edit"), SecretLinks("edit")]
@@ -201,17 +200,9 @@ class RDMRecordPermissionPolicy(RecordPermissionPolicy):
         ),
     ]
     # Allow publishing a new record or changes to an existing record.
-    can_publish = [
-        IfConfig(
-            "RDM_RECORD_ALWAYS_IN_COMMUNITY",
-            then_=[
-                IfAtleastOneCommunity(
-                    then_=can_review, else_=[Administration(), SystemProcess()]
-                )
-            ],
-            else_=can_review,
-        )
-    ]
+    can_publish = can_review
+    # Permission to allow special users to publish a record in special cases
+    can_publish_elevated = [Administration(), SystemProcess()]
     # Allow lifting a record or draft.
     can_lift_embargo = can_manage
 
@@ -221,25 +212,16 @@ class RDMRecordPermissionPolicy(RecordPermissionPolicy):
     # Who can add record to a community
     can_add_community = can_manage
     # Who can remove a community from a record
-    can_remove_community_ = [
+    can_remove_community = [
         RecordOwners(),
         CommunityCurators(),
+        Administration(),
         SystemProcess(),
     ]
-    can_remove_community = [
-        IfConfig(
-            "RDM_RECORD_ALWAYS_IN_COMMUNITY",
-            then_=[
-                IfOneCommunity(
-                    then_=[Administration(), SystemProcess()],
-                    else_=can_remove_community_,
-                )
-            ],
-            else_=can_remove_community_,
-        )
-    ]
+    # Permission to allow special users to remove community in special cases
+    can_remove_community_elevated = [Administration(), SystemProcess()]
     # Who can remove records from a community
-    can_remove_record = [CommunityCurators()]
+    can_remove_record = [CommunityCurators(), Administration()]
     # Who can add records to a community in bulk
     can_bulk_add = [SystemProcess()]
 
diff --git a/invenio_rdm_records/services/services.py b/invenio_rdm_records/services/services.py
index 616cb9493..eac5b4814 100644
--- a/invenio_rdm_records/services/services.py
+++ b/invenio_rdm_records/services/services.py
@@ -415,14 +415,17 @@ def publish(self, identity, id_, uow=None, expand=False):
         # Get the draft
         draft = self.draft_cls.pid.resolve(id_, registered_only=False)
 
-        # If config is true and there are no communities selected
         is_community_required = current_app.config["RDM_RECORD_ALWAYS_IN_COMMUNITY"]
         is_community_missing = len(draft.parent.communities.ids) == 0
-        # Then, check for permissions to upload without community
+        # If community is required for a record and there are no communities selected
+        # Then, check for permissions to publish without community
+        can_publish_without_community = self.check_permission(
+            identity, "publish_elevated", record=draft
+        )
         if (
             is_community_required
             and is_community_missing
-            and not self.check_permission(identity, "publish", record=draft)
+            and not can_publish_without_community
         ):
             raise CommunityNotSelectedError()
 

From a605015778c9fdb7d0dc358270c26a5e4ef5fb66 Mon Sep 17 00:00:00 2001
From: Saksham <sakshamarora1001@gmail.com>
Date: Wed, 9 Oct 2024 09:50:18 +0200
Subject: [PATCH 45/76] services: Allow only admin to bypass feature flag

---
 invenio_rdm_records/config.py                 |   2 +-
 invenio_rdm_records/resources/config.py       |   6 +-
 .../services/communities/service.py           |  34 +--
 invenio_rdm_records/services/errors.py        |   6 +-
 invenio_rdm_records/services/generators.py    |  16 ++
 invenio_rdm_records/services/permissions.py   |  37 ++-
 invenio_rdm_records/services/services.py      |  31 +--
 tests/conftest.py                             |  53 +---
 tests/resources/test_resources_communities.py | 250 +++++++++---------
 9 files changed, 208 insertions(+), 227 deletions(-)

diff --git a/invenio_rdm_records/config.py b/invenio_rdm_records/config.py
index e7ebfc32c..53d634f22 100644
--- a/invenio_rdm_records/config.py
+++ b/invenio_rdm_records/config.py
@@ -134,7 +134,7 @@ def always_valid(identifier):
 #
 # Record communities
 #
-RDM_RECORD_ALWAYS_IN_COMMUNITY = False
+RDM_COMMUNITY_REQUIRED_TO_PUBLISH = False
 """Enforces at least one community per record."""
 
 #
diff --git a/invenio_rdm_records/resources/config.py b/invenio_rdm_records/resources/config.py
index 6675de01c..192ab61ad 100644
--- a/invenio_rdm_records/resources/config.py
+++ b/invenio_rdm_records/resources/config.py
@@ -36,7 +36,7 @@
 
 from ..services.errors import (
     AccessRequestExistsError,
-    CommunityNotSelectedError,
+    CommunityRequiredError,
     GrantExistsError,
     InvalidAccessRestrictions,
     RecordDeletedException,
@@ -257,10 +257,10 @@ class RDMRecordResourceConfig(RecordResourceConfig, ConfiguratorMixin):
                 description=e.description,
             )
         ),
-        CommunityNotSelectedError: create_error_handler(
+        CommunityRequiredError: create_error_handler(
             HTTPJSONException(
                 code=400,
-                description="Cannot publish without selecting a community.",
+                description=_("Cannot publish without selecting a community."),
             )
         ),
     }
diff --git a/invenio_rdm_records/services/communities/service.py b/invenio_rdm_records/services/communities/service.py
index c2c57eaa8..613d168c8 100644
--- a/invenio_rdm_records/services/communities/service.py
+++ b/invenio_rdm_records/services/communities/service.py
@@ -209,26 +209,20 @@ def _remove(self, identity, community_id, record):
         if community_id not in record.parent.communities.ids:
             raise RecordCommunityMissing(record.id, community_id)
 
-        # Check for permission to remove a community from a record
-        self.require_permission(
-            identity, "remove_community", record=record, community_id=community_id
-        )
-        is_community_required = current_app.config["RDM_RECORD_ALWAYS_IN_COMMUNITY"]
-        is_last_community = len(record.parent.communities.ids) == 1
-        # If community is required for a record and if it is the last community to remove
-        # Then, only users with special permissions can remove
-        can_remove_last_community = self.check_permission(
-            identity,
-            "remove_community_elevated",
-            record=record,
-            community_id=community_id,
-        )
-        if (
-            is_community_required
-            and is_last_community
-            and not can_remove_last_community
-        ):
-            raise CannotRemoveCommunityError()
+        try:
+            self.require_permission(
+                identity, "remove_community", record=record, community_id=community_id
+            )
+            # By default, admin/superuser has permission to do everything, so PermissionDeniedError won't be raised for admin in any case
+        except PermissionDeniedError as exc:
+            # If permission is denied, determine which error to raise, based on config
+            community_required = current_app.config["RDM_COMMUNITY_REQUIRED_TO_PUBLISH"]
+            is_last_community = len(record.parent.communities.ids) <= 1
+            if community_required and is_last_community:
+                raise CannotRemoveCommunityError()
+            else:
+                # If the config wasn't enabled, then raise the PermissionDeniedError
+                raise exc
 
         # Default community is deleted when the exact same community is removed from the record
         record.parent.communities.remove(community_id)
diff --git a/invenio_rdm_records/services/errors.py b/invenio_rdm_records/services/errors.py
index 33c6e5d27..916cd25a0 100644
--- a/invenio_rdm_records/services/errors.py
+++ b/invenio_rdm_records/services/errors.py
@@ -210,13 +210,13 @@ class RecordSubmissionClosedCommunityError(PermissionDenied):
     )
 
 
-class CommunityNotSelectedError(Exception):
+class CommunityRequiredError(Exception):
     """Error thrown when a record is being created/updated with less than 1 community."""
 
-    description = _("Cannot publish without selecting a community.")
+    description = _("Cannot publish without a community.")
 
 
 class CannotRemoveCommunityError(Exception):
     """Error thrown when the last community is being removed from the record."""
 
-    description = _("Cannot remove. A record should be part of at least 1 community.")
+    description = _("A record should be part of at least 1 community.")
diff --git a/invenio_rdm_records/services/generators.py b/invenio_rdm_records/services/generators.py
index 270476b27..154fddc09 100644
--- a/invenio_rdm_records/services/generators.py
+++ b/invenio_rdm_records/services/generators.py
@@ -414,3 +414,19 @@ def needs(self, request=None, **kwargs):
             return [AccessRequestTokenNeed(request["payload"]["token"])]
 
         return []
+
+
+class IfOneCommunity(ConditionalGenerator):
+    """Conditional generator for records always in communities case."""
+
+    def _condition(self, record=None, **kwargs):
+        """Check if the record is associated with one community."""
+        return bool(record and len(record.parent.communities.ids) == 1)
+
+
+class IfAtLeastOneCommunity(ConditionalGenerator):
+    """Conditional generator for records always in communities case."""
+
+    def _condition(self, record=None, **kwargs):
+        """Check if the record is associated with at least one community."""
+        return bool(record and record.parent.communities.ids)
diff --git a/invenio_rdm_records/services/permissions.py b/invenio_rdm_records/services/permissions.py
index de9681569..5cc3243cb 100644
--- a/invenio_rdm_records/services/permissions.py
+++ b/invenio_rdm_records/services/permissions.py
@@ -30,11 +30,13 @@
     AccessGrant,
     CommunityInclusionReviewers,
     GuestAccessRequestToken,
+    IfAtLeastOneCommunity,
     IfCreate,
     IfDeleted,
     IfExternalDOIRecord,
     IfFileIsLocal,
     IfNewRecord,
+    IfOneCommunity,
     IfRecordDeleted,
     IfRequestType,
     IfRestricted,
@@ -66,7 +68,6 @@ class RDMRecordPermissionPolicy(RecordPermissionPolicy):
         RecordOwners(),
         RecordCommunitiesAction("curate"),
         AccessGrant("manage"),
-        Administration(),
         SystemProcess(),
     ]
     can_curate = can_manage + [AccessGrant("edit"), SecretLinks("edit")]
@@ -200,9 +201,18 @@ class RDMRecordPermissionPolicy(RecordPermissionPolicy):
         ),
     ]
     # Allow publishing a new record or changes to an existing record.
-    can_publish = can_review
-    # Permission to allow special users to publish a record in special cases
-    can_publish_elevated = [Administration(), SystemProcess()]
+    can_publish = [
+        IfConfig(
+            "RDM_COMMUNITY_REQUIRED_TO_PUBLISH",
+            then_=[
+                IfAtLeastOneCommunity(
+                    then_=can_review,
+                    else_=[Administration(), SystemProcess()],
+                ),
+            ],
+            else_=can_review,
+        )
+    ]
     # Allow lifting a record or draft.
     can_lift_embargo = can_manage
 
@@ -212,16 +222,25 @@ class RDMRecordPermissionPolicy(RecordPermissionPolicy):
     # Who can add record to a community
     can_add_community = can_manage
     # Who can remove a community from a record
-    can_remove_community = [
+    can_remove_community_ = [
         RecordOwners(),
         CommunityCurators(),
-        Administration(),
         SystemProcess(),
     ]
-    # Permission to allow special users to remove community in special cases
-    can_remove_community_elevated = [Administration(), SystemProcess()]
+    can_remove_community = [
+        IfConfig(
+            "RDM_COMMUNITY_REQUIRED_TO_PUBLISH",
+            then_=[
+                IfOneCommunity(
+                    then_=[Administration(), SystemProcess()],
+                    else_=can_remove_community_,
+                ),
+            ],
+            else_=can_remove_community_,
+        ),
+    ]
     # Who can remove records from a community
-    can_remove_record = [CommunityCurators(), Administration()]
+    can_remove_record = [CommunityCurators(), Administration(), SystemProcess()]
     # Who can add records to a community in bulk
     can_bulk_add = [SystemProcess()]
 
diff --git a/invenio_rdm_records/services/services.py b/invenio_rdm_records/services/services.py
index eac5b4814..e99d0761a 100644
--- a/invenio_rdm_records/services/services.py
+++ b/invenio_rdm_records/services/services.py
@@ -38,7 +38,7 @@
 
 from ..records.systemfields.deletion_status import RecordDeletionStatusEnum
 from .errors import (
-    CommunityNotSelectedError,
+    CommunityRequiredError,
     DeletionStatusException,
     EmbargoNotLiftedError,
     RecordDeletedException,
@@ -412,22 +412,19 @@ def publish(self, identity, id_, uow=None, expand=False):
 
         Check for permissions to publish a draft and then call invenio_drafts_resourcs.services.records.services.publish()
         """
-        # Get the draft
-        draft = self.draft_cls.pid.resolve(id_, registered_only=False)
-
-        is_community_required = current_app.config["RDM_RECORD_ALWAYS_IN_COMMUNITY"]
-        is_community_missing = len(draft.parent.communities.ids) == 0
-        # If community is required for a record and there are no communities selected
-        # Then, check for permissions to publish without community
-        can_publish_without_community = self.check_permission(
-            identity, "publish_elevated", record=draft
-        )
-        if (
-            is_community_required
-            and is_community_missing
-            and not can_publish_without_community
-        ):
-            raise CommunityNotSelectedError()
+        try:
+            draft = self.draft_cls.pid.resolve(id_, registered_only=False)
+            self.require_permission(identity, "publish", record=draft)
+            # By default, admin/superuser has permission to do everything, so PermissionDeniedError won't be raised for admin in any case
+        except PermissionDeniedError as exc:
+            # If user doesn't have permission to publish, determine which error to raise, based on config
+            community_required = current_app.config["RDM_COMMUNITY_REQUIRED_TO_PUBLISH"]
+            is_community_missing = len(draft.parent.communities.ids) < 1
+            if community_required and is_community_missing:
+                raise CommunityRequiredError()
+            else:
+                # If the config wasn't enabled, then raise the PermissionDeniedError
+                raise exc
 
         return super().publish(identity, id_, uow=uow, expand=expand)
 
diff --git a/tests/conftest.py b/tests/conftest.py
index 6a89872c3..2f54cfa3e 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -2027,20 +2027,22 @@ def create_record(
             """Creates new record that belongs to the same community."""
             # create draft
             draft = current_rdm_records_service.create(uploader.identity, record_dict)
-            # publish and get record
-            result_item = current_rdm_records_service.publish(
-                uploader.identity, draft.id
-            )
-            record = result_item._record
+            record = draft._record
             if community:
                 # add the record to the community
                 community_record = community._record
                 record.parent.communities.add(community_record, default=False)
                 record.parent.commit()
                 db.session.commit()
-                current_rdm_records_service.indexer.index(
-                    record, arguments={"refresh": True}
-                )
+
+            # publish and get record
+            result_item = current_rdm_records_service.publish(
+                uploader.identity, draft.id
+            )
+            record = result_item._record
+            current_rdm_records_service.indexer.index(
+                record, arguments={"refresh": True}
+            )
 
             return record
 
@@ -2094,41 +2096,6 @@ def create_record(
     return RecordFactory()
 
 
-@pytest.fixture()
-def record_required_community(db, uploader, minimal_record, community):
-    """Creates a record that belongs to a community before publishing."""
-
-    class Record:
-        """Test record class."""
-
-        def create_record(
-            self,
-            record_dict=minimal_record,
-            uploader=uploader,
-            community=community,
-        ):
-            """Creates new record that belongs to the same community."""
-            # create draft
-            draft = current_rdm_records_service.create(uploader.identity, record_dict)
-            record = draft._record
-            # add the record to the community
-            community_record = community._record
-            record.parent.communities.add(community_record, default=False)
-            record.parent.commit()
-            db.session.commit()
-            current_rdm_records_service.indexer.index(
-                record, arguments={"refresh": True}
-            )
-
-            # publish and get record
-            community_record = current_rdm_records_service.publish(
-                uploader.identity, draft.id
-            )
-            return community_record
-
-    return Record()
-
-
 @pytest.fixture(scope="session")
 def headers():
     """Default headers for making requests."""
diff --git a/tests/resources/test_resources_communities.py b/tests/resources/test_resources_communities.py
index 895479e15..db56027e2 100644
--- a/tests/resources/test_resources_communities.py
+++ b/tests/resources/test_resources_communities.py
@@ -20,7 +20,7 @@
 from invenio_rdm_records.records.api import RDMDraft, RDMRecord
 from invenio_rdm_records.requests.community_inclusion import CommunityInclusion
 from invenio_rdm_records.services.errors import (
-    CommunityNotSelectedError,
+    CommunityRequiredError,
     InvalidAccessRestrictions,
 )
 
@@ -899,24 +899,6 @@ def test_add_record_to_restricted_community_submission_open_member(
 # -------------------------------------
 
 
-@contextmanager
-def ensure_record_community_exists_config(app):
-    """
-    Context manager to ensure record community exists config
-    is set to True during a specific code block.
-    Parameters:
-    app: app fixture
-    Usage:
-    with ensure_record_community_exists_config(app):
-    # code block that requires the flag to be set
-    """
-    try:
-        app.config["RDM_RECORD_ALWAYS_IN_COMMUNITY"] = True
-        yield
-    finally:
-        app.config["RDM_RECORD_ALWAYS_IN_COMMUNITY"] = False
-
-
 def test_restricted_record_creation(
     app,
     record_community,
@@ -925,26 +907,28 @@ def test_restricted_record_creation(
     community_owner,
     test_user,
     superuser,
+    monkeypatch,
 ):
-    """Verify CommunityNotSelectedError is raised when direct publish a record"""
+    """Verify CommunityRequiredError is raised when direct publish a record"""
     # You can directly publish a record when the config is disabled
-    rec = record_community.create_record()
+    monkeypatch.setitem(app.config, "RDM_COMMUNITY_REQUIRED_TO_PUBLISH", False)
+    rec = record_community.create_record(community=None)
     assert rec.id
-    with ensure_record_community_exists_config(app):
-        # You can't directly publish
-        users = [
-            curator,
-            test_user,
-            uploader,
-            community_owner,
-        ]
-        for user in users:
-            with pytest.raises(CommunityNotSelectedError):
-                record_community.create_record(uploader=user)
+    monkeypatch.setitem(app.config, "RDM_COMMUNITY_REQUIRED_TO_PUBLISH", True)
+    # You can't directly publish
+    users = [
+        curator,
+        test_user,
+        uploader,
+        community_owner,
+    ]
+    for user in users:
+        with pytest.raises(CommunityRequiredError):
+            record_community.create_record(uploader=user, community=None)
 
-        # Super user can!
-        super_user_rec = record_community.create_record(uploader=superuser)
-        assert super_user_rec.id
+    # Super user can!
+    super_user_rec = record_community.create_record(uploader=superuser, community=None)
+    assert super_user_rec.id
 
 
 def test_republish_with_mulitple_communities(
@@ -954,45 +938,53 @@ def test_republish_with_mulitple_communities(
     client,
     minimal_record,
     open_review_community,
-    record_required_community,
+    record_community,
     community2,
     uploader,
+    monkeypatch,
 ):
     """Verify new version of record with multiple communities can be re-published"""
+    monkeypatch.setitem(app.config, "RDM_COMMUNITY_REQUIRED_TO_PUBLISH", True)
     client = uploader.login(client)
     comm = [
         community2,
         open_review_community,
     ]
-    with ensure_record_community_exists_config(app):
-        record = record_required_community.create_record()
-        record_pid = record._record.pid.pid_value
-        for com in comm:
-            _add_to_community(db, record._record, com)
-        assert len(record._record.parent.communities.ids) == 3
-        response = client.post(
-            f"/records/{record_pid}/versions",
-            headers=headers,
-        )
-        assert response.is_json
-        assert response.status_code == 201
-        current_rdm_records_service.update_draft(
-            uploader.identity, response.json["id"], minimal_record
-        )
-        result_item = current_rdm_records_service.publish(
-            uploader.identity, response.json["id"]
-        )
-        new_record_pid = result_item._record.pid.pid_value
+    record = record_community.create_record()
+    record_pid = record.pid.pid_value
+    for com in comm:
+        _add_to_community(db, record, com)
+    assert len(record.parent.communities.ids) == 3
+    response = client.post(
+        f"/records/{record_pid}/versions",
+        headers=headers,
+    )
+    assert response.is_json
+    assert response.status_code == 201
+    current_rdm_records_service.update_draft(
+        uploader.identity, response.json["id"], minimal_record
+    )
+    result_item = current_rdm_records_service.publish(
+        uploader.identity, response.json["id"]
+    )
+    new_record_pid = result_item._record.pid.pid_value
 
-        new_record = client.get(f"/records/{new_record_pid}", headers=headers)
-        assert len(new_record.json["parent"]["communities"]["ids"]) == 3
+    new_record = client.get(f"/records/{new_record_pid}", headers=headers)
+    assert len(new_record.json["parent"]["communities"]["ids"]) == 3
 
 
 def test_remove_last_existing_non_existing_community(
-    app, client, uploader, record_required_community, headers, community
+    app,
+    client,
+    uploader,
+    record_community,
+    headers,
+    community,
+    monkeypatch,
 ):
     """Test removal of an existing and non-existing community from the record,
     while ensuring at least one community exists."""
+    monkeypatch.setitem(app.config, "RDM_COMMUNITY_REQUIRED_TO_PUBLISH", True)
     data = {
         "communities": [
             {"id": "wrong-id"},
@@ -1002,34 +994,35 @@ def test_remove_last_existing_non_existing_community(
     }
 
     client = uploader.login(client)
-    record = record_required_community.create_record()
-    record_pid = record._record.pid.pid_value
-    with ensure_record_community_exists_config(app):
-        response = client.delete(
-            f"/records/{record_pid}/communities",
-            headers=headers,
-            json=data,
-        )
-        assert response.is_json
-        assert response.status_code == 400
-        # Should get 3 errors: Can't remove community, 2 bad IDs
-        assert len(response.json["errors"]) == 3
-        record_saved = client.get(f"/records/{record_pid}", headers=headers)
-        assert record_saved.json["parent"]["communities"]
+    record = record_community.create_record()
+    record_pid = record.pid.pid_value
+    response = client.delete(
+        f"/records/{record_pid}/communities",
+        headers=headers,
+        json=data,
+    )
+    assert response.is_json
+    assert response.status_code == 400
+    # Should get 3 errors: Can't remove community, 2 bad IDs
+    assert len(response.json["errors"]) == 3
+    record_saved = client.get(f"/records/{record_pid}", headers=headers)
+    assert record_saved.json["parent"]["communities"]
 
 
 def test_remove_last_community_api_error_handling(
-    record_required_community,
+    record_community,
     community,
     uploader,
     headers,
     curator,
     client,
     app,
+    monkeypatch,
 ):
     """Testing error message when trying to remove last community."""
-    record = record_required_community.create_record()
-    record_pid = record._record.pid.pid_value
+    monkeypatch.setitem(app.config, "RDM_COMMUNITY_REQUIRED_TO_PUBLISH", True)
+    record = record_community.create_record()
+    record_pid = record.pid.pid_value
     data = {"communities": [{"id": community.id}]}
     for user in [uploader, curator]:
         client = user.login(client)
@@ -1041,85 +1034,80 @@ def test_remove_last_community_api_error_handling(
         assert (
             len(response.json["hits"]["hits"][0]["parent"]["communities"]["ids"]) == 1
         )
-        with ensure_record_community_exists_config(app):
-            response = client.delete(
-                f"/records/{record_pid}/communities",
-                headers=headers,
-                json=data,
-            )
-            assert response.is_json
-            assert response.status_code == 400
-
-            record_saved = client.get(f"/records/{record_pid}", headers=headers)
-            assert record_saved.json["parent"]["communities"]
-            assert len(response.json["errors"]) == 1
-
-            client = user.logout(client)
-            # check communities number
-            response = client.get(
-                f"/communities/{community.id}/records",
-                headers=headers,
-                json=data,
-            )
-            assert (
-                len(response.json["hits"]["hits"][0]["parent"]["communities"]["ids"])
-                == 1
-            )
+        response = client.delete(
+            f"/records/{record_pid}/communities",
+            headers=headers,
+            json=data,
+        )
+        assert response.is_json
+        assert response.status_code == 400
+
+        record_saved = client.get(f"/records/{record_pid}", headers=headers)
+        assert record_saved.json["parent"]["communities"]
+        assert len(response.json["errors"]) == 1
+
+        client = user.logout(client)
+        # check communities number
+        response = client.get(
+            f"/communities/{community.id}/records",
+            headers=headers,
+            json=data,
+        )
+        assert (
+            len(response.json["hits"]["hits"][0]["parent"]["communities"]["ids"]) == 1
+        )
 
 
 def test_remove_record_last_community_with_multiple_communities(
     closed_review_community,
     open_review_community,
-    record_required_community,
+    record_community,
     community2,
     uploader,
     headers,
     client,
     app,
     db,
+    monkeypatch,
 ):
     """Testing correct removal of multiple communities"""
+    monkeypatch.setitem(app.config, "RDM_COMMUNITY_REQUIRED_TO_PUBLISH", True)
     client = uploader.login(client)
 
-    record = record_required_community.create_record()
-    record_pid = record._record.pid.pid_value
+    record = record_community.create_record()
+    record_pid = record.pid.pid_value
     comm = [
         community2,
         open_review_community,
         closed_review_community,
     ]  # one more in the rec fixture so it's 4
     for com in comm:
-        _add_to_community(db, record._record, com)
-    assert len(record._record.parent.communities.ids) == 4
+        _add_to_community(db, record, com)
+    assert len(record.parent.communities.ids) == 4
 
-    with ensure_record_community_exists_config(app):
-        data = {
-            "communities": [{"id": x} for x in record._record.parent.communities.ids]
-        }
+    data = {"communities": [{"id": x} for x in record.parent.communities.ids]}
 
-        response = client.delete(
-            f"/records/{record_pid}/communities",
-            headers=headers,
-            json=data,
-        )
-        # You get res 200 with error msg if all communities you are deleting
-        assert response.status_code == 200
-        assert "error" in str(response.data)
+    response = client.delete(
+        f"/records/{record_pid}/communities",
+        headers=headers,
+        json=data,
+    )
+    # You get res 200 with error msg if all communities you are deleting
+    assert response.status_code == 200
+    assert "error" in str(response.data)
 
-        rec_com_left = client.get(f"/records/{record_pid}", headers=headers)
-        assert len(rec_com_left.json["parent"]["communities"]["ids"]) == 1
+    rec_com_left = client.get(f"/records/{record_pid}", headers=headers)
+    assert len(rec_com_left.json["parent"]["communities"]["ids"]) == 1
 
-        # You get res 400 with error msg if you Delete the last one only.
-        response = client.delete(
-            f"/records/{record_pid}/communities",
-            headers=headers,
-            json={
-                "communities": [{"id": str(record._record.parent.communities.ids[0])}]
-            },
-        )
-        assert response.status_code == 400
-        assert "error" in str(response.data)
+    # You get res 400 with error msg if you Delete the last one only.
+    response = client.delete(
+        f"/records/{record_pid}/communities",
+        headers=headers,
+        json={"communities": [{"id": str(record.parent.communities.ids[0])}]},
+    )
+    assert response.status_code == 400
+    assert "error" in str(response.data)
 
-        record_saved = client.get(f"/records/{record_pid}", headers=headers)
-        # check that only one community ID is associated with the record
-        assert len(record_saved.json["parent"]["communities"]["ids"]) == 1
+    record_saved = client.get(f"/records/{record_pid}", headers=headers)
+    # check that only one community ID is associated with the record
+    assert len(record_saved.json["parent"]["communities"]["ids"]) == 1

From 1a1a8dc60a0832135b7674c93f22debce6cb226b Mon Sep 17 00:00:00 2001
From: Alex Ioannidis <a.ioannidis@cern.ch>
Date: Fri, 27 Sep 2024 10:31:58 +0200
Subject: [PATCH 46/76] mappings: disable doc_values for geo_shape fields

* Addresses #1807.
* Fixes that multiple values for ``metadata.locaations.features``.
---
 .../mappings/os-v1/rdmrecords/records/record-v7.0.0.json       | 3 ++-
 .../mappings/os-v2/rdmrecords/records/record-v7.0.0.json       | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/invenio_rdm_records/records/mappings/os-v1/rdmrecords/records/record-v7.0.0.json b/invenio_rdm_records/records/mappings/os-v1/rdmrecords/records/record-v7.0.0.json
index 3fd7e98d4..3e3cd6600 100644
--- a/invenio_rdm_records/records/mappings/os-v1/rdmrecords/records/record-v7.0.0.json
+++ b/invenio_rdm_records/records/mappings/os-v1/rdmrecords/records/record-v7.0.0.json
@@ -1074,7 +1074,8 @@
                     "type": "geo_point"
                   },
                   "geometry": {
-                    "type": "geo_shape"
+                    "type": "geo_shape",
+                    "doc_values": false
                   },
                   "place": {
                     "type": "text"
diff --git a/invenio_rdm_records/records/mappings/os-v2/rdmrecords/records/record-v7.0.0.json b/invenio_rdm_records/records/mappings/os-v2/rdmrecords/records/record-v7.0.0.json
index 0719293e9..9291eec90 100644
--- a/invenio_rdm_records/records/mappings/os-v2/rdmrecords/records/record-v7.0.0.json
+++ b/invenio_rdm_records/records/mappings/os-v2/rdmrecords/records/record-v7.0.0.json
@@ -1074,7 +1074,8 @@
                     "type": "geo_point"
                   },
                   "geometry": {
-                    "type": "geo_shape"
+                    "type": "geo_shape",
+                    "doc_values": false
                   },
                   "place": {
                     "type": "text"

From 39128db78806c14ae844e07b192ba58b97ed2282 Mon Sep 17 00:00:00 2001
From: alejandromumo <manuel.alejandro.de.oliveira.da.costa@cern.ch>
Date: Mon, 30 Sep 2024 14:47:06 +0200
Subject: [PATCH 47/76] collections: refactored backend

---
 invenio_rdm_records/collections/__init__.py |   3 +
 invenio_rdm_records/collections/api.py      | 291 ++++++++++----------
 invenio_rdm_records/collections/errors.py   |  19 ++
 invenio_rdm_records/collections/models.py   |  51 +++-
 invenio_rdm_records/collections/results.py  | 157 +++++++++++
 invenio_rdm_records/collections/service.py  |  90 ++----
 tests/collections/test_collections_api.py   |  73 ++++-
 tests/services/test_collections_service.py  |   2 +-
 8 files changed, 458 insertions(+), 228 deletions(-)
 create mode 100644 invenio_rdm_records/collections/errors.py
 create mode 100644 invenio_rdm_records/collections/results.py

diff --git a/invenio_rdm_records/collections/__init__.py b/invenio_rdm_records/collections/__init__.py
index abd0193da..2b152bb95 100644
--- a/invenio_rdm_records/collections/__init__.py
+++ b/invenio_rdm_records/collections/__init__.py
@@ -6,11 +6,14 @@
 # it under the terms of the MIT License; see LICENSE file for more details.
 """Collections entrypoint."""
 
+from .errors import CollectionNotFound, CollectionTreeNotFound
 from .models import Collection, CollectionTree
 from .searchapp import search_app_context
 
 __all__ = (
     "Collection",
+    "CollectionNotFound",
     "CollectionTree",
+    "CollectionTreeNotFound",
     "search_app_context",
 )
diff --git a/invenio_rdm_records/collections/api.py b/invenio_rdm_records/collections/api.py
index 68200f60a..4b46326f4 100644
--- a/invenio_rdm_records/collections/api.py
+++ b/invenio_rdm_records/collections/api.py
@@ -5,26 +5,68 @@
 # Invenio-RDM is free software; you can redistribute it and/or modify
 # it under the terms of the MIT License; see LICENSE file for more details.
 """Collections programmatic API."""
-
-from types import ClassMethodDescriptorType
-
-from invenio_db import db
-from sqlalchemy import select
+from luqum.parser import parser as luqum_parser
 from werkzeug.utils import cached_property
 
+from .errors import CollectionNotFound, CollectionTreeNotFound, InvalidQuery
 from .models import Collection as CollectionModel
 from .models import CollectionTree as CollectionTreeModel
 
 
+class ModelField:
+    """Model Field Descriptor."""
+
+    def __init__(self, attr_name):
+        """Initialize the descriptor."""
+        self._attr_name = attr_name
+
+    @property
+    def attr_name(self):
+        """The name of the SQLAlchemy field on the model.
+
+        Defaults to the attribute name used on the class.
+        """
+        return self._attr_name
+
+    def __get__(self, obj, objtype=None):
+        """Descriptor method to get the object."""
+        if obj is None:
+            return self
+
+        # Try instance access
+        try:
+            return getattr(obj.model, self.attr_name)
+        except AttributeError:
+            return None
+
+
 class Collection:
     """Collection Object."""
 
     model_cls = CollectionModel
 
+    id = ModelField("id")
+    path = ModelField("path")
+    ctree_id = ModelField("collection_tree_id")
+    order = ModelField("order")
+    title = ModelField("title")
+    slug = ModelField("slug")
+    depth = ModelField("depth")
+    search_query = ModelField("search_query")
+    num_records = ModelField("num_records")
+
     def __init__(self, model=None):
         """Instantiate a Collection object."""
         self.model = model
 
+    @classmethod
+    def validate_query(cls, query):
+        """Validate the collection query."""
+        try:
+            luqum_parser.parse(query)
+        except Exception:
+            raise InvalidQuery()
+
     @classmethod
     def create(cls, slug, title, query, ctree=None, parent=None, order=None):
         """Create a new collection."""
@@ -38,6 +80,7 @@ def create(cls, slug, title, query, ctree=None, parent=None, order=None):
         else:
             raise ValueError("Either parent or ctree must be set.")
 
+        Collection.validate_query(query)
         return cls(
             cls.model_cls.create(
                 slug=slug,
@@ -50,34 +93,30 @@ def create(cls, slug, title, query, ctree=None, parent=None, order=None):
         )
 
     @classmethod
-    def resolve(cls, id_, ctree_id=None, use_slug=False):
+    def resolve(cls, id_=None, slug=None, ctree_id=None):
         """Resolve a collection by ID or slug.
 
         To resolve by slug, the collection tree ID must be provided.
         """
-        if not use_slug:
-            return cls.get(id_)
-        if not ctree_id:
+        res = None
+        if id_:
+            res = cls(cls.model_cls.get(id_))
+        elif slug and ctree_id:
+            res = cls(cls.model_cls.get_by_slug(slug, ctree_id))
+        else:
             raise ValueError(
-                "Collection tree ID is required to resolve a collection by slug."
+                "Either ID or slug and collection tree ID must be provided."
             )
-        return cls.get_by_slug(id_, ctree_id)
 
-    @classmethod
-    def get(cls, id_):
-        """Get a collection by ID."""
-        model = cls.model_cls.get(id_)
-        if not model:
-            return None
-        return cls(model)
+        if res.model is None:
+            raise CollectionNotFound()
+        return res
 
     @classmethod
-    def get_by_slug(cls, slug, ctree_id):
-        """Get a collection by slug."""
-        model = cls.model_cls.get_by_slug(slug, ctree_id)
-        if not model:
-            return None
-        return cls(model)
+    def resolve_many(cls, ids_=None):
+        """Resolve many collections by ID."""
+        _ids = ids_ or []
+        return [cls(c) for c in cls.model_cls.read_many(_ids)]
 
     def add(
         self,
@@ -95,36 +134,6 @@ def add(
             order=order,
         )
 
-    @property
-    def id(self):
-        """Get the collection ID."""
-        return self.model.id
-
-    @property
-    def path(self):
-        """Get the collection path."""
-        return self.model.path
-
-    @property
-    def ctree_id(self):
-        """Get the collection tree ID."""
-        return self.model.tree_id
-
-    @property
-    def order(self):
-        """Get the collection order."""
-        return self.model.order
-
-    @property
-    def title(self):
-        """Get the collection title."""
-        return self.model.title
-
-    @property
-    def ctree_title(self):
-        """Get the collection tree title."""
-        return self.model.collection_tree.title
-
     @property
     def collection_tree(self):
         """Get the collection tree object.
@@ -133,16 +142,6 @@ def collection_tree(self):
         """
         return CollectionTree(self.model.collection_tree)
 
-    @property
-    def depth(self):
-        """Get the collection depth in its tree."""
-        return self.model.depth
-
-    @property
-    def slug(self):
-        """Get the collection slug."""
-        return self.model.slug
-
     @cached_property
     def community(self):
         """Get the community object."""
@@ -155,22 +154,13 @@ def query(self):
         for _a in self.ancestors:
             q += f"({_a.model.search_query}) AND "
         q += f"({self.model.search_query})"
+        Collection.validate_query(q)
         return q
 
     @cached_property
     def ancestors(self):
         """Get the collection ancestors."""
-        if not self.model:
-            return None
-
-        cps = self.path.split(",")
-        ret = []
-        for cid in cps:
-            if not cid:
-                continue
-            cl = Collection.get(cid)
-            ret.append(cl)
-        return list(sorted(ret, key=lambda x: (x.path, x.order)))
+        return Collection.resolve_many(self.split_path_to_ids())
 
     @cached_property
     def sub_collections(self):
@@ -178,27 +168,25 @@ def sub_collections(self):
         return self.get_subcollections()
 
     @cached_property
-    def direct_subcollections(self):
+    def children(self):
         """Fetch only direct descendants."""
-        return self.get_direct_subcollections()
+        return self.get_children()
+
+    def split_path_to_ids(self):
+        """Return the path as a list of integers."""
+        if not self.model:
+            return None
+        return [int(part) for part in self.path.split(",") if part.strip()]
 
-    def get_direct_subcollections(self):
+    def get_children(self):
         """Get the collection first level (direct) children.
 
         More preformant query to retrieve descendants, executes an exact match query.
         """
         if not self.model:
             return None
-        stmt = (
-            select(self.model_cls)
-            .filter(
-                self.model_cls.path == f"{self.path}{self.id},",
-                self.model_cls.tree_id == self.ctree_id,
-            )
-            .order_by(self.model_cls.path, self.model_cls.order)
-        )
-        ret = db.session.execute(stmt).scalars().all()
-        return [type(self)(r) for r in ret]
+        res = self.model_cls.get_children(self.model)
+        return [type(self)(r) for r in res]
 
     def get_subcollections(self, max_depth=3):
         """Get the collection subcollections.
@@ -208,16 +196,8 @@ def get_subcollections(self, max_depth=3):
         if not self.model:
             return None
 
-        stmt = (
-            select(self.model_cls)
-            .filter(
-                self.model_cls.path.like(f"{self.path}{self.id},%"),
-                self.model_cls.depth < self.model.depth + max_depth,
-            )
-            .order_by(self.model_cls.path, self.model_cls.order)
-        )
-        ret = db.session.execute(stmt).scalars().all()
-        return [type(self)(r) for r in ret]
+        res = self.model_cls.get_subcollections(self.model, max_depth)
+        return [type(self)(r) for r in res]
 
     @classmethod
     def dump(cls, collection):
@@ -229,10 +209,11 @@ def dump(cls, collection):
             "order": collection.order,
             "id": collection.id,
             "query": collection.query,
+            "num_records": collection.num_records,
         }
         return res
 
-    def to_dict(self) -> dict:
+    def to_dict(self, max_depth=2) -> dict:
         """Return a dictionary representation of the collection.
 
         Uses an adjacency list.
@@ -242,13 +223,13 @@ def to_dict(self) -> dict:
             self.id: {**Collection.dump(self), "children": set()},
         }
 
-        for _c in self.sub_collections:
+        for _c in self.get_subcollections(max_depth=max_depth):
             # Add the collection itself to the dictionary
             if _c.id not in ret:
                 ret[_c.id] = {**Collection.dump(_c), "children": set()}
 
             # Find the parent ID from the collection's path (last valid ID in the path)
-            path_parts = [int(part) for part in _c.path.split(",") if part.strip()]
+            path_parts = _c.split_path_to_ids()
             if path_parts:
                 parent_id = path_parts[-1]
                 # Add the collection as a child of its parent
@@ -265,12 +246,24 @@ def __repr__(self) -> str:
         else:
             return "Collection (None)"
 
+    def __eq__(self, value: object) -> bool:
+        """Check if the value is equal to the collection."""
+        return isinstance(value, Collection) and value.id == self.id
+
 
 class CollectionTree:
     """Collection Tree Object."""
 
     model_cls = CollectionTreeModel
 
+    id = ModelField("id")
+    title = ModelField("title")
+    slug = ModelField("slug")
+    community_id = ModelField("community_id")
+    order = ModelField("order")
+    community = ModelField("community")
+    collections = ModelField("collections")
+
     def __init__(self, model):
         """Instantiate a CollectionTree object."""
         self.model = model
@@ -285,62 +278,54 @@ def create(cls, title, slug, community_id=None, order=None):
         )
 
     @classmethod
-    def resolve(cls, id_, community_id=None, use_slug=False):
+    def resolve(cls, id_=None, slug=None, community_id=None):
         """Resolve a CollectionTree."""
-        if not use_slug:
-            return cls.get(id_)
-
-        if not community_id:
-            raise ValueError(
-                "Community ID is required to resolve a collection tree by slug."
-            )
-        return cls.get_by_slug(id_, community_id)
-
-    @classmethod
-    def get(cls, id_):
-        """Get a collection tree by ID."""
-        model = cls.model_cls.get(id_)
-        if not model:
-            return None
-        return cls(model)
-
-    @classmethod
-    def get_by_slug(cls, slug, community_id):
-        """Get a collection tree by slug.
-
-        Community ID is required to avoid ambiguity.
-        """
-        model = cls.model_cls.get_by_slug(slug, community_id)
-        if not model:
-            return None
-        return cls(model)
+        res = None
+        if id_:
+            res = cls(cls.model_cls.get(id_))
+        elif slug and community_id:
+            res = cls(cls.model_cls.get_by_slug(slug, community_id))
+        else:
+            raise ValueError("Either ID or slug and community ID must be provided.")
 
-    @property
-    def id(self):
-        """Get the collection tree ID."""
-        return self.model.id
+        if res.model is None:
+            raise CollectionTreeNotFound()
+        return res
 
-    @property
-    def title(self):
-        """Get the collection tree title."""
-        return self.model.title
+    def dump(self):
+        """Transform the collection tree into a dictionary."""
+        if not self.model:
+            return {}
+
+        return {
+            "title": self.title,
+            "slug": self.slug,
+            "community_id": str(self.community_id),
+            "order": self.order,
+            "id": self.id,
+        }
 
-    @property
-    def slug(self):
-        """Get the collection tree slug."""
-        return self.model.slug
+    def to_dict(self, max_depth=2):
+        """Return a dictionary representation of the collection tree."""
+        collections = self.root_collections
+        return {
+            **self.dump(),
+            "collections": [c.to_dict(max_depth=max_depth) for c in collections],
+        }
 
-    @property
-    def community_id(self):
-        """Get the community ID."""
-        return self.model.community_id
+    @cached_property
+    def root_collections(self):
+        """Get the collections under this tree."""
+        return self.get_collections(max_depth=1)
 
-    @property
-    def order(self):
-        """Get the collection tree order."""
-        return self.model.order
+    def get_collections(self, max_depth=0):
+        """Get the collections under this tree, up to a depth."""
+        return [
+            Collection(c)
+            for c in CollectionTreeModel.get_collections(self.model, max_depth)
+        ]
 
-    @property
-    def community(self):
-        """Get the community object."""
-        return self.model.community
+    @classmethod
+    def get_community_trees(cls, community_id):
+        """Get all the collection trees for a community."""
+        return [cls(c) for c in cls.model_cls.get_community_trees(community_id)]
diff --git a/invenio_rdm_records/collections/errors.py b/invenio_rdm_records/collections/errors.py
new file mode 100644
index 000000000..db6d84dda
--- /dev/null
+++ b/invenio_rdm_records/collections/errors.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""Errors for collections module."""
+
+class CollectionError(Exception):
+    """Base class for collection errors."""
+
+class CollectionNotFound(CollectionError):
+    """Collection not found error."""
+
+class CollectionTreeNotFound(CollectionError):
+    """Collection tree not found error."""
+
+class InvalidQuery(CollectionError):
+    """Query syntax is invalid."""
diff --git a/invenio_rdm_records/collections/models.py b/invenio_rdm_records/collections/models.py
index d3f510dba..64219632b 100644
--- a/invenio_rdm_records/collections/models.py
+++ b/invenio_rdm_records/collections/models.py
@@ -6,12 +6,12 @@
 # it under the terms of the MIT License; see LICENSE file for more details.
 """Collections models."""
 
+from traitlets import default
 from invenio_communities.communities.records.models import CommunityMetadata
 from invenio_db import db
 from invenio_records.models import Timestamp
 from sqlalchemy import UniqueConstraint
 from sqlalchemy_utils.types import UUIDType
-from traitlets import ClassBasedTraitType
 
 
 # CollectionTree Table
@@ -66,6 +66,21 @@ def get_by_slug(cls, slug, community_id):
             cls.slug == slug, cls.community_id == community_id
         ).one_or_none()
 
+    @classmethod
+    def get_community_trees(cls, community_id):
+        """Get all collection trees of a community."""
+        return cls.query.filter(cls.community_id == community_id).all()
+
+    @classmethod
+    def get_collections(cls, model, max_depth):
+        """Get collections under a tree."""
+        stmt = Collection.query.filter(
+            Collection.tree_id == model.id, Collection.depth < max_depth
+        ).order_by(Collection.path, Collection.order)
+
+        # Execute the query and get the result
+        return stmt.all()
+
 
 # Collection Table
 class Collection(db.Model, Timestamp):
@@ -97,7 +112,7 @@ class Collection(db.Model, Timestamp):
     order = db.Column(db.Integer, nullable=True)
     # TODO index depth
     depth = db.Column(db.Integer, nullable=False)
-    num_records = db.Column(db.Integer, nullable=True)
+    num_records = db.Column(db.Integer, nullable=False, default=0)
 
     # Relationship to CollectionTree
     collection_tree = db.relationship("CollectionTree", back_populates="collections")
@@ -143,3 +158,35 @@ def get(cls, id_):
     def get_by_slug(cls, slug, tree_id):
         """Get a collection by slug."""
         return cls.query.filter(cls.slug == slug, cls.tree_id == tree_id).one_or_none()
+
+    @classmethod
+    def read_many(cls, ids):
+        """Get many collections by ID."""
+        return cls.query.filter(cls.id.in_(ids)).order_by(cls.path, cls.order).all()
+
+    @classmethod
+    def get_children(cls, model):
+        """Get children collections of a collection."""
+        # Assuming self.model_cls is the ORM model
+        stmt = cls.query.filter(
+            cls.path == f"{model.path}{model.id},", cls.tree_id == model.tree_id
+        ).order_by(cls.path, cls.order)
+
+        # Execute the query and get the result
+        return stmt.all()
+
+    @classmethod
+    def get_subcollections(cls, model, max_depth):
+        """Get subcollections of a collection.
+
+        This query will return all subcollections of a collection up to a certain depth.
+        It can be used for max_depth=1, however it is more efficient to use get_children.
+        """
+        stmt = cls.query.filter(
+            cls.path.like(f"{model.path}{model.id},%"),
+            cls.tree_id == model.tree_id,
+            cls.depth < model.depth + max_depth,
+        ).order_by(cls.path, cls.order)
+
+        # Execute the query and get the result
+        return stmt.all()
diff --git a/invenio_rdm_records/collections/results.py b/invenio_rdm_records/collections/results.py
new file mode 100644
index 000000000..58885b700
--- /dev/null
+++ b/invenio_rdm_records/collections/results.py
@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""Collections service items."""
+
+
+class CollectionItem:
+    """Collection item."""
+
+    def __init__(self, collection):
+        """Instantiate a Collection object.
+
+        Optionally pass a community to cache its information in the collection's instance.
+        """
+        self._collection = collection
+
+    def to_dict(self, max_depth=2, include_breadcrumbs=False):
+        """Serialize the collection to a dictionary and add links."""
+        res = {**self._collection.to_dict(max_depth=max_depth)}
+
+        # Add links for all the collections
+        tree = self._collection.collection_tree
+        community = self._collection.community
+        for id_, coll in res.items():
+            if id_ == "root":
+                continue
+            res[id_]["links"] = {
+                "search": CollectionItem.search_link(community.slug),
+                "self_html": CollectionItem.html_link(
+                    community.slug, tree.slug, coll["slug"]
+                ),
+            }
+
+        # Add breadcrumbs
+        if include_breadcrumbs:
+            res[self._collection.id]["breadcrumbs"] = self.breadcrumbs
+
+        return res
+
+    @classmethod
+    def search_link(cls, community_slug):
+        """Get the search link."""
+        if community_slug:
+            return f"/api/communities/{community_slug}/records"
+        return "/api/records"
+
+    @classmethod
+    def html_link(cls, community_slug, tree_slug, collection_slug):
+        """Get the HTML link."""
+        if community_slug:
+            return f"/communities/{community_slug}/collections/{tree_slug}/{collection_slug}"
+        return f"/collections/{tree_slug}/{collection_slug}"
+
+    @property
+    def title(self):
+        """Get the collection title."""
+        return self._collection.title
+
+    @property
+    def query(self):
+        """Get the collection query."""
+        return self._collection.query
+
+    @property
+    def collection_tree(self):
+        """Get the collection tree."""
+        return self._collection.collection_tree
+
+    @property
+    def children(self):
+        """Get the collection children."""
+        return self._collection.children
+
+    @property
+    def community(self):
+        """Get the collection community."""
+        return self._collection.community
+
+    @property
+    def breadcrumbs(self):
+        """Get the collection ancestors."""
+        res = []
+        community_slug = self.community.slug
+        tree_slug = self.collection_tree.slug
+        for anc in self._collection.ancestors:
+            _a = {
+                "title": anc.title,
+                "link": CollectionItem.html_link(community_slug, tree_slug, anc.slug),
+            }
+            res.append(_a)
+        res.append(
+            {
+                "title": self.title,
+                "link": CollectionItem.html_link(
+                    community_slug, tree_slug, self._collection.slug
+                ),
+            }
+        )
+        return res
+
+
+class CollectionList:
+    """Collection list item."""
+
+    def __init__(self, collections):
+        """Instantiate a Collection list item."""
+        self._collections = collections
+
+    def to_dict(self, max_depth=2):
+        """Serialize the collection list to a dictionary."""
+        res = []
+        for collection in self._collections:
+            _r = collection.to_dict(max_depth=2)
+            _r["links"] = CollectionItem(collection).links
+            res.append(_r)
+        return res
+
+    def __iter__(self):
+        """Iterate over the collections."""
+        return iter(self._collections)
+
+
+class CollectionTreeItem:
+    """Collection tree item."""
+
+    def __init__(self, tree, max_depth=2):
+        """Instantiate a Collection tree object."""
+        self._tree = tree
+        self._max_depth = max_depth
+
+    def to_dict(self):
+        """Serialize the collection tree to a dictionary."""
+        return self._tree.to_dict(max_depth=self._max_depth)
+
+
+class CollectionTreeList:
+    """Collection tree list item."""
+
+    def __init__(self, trees, max_depth=1):
+        """Instantiate a Collection tree list item."""
+        self._trees = trees
+        self._max_depth = max_depth
+
+    def to_dict(self):
+        """Serialize the collection tree list to a dictionary."""
+        res = {}
+        for tree in self._trees:
+            # Only root collections
+            res[tree.id] = CollectionTreeItem(tree, self._max_depth).to_dict()
+        return res
+
+    def __iter__(self):
+        """Iterate over the collection trees."""
+        return iter(self._trees)
diff --git a/invenio_rdm_records/collections/service.py b/invenio_rdm_records/collections/service.py
index 98b7f9e8f..6ebb5624f 100644
--- a/invenio_rdm_records/collections/service.py
+++ b/invenio_rdm_records/collections/service.py
@@ -10,53 +10,7 @@
 from invenio_records_resources.services.uow import ModelCommitOp, unit_of_work
 
 from .api import Collection, CollectionTree
-
-
-class CollectionItem:
-    """Collection item."""
-
-    def __init__(self, collection):
-        """Instantiate a Collection object."""
-        self._collection = collection
-
-    def to_dict(self):
-        """Serialize the collection to a dictionary and add links."""
-        res = {**self._collection.to_dict()}
-        res[self._collection.id]["links"] = self.links
-
-        return res
-
-    @property
-    def links(self):
-        """Return the links of the collection."""
-        self_html = None
-        search = None
-        tree_slug = self._collection.collection_tree.slug
-        if self._collection.community:
-            self_html = f"/communities/{self._collection.community.slug}/{tree_slug}/{self._collection.slug}"
-            search = f"/api/communities/{self._collection.community.slug}/records"
-        else:
-            self_html = f"/collections/{tree_slug}/{self._collection.slug}"
-            search = "/api/records"
-        return {
-            "search": search,
-            "self_html": self_html,
-        }
-
-    @property
-    def community(self):
-        """Get the collection community."""
-        return self._collection.community
-
-    @property
-    def query(self):
-        """Get the collection query."""
-        return self._collection.query
-
-    @property
-    def title(self):
-        """Get the collection title."""
-        return self._collection.title
+from .results import CollectionItem, CollectionTreeList
 
 
 class CollectionsService:
@@ -72,20 +26,28 @@ def create(
         current_communities.service.require_permission(
             identity, "update", community_id=community_id
         )
-        ctree = CollectionTree.get_by_slug(tree_slug, community_id)
-        if not ctree:
-            raise ValueError(f"Collection tree {tree_slug} not found.")
+        ctree = CollectionTree.resolve(slug=tree_slug, community_id=community_id)
         collection = self.collection_cls.create(
             slug=slug, title=title, query=query, ctree=ctree, **kwargs
         )
         uow.register(ModelCommitOp(collection.model))
         return CollectionItem(collection)
 
-    def read(self, identity, id_):
-        """Get a collection by ID or slug."""
-        collection = self.collection_cls.resolve(id_)
-        if not collection:
-            raise ValueError(f"Collection {id_} not found.")
+    def read(
+        self, identity, /, *, id_=None, slug=None, community_id=None, tree_slug=None
+    ):
+        """Get a collection by ID or slug.
+
+        To resolve by slug, the collection tree ID and community ID must be provided.
+        """
+        if id_:
+            collection = self.collection_cls.resolve(id_)
+        elif slug and tree_slug and community_id:
+            ctree = CollectionTree.resolve(slug=tree_slug, community_id=community_id)
+            collection = self.collection_cls.resolve(slug=slug, ctree_id=ctree.id)
+        else:
+            raise ValueError("ID or slug and tree_slug and community_id must be provided.")
+
         if collection.community:
             current_communities.service.require_permission(
                 identity, "read", community_id=collection.community.id
@@ -93,21 +55,15 @@ def read(self, identity, id_):
 
         return CollectionItem(collection)
 
-    def read_slug(self, identity, community_id, tree_slug, slug):
-        """Get a collection by slug."""
+    def list_trees(self, identity, community_id, max_depth=2):
+        """Get the trees of a community."""
         current_communities.service.require_permission(
             identity, "read", community_id=community_id
         )
-
-        ctree = CollectionTree.get_by_slug(tree_slug, community_id)
-        if not ctree:
-            raise ValueError(f"Collection tree {tree_slug} not found.")
-
-        collection = self.collection_cls.resolve(slug, ctree.id, use_slug=True)
-        if not collection:
-            raise ValueError(f"Collection {slug} not found.")
-
-        return CollectionItem(collection)
+        if not community_id:
+            raise ValueError("Community ID must be provided.")
+        res = CollectionTree.get_community_trees(community_id)
+        return CollectionTreeList(res, max_depth=max_depth)
 
     @unit_of_work()
     def add(self, identity, collection, slug, title, query, uow=None, **kwargs):
diff --git a/tests/collections/test_collections_api.py b/tests/collections/test_collections_api.py
index d6b715a80..f87ea3992 100644
--- a/tests/collections/test_collections_api.py
+++ b/tests/collections/test_collections_api.py
@@ -26,9 +26,10 @@ def test_create(running_app, db, community, community_owner):
         ctree=tree,
     )
 
-    assert collection.id
-    assert collection.title == "My Collection"
-    assert collection.collection_tree.id == tree.id
+    read_c = Collection.resolve(collection.id)
+    assert read_c.id == collection.id
+    assert read_c.title == "My Collection"
+    assert read_c.collection_tree.id == tree.id
 
     # Use collection tree id
     collection = Collection.create(
@@ -37,7 +38,9 @@ def test_create(running_app, db, community, community_owner):
         slug="my-collection-2",
         ctree=tree.id,
     )
-    assert collection.id
+
+    read_c = Collection.resolve(collection.id)
+    assert read_c.id == collection.id
     assert collection.title == "My Collection 2"
     assert collection.collection_tree.id == tree.id
 
@@ -64,9 +67,69 @@ def test_as_dict(running_app, db):
     )
 
     c3 = Collection.create(title="3", query="*", slug="my-collection-3", parent=c2)
-    res = c1.to_dict()
+    res = c1.to_dict(max_depth=3)
     assert all(k in res for k in (c1.id, c2.id, c3.id))
     assert res[c1.id]["title"] == "My Collection"
     assert res[c1.id]["children"] == [c2.id]
     assert res[c2.id]["children"] == [c3.id]
     assert res[c3.id]["children"] == []
+
+    res = c1.to_dict(max_depth=2)
+    assert all(k in res for k in (c1.id, c2.id))
+    assert res[c1.id]["title"] == "My Collection"
+    assert res[c1.id]["children"] == [c2.id]
+    assert res[c2.id]["children"] == []
+    assert c3.id not in res
+
+
+def test_query_build(running_app, db):
+    """Test query building."""
+
+    tree = CollectionTree.create(
+        title="Tree 1",
+        order=10,
+        slug="tree-1",
+    )
+    c1 = Collection.create(
+        title="My Collection",
+        query="metadata.title:hello",
+        slug="my-collection",
+        ctree=tree,
+    )
+    c2 = Collection.create(
+        title="My Collection 2",
+        query="metadata.creators.name:john",
+        slug="my-collection-2",
+        parent=c1,
+    )
+    assert c2.query == "(metadata.title:hello) AND (metadata.creators.name:john)"
+
+
+def test_children(running_app, db):
+    """Test children property."""
+    tree = CollectionTree.create(
+        title="Tree 1",
+        order=10,
+        slug="tree-1",
+    )
+    c1 = Collection.create(
+        title="My Collection",
+        query="*:*",
+        slug="my-collection",
+        ctree=tree,
+    )
+    c2 = Collection.create(
+        title="My Collection 2",
+        query="*:*",
+        slug="my-collection-2",
+        parent=c1,
+    )
+    c3 = Collection.create(
+        title="My Collection 3",
+        query="*:*",
+        slug="my-collection-3",
+        parent=c2,
+    )
+    assert c1.children == [c2]
+    assert c2.children == [c3]
+    assert c3.children == []
diff --git a/tests/services/test_collections_service.py b/tests/services/test_collections_service.py
index cf97becdb..ab5baee56 100644
--- a/tests/services/test_collections_service.py
+++ b/tests/services/test_collections_service.py
@@ -52,5 +52,5 @@ def test_collections_service_read(
     collections = add_collections()
     c0 = collections[0]
     c1 = collections[1]
-    res = collections_service.read(community_owner.identity, c0.id)
+    res = collections_service.read(community_owner.identity, id_=c0.id)
     assert res._collection.id == c0.id

From 2e9ebdcb9162cc69f5b35ccdf3abab8c48c374ac Mon Sep 17 00:00:00 2001
From: alejandromumo <manuel.alejandro.de.oliveira.da.costa@cern.ch>
Date: Wed, 9 Oct 2024 11:37:21 +0200
Subject: [PATCH 48/76] collections: added logos as static files

---
 invenio_rdm_records/collections/__init__.py |  3 ++-
 invenio_rdm_records/collections/errors.py   |  3 +++
 invenio_rdm_records/collections/models.py   |  4 +++-
 invenio_rdm_records/collections/service.py  | 19 ++++++++++++++++++-
 4 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/invenio_rdm_records/collections/__init__.py b/invenio_rdm_records/collections/__init__.py
index 2b152bb95..5ec831f0b 100644
--- a/invenio_rdm_records/collections/__init__.py
+++ b/invenio_rdm_records/collections/__init__.py
@@ -6,7 +6,7 @@
 # it under the terms of the MIT License; see LICENSE file for more details.
 """Collections entrypoint."""
 
-from .errors import CollectionNotFound, CollectionTreeNotFound
+from .errors import CollectionNotFound, CollectionTreeNotFound, LogoNotFoundError
 from .models import Collection, CollectionTree
 from .searchapp import search_app_context
 
@@ -15,5 +15,6 @@
     "CollectionNotFound",
     "CollectionTree",
     "CollectionTreeNotFound",
+    "LogoNotFoundError",
     "search_app_context",
 )
diff --git a/invenio_rdm_records/collections/errors.py b/invenio_rdm_records/collections/errors.py
index db6d84dda..5db7131c9 100644
--- a/invenio_rdm_records/collections/errors.py
+++ b/invenio_rdm_records/collections/errors.py
@@ -17,3 +17,6 @@ class CollectionTreeNotFound(CollectionError):
 
 class InvalidQuery(CollectionError):
     """Query syntax is invalid."""
+
+class LogoNotFoundError(CollectionError):
+    """Logo not found error."""
diff --git a/invenio_rdm_records/collections/models.py b/invenio_rdm_records/collections/models.py
index 64219632b..3a6df4d93 100644
--- a/invenio_rdm_records/collections/models.py
+++ b/invenio_rdm_records/collections/models.py
@@ -69,7 +69,9 @@ def get_by_slug(cls, slug, community_id):
     @classmethod
     def get_community_trees(cls, community_id):
         """Get all collection trees of a community."""
-        return cls.query.filter(cls.community_id == community_id).all()
+        return (
+            cls.query.filter(cls.community_id == community_id).order_by(cls.order).all()
+        )
 
     @classmethod
     def get_collections(cls, model, max_depth):
diff --git a/invenio_rdm_records/collections/service.py b/invenio_rdm_records/collections/service.py
index 6ebb5624f..af95b76e4 100644
--- a/invenio_rdm_records/collections/service.py
+++ b/invenio_rdm_records/collections/service.py
@@ -6,10 +6,14 @@
 # it under the terms of the MIT License; see LICENSE file for more details.
 """Collections service."""
 
+import os
+
+from flask import current_app, url_for
 from invenio_communities.proxies import current_communities
 from invenio_records_resources.services.uow import ModelCommitOp, unit_of_work
 
 from .api import Collection, CollectionTree
+from .errors import LogoNotFoundError
 from .results import CollectionItem, CollectionTreeList
 
 
@@ -46,7 +50,9 @@ def read(
             ctree = CollectionTree.resolve(slug=tree_slug, community_id=community_id)
             collection = self.collection_cls.resolve(slug=slug, ctree_id=ctree.id)
         else:
-            raise ValueError("ID or slug and tree_slug and community_id must be provided.")
+            raise ValueError(
+                "ID or slug and tree_slug and community_id must be provided."
+            )
 
         if collection.community:
             current_communities.service.require_permission(
@@ -76,3 +82,14 @@ def add(self, identity, collection, slug, title, query, uow=None, **kwargs):
         )
         uow.register(ModelCommitOp(new_collection.model))
         return CollectionItem(new_collection)
+
+    def read_logo(self, identity, slug):
+        """Read a collection logo.
+
+        TODO: implement logos as files in the database. For now, we just check if the file exists as a static file.
+        """
+        logo_path = f"images/collections/{slug}.jpg"
+        _exists = os.path.exists(os.path.join(current_app.static_folder, logo_path))
+        if _exists:
+            return url_for("static", filename=logo_path)
+        raise LogoNotFoundError()

From e3c6ff3297044be1954890df73e78d3cb453e9ba Mon Sep 17 00:00:00 2001
From: alejandromumo <manuel.alejandro.de.oliveira.da.costa@cern.ch>
Date: Wed, 9 Oct 2024 16:03:17 +0200
Subject: [PATCH 49/76] collections: improved backend

* Added service config
* Added links for collections
* Added schema for collections
* Refactored collections to_dict
* Added depth parameter to collections API
* Added cached_property to children property in Collection class.
* Added tests for collections service.
* Added tests for collections API.
---
 invenio_rdm_records/collections/api.py     | 129 ++++---------
 invenio_rdm_records/collections/config.py  |  43 +++++
 invenio_rdm_records/collections/errors.py  |   5 +
 invenio_rdm_records/collections/links.py   |  31 +++
 invenio_rdm_records/collections/models.py  |   1 -
 invenio_rdm_records/collections/results.py | 185 ++++++++++--------
 invenio_rdm_records/collections/schema.py  |  21 +++
 invenio_rdm_records/collections/service.py |  76 ++++++--
 invenio_rdm_records/ext.py                 |   6 +-
 tests/collections/test_collections_api.py  |  36 ++--
 tests/services/test_collections_service.py | 207 ++++++++++++++++++++-
 11 files changed, 518 insertions(+), 222 deletions(-)
 create mode 100644 invenio_rdm_records/collections/config.py
 create mode 100644 invenio_rdm_records/collections/links.py
 create mode 100644 invenio_rdm_records/collections/schema.py

diff --git a/invenio_rdm_records/collections/api.py b/invenio_rdm_records/collections/api.py
index 4b46326f4..5bc7d651e 100644
--- a/invenio_rdm_records/collections/api.py
+++ b/invenio_rdm_records/collections/api.py
@@ -55,9 +55,10 @@ class Collection:
     search_query = ModelField("search_query")
     num_records = ModelField("num_records")
 
-    def __init__(self, model=None):
+    def __init__(self, model=None, max_depth=2):
         """Instantiate a Collection object."""
         self.model = model
+        self.max_depth = max_depth
 
     @classmethod
     def validate_query(cls, query):
@@ -68,7 +69,7 @@ def validate_query(cls, query):
             raise InvalidQuery()
 
     @classmethod
-    def create(cls, slug, title, query, ctree=None, parent=None, order=None):
+    def create(cls, slug, title, query, ctree=None, parent=None, order=None, depth=2):
         """Create a new collection."""
         _ctree = None
         if parent:
@@ -89,20 +90,21 @@ def create(cls, slug, title, query, ctree=None, parent=None, order=None):
                 search_query=query,
                 order=order,
                 ctree_or_id=_ctree,
-            )
+            ),
+            depth,
         )
 
     @classmethod
-    def resolve(cls, id_=None, slug=None, ctree_id=None):
+    def resolve(cls, id_=None, slug=None, ctree_id=None, depth=2):
         """Resolve a collection by ID or slug.
 
         To resolve by slug, the collection tree ID must be provided.
         """
         res = None
         if id_:
-            res = cls(cls.model_cls.get(id_))
+            res = cls(cls.model_cls.get(id_), depth)
         elif slug and ctree_id:
-            res = cls(cls.model_cls.get_by_slug(slug, ctree_id))
+            res = cls(cls.model_cls.get_by_slug(slug, ctree_id), depth)
         else:
             raise ValueError(
                 "Either ID or slug and collection tree ID must be provided."
@@ -113,25 +115,15 @@ def resolve(cls, id_=None, slug=None, ctree_id=None):
         return res
 
     @classmethod
-    def resolve_many(cls, ids_=None):
+    def resolve_many(cls, ids_=None, depth=2):
         """Resolve many collections by ID."""
         _ids = ids_ or []
-        return [cls(c) for c in cls.model_cls.read_many(_ids)]
-
-    def add(
-        self,
-        slug,
-        title,
-        query,
-        order=None,
-    ):
+        return [cls(c, depth) for c in cls.model_cls.read_many(_ids)]
+
+    def add(self, slug, title, query, order=None, depth=2):
         """Add a subcollection to the collection."""
         return self.create(
-            slug=slug,
-            title=title,
-            query=query,
-            parent=self,
-            order=order,
+            slug=slug, title=title, query=query, parent=self, order=order, depth=depth
         )
 
     @property
@@ -164,7 +156,16 @@ def ancestors(self):
 
     @cached_property
     def sub_collections(self):
-        """Fetch all the descendants."""
+        """Fetch descendants.
+
+        If the max_depth is 1, fetch only direct descendants.
+        """
+        if self.max_depth == 0:
+            return []
+
+        if self.max_depth == 1:
+            return self.get_children()
+
         return self.get_subcollections()
 
     @cached_property
@@ -188,7 +189,7 @@ def get_children(self):
         res = self.model_cls.get_children(self.model)
         return [type(self)(r) for r in res]
 
-    def get_subcollections(self, max_depth=3):
+    def get_subcollections(self):
         """Get the collection subcollections.
 
         This query executes a LIKE query on the path column.
@@ -196,49 +197,9 @@ def get_subcollections(self, max_depth=3):
         if not self.model:
             return None
 
-        res = self.model_cls.get_subcollections(self.model, max_depth)
+        res = self.model_cls.get_subcollections(self.model, self.max_depth)
         return [type(self)(r) for r in res]
 
-    @classmethod
-    def dump(cls, collection):
-        """Transform the collection into a dictionary."""
-        res = {
-            "title": collection.title,
-            "slug": collection.slug,
-            "depth": collection.depth,
-            "order": collection.order,
-            "id": collection.id,
-            "query": collection.query,
-            "num_records": collection.num_records,
-        }
-        return res
-
-    def to_dict(self, max_depth=2) -> dict:
-        """Return a dictionary representation of the collection.
-
-        Uses an adjacency list.
-        """
-        ret = {
-            "root": self.id,
-            self.id: {**Collection.dump(self), "children": set()},
-        }
-
-        for _c in self.get_subcollections(max_depth=max_depth):
-            # Add the collection itself to the dictionary
-            if _c.id not in ret:
-                ret[_c.id] = {**Collection.dump(_c), "children": set()}
-
-            # Find the parent ID from the collection's path (last valid ID in the path)
-            path_parts = _c.split_path_to_ids()
-            if path_parts:
-                parent_id = path_parts[-1]
-                # Add the collection as a child of its parent
-                ret[parent_id]["children"].add(_c.id)
-        for k, v in ret.items():
-            if isinstance(v, dict):
-                v["children"] = list(v["children"])
-        return ret
-
     def __repr__(self) -> str:
         """Return a string representation of the collection."""
         if self.model:
@@ -264,9 +225,10 @@ class CollectionTree:
     community = ModelField("community")
     collections = ModelField("collections")
 
-    def __init__(self, model):
+    def __init__(self, model=None, max_depth=2):
         """Instantiate a CollectionTree object."""
         self.model = model
+        self.max_depth = max_depth
 
     @classmethod
     def create(cls, title, slug, community_id=None, order=None):
@@ -292,40 +254,13 @@ def resolve(cls, id_=None, slug=None, community_id=None):
             raise CollectionTreeNotFound()
         return res
 
-    def dump(self):
-        """Transform the collection tree into a dictionary."""
-        if not self.model:
-            return {}
-
-        return {
-            "title": self.title,
-            "slug": self.slug,
-            "community_id": str(self.community_id),
-            "order": self.order,
-            "id": self.id,
-        }
-
-    def to_dict(self, max_depth=2):
-        """Return a dictionary representation of the collection tree."""
-        collections = self.root_collections
-        return {
-            **self.dump(),
-            "collections": [c.to_dict(max_depth=max_depth) for c in collections],
-        }
-
     @cached_property
-    def root_collections(self):
+    def collections(self):
         """Get the collections under this tree."""
-        return self.get_collections(max_depth=1)
-
-    def get_collections(self, max_depth=0):
-        """Get the collections under this tree, up to a depth."""
-        return [
-            Collection(c)
-            for c in CollectionTreeModel.get_collections(self.model, max_depth)
-        ]
+        root_collections = CollectionTreeModel.get_collections(self.model, 1)
+        return [Collection(c, self.max_depth) for c in root_collections]
 
     @classmethod
-    def get_community_trees(cls, community_id):
+    def get_community_trees(cls, community_id, depth=2):
         """Get all the collection trees for a community."""
-        return [cls(c) for c in cls.model_cls.get_community_trees(community_id)]
+        return [cls(c, depth) for c in cls.model_cls.get_community_trees(community_id)]
diff --git a/invenio_rdm_records/collections/config.py b/invenio_rdm_records/collections/config.py
new file mode 100644
index 000000000..835a19a36
--- /dev/null
+++ b/invenio_rdm_records/collections/config.py
@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""Collections service config."""
+
+from invenio_communities.permissions import CommunityPermissionPolicy
+from invenio_records_resources.services import ConditionalLink
+from invenio_records_resources.services.base import ServiceConfig
+from invenio_records_resources.services.base.config import ConfiguratorMixin, FromConfig
+
+from .links import CollectionLink
+from .results import CollectionItem, CollectionList
+from .schema import CollectionSchema
+
+
+class CollectionServiceConfig(ServiceConfig, ConfiguratorMixin):
+    """Collections service configuration."""
+
+    result_item_cls = CollectionItem
+    result_list_cls = CollectionList
+    service_id = "collections"
+    permission_policy_cls = FromConfig(
+        "COMMUNITIES_PERMISSION_POLICY", default=CommunityPermissionPolicy
+    )
+    schema = CollectionSchema
+
+    links_item = {
+        "search": ConditionalLink(
+            cond=lambda coll, ctx: coll.community,
+            if_=CollectionLink("/api/communities/{community}/records"),
+            else_="/api/records",
+        ),
+        "self_html": ConditionalLink(
+            cond=lambda coll, ctx: coll.community,
+            if_=CollectionLink(
+                "/communities/{community}/collections/{tree}/{collection}"
+            ),
+            else_=CollectionLink("/collections/{tree}/{collection}"),
+        ),
+    }
diff --git a/invenio_rdm_records/collections/errors.py b/invenio_rdm_records/collections/errors.py
index 5db7131c9..e14a457d9 100644
--- a/invenio_rdm_records/collections/errors.py
+++ b/invenio_rdm_records/collections/errors.py
@@ -6,17 +6,22 @@
 # it under the terms of the MIT License; see LICENSE file for more details.
 """Errors for collections module."""
 
+
 class CollectionError(Exception):
     """Base class for collection errors."""
 
+
 class CollectionNotFound(CollectionError):
     """Collection not found error."""
 
+
 class CollectionTreeNotFound(CollectionError):
     """Collection tree not found error."""
 
+
 class InvalidQuery(CollectionError):
     """Query syntax is invalid."""
 
+
 class LogoNotFoundError(CollectionError):
     """Logo not found error."""
diff --git a/invenio_rdm_records/collections/links.py b/invenio_rdm_records/collections/links.py
new file mode 100644
index 000000000..7c4097ed8
--- /dev/null
+++ b/invenio_rdm_records/collections/links.py
@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""Collection links."""
+
+from invenio_records_resources.services.base.links import Link, LinksTemplate
+
+
+class CollectionLinkstemplate(LinksTemplate):
+    """Templates for generating links for a collection object."""
+
+    def __init__(self, links=None, context=None):
+        super().__init__(links, context)
+
+
+class CollectionLink(Link):
+    """Link variables setter for Collection links."""
+
+    @staticmethod
+    def vars(collection, vars):
+        """Variables for the URI template."""
+        vars.update(
+            {
+                "community": collection.community.slug,
+                "tree": collection.collection_tree.slug,
+                "collection": collection.slug,
+            }
+        )
diff --git a/invenio_rdm_records/collections/models.py b/invenio_rdm_records/collections/models.py
index 3a6df4d93..4b0d325a2 100644
--- a/invenio_rdm_records/collections/models.py
+++ b/invenio_rdm_records/collections/models.py
@@ -6,7 +6,6 @@
 # it under the terms of the MIT License; see LICENSE file for more details.
 """Collections models."""
 
-from traitlets import default
 from invenio_communities.communities.records.models import CommunityMetadata
 from invenio_db import db
 from invenio_records.models import Timestamp
diff --git a/invenio_rdm_records/collections/results.py b/invenio_rdm_records/collections/results.py
index 58885b700..8a296810e 100644
--- a/invenio_rdm_records/collections/results.py
+++ b/invenio_rdm_records/collections/results.py
@@ -6,114 +6,127 @@
 # it under the terms of the MIT License; see LICENSE file for more details.
 """Collections service items."""
 
+from invenio_records_resources.services.base.results import (
+    ServiceItemResult,
+    ServiceListResult,
+)
 
-class CollectionItem:
+
+class CollectionItem(ServiceItemResult):
     """Collection item."""
 
-    def __init__(self, collection):
+    def __init__(self, identity, collection, schema, links_tpl):
         """Instantiate a Collection object.
 
         Optionally pass a community to cache its information in the collection's instance.
         """
+        self._identity = identity
         self._collection = collection
+        self._schema = schema
+        self._links_tpl = links_tpl
 
-    def to_dict(self, max_depth=2, include_breadcrumbs=False):
-        """Serialize the collection to a dictionary and add links."""
-        res = {**self._collection.to_dict(max_depth=max_depth)}
-
-        # Add links for all the collections
-        tree = self._collection.collection_tree
-        community = self._collection.community
-        for id_, coll in res.items():
-            if id_ == "root":
-                continue
-            res[id_]["links"] = {
-                "search": CollectionItem.search_link(community.slug),
-                "self_html": CollectionItem.html_link(
-                    community.slug, tree.slug, coll["slug"]
-                ),
-            }
-
-        # Add breadcrumbs
-        if include_breadcrumbs:
-            res[self._collection.id]["breadcrumbs"] = self.breadcrumbs
-
-        return res
+    def to_dict(self):
+        """Serialize the collection to a dictionary and add links.
 
-    @classmethod
-    def search_link(cls, community_slug):
-        """Get the search link."""
-        if community_slug:
-            return f"/api/communities/{community_slug}/records"
-        return "/api/records"
+        It will output a dictionary with the following structure:
 
-    @classmethod
-    def html_link(cls, community_slug, tree_slug, collection_slug):
-        """Get the HTML link."""
-        if community_slug:
-            return f"/communities/{community_slug}/collections/{tree_slug}/{collection_slug}"
-        return f"/collections/{tree_slug}/{collection_slug}"
+        .. code-block:: python
 
-    @property
-    def title(self):
-        """Get the collection title."""
-        return self._collection.title
-
-    @property
-    def query(self):
-        """Get the collection query."""
-        return self._collection.query
-
-    @property
-    def collection_tree(self):
-        """Get the collection tree."""
-        return self._collection.collection_tree
+            {
+                "root": 1,
+                1: {
+                    "id": 1,
+                    "title": "Root",
+                    "slug": "root",
+                    "depth": 0,
+                    "order": 1,
+                    "query": "",
+                    "num_records": 0,
+                    "children": [2],
+                    "links": {
+                        "self_html": "..."
+                        "search": "..."
+                    }
+                },
+                2: {
+                    "id": 2,
+                    "title": "Subcollection",
+                    "slug": "subcollection",
+                    "depth": 1,
+                    "order": 1,
+                    "query": "",
+                    "num_records": 0,
+                    "children": [],
+                    "links": {
+                        "self_html": "..."
+                        "search": "..."
+                    }
+                }
+        """
+        res = {
+            "root": self._collection.id,
+            self._collection.id: {
+                **self._schema.dump(self._collection),
+                "children": list(),
+                "links": self._links_tpl.expand(self._identity, self._collection),
+            },
+        }
+
+        for _c in self._collection.sub_collections:
+            if _c.id not in res:
+                # Add the subcollection to the dictionary
+                res[_c.id] = {
+                    **self._schema.dump(_c),
+                    "children": list(),
+                    "links": self._links_tpl.expand(self._identity, _c),
+                }
+            # Find the parent ID from the collection's path (last valid ID in the path)
+            path_parts = _c.split_path_to_ids()
+            if path_parts:
+                parent_id = path_parts[-1]
+                # Add the collection as a child of its parent
+                res[parent_id]["children"].append(_c.id)
 
-    @property
-    def children(self):
-        """Get the collection children."""
-        return self._collection.children
+        # Add breadcrumbs
+        res[self._collection.id]["breadcrumbs"] = self.breadcrumbs
 
-    @property
-    def community(self):
-        """Get the collection community."""
-        return self._collection.community
+        return res
 
     @property
     def breadcrumbs(self):
         """Get the collection ancestors."""
         res = []
-        community_slug = self.community.slug
-        tree_slug = self.collection_tree.slug
+        community_slug = self._collection.community.slug
+        tree_slug = self._collection.collection_tree.slug
         for anc in self._collection.ancestors:
             _a = {
                 "title": anc.title,
-                "link": CollectionItem.html_link(community_slug, tree_slug, anc.slug),
+                "link": self._links_tpl.expand(self._identity, anc)["self_html"],
             }
             res.append(_a)
         res.append(
             {
-                "title": self.title,
-                "link": CollectionItem.html_link(
-                    community_slug, tree_slug, self._collection.slug
-                ),
+                "title": self._collection.title,
+                "link": self._links_tpl.expand(self._identity, self._collection)[
+                    "self_html"
+                ],
             }
         )
         return res
 
 
-class CollectionList:
+class CollectionList(ServiceListResult):
     """Collection list item."""
 
     def __init__(self, collections):
         """Instantiate a Collection list item."""
         self._collections = collections
 
-    def to_dict(self, max_depth=2):
+    def to_dict(self):
         """Serialize the collection list to a dictionary."""
         res = []
         for collection in self._collections:
-            _r = collection.to_dict(max_depth=2)
+            _r = collection.to_dict()
             _r["links"] = CollectionItem(collection).links
             res.append(_r)
         return res
@@ -126,30 +139,54 @@ def __iter__(self):
 class CollectionTreeItem:
     """Collection tree item."""
 
-    def __init__(self, tree, max_depth=2):
+    def __init__(self, identity, tree, collection_link_tpl, collection_schema):
         """Instantiate a Collection tree object."""
+        self._identity = identity
         self._tree = tree
-        self._max_depth = max_depth
+        self._collection_link_tpl = collection_link_tpl
+        self._collection_schema = collection_schema
 
     def to_dict(self):
         """Serialize the collection tree to a dictionary."""
-        return self._tree.to_dict(max_depth=self._max_depth)
+        return {
+            "title": self._tree.title,
+            "slug": self._tree.slug,
+            "community_id": str(self._tree.community_id),
+            "order": self._tree.order,
+            "id": self._tree.id,
+            "collections": [
+                CollectionItem(
+                    self._identity,
+                    c,
+                    self._collection_schema,
+                    self._collection_link_tpl,
+                ).to_dict()
+                for c in self._tree.collections
+            ],
+        }
 
 
 class CollectionTreeList:
     """Collection tree list item."""
 
-    def __init__(self, trees, max_depth=1):
+    def __init__(self, identity, trees, collection_schema, collection_link_tpl):
         """Instantiate a Collection tree list item."""
+        self._identity = identity
         self._trees = trees
-        self._max_depth = max_depth
+        self._collection_link_tpl = collection_link_tpl
+        self._collection_schema = collection_schema
 
     def to_dict(self):
         """Serialize the collection tree list to a dictionary."""
         res = {}
         for tree in self._trees:
             # Only root collections
-            res[tree.id] = CollectionTreeItem(tree, self._max_depth).to_dict()
+            res[tree.id] = CollectionTreeItem(
+                self._identity,
+                tree,
+                self._collection_schema,
+                self._collection_link_tpl,
+            ).to_dict()
         return res
 
     def __iter__(self):
diff --git a/invenio_rdm_records/collections/schema.py b/invenio_rdm_records/collections/schema.py
new file mode 100644
index 000000000..bcddf0799
--- /dev/null
+++ b/invenio_rdm_records/collections/schema.py
@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""Collections schema."""
+
+from marshmallow import Schema, fields
+
+
+class CollectionSchema(Schema):
+    """Collection schema."""
+
+    slug = fields.Str()
+    title = fields.Str()
+    depth = fields.Int()
+    order = fields.Int()
+    id = fields.Int()
+    query = fields.Str()
+    num_records = fields.Int()
diff --git a/invenio_rdm_records/collections/service.py b/invenio_rdm_records/collections/service.py
index af95b76e4..1d5c74c19 100644
--- a/invenio_rdm_records/collections/service.py
+++ b/invenio_rdm_records/collections/service.py
@@ -9,79 +9,117 @@
 import os
 
 from flask import current_app, url_for
-from invenio_communities.proxies import current_communities
+from invenio_records_resources.services.base import Service
 from invenio_records_resources.services.uow import ModelCommitOp, unit_of_work
 
 from .api import Collection, CollectionTree
 from .errors import LogoNotFoundError
+from .links import CollectionLinkstemplate
 from .results import CollectionItem, CollectionTreeList
 
 
-class CollectionsService:
+class CollectionsService(Service):
     """Collections service."""
 
+    def __init__(self, config):
+        """Instantiate the service with the given config."""
+        self.config = config
+
     collection_cls = Collection
 
+    @property
+    def collection_schema(self):
+        """Get the collection schema."""
+        return self.config.schema()
+
+    @property
+    def links_item_tpl(self):
+        """Get the item links template."""
+        return CollectionLinkstemplate(
+            links=self.config.links_item,
+            context={
+                "permission_policy_cls": self.config.permission_policy_cls,
+            },
+        )
+
     @unit_of_work()
     def create(
         self, identity, community_id, tree_slug, slug, title, query, uow=None, **kwargs
     ):
         """Create a new collection."""
-        current_communities.service.require_permission(
-            identity, "update", community_id=community_id
-        )
+        self.require_permission(identity, "update", community_id=community_id)
         ctree = CollectionTree.resolve(slug=tree_slug, community_id=community_id)
         collection = self.collection_cls.create(
             slug=slug, title=title, query=query, ctree=ctree, **kwargs
         )
         uow.register(ModelCommitOp(collection.model))
-        return CollectionItem(collection)
+        return CollectionItem(
+            identity, collection, self.collection_schema, self.links_item_tpl
+        )
 
     def read(
-        self, identity, /, *, id_=None, slug=None, community_id=None, tree_slug=None
+        self,
+        identity,
+        /,
+        *,
+        id_=None,
+        slug=None,
+        community_id=None,
+        tree_slug=None,
+        depth=2,
+        **kwargs,
     ):
         """Get a collection by ID or slug.
 
         To resolve by slug, the collection tree ID and community ID must be provided.
         """
         if id_:
-            collection = self.collection_cls.resolve(id_)
+            collection = self.collection_cls.resolve(id_, depth=depth)
         elif slug and tree_slug and community_id:
             ctree = CollectionTree.resolve(slug=tree_slug, community_id=community_id)
-            collection = self.collection_cls.resolve(slug=slug, ctree_id=ctree.id)
+            collection = self.collection_cls.resolve(
+                slug=slug, ctree_id=ctree.id, depth=depth
+            )
         else:
             raise ValueError(
                 "ID or slug and tree_slug and community_id must be provided."
             )
 
         if collection.community:
-            current_communities.service.require_permission(
+            self.require_permission(
                 identity, "read", community_id=collection.community.id
             )
 
-        return CollectionItem(collection)
+        return CollectionItem(
+            identity,
+            collection,
+            self.collection_schema,
+            self.links_item_tpl,
+        )
 
-    def list_trees(self, identity, community_id, max_depth=2):
+    def list_trees(self, identity, community_id, **kwargs):
         """Get the trees of a community."""
-        current_communities.service.require_permission(
-            identity, "read", community_id=community_id
-        )
+        self.require_permission(identity, "read", community_id=community_id)
         if not community_id:
             raise ValueError("Community ID must be provided.")
-        res = CollectionTree.get_community_trees(community_id)
-        return CollectionTreeList(res, max_depth=max_depth)
+        res = CollectionTree.get_community_trees(community_id, **kwargs)
+        return CollectionTreeList(
+            identity, res, self.links_item_tpl, self.collection_schema
+        )
 
     @unit_of_work()
     def add(self, identity, collection, slug, title, query, uow=None, **kwargs):
         """Add a subcollection to a collection."""
-        current_communities.service.require_permission(
+        self.require_permission(
             identity, "update", community_id=collection.community.id
         )
         new_collection = self.collection_cls.create(
             parent=collection, slug=slug, title=title, query=query, **kwargs
         )
         uow.register(ModelCommitOp(new_collection.model))
-        return CollectionItem(new_collection)
+        return CollectionItem(
+            identity, new_collection, self.collection_schema, self.links_item_tpl
+        )
 
     def read_logo(self, identity, slug):
         """Read a collection logo.
diff --git a/invenio_rdm_records/ext.py b/invenio_rdm_records/ext.py
index 470c9ef42..f96743995 100644
--- a/invenio_rdm_records/ext.py
+++ b/invenio_rdm_records/ext.py
@@ -19,6 +19,7 @@
 from invenio_records_resources.resources.files import FileResource
 
 from . import config
+from .collections.config import CollectionServiceConfig
 from .collections.service import CollectionsService
 from .oaiserver.resources.config import OAIPMHServerResourceConfig
 from .oaiserver.resources.resources import OAIPMHServerResource
@@ -159,6 +160,7 @@ class ServiceConfigs:
             record_communities = RDMRecordCommunitiesConfig.build(app)
             community_records = RDMCommunityRecordsConfig.build(app)
             record_requests = RDMRecordRequestsConfig.build(app)
+            collections = CollectionServiceConfig.build(app)
 
         return ServiceConfigs
 
@@ -204,7 +206,9 @@ def init_services(self, app):
             config=service_configs.oaipmh_server,
         )
 
-        self.collections_service = CollectionsService()
+        self.collections_service = CollectionsService(
+            config=service_configs.collections
+        )
 
     def init_resource(self, app):
         """Initialize resources."""
diff --git a/tests/collections/test_collections_api.py b/tests/collections/test_collections_api.py
index f87ea3992..e3f172b35 100644
--- a/tests/collections/test_collections_api.py
+++ b/tests/collections/test_collections_api.py
@@ -18,7 +18,6 @@ def test_create(running_app, db, community, community_owner):
         slug="tree-1",
     )
 
-    # Use ORM object (collection tree)
     collection = Collection.create(
         title="My Collection",
         query="*:*",
@@ -45,46 +44,33 @@ def test_create(running_app, db, community, community_owner):
     assert collection.collection_tree.id == tree.id
 
 
-def test_as_dict(running_app, db):
-    """Test collection as dict."""
+def test_resolve(running_app, db, community):
+    """Test collection resolution."""
     tree = CollectionTree.create(
         title="Tree 1",
         order=10,
+        community_id=community.id,
         slug="tree-1",
     )
-    c1 = Collection.create(
+
+    collection = Collection.create(
         title="My Collection",
         query="*:*",
         slug="my-collection",
         ctree=tree,
     )
 
-    c2 = Collection.create(
-        title="My Collection 2",
-        query="*:*",
-        slug="my-collection-2",
-        parent=c1,
-    )
-
-    c3 = Collection.create(title="3", query="*", slug="my-collection-3", parent=c2)
-    res = c1.to_dict(max_depth=3)
-    assert all(k in res for k in (c1.id, c2.id, c3.id))
-    assert res[c1.id]["title"] == "My Collection"
-    assert res[c1.id]["children"] == [c2.id]
-    assert res[c2.id]["children"] == [c3.id]
-    assert res[c3.id]["children"] == []
+    # Read by ID
+    read_by_id = Collection.resolve(collection.id)
+    assert read_by_id.id == collection.id
 
-    res = c1.to_dict(max_depth=2)
-    assert all(k in res for k in (c1.id, c2.id))
-    assert res[c1.id]["title"] == "My Collection"
-    assert res[c1.id]["children"] == [c2.id]
-    assert res[c2.id]["children"] == []
-    assert c3.id not in res
+    # Read by slug
+    read_by_slug = Collection.resolve(slug="my-collection", ctree_id=tree.id)
+    assert read_by_slug.id == read_by_id.id == collection.id
 
 
 def test_query_build(running_app, db):
     """Test query building."""
-
     tree = CollectionTree.create(
         title="Tree 1",
         order=10,
diff --git a/tests/services/test_collections_service.py b/tests/services/test_collections_service.py
index ab5baee56..8f7888a0d 100644
--- a/tests/services/test_collections_service.py
+++ b/tests/services/test_collections_service.py
@@ -4,8 +4,9 @@
 #
 # Invenio-RDM is free software; you can redistribute it and/or modify
 # it under the terms of the MIT License; see LICENSE file for more details.
-""" TODO """
+"""Test suite for the collections service."""
 
+import dictdiffer
 import pytest
 
 from invenio_rdm_records.collections.api import Collection, CollectionTree
@@ -31,11 +32,14 @@ def _inner():
             slug="tree-1",
         )
         c1 = Collection.create(
-            title="Collection 1", query="*:*", slug="collection-1", ctree=tree
+            title="Collection 1",
+            query="metadata.title:foo",
+            slug="collection-1",
+            ctree=tree,
         )
         c2 = Collection.create(
             title="Collection 2",
-            query="*:*",
+            query="metadata.title:bar",
             slug="collection-2",
             ctree=tree,
             parent=c1,
@@ -45,12 +49,205 @@ def _inner():
     return _inner
 
 
-def test_collections_service_read(
-    running_app, db, add_collections, collections_service, community_owner
+def test_collections_read(
+    running_app, db, add_collections, collections_service, community, community_owner
 ):
     """Test collections service."""
     collections = add_collections()
     c0 = collections[0]
     c1 = collections[1]
+
+    # Read by id
     res = collections_service.read(community_owner.identity, id_=c0.id)
     assert res._collection.id == c0.id
+
+    # Read by slug
+    res = collections_service.read(
+        community_owner.identity,
+        community_id=community.id,
+        tree_slug=c0.collection_tree.slug,
+        slug=c0.slug,
+    )
+    assert res._collection.id == c0.id
+
+
+def test_collections_create(
+    running_app, db, collections_service, community, community_owner
+):
+    """Test collection creation via service."""
+    tree = CollectionTree.create(
+        title="Tree 1",
+        order=10,
+        community_id=community.id,
+        slug="tree-1",
+    )
+    collection = collections_service.create(
+        community_owner.identity,
+        community.id,
+        tree_slug="tree-1",
+        slug="my-collection",
+        title="My Collection",
+        query="*:*",
+    )
+
+    # Get the API object, just for the sake of testing
+    collection = collection._collection
+
+    assert collection.title == "My Collection"
+    assert collection.collection_tree.id == tree.id
+
+    read_collection = collections_service.read(
+        community_owner.identity, id_=collection.id
+    )
+    assert read_collection._collection.id == collection.id
+    assert read_collection._collection.title == "My Collection"
+
+
+def test_collections_add(
+    running_app, db, collections_service, add_collections, community_owner
+):
+    """Test adding a collection to another via service."""
+    collections = add_collections()
+    c1 = collections[0]
+    c2 = collections[1]
+
+    c3 = collections_service.add(
+        community_owner.identity,
+        collection=c2,
+        slug="collection-3",
+        title="Collection 3",
+        query="metadata.title:baz",
+    )
+
+    # Get the API object, just for the sake of testing
+    c3 = c3._collection
+
+    # Read the collection
+    res = collections_service.read(community_owner.identity, id_=c3.id)
+    assert res._collection.id == c3.id
+    assert res._collection.title == "Collection 3"
+
+    # Read the parent collection
+    res = collections_service.read(community_owner.identity, id_=c2.id)
+    assert res.to_dict()[c2.id]["children"] == [c3.id]
+
+
+def test_collections_results(
+    running_app, db, add_collections, collections_service, community_owner
+):
+    """Test collection results.
+
+    The goal is to test the format returned by the service, based on the required depth.
+    """
+    collections = add_collections()
+    c0 = collections[0]
+    c1 = collections[1]
+    c3 = collections_service.add(
+        community_owner.identity,
+        c1,
+        slug="collection-3",
+        title="Collection 3",
+        query="metadata.title:baz",
+    )
+    # Read the collection tree up to depth 2
+    res = collections_service.read(community_owner.identity, id_=c0.id, depth=2)
+    r_dict = res.to_dict()
+
+    expected = {
+        "root": c0.id,
+        c0.id: {
+            "breadcrumbs": [
+                {
+                    "link": "/communities/blr/collections/tree-1/collection-1",
+                    "title": "Collection 1",
+                }
+            ],
+            "children": [c1.id],
+            "depth": 0,
+            "id": c0.id,
+            "links": {
+                "search": "/api/communities/blr/records",
+                "self_html": "/communities/blr/collections/tree-1/collection-1",
+            },
+            "num_records": 0,
+            "order": c0.order,
+            "query": "(metadata.title:foo)",
+            "slug": "collection-1",
+            "title": "Collection 1",
+        },
+        c1.id: {
+            "children": [],
+            "depth": 1,
+            "id": c1.id,
+            "links": {
+                "search": "/api/communities/blr/records",
+                "self_html": "/communities/blr/collections/tree-1/collection-2",
+            },
+            "num_records": 0,
+            "order": c1.order,
+            "query": "(metadata.title:foo) AND (metadata.title:bar)",
+            "slug": "collection-2",
+            "title": "Collection 2",
+        },
+    }
+    assert not list(dictdiffer.diff(expected, r_dict))
+
+    # Read the collection tree up to depth 3
+    res = collections_service.read(community_owner.identity, id_=c0.id, depth=3)
+    r_dict = res.to_dict()
+
+    # Get the API object, just for the sake of testing
+    c3 = c3._collection
+    expected = {
+        "root": c0.id,
+        c0.id: {
+            "breadcrumbs": [
+                {
+                    "link": "/communities/blr/collections/tree-1/collection-1",
+                    "title": "Collection 1",
+                }
+            ],
+            "children": [c1.id],
+            "depth": 0,
+            "id": c0.id,
+            "links": {
+                "search": "/api/communities/blr/records",
+                "self_html": "/communities/blr/collections/tree-1/collection-1",
+            },
+            "num_records": 0,
+            "order": c0.order,
+            "query": "(metadata.title:foo)",
+            "slug": "collection-1",
+            "title": "Collection 1",
+        },
+        c1.id: {
+            "children": [c3.id],
+            "depth": 1,
+            "id": c1.id,
+            "links": {
+                "search": "/api/communities/blr/records",
+                "self_html": "/communities/blr/collections/tree-1/collection-2",
+            },
+            "num_records": 0,
+            "order": c1.order,
+            "query": "(metadata.title:foo) AND (metadata.title:bar)",
+            "slug": "collection-2",
+            "title": "Collection 2",
+        },
+        c3.id: {
+            "children": [],
+            "depth": 2,
+            "id": c3.id,
+            "links": {
+                "search": "/api/communities/blr/records",
+                "self_html": "/communities/blr/collections/tree-1/collection-3",
+            },
+            "num_records": 0,
+            "order": c3.order,
+            "query": "(metadata.title:foo) AND (metadata.title:bar) AND (metadata.title:baz)",
+            "slug": "collection-3",
+            "title": "Collection 3",
+        },
+    }
+
+    assert not list(dictdiffer.diff(expected, r_dict))

From 38b8bd0df88b501f1b09b07219fd3ab9aa532d7c Mon Sep 17 00:00:00 2001
From: alejandromumo <manuel.alejandro.de.oliveira.da.costa@cern.ch>
Date: Tue, 15 Oct 2024 10:26:37 +0200
Subject: [PATCH 50/76] collections: use ModelField from invenio-records

* collections: make model queries lazy
---
 invenio_rdm_records/collections/api.py    | 44 +++++------------------
 invenio_rdm_records/collections/models.py | 22 +++---------
 2 files changed, 14 insertions(+), 52 deletions(-)

diff --git a/invenio_rdm_records/collections/api.py b/invenio_rdm_records/collections/api.py
index 5bc7d651e..5080acbbb 100644
--- a/invenio_rdm_records/collections/api.py
+++ b/invenio_rdm_records/collections/api.py
@@ -5,6 +5,7 @@
 # Invenio-RDM is free software; you can redistribute it and/or modify
 # it under the terms of the MIT License; see LICENSE file for more details.
 """Collections programmatic API."""
+from invenio_records.systemfields import ModelField
 from luqum.parser import parser as luqum_parser
 from werkzeug.utils import cached_property
 
@@ -13,47 +14,20 @@
 from .models import CollectionTree as CollectionTreeModel
 
 
-class ModelField:
-    """Model Field Descriptor."""
-
-    def __init__(self, attr_name):
-        """Initialize the descriptor."""
-        self._attr_name = attr_name
-
-    @property
-    def attr_name(self):
-        """The name of the SQLAlchemy field on the model.
-
-        Defaults to the attribute name used on the class.
-        """
-        return self._attr_name
-
-    def __get__(self, obj, objtype=None):
-        """Descriptor method to get the object."""
-        if obj is None:
-            return self
-
-        # Try instance access
-        try:
-            return getattr(obj.model, self.attr_name)
-        except AttributeError:
-            return None
-
-
 class Collection:
     """Collection Object."""
 
     model_cls = CollectionModel
 
-    id = ModelField("id")
-    path = ModelField("path")
+    id = ModelField()
+    path = ModelField()
     ctree_id = ModelField("collection_tree_id")
-    order = ModelField("order")
-    title = ModelField("title")
-    slug = ModelField("slug")
-    depth = ModelField("depth")
-    search_query = ModelField("search_query")
-    num_records = ModelField("num_records")
+    order = ModelField()
+    title = ModelField()
+    slug = ModelField()
+    depth = ModelField()
+    search_query = ModelField()
+    num_records = ModelField()
 
     def __init__(self, model=None, max_depth=2):
         """Instantiate a Collection object."""
diff --git a/invenio_rdm_records/collections/models.py b/invenio_rdm_records/collections/models.py
index 4b0d325a2..740242b1b 100644
--- a/invenio_rdm_records/collections/models.py
+++ b/invenio_rdm_records/collections/models.py
@@ -68,20 +68,15 @@ def get_by_slug(cls, slug, community_id):
     @classmethod
     def get_community_trees(cls, community_id):
         """Get all collection trees of a community."""
-        return (
-            cls.query.filter(cls.community_id == community_id).order_by(cls.order).all()
-        )
+        return cls.query.filter(cls.community_id == community_id).order_by(cls.order)
 
     @classmethod
     def get_collections(cls, model, max_depth):
         """Get collections under a tree."""
-        stmt = Collection.query.filter(
+        return Collection.query.filter(
             Collection.tree_id == model.id, Collection.depth < max_depth
         ).order_by(Collection.path, Collection.order)
 
-        # Execute the query and get the result
-        return stmt.all()
-
 
 # Collection Table
 class Collection(db.Model, Timestamp):
@@ -163,19 +158,15 @@ def get_by_slug(cls, slug, tree_id):
     @classmethod
     def read_many(cls, ids):
         """Get many collections by ID."""
-        return cls.query.filter(cls.id.in_(ids)).order_by(cls.path, cls.order).all()
+        return cls.query.filter(cls.id.in_(ids)).order_by(cls.path, cls.order)
 
     @classmethod
     def get_children(cls, model):
         """Get children collections of a collection."""
-        # Assuming self.model_cls is the ORM model
-        stmt = cls.query.filter(
+        return cls.query.filter(
             cls.path == f"{model.path}{model.id},", cls.tree_id == model.tree_id
         ).order_by(cls.path, cls.order)
 
-        # Execute the query and get the result
-        return stmt.all()
-
     @classmethod
     def get_subcollections(cls, model, max_depth):
         """Get subcollections of a collection.
@@ -183,11 +174,8 @@ def get_subcollections(cls, model, max_depth):
         This query will return all subcollections of a collection up to a certain depth.
         It can be used for max_depth=1, however it is more efficient to use get_children.
         """
-        stmt = cls.query.filter(
+        return cls.query.filter(
             cls.path.like(f"{model.path}{model.id},%"),
             cls.tree_id == model.tree_id,
             cls.depth < model.depth + max_depth,
         ).order_by(cls.path, cls.order)
-
-        # Execute the query and get the result
-        return stmt.all()

From 8ef05a121a4087f06cd3e2d4364bbad8f74a962f Mon Sep 17 00:00:00 2001
From: alejandromumo <manuel.alejandro.de.oliveira.da.costa@cern.ch>
Date: Tue, 15 Oct 2024 11:07:48 +0200
Subject: [PATCH 51/76] collections: use dsl.Q to build queries

---
 invenio_rdm_records/collections/api.py     | 6 ++++--
 invenio_rdm_records/collections/results.py | 6 ++----
 invenio_rdm_records/collections/service.py | 2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/invenio_rdm_records/collections/api.py b/invenio_rdm_records/collections/api.py
index 5080acbbb..7ac66f2a8 100644
--- a/invenio_rdm_records/collections/api.py
+++ b/invenio_rdm_records/collections/api.py
@@ -69,7 +69,7 @@ def create(cls, slug, title, query, ctree=None, parent=None, order=None, depth=2
         )
 
     @classmethod
-    def resolve(cls, id_=None, slug=None, ctree_id=None, depth=2):
+    def resolve(cls, *, id_=None, slug=None, ctree_id=None, depth=2):
         """Resolve a collection by ID or slug.
 
         To resolve by slug, the collection tree ID must be provided.
@@ -120,6 +120,8 @@ def query(self):
         for _a in self.ancestors:
             q += f"({_a.model.search_query}) AND "
         q += f"({self.model.search_query})"
+
+        # Query must be validated because it is not being built using dsl
         Collection.validate_query(q)
         return q
 
@@ -129,7 +131,7 @@ def ancestors(self):
         return Collection.resolve_many(self.split_path_to_ids())
 
     @cached_property
-    def sub_collections(self):
+    def subcollections(self):
         """Fetch descendants.
 
         If the max_depth is 1, fetch only direct descendants.
diff --git a/invenio_rdm_records/collections/results.py b/invenio_rdm_records/collections/results.py
index 8a296810e..e6bc6a80b 100644
--- a/invenio_rdm_records/collections/results.py
+++ b/invenio_rdm_records/collections/results.py
@@ -72,7 +72,7 @@ def to_dict(self):
             },
         }
 
-        for _c in self._collection.sub_collections:
+        for _c in self._collection.subcollections:
             if _c.id not in res:
                 # Add the subcollection to the dictionary
                 res[_c.id] = {
@@ -87,7 +87,7 @@ def to_dict(self):
                 # Add the collection as a child of its parent
                 res[parent_id]["children"].append(_c.id)
 
-        # Add breadcrumbs
+        # Add breadcrumbs, sorted from root to leaf and taking into account the `order` field
         res[self._collection.id]["breadcrumbs"] = self.breadcrumbs
 
         return res
@@ -96,8 +96,6 @@ def to_dict(self):
     def breadcrumbs(self):
         """Get the collection ancestors."""
         res = []
-        community_slug = self._collection.community.slug
-        tree_slug = self._collection.collection_tree.slug
         for anc in self._collection.ancestors:
             _a = {
                 "title": anc.title,
diff --git a/invenio_rdm_records/collections/service.py b/invenio_rdm_records/collections/service.py
index 1d5c74c19..5662b4906 100644
--- a/invenio_rdm_records/collections/service.py
+++ b/invenio_rdm_records/collections/service.py
@@ -59,9 +59,9 @@ def create(
 
     def read(
         self,
-        identity,
         /,
         *,
+        identity=None,
         id_=None,
         slug=None,
         community_id=None,

From 3e749537197666dd79e1ef30cc4d8e9b175ea2e7 Mon Sep 17 00:00:00 2001
From: alejandromumo <manuel.alejandro.de.oliveira.da.costa@cern.ch>
Date: Tue, 15 Oct 2024 15:53:59 +0200
Subject: [PATCH 52/76] collections: added collection id to record search

* collections: remove query from serialization.
---
 invenio_rdm_records/collections/api.py        | 16 ++++----
 invenio_rdm_records/collections/links.py      |  1 +
 invenio_rdm_records/collections/results.py    |  5 +++
 invenio_rdm_records/collections/schema.py     |  1 -
 invenio_rdm_records/collections/service.py    |  2 +-
 invenio_rdm_records/resources/args.py         |  6 +++
 invenio_rdm_records/resources/config.py       |  4 +-
 .../services/community_records/service.py     | 10 ++++-
 tests/collections/test_collections_api.py     | 10 +++--
 tests/services/test_collections_service.py    | 23 ++++++-----
 .../test_service_community_records.py         | 38 +++++++++++++++++++
 11 files changed, 88 insertions(+), 28 deletions(-)

diff --git a/invenio_rdm_records/collections/api.py b/invenio_rdm_records/collections/api.py
index 7ac66f2a8..fc2bc3753 100644
--- a/invenio_rdm_records/collections/api.py
+++ b/invenio_rdm_records/collections/api.py
@@ -116,14 +116,14 @@ def community(self):
     @property
     def query(self):
         """Get the collection query."""
-        q = ""
-        for _a in self.ancestors:
-            q += f"({_a.model.search_query}) AND "
-        q += f"({self.model.search_query})"
-
-        # Query must be validated because it is not being built using dsl
-        Collection.validate_query(q)
-        return q
+        import operator
+        from functools import reduce
+
+        from invenio_search.engine import dsl
+
+        queries = [dsl.Q("query_string", query=a.search_query) for a in self.ancestors]
+        queries.append(dsl.Q("query_string", query=self.search_query))
+        return reduce(operator.and_, queries)
 
     @cached_property
     def ancestors(self):
diff --git a/invenio_rdm_records/collections/links.py b/invenio_rdm_records/collections/links.py
index 7c4097ed8..ecff01f29 100644
--- a/invenio_rdm_records/collections/links.py
+++ b/invenio_rdm_records/collections/links.py
@@ -13,6 +13,7 @@ class CollectionLinkstemplate(LinksTemplate):
     """Templates for generating links for a collection object."""
 
     def __init__(self, links=None, context=None):
+        """Initialize the links template."""
         super().__init__(links, context)
 
 
diff --git a/invenio_rdm_records/collections/results.py b/invenio_rdm_records/collections/results.py
index e6bc6a80b..d0acb50af 100644
--- a/invenio_rdm_records/collections/results.py
+++ b/invenio_rdm_records/collections/results.py
@@ -112,6 +112,11 @@ def breadcrumbs(self):
         )
         return res
 
+    @property
+    def query(self):
+        """Get the collection query."""
+        return self._collection.query
+
 
 class CollectionList(ServiceListResult):
     """Collection list item."""
diff --git a/invenio_rdm_records/collections/schema.py b/invenio_rdm_records/collections/schema.py
index bcddf0799..f1241c778 100644
--- a/invenio_rdm_records/collections/schema.py
+++ b/invenio_rdm_records/collections/schema.py
@@ -17,5 +17,4 @@ class CollectionSchema(Schema):
     depth = fields.Int()
     order = fields.Int()
     id = fields.Int()
-    query = fields.Str()
     num_records = fields.Int()
diff --git a/invenio_rdm_records/collections/service.py b/invenio_rdm_records/collections/service.py
index 5662b4906..a526200b7 100644
--- a/invenio_rdm_records/collections/service.py
+++ b/invenio_rdm_records/collections/service.py
@@ -74,7 +74,7 @@ def read(
         To resolve by slug, the collection tree ID and community ID must be provided.
         """
         if id_:
-            collection = self.collection_cls.resolve(id_, depth=depth)
+            collection = self.collection_cls.resolve(id_=id_, depth=depth)
         elif slug and tree_slug and community_id:
             ctree = CollectionTree.resolve(slug=tree_slug, community_id=community_id)
             collection = self.collection_cls.resolve(
diff --git a/invenio_rdm_records/resources/args.py b/invenio_rdm_records/resources/args.py
index 90531a126..1d9e73fbf 100644
--- a/invenio_rdm_records/resources/args.py
+++ b/invenio_rdm_records/resources/args.py
@@ -19,3 +19,9 @@ class RDMSearchRequestArgsSchema(SearchRequestArgsSchema):
     locale = fields.Str()
     status = fields.Str()
     include_deleted = fields.Bool()
+
+
+class CommunityRecordsSearchRequestArgsSchema(SearchRequestArgsSchema):
+    """Extend schema with collection_id field."""
+
+    collection_id = fields.Integer()
diff --git a/invenio_rdm_records/resources/config.py b/invenio_rdm_records/resources/config.py
index 192ab61ad..522372ee5 100644
--- a/invenio_rdm_records/resources/config.py
+++ b/invenio_rdm_records/resources/config.py
@@ -46,7 +46,7 @@
     ReviewStateError,
     ValidationErrorWithMessageAsList,
 )
-from .args import RDMSearchRequestArgsSchema
+from .args import CommunityRecordsSearchRequestArgsSchema, RDMSearchRequestArgsSchema
 from .deserializers import ROCrateJSONDeserializer
 from .deserializers.errors import DeserializerError
 from .errors import HTTPJSONException, HTTPJSONValidationWithMessageAsListException
@@ -558,6 +558,8 @@ class RDMCommunityRecordsResourceConfig(RecordResourceConfig, ConfiguratorMixin)
         default=record_serializers,
     )
 
+    request_search_args = CommunityRecordsSearchRequestArgsSchema
+
 
 class RDMRecordCommunitiesResourceConfig(CommunityResourceConfig, ConfiguratorMixin):
     """Record communities resource config."""
diff --git a/invenio_rdm_records/services/community_records/service.py b/invenio_rdm_records/services/community_records/service.py
index 8ed6c4c85..04be70ab3 100644
--- a/invenio_rdm_records/services/community_records/service.py
+++ b/invenio_rdm_records/services/community_records/service.py
@@ -18,7 +18,7 @@
 from invenio_records_resources.services.uow import unit_of_work
 from invenio_search.engine import dsl
 
-from ...proxies import current_record_communities_service
+from ...proxies import current_rdm_records, current_record_communities_service
 from ...records.systemfields.deletion_status import RecordDeletionStatusEnum
 
 
@@ -67,6 +67,14 @@ def search(
         if extra_filter is not None:
             community_filter = community_filter & extra_filter
 
+        # Search in a specific collection
+        if "collection_id" in params:
+            collections_service = current_rdm_records.collections_service
+            collection = collections_service.read(
+                identity=identity, id_=params["collection_id"]
+            )
+            community_filter &= collection.query
+
         search = self._search(
             "search",
             identity,
diff --git a/tests/collections/test_collections_api.py b/tests/collections/test_collections_api.py
index e3f172b35..eff421b3b 100644
--- a/tests/collections/test_collections_api.py
+++ b/tests/collections/test_collections_api.py
@@ -6,6 +6,8 @@
 # it under the terms of the MIT License; see LICENSE file for more details.
 """Test suite for the collections programmatic API."""
 
+from invenio_search.engine import dsl
+
 from invenio_rdm_records.collections.api import Collection, CollectionTree
 
 
@@ -25,7 +27,7 @@ def test_create(running_app, db, community, community_owner):
         ctree=tree,
     )
 
-    read_c = Collection.resolve(collection.id)
+    read_c = Collection.resolve(id_=collection.id)
     assert read_c.id == collection.id
     assert read_c.title == "My Collection"
     assert read_c.collection_tree.id == tree.id
@@ -38,7 +40,7 @@ def test_create(running_app, db, community, community_owner):
         ctree=tree.id,
     )
 
-    read_c = Collection.resolve(collection.id)
+    read_c = Collection.resolve(id_=collection.id)
     assert read_c.id == collection.id
     assert collection.title == "My Collection 2"
     assert collection.collection_tree.id == tree.id
@@ -61,7 +63,7 @@ def test_resolve(running_app, db, community):
     )
 
     # Read by ID
-    read_by_id = Collection.resolve(collection.id)
+    read_by_id = Collection.resolve(id_=collection.id)
     assert read_by_id.id == collection.id
 
     # Read by slug
@@ -88,7 +90,7 @@ def test_query_build(running_app, db):
         slug="my-collection-2",
         parent=c1,
     )
-    assert c2.query == "(metadata.title:hello) AND (metadata.creators.name:john)"
+    assert c2.query == c1.query & dsl.Q("query_string", query=c2.search_query)
 
 
 def test_children(running_app, db):
diff --git a/tests/services/test_collections_service.py b/tests/services/test_collections_service.py
index 8f7888a0d..428617b6e 100644
--- a/tests/services/test_collections_service.py
+++ b/tests/services/test_collections_service.py
@@ -58,12 +58,12 @@ def test_collections_read(
     c1 = collections[1]
 
     # Read by id
-    res = collections_service.read(community_owner.identity, id_=c0.id)
+    res = collections_service.read(identity=community_owner.identity, id_=c0.id)
     assert res._collection.id == c0.id
 
     # Read by slug
     res = collections_service.read(
-        community_owner.identity,
+        identity=community_owner.identity,
         community_id=community.id,
         tree_slug=c0.collection_tree.slug,
         slug=c0.slug,
@@ -97,7 +97,7 @@ def test_collections_create(
     assert collection.collection_tree.id == tree.id
 
     read_collection = collections_service.read(
-        community_owner.identity, id_=collection.id
+        identity=community_owner.identity, id_=collection.id
     )
     assert read_collection._collection.id == collection.id
     assert read_collection._collection.title == "My Collection"
@@ -123,12 +123,12 @@ def test_collections_add(
     c3 = c3._collection
 
     # Read the collection
-    res = collections_service.read(community_owner.identity, id_=c3.id)
+    res = collections_service.read(identity=community_owner.identity, id_=c3.id)
     assert res._collection.id == c3.id
     assert res._collection.title == "Collection 3"
 
     # Read the parent collection
-    res = collections_service.read(community_owner.identity, id_=c2.id)
+    res = collections_service.read(identity=community_owner.identity, id_=c2.id)
     assert res.to_dict()[c2.id]["children"] == [c3.id]
 
 
@@ -150,7 +150,9 @@ def test_collections_results(
         query="metadata.title:baz",
     )
     # Read the collection tree up to depth 2
-    res = collections_service.read(community_owner.identity, id_=c0.id, depth=2)
+    res = collections_service.read(
+        identity=community_owner.identity, id_=c0.id, depth=2
+    )
     r_dict = res.to_dict()
 
     expected = {
@@ -171,7 +173,6 @@ def test_collections_results(
             },
             "num_records": 0,
             "order": c0.order,
-            "query": "(metadata.title:foo)",
             "slug": "collection-1",
             "title": "Collection 1",
         },
@@ -185,7 +186,6 @@ def test_collections_results(
             },
             "num_records": 0,
             "order": c1.order,
-            "query": "(metadata.title:foo) AND (metadata.title:bar)",
             "slug": "collection-2",
             "title": "Collection 2",
         },
@@ -193,7 +193,9 @@ def test_collections_results(
     assert not list(dictdiffer.diff(expected, r_dict))
 
     # Read the collection tree up to depth 3
-    res = collections_service.read(community_owner.identity, id_=c0.id, depth=3)
+    res = collections_service.read(
+        identity=community_owner.identity, id_=c0.id, depth=3
+    )
     r_dict = res.to_dict()
 
     # Get the API object, just for the sake of testing
@@ -216,7 +218,6 @@ def test_collections_results(
             },
             "num_records": 0,
             "order": c0.order,
-            "query": "(metadata.title:foo)",
             "slug": "collection-1",
             "title": "Collection 1",
         },
@@ -230,7 +231,6 @@ def test_collections_results(
             },
             "num_records": 0,
             "order": c1.order,
-            "query": "(metadata.title:foo) AND (metadata.title:bar)",
             "slug": "collection-2",
             "title": "Collection 2",
         },
@@ -244,7 +244,6 @@ def test_collections_results(
             },
             "num_records": 0,
             "order": c3.order,
-            "query": "(metadata.title:foo) AND (metadata.title:bar) AND (metadata.title:baz)",
             "slug": "collection-3",
             "title": "Collection 3",
         },
diff --git a/tests/services/test_service_community_records.py b/tests/services/test_service_community_records.py
index 9a8695a7d..06c2db7b0 100644
--- a/tests/services/test_service_community_records.py
+++ b/tests/services/test_service_community_records.py
@@ -7,10 +7,13 @@
 
 """Test community records service."""
 
+from copy import deepcopy
+
 import pytest
 from invenio_records_resources.services.errors import PermissionDeniedError
 from marshmallow import ValidationError
 
+from invenio_rdm_records.collections.api import Collection, CollectionTree
 from invenio_rdm_records.proxies import (
     current_community_records_service,
     current_rdm_records_service,
@@ -156,3 +159,38 @@ def test_search_community_records(
             community_id=str(community.id),
         )
         assert results.to_dict()["hits"]["total"] == 3
+
+
+def test_search_community_records_in_collections(
+    community, record_community, service, uploader, minimal_record
+):
+    """Test search for records in a community collection."""
+    rec1 = deepcopy(minimal_record)
+    rec1["metadata"]["title"] = "Another record"
+    record_community.create_record(record_dict=rec1)
+    record_community.create_record(record_dict=minimal_record)
+    ctree = CollectionTree.create(
+        title="Tree 1",
+        order=10,
+        community_id=community.id,
+        slug="tree-1",
+    )
+    collection = Collection.create(
+        title="My Collection",
+        query='metadata.title:"Another record"',
+        slug="my-collection",
+        ctree=ctree,
+    )
+    all_results = service.search(
+        uploader.identity,
+        community_id=str(community.id),
+    )
+    assert all_results.total == 2
+
+    results = service.search(
+        uploader.identity,
+        community_id=str(community.id),
+        params={"collection_id": str(collection.id)},
+    )
+    assert results.total == 1
+    assert results.to_dict()["hits"]["hits"][0]["metadata"]["title"] == "Another record"

From a7e636d44ad1ba3b0b08f55564f32cfdbe9f412b Mon Sep 17 00:00:00 2001
From: Alex Ioannidis <a.ioannidis@cern.ch>
Date: Wed, 16 Oct 2024 13:28:41 +0200
Subject: [PATCH 53/76] =?UTF-8?q?=F0=9F=93=A6=20release:=20v15.3.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 CHANGES.rst                                      | 7 +++++++
 invenio_rdm_records/__init__.py                  | 2 +-
 tests/services/test_service_community_records.py | 2 +-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index dedb03d1f..166b26690 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -12,6 +12,13 @@
 Changes
 =======
 
+Version v15.3.0 (released 2024-10-16)
+
+- collections: display pages and REST API
+- deposit: add feature flag for required community submission flow
+- mappings: disable doc_values for geo_shape fields (#1807)
+    * Fixes multiple values for ``metadata.locaations.features``.
+
 Version v15.2.0 (released 2024-10-10)
 
 - webpack: update axios and react-searchkit(due to axios) major versions
diff --git a/invenio_rdm_records/__init__.py b/invenio_rdm_records/__init__.py
index 5cafefe69..5999b7031 100644
--- a/invenio_rdm_records/__init__.py
+++ b/invenio_rdm_records/__init__.py
@@ -11,6 +11,6 @@
 
 from .ext import InvenioRDMRecords
 
-__version__ = "15.2.0"
+__version__ = "15.3.0"
 
 __all__ = ("__version__", "InvenioRDMRecords")
diff --git a/tests/services/test_service_community_records.py b/tests/services/test_service_community_records.py
index 06c2db7b0..9c48c40ae 100644
--- a/tests/services/test_service_community_records.py
+++ b/tests/services/test_service_community_records.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2023 CERN.
+# Copyright (C) 2023-2024 CERN.
 #
 # Invenio-RDM-Records is free software; you can redistribute it and/or modify
 # it under the terms of the MIT License; see LICENSE file for more details.

From e28449b0b5e2f9d1ddfe92331a072c1e4d5ab180 Mon Sep 17 00:00:00 2001
From: Fatimah Zulfiqar <fatimah0zulfiqar@gmail.com>
Date: Wed, 16 Oct 2024 15:32:26 +0200
Subject: [PATCH 54/76] community: added props to make CommunitySelectionSearch
 reusable

---
 .../CommunitySelectionSearch.js               | 19 +++++++++++++++----
 .../CommunitySelectionModal/index.js          |  2 ++
 .../src/deposit/components/index.js           |  1 +
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/CommunitySelectionSearch.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/CommunitySelectionSearch.js
index 93797c2e6..9ff403ebd 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/CommunitySelectionSearch.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/CommunitySelectionSearch.js
@@ -43,11 +43,15 @@ export class CommunitySelectionSearch extends Component {
         toggleText,
       },
     } = this.state;
+
     const {
       apiConfigs: { allCommunities, myCommunities },
       record,
       isInitialSubmission,
+      CommunityListItem,
+      pagination,
     } = this.props;
+
     const searchApi = new InvenioSearchApi(selectedSearchApi);
     const overriddenComponents = {
       [`${selectedAppId}.ResultsList.item`]: parametrize(CommunityListItem, {
@@ -55,6 +59,7 @@ export class CommunitySelectionSearch extends Component {
         isInitialSubmission: isInitialSubmission,
       }),
     };
+
     return (
       <OverridableContext.Provider value={overriddenComponents}>
         <ReactSearchKit
@@ -75,7 +80,7 @@ export class CommunitySelectionSearch extends Component {
                 floated="left"
                 className="pt-0 pl-0"
               >
-                <Menu role="tablist" compact>
+                <Menu role="tablist" className="theme-primary-menu" compact>
                   <Menu.Item
                     as="button"
                     role="tab"
@@ -144,9 +149,11 @@ export class CommunitySelectionSearch extends Component {
               </ResultsLoader>
             </Modal.Content>
 
-            <Modal.Content className="text-align-center">
-              <Pagination />
-            </Modal.Content>
+            {pagination && (
+              <Modal.Content className="text-align-center">
+                <Pagination />
+              </Modal.Content>
+            )}
           </>
         </ReactSearchKit>
       </OverridableContext.Provider>
@@ -169,10 +176,14 @@ CommunitySelectionSearch.propTypes = {
   }),
   record: PropTypes.object.isRequired,
   isInitialSubmission: PropTypes.bool,
+  CommunityListItem: PropTypes.elementType,
+  pagination: PropTypes.bool,
 };
 
 CommunitySelectionSearch.defaultProps = {
   isInitialSubmission: true,
+  pagination: true,
+  CommunityListItem: CommunityListItem,
   apiConfigs: {
     allCommunities: {
       initialQueryState: { size: 5, page: 1, sortBy: "bestmatch" },
diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/index.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/index.js
index a473e8e66..98f9edd29 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/index.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/index.js
@@ -9,4 +9,6 @@ export {
   CommunitySelectionModalComponent,
 } from "./CommunitySelectionModal";
 
+export { CommunitySelectionSearch } from "./CommunitySelectionSearch";
+
 export { CommunityContext } from "./CommunityContext";
diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/index.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/index.js
index 52d27ef3a..2dbb86ca0 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/index.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/index.js
@@ -8,6 +8,7 @@
 export {
   CommunitySelectionModal,
   CommunitySelectionModalComponent,
+  CommunitySelectionSearch,
 } from "./CommunitySelectionModal";
 export { DepositStatusBox } from "./DepositStatus";
 export { RDMUploadProgressNotifier } from "./UploadProgressNotifier";

From 2c861decf3560f3e75d16709a3c8024c6928a403 Mon Sep 17 00:00:00 2001
From: Nicola Tarocco <ntarocco@gmail.com>
Date: Fri, 11 Oct 2024 22:38:56 +0200
Subject: [PATCH 55/76] DOI: fix wrong parent DOI link

---
 invenio_rdm_records/services/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/invenio_rdm_records/services/config.py b/invenio_rdm_records/services/config.py
index 7a9768c37..5c4e796ca 100644
--- a/invenio_rdm_records/services/config.py
+++ b/invenio_rdm_records/services/config.py
@@ -381,7 +381,7 @@ def expand(self, obj, context):
         when=is_record_or_draft_and_has_parent_doi,
     ),
     else_=RecordPIDLink(
-        "https://doi.org/{+pid_doi}", when=is_record_or_draft_and_has_parent_doi
+        "https://doi.org/{+parent_pid_doi}", when=is_record_or_draft_and_has_parent_doi
     ),
 )
 parent_doi_html_link = RecordPIDLink(

From aa67a1a9c6247dc0a0c63f1fe0b79acb63958d58 Mon Sep 17 00:00:00 2001
From: Alex Ioannidis <a.ioannidis@cern.ch>
Date: Thu, 17 Oct 2024 17:53:28 +0200
Subject: [PATCH 56/76] =?UTF-8?q?=F0=9F=93=A6=20release:=20v15.4.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 CHANGES.rst                                                  | 5 +++++
 invenio_rdm_records/__init__.py                              | 2 +-
 .../CommunitySelectionModal/CommunitySelectionSearch.js      | 2 +-
 .../src/deposit/components/CommunitySelectionModal/index.js  | 2 +-
 .../js/invenio_rdm_records/src/deposit/components/index.js   | 2 +-
 5 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index 166b26690..d84c1d41d 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -12,6 +12,11 @@
 Changes
 =======
 
+Version v15.4.0 (released 2024-10-17)
+
+- DOI: fix wrong parent DOI link
+- community: added props to make CommunitySelectionSearch reusable
+
 Version v15.3.0 (released 2024-10-16)
 
 - collections: display pages and REST API
diff --git a/invenio_rdm_records/__init__.py b/invenio_rdm_records/__init__.py
index 5999b7031..4b62fad62 100644
--- a/invenio_rdm_records/__init__.py
+++ b/invenio_rdm_records/__init__.py
@@ -11,6 +11,6 @@
 
 from .ext import InvenioRDMRecords
 
-__version__ = "15.3.0"
+__version__ = "15.4.0"
 
 __all__ = ("__version__", "InvenioRDMRecords")
diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/CommunitySelectionSearch.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/CommunitySelectionSearch.js
index 9ff403ebd..c7721d3a6 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/CommunitySelectionSearch.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/CommunitySelectionSearch.js
@@ -1,5 +1,5 @@
 // This file is part of Invenio-RDM-Records
-// Copyright (C) 2020-2023 CERN.
+// Copyright (C) 2020-2024 CERN.
 // Copyright (C) 2020-2022 Northwestern University.
 //
 // Invenio-RDM-Records is free software; you can redistribute it and/or modify it
diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/index.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/index.js
index 98f9edd29..2884158c4 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/index.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/index.js
@@ -1,5 +1,5 @@
 // This file is part of Invenio-RDM-Records
-// Copyright (C) 2020-2023 CERN.
+// Copyright (C) 2020-2024 CERN.
 //
 // Invenio-RDM-Records is free software; you can redistribute it and/or modify it
 // under the terms of the MIT License; see LICENSE file for more details.
diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/index.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/index.js
index 2dbb86ca0..4b4ca8521 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/index.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/index.js
@@ -1,5 +1,5 @@
 // This file is part of Invenio-RDM-Records
-// Copyright (C) 2020-2023 CERN.
+// Copyright (C) 2020-2024 CERN.
 // Copyright (C) 2020-2022 Northwestern University.
 //
 // Invenio-RDM-Records is free software; you can redistribute it and/or modify it

From cbf7fdf655dfe9572157e1096b138feb7dc189f2 Mon Sep 17 00:00:00 2001
From: Pablo Tamarit <pablo.tamarit@cern.ch>
Date: Fri, 18 Oct 2024 10:26:31 +0200
Subject: [PATCH 57/76] community: added autofocus prop to
 CommunitySelectionSearch

---
 .../CommunitySelectionModal/CommunitySelectionSearch.js      | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/CommunitySelectionSearch.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/CommunitySelectionSearch.js
index c7721d3a6..dd336451a 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/CommunitySelectionSearch.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/CommunitySelectionSearch.js
@@ -50,6 +50,7 @@ export class CommunitySelectionSearch extends Component {
       isInitialSubmission,
       CommunityListItem,
       pagination,
+      autofocus,
     } = this.props;
 
     const searchApi = new InvenioSearchApi(selectedSearchApi);
@@ -125,7 +126,7 @@ export class CommunitySelectionSearch extends Component {
               >
                 <SearchBar
                   placeholder={toggleText}
-                  autofocus
+                  autofocus={autofocus}
                   actionProps={{
                     "icon": "search",
                     "content": null,
@@ -178,11 +179,13 @@ CommunitySelectionSearch.propTypes = {
   isInitialSubmission: PropTypes.bool,
   CommunityListItem: PropTypes.elementType,
   pagination: PropTypes.bool,
+  autofocus: PropTypes.bool,
 };
 
 CommunitySelectionSearch.defaultProps = {
   isInitialSubmission: true,
   pagination: true,
+  autofocus: true,
   CommunityListItem: CommunityListItem,
   apiConfigs: {
     allCommunities: {

From 970aa7bc7f399b6bd2696aff7eb82a49f1c659c5 Mon Sep 17 00:00:00 2001
From: Pablo Tamarit <pablo.tamarit@cern.ch>
Date: Fri, 18 Oct 2024 11:06:02 +0200
Subject: [PATCH 58/76] =?UTF-8?q?=F0=9F=93=A6=20release:=20v15.5.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 CHANGES.rst                     | 4 ++++
 invenio_rdm_records/__init__.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index d84c1d41d..f5ab1ab23 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -12,6 +12,10 @@
 Changes
 =======
 
+Version v15.5.0 (released 2024-10-18)
+
+- community: added autofocus prop to CommunitySelectionSearch
+
 Version v15.4.0 (released 2024-10-17)
 
 - DOI: fix wrong parent DOI link
diff --git a/invenio_rdm_records/__init__.py b/invenio_rdm_records/__init__.py
index 4b62fad62..f82d2f903 100644
--- a/invenio_rdm_records/__init__.py
+++ b/invenio_rdm_records/__init__.py
@@ -11,6 +11,6 @@
 
 from .ext import InvenioRDMRecords
 
-__version__ = "15.4.0"
+__version__ = "15.5.0"
 
 __all__ = ("__version__", "InvenioRDMRecords")

From 86c2e17b8e467f9e4b5cb07ea285d06881006bcd Mon Sep 17 00:00:00 2001
From: Pablo Tamarit <pablo.tamarit@cern.ch>
Date: Fri, 18 Oct 2024 20:07:29 +0200
Subject: [PATCH 59/76] community: added myCommunitiesEnabled prop to
 CommunitySelectionSearch

---
 .../CommunitySelectionSearch.js               | 95 ++++++++++---------
 1 file changed, 50 insertions(+), 45 deletions(-)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/CommunitySelectionSearch.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/CommunitySelectionSearch.js
index dd336451a..bc086e15b 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/CommunitySelectionSearch.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/components/CommunitySelectionModal/CommunitySelectionSearch.js
@@ -50,6 +50,7 @@ export class CommunitySelectionSearch extends Component {
       isInitialSubmission,
       CommunityListItem,
       pagination,
+      myCommunitiesEnabled,
       autofocus,
     } = this.props;
 
@@ -72,55 +73,57 @@ export class CommunitySelectionSearch extends Component {
           defaultSortingOnEmptyQueryString={{ sortBy: "bestmatch" }}
         >
           <>
-            <Modal.Content as={Grid} className="m-0 pb-0">
+            <Modal.Content as={Grid} className="m-0 pb-0 centered">
+              {myCommunitiesEnabled && (
+                <Grid.Column
+                  mobile={16}
+                  tablet={8}
+                  computer={8}
+                  textAlign="left"
+                  floated="left"
+                  className="pt-0 pl-0"
+                >
+                  <Menu role="tablist" className="theme-primary-menu" compact>
+                    <Menu.Item
+                      as="button"
+                      role="tab"
+                      id="all-communities-tab"
+                      aria-selected={selectedAppId === allCommunities.appId}
+                      aria-controls={allCommunities.appId}
+                      name="All"
+                      active={selectedAppId === allCommunities.appId}
+                      onClick={() =>
+                        this.setState({
+                          selectedConfig: allCommunities,
+                        })
+                      }
+                    >
+                      {i18next.t("All")}
+                    </Menu.Item>
+                    <Menu.Item
+                      as="button"
+                      role="tab"
+                      id="my-communities-tab"
+                      aria-selected={selectedAppId === myCommunities.appId}
+                      aria-controls={myCommunities.appId}
+                      name="My communities"
+                      active={selectedAppId === myCommunities.appId}
+                      onClick={() =>
+                        this.setState({
+                          selectedConfig: myCommunities,
+                        })
+                      }
+                    >
+                      {i18next.t("My communities")}
+                    </Menu.Item>
+                  </Menu>
+                </Grid.Column>
+              )}
               <Grid.Column
                 mobile={16}
                 tablet={8}
                 computer={8}
-                textAlign="left"
-                floated="left"
-                className="pt-0 pl-0"
-              >
-                <Menu role="tablist" className="theme-primary-menu" compact>
-                  <Menu.Item
-                    as="button"
-                    role="tab"
-                    id="all-communities-tab"
-                    aria-selected={selectedAppId === allCommunities.appId}
-                    aria-controls={allCommunities.appId}
-                    name="All"
-                    active={selectedAppId === allCommunities.appId}
-                    onClick={() =>
-                      this.setState({
-                        selectedConfig: allCommunities,
-                      })
-                    }
-                  >
-                    {i18next.t("All")}
-                  </Menu.Item>
-                  <Menu.Item
-                    as="button"
-                    role="tab"
-                    id="my-communities-tab"
-                    aria-selected={selectedAppId === myCommunities.appId}
-                    aria-controls={myCommunities.appId}
-                    name="My communities"
-                    active={selectedAppId === myCommunities.appId}
-                    onClick={() =>
-                      this.setState({
-                        selectedConfig: myCommunities,
-                      })
-                    }
-                  >
-                    {i18next.t("My communities")}
-                  </Menu.Item>
-                </Menu>
-              </Grid.Column>
-              <Grid.Column
-                mobile={16}
-                tablet={8}
-                computer={8}
-                floated="right"
+                floated={myCommunitiesEnabled ? "right" : "null"}
                 verticalAlign="middle"
                 className="pt-0 pr-0 pl-0"
               >
@@ -179,12 +182,14 @@ CommunitySelectionSearch.propTypes = {
   isInitialSubmission: PropTypes.bool,
   CommunityListItem: PropTypes.elementType,
   pagination: PropTypes.bool,
+  myCommunitiesEnabled: PropTypes.bool,
   autofocus: PropTypes.bool,
 };
 
 CommunitySelectionSearch.defaultProps = {
   isInitialSubmission: true,
   pagination: true,
+  myCommunitiesEnabled: true,
   autofocus: true,
   CommunityListItem: CommunityListItem,
   apiConfigs: {

From de1493e72c1713d57e1d83fc637021746450470f Mon Sep 17 00:00:00 2001
From: Pablo Tamarit <pablo.tamarit@cern.ch>
Date: Fri, 18 Oct 2024 20:08:35 +0200
Subject: [PATCH 60/76] =?UTF-8?q?=F0=9F=93=A6=20release:=20v15.6.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 CHANGES.rst                     | 4 ++++
 invenio_rdm_records/__init__.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index f5ab1ab23..fcc2cc46f 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -12,6 +12,10 @@
 Changes
 =======
 
+Version v15.6.0 (released 2024-10-18)
+
+- community: added myCommunitiesEnabled prop to CommunitySelectionSearch
+
 Version v15.5.0 (released 2024-10-18)
 
 - community: added autofocus prop to CommunitySelectionSearch
diff --git a/invenio_rdm_records/__init__.py b/invenio_rdm_records/__init__.py
index f82d2f903..eb38213ac 100644
--- a/invenio_rdm_records/__init__.py
+++ b/invenio_rdm_records/__init__.py
@@ -11,6 +11,6 @@
 
 from .ext import InvenioRDMRecords
 
-__version__ = "15.5.0"
+__version__ = "15.6.0"
 
 __all__ = ("__version__", "InvenioRDMRecords")

From 120894386ff7c89870b30cde3a19586e5accb43f Mon Sep 17 00:00:00 2001
From: Guillaume Viger <fenekku@fenekku.com>
Date: Tue, 22 Oct 2024 11:37:02 -0400
Subject: [PATCH 61/76] access notification: provide correct draft preview link

closes inveniosoftware/invenio-app-rdm#2827
---
 .../grant-user-access.create.jinja            | 53 ++++++++++---------
 tests/services/test_service_access.py         | 33 ++++++++++++
 2 files changed, 62 insertions(+), 24 deletions(-)

diff --git a/invenio_rdm_records/templates/semantic-ui/invenio_notifications/grant-user-access.create.jinja b/invenio_rdm_records/templates/semantic-ui/invenio_notifications/grant-user-access.create.jinja
index 50974e9cd..b5fe7db36 100644
--- a/invenio_rdm_records/templates/semantic-ui/invenio_notifications/grant-user-access.create.jinja
+++ b/invenio_rdm_records/templates/semantic-ui/invenio_notifications/grant-user-access.create.jinja
@@ -4,7 +4,12 @@
 {% set permission = notification.context.permission %}
 
 {% set record_title = record.metadata.title %}
-{% set record_link = record.links.self_html %}
+{# Determine shared link #}
+{%- if not record.is_published and permission == "preview" -%}
+    {% set shared_link = record.links.record_html + "?preview=1" %}
+{%- else -%}
+    {% set shared_link = record.links.self_html %}
+{%- endif -%}
 {% set account_settings_link = "{ui}/account/settings/notifications".format(
     ui=config.SITE_UI_URL
     )
@@ -49,11 +54,11 @@
         {% endif %}
     </tr>
     <tr>
-        {%- if record.is_published -%}
-            <td><a href="{{ record_link }}" class="button">{{ _("View the record")}}</a></td>
-        {%- else -%}
-            <td><a href="{{ record_link }}" class="button">{{ _("View the draft")}}</a></td>
-        {%- endif -%}
+        <td>
+            <a href="{{ shared_link }}" class="button">
+                {{ _("View the record") if record.is_published else _("View the draft") }}
+            </a>
+        </td>
     </tr>
     <tr>
         <td><strong>_</strong></td>
@@ -64,29 +69,29 @@
 </table>
 {%- endblock html_body -%}
 
+{#
+    Because of whitespace interpretation for plain text we have to: indent, format and strip jinja blocks (-)
+    just so to get the right output. This is unfortunate for code readability but required for output.
+#}
 {%- block plain_body -%}
-    {%- if record.is_published -%}
-        {{ _("You have now permission to {permission} all versions of the record '{record_title}'.").format(record_title=record_title, permission=permission)}}
+{%- if record.is_published -%}
+    {{ _("You have now permission to {permission} all versions of the record '{record_title}'.").format(record_title=record_title, permission=permission)}}
+{%- else -%}
+    {%- if record_title -%}
+        {{ _("You have now permission to {permission} the draft '{record_title}'.").format(record_title=record_title, permission=permission)}}
     {%- else -%}
-        {%- if record_title -%}
-            {{ _("You have now permission to {permission} the draft '{record_title}'.").format(record_title=record_title, permission=permission)}}
-        {%- else -%}
-            {{ _("You have now permission to {permission} the draft.").format(permission=permission)}}
-        {%- endif -%}
+        {{ _("You have now permission to {permission} the draft.").format(permission=permission)}}
     {%- endif -%}
+{%- endif -%}
 
 {% if message %}
-<br>
-<br>
-{{ _("Message:")}}
-{{message}}
-{% endif %}
 
-{%- if record.is_published -%}
-    {{ _("View the record: ") }}{{ record_link }}
-{%- else -%}
-    {{ _("View the draft: ") }}{{ record_link }}
-{%- endif -%}
+{{ _("Message:") }}
+
+{{ message }}
+{%- endif %}
+
+{{ _("View the record: ") if record.is_published else _("View the draft: ") }}{{ shared_link }}
 
-{{ _("This is an auto-generated message. To manage notifications, visit your account settings: ")}}{{ account_settings_link }}
+{{ _("This is an auto-generated message. To manage notifications, visit your account settings: ")}}{{ account_settings_link }} .
 {%- endblock plain_body -%}
diff --git a/tests/services/test_service_access.py b/tests/services/test_service_access.py
index 820a3d94e..83aac3dbc 100644
--- a/tests/services/test_service_access.py
+++ b/tests/services/test_service_access.py
@@ -679,3 +679,36 @@ def test_delete_grant_by_subject_permissions(
     # assert that now user can not create a new version
     with pytest.raises(PermissionDeniedError):
         records_service.new_version(user_with_grant.identity, id_=record.id)
+
+
+def test_preview_draft_link_in_email(
+    running_app, uploader, minimal_record, community_owner
+):
+    # services
+    records_service = current_rdm_records.records_service
+    access_service = records_service.access
+    # instances
+    draft = records_service.create(uploader.identity, minimal_record)
+
+    # community_owner is not used because this has anything to do with
+    # communities. It is used simply because it is a user with "visibility": "public"
+    # like it is in test_resources_user_access.py
+    user_id = str(community_owner.id)
+    grant_payload = {
+        "grants": [
+            {
+                "subject": {"type": "user", "id": user_id},
+                "permission": "preview",
+                "notify": True,
+            }
+        ]
+    }
+    mail = running_app.app.extensions.get("mail")
+
+    with mail.record_messages() as outbox:
+        # This triggers an email notification because of "notify": True
+        access_service.bulk_create_grants(uploader.identity, draft.id, grant_payload)
+
+        sent_mail = outbox[0]
+        assert f"/records/{draft.id}?preview=1" in sent_mail.html
+        assert f"/records/{draft.id}?preview=1" in sent_mail.body

From ebb77d1ec0d24f87adf1b95583bb060fb10c8f7f Mon Sep 17 00:00:00 2001
From: Martin Obersteiner <obersteiner@tugraz.at>
Date: Tue, 24 Sep 2024 11:26:23 +0200
Subject: [PATCH 62/76] services: make file-service components configurable

---
 invenio_rdm_records/services/config.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/invenio_rdm_records/services/config.py b/invenio_rdm_records/services/config.py
index 5c4e796ca..b95a54d52 100644
--- a/invenio_rdm_records/services/config.py
+++ b/invenio_rdm_records/services/config.py
@@ -341,6 +341,10 @@ class RDMFileRecordServiceConfig(FileServiceConfig, ConfiguratorMixin):
 
     file_schema = FileSchema
 
+    components = FromConfig(
+        "RDM_FILES_SERVICE_COMPONENTS", default=FileServiceConfig.components
+    )
+
 
 class ThumbnailLinks(RecordLink):
     """RDM thumbnail links dictionary."""
@@ -751,6 +755,10 @@ class RDMFileDraftServiceConfig(FileServiceConfig, ConfiguratorMixin):
 
     file_schema = FileSchema
 
+    components = FromConfig(
+        "RDM_DRAFT_FILES_SERVICE_COMPONENTS", default=FileServiceConfig.components
+    )
+
 
 class RDMMediaFileDraftServiceConfig(FileServiceConfig, ConfiguratorMixin):
     """Configuration for draft media files."""

From 9620bb580415f96158665ec12f613a09262ec29c Mon Sep 17 00:00:00 2001
From: alejandromumo <manuel.alejandro.de.oliveira.da.costa@cern.ch>
Date: Mon, 21 Oct 2024 16:05:20 +0200
Subject: [PATCH 63/76] collections: added task to compute num of records

* added task to compute number of records for all the collections
* added "collection_id" parameter to record search
* added service methods to read collections (many, all)
* added tests
* collections: refactor 'resolve' to 'read'
* collections: rename 'search_records' method
* collections: update read method signature
---
 invenio_rdm_records/collections/api.py      | 30 +++++--
 invenio_rdm_records/collections/models.py   | 20 ++++-
 invenio_rdm_records/collections/results.py  | 22 +++--
 invenio_rdm_records/collections/schema.py   |  5 +-
 invenio_rdm_records/collections/service.py  | 93 +++++++++++++++++++--
 invenio_rdm_records/collections/tasks.py    | 31 +++++++
 setup.cfg                                   |  1 +
 tests/collections/test_collections_api.py   |  8 +-
 tests/collections/test_collections_tasks.py | 44 ++++++++++
 tests/services/test_collections_service.py  | 79 +++++++++++++++++
 10 files changed, 304 insertions(+), 29 deletions(-)
 create mode 100644 invenio_rdm_records/collections/tasks.py
 create mode 100644 tests/collections/test_collections_tasks.py

diff --git a/invenio_rdm_records/collections/api.py b/invenio_rdm_records/collections/api.py
index fc2bc3753..10a686935 100644
--- a/invenio_rdm_records/collections/api.py
+++ b/invenio_rdm_records/collections/api.py
@@ -69,10 +69,10 @@ def create(cls, slug, title, query, ctree=None, parent=None, order=None, depth=2
         )
 
     @classmethod
-    def resolve(cls, *, id_=None, slug=None, ctree_id=None, depth=2):
-        """Resolve a collection by ID or slug.
+    def read(cls, *, id_=None, slug=None, ctree_id=None, depth=2):
+        """Read a collection by ID or slug.
 
-        To resolve by slug, the collection tree ID must be provided.
+        To read by slug, the collection tree ID must be provided.
         """
         res = None
         if id_:
@@ -89,10 +89,21 @@ def resolve(cls, *, id_=None, slug=None, ctree_id=None, depth=2):
         return res
 
     @classmethod
-    def resolve_many(cls, ids_=None, depth=2):
-        """Resolve many collections by ID."""
-        _ids = ids_ or []
-        return [cls(c, depth) for c in cls.model_cls.read_many(_ids)]
+    def read_many(cls, ids_, depth=2):
+        """Read many collections by ID."""
+        return [cls(c, depth) for c in cls.model_cls.read_many(ids_)]
+
+    @classmethod
+    def read_all(cls, depth=2):
+        """Read all collections."""
+        return [cls(c, depth) for c in cls.model_cls.read_all()]
+
+    def update(self, **kwargs):
+        """Update the collection."""
+        if "search_query" in kwargs:
+            Collection.validate_query(kwargs["search_query"])
+        self.model.update(**kwargs)
+        return self
 
     def add(self, slug, title, query, order=None, depth=2):
         """Add a subcollection to the collection."""
@@ -128,7 +139,10 @@ def query(self):
     @cached_property
     def ancestors(self):
         """Get the collection ancestors."""
-        return Collection.resolve_many(self.split_path_to_ids())
+        ids_ = self.split_path_to_ids()
+        if not ids_:
+            return []
+        return Collection.read_many(ids_)
 
     @cached_property
     def subcollections(self):
diff --git a/invenio_rdm_records/collections/models.py b/invenio_rdm_records/collections/models.py
index 740242b1b..5d9c0820a 100644
--- a/invenio_rdm_records/collections/models.py
+++ b/invenio_rdm_records/collections/models.py
@@ -156,9 +156,25 @@ def get_by_slug(cls, slug, tree_id):
         return cls.query.filter(cls.slug == slug, cls.tree_id == tree_id).one_or_none()
 
     @classmethod
-    def read_many(cls, ids):
+    def read_many(cls, ids_):
         """Get many collections by ID."""
-        return cls.query.filter(cls.id.in_(ids)).order_by(cls.path, cls.order)
+        return cls.query.filter(cls.id.in_(ids_)).order_by(cls.path, cls.order)
+
+    @classmethod
+    def read_all(cls):
+        """Get all collections.
+
+        The collections are ordered by ``path`` and ``order``, which means:
+
+        - By path: the collections are ordered in a breadth-first manner (first come the root collection, then the next level, and so on)
+        - By order: between the same level collections, they are ordered by the specified order field.
+        """
+        return cls.query.order_by(cls.path, cls.order)
+
+    def update(self, **kwargs):
+        """Update a collection."""
+        for key, value in kwargs.items():
+            setattr(self, key, value)
 
     @classmethod
     def get_children(cls, model):
diff --git a/invenio_rdm_records/collections/results.py b/invenio_rdm_records/collections/results.py
index d0acb50af..21906002c 100644
--- a/invenio_rdm_records/collections/results.py
+++ b/invenio_rdm_records/collections/results.py
@@ -66,7 +66,9 @@ def to_dict(self):
         res = {
             "root": self._collection.id,
             self._collection.id: {
-                **self._schema.dump(self._collection),
+                **self._schema.dump(
+                    self._collection, context={"identity": self._identity}
+                ),
                 "children": list(),
                 "links": self._links_tpl.expand(self._identity, self._collection),
             },
@@ -76,7 +78,7 @@ def to_dict(self):
             if _c.id not in res:
                 # Add the subcollection to the dictionary
                 res[_c.id] = {
-                    **self._schema.dump(_c),
+                    **self._schema.dump(_c, context={"identity": self._identity}),
                     "children": list(),
                     "links": self._links_tpl.expand(self._identity, _c),
                 }
@@ -121,22 +123,30 @@ def query(self):
 class CollectionList(ServiceListResult):
     """Collection list item."""
 
-    def __init__(self, collections):
+    def __init__(self, identity, collections, schema, links_tpl, links_item_tpl):
         """Instantiate a Collection list item."""
+        self._identity = identity
         self._collections = collections
+        self._schema = schema
+        self._links_tpl = links_tpl
+        self._links_item_tpl = links_item_tpl
 
     def to_dict(self):
         """Serialize the collection list to a dictionary."""
         res = []
         for collection in self._collections:
-            _r = collection.to_dict()
-            _r["links"] = CollectionItem(collection).links
+            _r = CollectionItem(
+                self._identity, collection, self._schema, self._links_item_tpl
+            ).to_dict()
             res.append(_r)
         return res
 
     def __iter__(self):
         """Iterate over the collections."""
-        return iter(self._collections)
+        return (
+            CollectionItem(self._identity, x, self._schema, self._links_item_tpl)
+            for x in self._collections
+        )
 
 
 class CollectionTreeItem:
diff --git a/invenio_rdm_records/collections/schema.py b/invenio_rdm_records/collections/schema.py
index f1241c778..f2368368b 100644
--- a/invenio_rdm_records/collections/schema.py
+++ b/invenio_rdm_records/collections/schema.py
@@ -14,7 +14,8 @@ class CollectionSchema(Schema):
 
     slug = fields.Str()
     title = fields.Str()
-    depth = fields.Int()
+    depth = fields.Int(dump_only=True)
     order = fields.Int()
-    id = fields.Int()
+    id = fields.Int(dump_only=True)
     num_records = fields.Int()
+    search_query = fields.Str(load_only=True)
diff --git a/invenio_rdm_records/collections/service.py b/invenio_rdm_records/collections/service.py
index a526200b7..15484aca6 100644
--- a/invenio_rdm_records/collections/service.py
+++ b/invenio_rdm_records/collections/service.py
@@ -9,13 +9,19 @@
 import os
 
 from flask import current_app, url_for
+from invenio_records_resources.services import ServiceSchemaWrapper
 from invenio_records_resources.services.base import Service
 from invenio_records_resources.services.uow import ModelCommitOp, unit_of_work
 
+from invenio_rdm_records.proxies import (
+    current_community_records_service,
+    current_rdm_records_service,
+)
+
 from .api import Collection, CollectionTree
 from .errors import LogoNotFoundError
 from .links import CollectionLinkstemplate
-from .results import CollectionItem, CollectionTreeList
+from .results import CollectionItem, CollectionList, CollectionTreeList
 
 
 class CollectionsService(Service):
@@ -30,7 +36,7 @@ def __init__(self, config):
     @property
     def collection_schema(self):
         """Get the collection schema."""
-        return self.config.schema()
+        return ServiceSchemaWrapper(self, schema=self.config.schema)
 
     @property
     def links_item_tpl(self):
@@ -46,7 +52,11 @@ def links_item_tpl(self):
     def create(
         self, identity, community_id, tree_slug, slug, title, query, uow=None, **kwargs
     ):
-        """Create a new collection."""
+        """Create a new collection.
+
+        The created collection will be added to the collection tree as a root collection (no parent).
+        If a parent is needed, use the ``add`` method.
+        """
         self.require_permission(identity, "update", community_id=community_id)
         ctree = CollectionTree.resolve(slug=tree_slug, community_id=community_id)
         collection = self.collection_cls.create(
@@ -59,9 +69,8 @@ def create(
 
     def read(
         self,
-        /,
-        *,
         identity=None,
+        *,
         id_=None,
         slug=None,
         community_id=None,
@@ -74,10 +83,10 @@ def read(
         To resolve by slug, the collection tree ID and community ID must be provided.
         """
         if id_:
-            collection = self.collection_cls.resolve(id_=id_, depth=depth)
+            collection = self.collection_cls.read(id_=id_, depth=depth)
         elif slug and tree_slug and community_id:
             ctree = CollectionTree.resolve(slug=tree_slug, community_id=community_id)
-            collection = self.collection_cls.resolve(
+            collection = self.collection_cls.read(
                 slug=slug, ctree_id=ctree.id, depth=depth
             )
         else:
@@ -121,6 +130,31 @@ def add(self, identity, collection, slug, title, query, uow=None, **kwargs):
             identity, new_collection, self.collection_schema, self.links_item_tpl
         )
 
+    @unit_of_work()
+    def update(self, identity, collection_or_id, data=None, uow=None):
+        """Update a collection."""
+        if isinstance(collection_or_id, int):
+            collection = self.collection_cls.read(id_=collection_or_id)
+        else:
+            collection = collection_or_id
+        self.require_permission(
+            identity, "update", community_id=collection.community.id
+        )
+
+        data = data or {}
+
+        valid_data, errors = self.collection_schema.load(
+            data, context={"identity": identity}, raise_errors=True
+        )
+
+        res = collection.update(**valid_data)
+
+        uow.register(ModelCommitOp(res.model))
+
+        return CollectionItem(
+            identity, collection, self.collection_schema, self.links_item_tpl
+        )
+
     def read_logo(self, identity, slug):
         """Read a collection logo.
 
@@ -131,3 +165,48 @@ def read_logo(self, identity, slug):
         if _exists:
             return url_for("static", filename=logo_path)
         raise LogoNotFoundError()
+
+    def read_many(self, identity, ids_, depth=2):
+        """Get many collections."""
+        self.require_permission(identity, "read")
+
+        if ids_ is None:
+            raise ValueError("IDs must be provided.")
+
+        if ids_ == []:
+            raise ValueError("Use read_all to get all collections.")
+
+        res = self.collection_cls.read_many(ids_, depth=depth)
+        return CollectionList(
+            identity, res, self.collection_schema, None, self.links_item_tpl
+        )
+
+    def read_all(self, identity, depth=2):
+        """Get all collections."""
+        self.require_permission(identity, "read")
+        res = self.collection_cls.read_all(depth=depth)
+        return CollectionList(
+            identity, res, self.collection_schema, None, self.links_item_tpl
+        )
+
+    def search_collection_records(self, identity, collection_or_id, params=None):
+        """Search records in a collection."""
+        params = params or {}
+
+        if isinstance(collection_or_id, int):
+            collection = self.collection_cls.read(id_=collection_or_id)
+        else:
+            collection = collection_or_id
+
+        params.update({"collection_id": collection.id})
+        if collection.community:
+            res = current_community_records_service.search(
+                identity,
+                community_id=collection.community.id,
+                params=params,
+            )
+        else:
+            raise NotImplementedError(
+                "Search for collections without community not supported."
+            )
+        return res
diff --git a/invenio_rdm_records/collections/tasks.py b/invenio_rdm_records/collections/tasks.py
new file mode 100644
index 000000000..951c9d2d2
--- /dev/null
+++ b/invenio_rdm_records/collections/tasks.py
@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""Collections celery tasks."""
+
+from celery import shared_task
+from flask import current_app
+from invenio_access.permissions import system_identity
+
+from invenio_rdm_records.proxies import current_rdm_records
+
+
+@shared_task(ignore_result=True)
+def update_collections_size():
+    """Calculate and update the size of all the collections."""
+    collections_service = current_rdm_records.collections_service
+    res = collections_service.read_all(system_identity, depth=0)
+    for citem in res:
+        try:
+            collection = citem._collection
+            res = collections_service.search_collection_records(
+                system_identity, collection
+            )
+            collections_service.update(
+                system_identity, collection, data={"num_records": res.total}
+            )
+        except Exception as e:
+            current_app.logger.exception(str(e))
diff --git a/setup.cfg b/setup.cfg
index 450fe0bf4..0d73d84c0 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -109,6 +109,7 @@ invenio_celery.tasks =
     invenio_rdm_records_access_requests = invenio_rdm_records.requests.access.tasks
     invenio_rdm_records_iiif = invenio_rdm_records.services.iiif.tasks
     invenio_rdm_records_user_moderation = invenio_rdm_records.requests.user_moderation.tasks
+    invenio_rdm_records_collections = invenio_rdm_records.collections.tasks
 invenio_db.models =
     invenio_rdm_records = invenio_rdm_records.records.models
     invenio_rdm_records_collections = invenio_rdm_records.collections.models
diff --git a/tests/collections/test_collections_api.py b/tests/collections/test_collections_api.py
index eff421b3b..ddb91e10b 100644
--- a/tests/collections/test_collections_api.py
+++ b/tests/collections/test_collections_api.py
@@ -27,7 +27,7 @@ def test_create(running_app, db, community, community_owner):
         ctree=tree,
     )
 
-    read_c = Collection.resolve(id_=collection.id)
+    read_c = Collection.read(id_=collection.id)
     assert read_c.id == collection.id
     assert read_c.title == "My Collection"
     assert read_c.collection_tree.id == tree.id
@@ -40,7 +40,7 @@ def test_create(running_app, db, community, community_owner):
         ctree=tree.id,
     )
 
-    read_c = Collection.resolve(id_=collection.id)
+    read_c = Collection.read(id_=collection.id)
     assert read_c.id == collection.id
     assert collection.title == "My Collection 2"
     assert collection.collection_tree.id == tree.id
@@ -63,11 +63,11 @@ def test_resolve(running_app, db, community):
     )
 
     # Read by ID
-    read_by_id = Collection.resolve(id_=collection.id)
+    read_by_id = Collection.read(id_=collection.id)
     assert read_by_id.id == collection.id
 
     # Read by slug
-    read_by_slug = Collection.resolve(slug="my-collection", ctree_id=tree.id)
+    read_by_slug = Collection.read(slug="my-collection", ctree_id=tree.id)
     assert read_by_slug.id == read_by_id.id == collection.id
 
 
diff --git a/tests/collections/test_collections_tasks.py b/tests/collections/test_collections_tasks.py
new file mode 100644
index 000000000..85bd63d99
--- /dev/null
+++ b/tests/collections/test_collections_tasks.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""Test celery tasks of collections."""
+
+from copy import deepcopy
+
+from invenio_rdm_records.collections.api import Collection, CollectionTree
+from invenio_rdm_records.collections.tasks import update_collections_size
+
+
+def test_update_collections_size(app, db, record_factory, minimal_record, community):
+    """Test update_collections_size task."""
+    tree = CollectionTree.create(
+        title="Tree 1",
+        order=10,
+        community_id=community.id,
+        slug="tree-1",
+    )
+
+    collection = Collection.create(
+        title="My Collection",
+        query="metadata.title:foo",
+        slug="my-collection",
+        ctree=tree,
+    )
+    update_collections_size()
+
+    # Check that the collections have been updated
+    collection = Collection.read(id_=collection.id)
+    assert collection.num_records == 0
+
+    # Add a record that matches the collection
+    rec = deepcopy(minimal_record)
+    rec["metadata"]["title"] = "foo"
+    record = record_factory.create_record(record_dict=rec, community=community)
+
+    update_collections_size()
+
+    collection = Collection.read(id_=collection.id)
+    assert collection.num_records == 1
diff --git a/tests/services/test_collections_service.py b/tests/services/test_collections_service.py
index 428617b6e..e56a80d69 100644
--- a/tests/services/test_collections_service.py
+++ b/tests/services/test_collections_service.py
@@ -250,3 +250,82 @@ def test_collections_results(
     }
 
     assert not list(dictdiffer.diff(expected, r_dict))
+
+
+def test_update(running_app, db, add_collections, collections_service, community_owner):
+    """Test updating a collection."""
+    collections = add_collections()
+    c0 = collections[0]
+
+    # Update by ID
+    collections_service.update(
+        community_owner.identity,
+        c0.id,
+        data={"slug": "New slug"},
+    )
+
+    res = collections_service.read(
+        identity=community_owner.identity,
+        id_=c0.id,
+    )
+
+    assert res.to_dict()[c0.id]["slug"] == "New slug"
+
+    # Update by object
+    collections_service.update(
+        community_owner.identity,
+        c0,
+        data={"slug": "New slug 2"},
+    )
+
+    res = collections_service.read(
+        identity=community_owner.identity,
+        id_=c0.id,
+    )
+    assert res.to_dict()[c0.id]["slug"] == "New slug 2"
+
+
+def test_read_many(
+    running_app, db, add_collections, collections_service, community_owner
+):
+    """Test reading multiple collections."""
+    collections = add_collections()
+    c0 = collections[0]
+    c1 = collections[1]
+
+    # Read two collections
+    res = collections_service.read_many(
+        community_owner.identity,
+        ids_=[c0.id, c1.id],
+        depth=0,
+    )
+
+    res = res.to_dict()
+    assert len(res) == 2
+    assert res[0]["root"] == c0.id
+    assert res[1]["root"] == c1.id
+
+
+def test_read_all(
+    running_app, db, add_collections, collections_service, community_owner
+):
+    """Test reading all collections."""
+    collections = add_collections()
+    c0 = collections[0]
+    c1 = collections[1]
+
+    # Read all collections
+    res = collections_service.read_all(community_owner.identity, depth=0)
+
+    res = res.to_dict()
+    assert len(res) == 2
+    assert res[0]["root"] == c0.id
+    assert res[1]["root"] == c1.id
+
+
+def test_read_invalid(running_app, db, collections_service, community_owner):
+    """Test reading a non-existing collection."""
+    with pytest.raises(ValueError):
+        collections_service.read(
+            identity=community_owner.identity,
+        )

From d2ccb60eddb3d64be6f44443218239cd9ede8943 Mon Sep 17 00:00:00 2001
From: alejandromumo <manuel.alejandro.de.oliveira.da.costa@cern.ch>
Date: Fri, 25 Oct 2024 17:10:12 +0100
Subject: [PATCH 64/76] collections: move records search into service

* Added resource for collections
* Moved records search from community records to collections service
---
 .../collections/resources/__init__.py         |  7 +++
 .../collections/resources/config.py           | 50 +++++++++++++++++++
 .../collections/resources/resource.py         | 43 ++++++++++++++++
 invenio_rdm_records/collections/searchapp.py  |  5 --
 .../collections/services/__init__.py          |  7 +++
 .../collections/{ => services}/config.py      |  6 +--
 .../collections/{ => services}/links.py       |  1 +
 .../collections/{ => services}/results.py     |  0
 .../collections/{ => services}/schema.py      |  0
 .../collections/{ => services}/service.py     |  7 ++-
 invenio_rdm_records/ext.py                    | 12 ++++-
 invenio_rdm_records/resources/args.py         |  6 ---
 invenio_rdm_records/resources/config.py       |  4 +-
 .../services/community_records/service.py     |  8 ---
 invenio_rdm_records/views.py                  |  6 +++
 setup.cfg                                     |  1 +
 tests/services/test_collections_service.py    |  5 ++
 .../test_service_community_records.py         | 35 -------------
 18 files changed, 135 insertions(+), 68 deletions(-)
 create mode 100644 invenio_rdm_records/collections/resources/__init__.py
 create mode 100644 invenio_rdm_records/collections/resources/config.py
 create mode 100644 invenio_rdm_records/collections/resources/resource.py
 create mode 100644 invenio_rdm_records/collections/services/__init__.py
 rename invenio_rdm_records/collections/{ => services}/config.py (86%)
 rename invenio_rdm_records/collections/{ => services}/links.py (96%)
 rename invenio_rdm_records/collections/{ => services}/results.py (100%)
 rename invenio_rdm_records/collections/{ => services}/schema.py (100%)
 rename invenio_rdm_records/collections/{ => services}/service.py (97%)

diff --git a/invenio_rdm_records/collections/resources/__init__.py b/invenio_rdm_records/collections/resources/__init__.py
new file mode 100644
index 000000000..d9b40da79
--- /dev/null
+++ b/invenio_rdm_records/collections/resources/__init__.py
@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""Collection resource module."""
diff --git a/invenio_rdm_records/collections/resources/config.py b/invenio_rdm_records/collections/resources/config.py
new file mode 100644
index 000000000..581dfea71
--- /dev/null
+++ b/invenio_rdm_records/collections/resources/config.py
@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""Collection resource config."""
+
+from flask_resources import (
+    HTTPJSONException,
+    JSONSerializer,
+    ResourceConfig,
+    ResponseHandler,
+    create_error_handler,
+)
+from invenio_records_resources.resources.records.args import SearchRequestArgsSchema
+from invenio_records_resources.resources.records.headers import etag_headers
+from marshmallow.fields import Integer
+
+from invenio_rdm_records.resources.serializers import UIJSONSerializer
+
+from ..errors import CollectionNotFound
+
+
+class CollectionsResourceConfig(ResourceConfig):
+    """Configuration for the Collection resource."""
+
+    blueprint_name = "collections"
+    url_prefix = "/collections"
+
+    routes = {
+        "search-records": "/<id>/records",
+    }
+
+    request_view_args = {"id": Integer()}
+    request_search_args = SearchRequestArgsSchema
+    error_handlers = {
+        CollectionNotFound: create_error_handler(
+            HTTPJSONException(
+                code=404,
+                description="Collection was not found.",
+            )
+        ),
+    }
+    response_handlers = {
+        "application/json": ResponseHandler(JSONSerializer(), headers=etag_headers),
+        "application/vnd.inveniordm.v1+json": ResponseHandler(
+            UIJSONSerializer(), headers=etag_headers
+        ),
+    }
diff --git a/invenio_rdm_records/collections/resources/resource.py b/invenio_rdm_records/collections/resources/resource.py
new file mode 100644
index 000000000..2b505e184
--- /dev/null
+++ b/invenio_rdm_records/collections/resources/resource.py
@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""Collection resource."""
+
+from flask import g
+from flask_resources import Resource, resource_requestctx, response_handler, route
+from invenio_records_resources.resources.records.resource import (
+    request_search_args,
+    request_view_args,
+)
+
+
+class CollectionsResource(Resource):
+    """Collection resource."""
+
+    def __init__(self, config, service):
+        """Instantiate the resource."""
+        super().__init__(config)
+        self.service = service
+
+    def create_url_rules(self):
+        """Create the URL rules for the record resource."""
+        routes = self.config.routes
+        return [
+            route("GET", routes["search-records"], self.search_records),
+        ]
+
+    @request_view_args
+    @request_search_args
+    @response_handler(many=True)
+    def search_records(self):
+        """Search records in a collection."""
+        id_ = resource_requestctx.view_args["id"]
+        records = self.service.search_collection_records(
+            g.identity,
+            id_,
+            params=resource_requestctx.args,
+        )
+        return records.to_dict(), 200
diff --git a/invenio_rdm_records/collections/searchapp.py b/invenio_rdm_records/collections/searchapp.py
index a48e2699a..2a05e26b1 100644
--- a/invenio_rdm_records/collections/searchapp.py
+++ b/invenio_rdm_records/collections/searchapp.py
@@ -23,10 +23,5 @@ def search_app_context():
             sort_options=current_app.config["RDM_SORT_OPTIONS"],
             headers={"Accept": "application/vnd.inveniordm.v1+json"},
             pagination_options=(10, 25, 50, 100),
-            # endpoint=/communities/eu/records
-            # endpoint=/api/records
-            # hidden_params=[
-            #     ["q", collection.query]
-            # ]
         )
     }
diff --git a/invenio_rdm_records/collections/services/__init__.py b/invenio_rdm_records/collections/services/__init__.py
new file mode 100644
index 000000000..10d60ea45
--- /dev/null
+++ b/invenio_rdm_records/collections/services/__init__.py
@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+"""Collection service module."""
diff --git a/invenio_rdm_records/collections/config.py b/invenio_rdm_records/collections/services/config.py
similarity index 86%
rename from invenio_rdm_records/collections/config.py
rename to invenio_rdm_records/collections/services/config.py
index 835a19a36..62d6264a4 100644
--- a/invenio_rdm_records/collections/config.py
+++ b/invenio_rdm_records/collections/services/config.py
@@ -28,11 +28,7 @@ class CollectionServiceConfig(ServiceConfig, ConfiguratorMixin):
     schema = CollectionSchema
 
     links_item = {
-        "search": ConditionalLink(
-            cond=lambda coll, ctx: coll.community,
-            if_=CollectionLink("/api/communities/{community}/records"),
-            else_="/api/records",
-        ),
+        "search": CollectionLink("/api/collections/{id}/records"),
         "self_html": ConditionalLink(
             cond=lambda coll, ctx: coll.community,
             if_=CollectionLink(
diff --git a/invenio_rdm_records/collections/links.py b/invenio_rdm_records/collections/services/links.py
similarity index 96%
rename from invenio_rdm_records/collections/links.py
rename to invenio_rdm_records/collections/services/links.py
index ecff01f29..4ba6f0064 100644
--- a/invenio_rdm_records/collections/links.py
+++ b/invenio_rdm_records/collections/services/links.py
@@ -28,5 +28,6 @@ def vars(collection, vars):
                 "community": collection.community.slug,
                 "tree": collection.collection_tree.slug,
                 "collection": collection.slug,
+                "id": collection.id,
             }
         )
diff --git a/invenio_rdm_records/collections/results.py b/invenio_rdm_records/collections/services/results.py
similarity index 100%
rename from invenio_rdm_records/collections/results.py
rename to invenio_rdm_records/collections/services/results.py
diff --git a/invenio_rdm_records/collections/schema.py b/invenio_rdm_records/collections/services/schema.py
similarity index 100%
rename from invenio_rdm_records/collections/schema.py
rename to invenio_rdm_records/collections/services/schema.py
diff --git a/invenio_rdm_records/collections/service.py b/invenio_rdm_records/collections/services/service.py
similarity index 97%
rename from invenio_rdm_records/collections/service.py
rename to invenio_rdm_records/collections/services/service.py
index 15484aca6..d7b312ba4 100644
--- a/invenio_rdm_records/collections/service.py
+++ b/invenio_rdm_records/collections/services/service.py
@@ -18,8 +18,8 @@
     current_rdm_records_service,
 )
 
-from .api import Collection, CollectionTree
-from .errors import LogoNotFoundError
+from ..api import Collection, CollectionTree
+from ..errors import LogoNotFoundError
 from .links import CollectionLinkstemplate
 from .results import CollectionItem, CollectionList, CollectionTreeList
 
@@ -198,12 +198,11 @@ def search_collection_records(self, identity, collection_or_id, params=None):
         else:
             collection = collection_or_id
 
-        params.update({"collection_id": collection.id})
         if collection.community:
             res = current_community_records_service.search(
                 identity,
                 community_id=collection.community.id,
-                params=params,
+                extra_filter=collection.query,
             )
         else:
             raise NotImplementedError(
diff --git a/invenio_rdm_records/ext.py b/invenio_rdm_records/ext.py
index f96743995..be975dd37 100644
--- a/invenio_rdm_records/ext.py
+++ b/invenio_rdm_records/ext.py
@@ -19,8 +19,10 @@
 from invenio_records_resources.resources.files import FileResource
 
 from . import config
-from .collections.config import CollectionServiceConfig
-from .collections.service import CollectionsService
+from .collections.resources.config import CollectionsResourceConfig
+from .collections.resources.resource import CollectionsResource
+from .collections.services.config import CollectionServiceConfig
+from .collections.services.service import CollectionsService
 from .oaiserver.resources.config import OAIPMHServerResourceConfig
 from .oaiserver.resources.resources import OAIPMHServerResource
 from .oaiserver.services.config import OAIPMHServerServiceConfig
@@ -278,6 +280,12 @@ def init_resource(self, app):
             config=RDMCommunityRecordsResourceConfig.build(app),
         )
 
+        # Collections
+        self.collections_resource = CollectionsResource(
+            service=self.collections_service,
+            config=CollectionsResourceConfig,
+        )
+
         # OAI-PMH
         self.oaipmh_server_resource = OAIPMHServerResource(
             service=self.oaipmh_server_service,
diff --git a/invenio_rdm_records/resources/args.py b/invenio_rdm_records/resources/args.py
index 1d9e73fbf..90531a126 100644
--- a/invenio_rdm_records/resources/args.py
+++ b/invenio_rdm_records/resources/args.py
@@ -19,9 +19,3 @@ class RDMSearchRequestArgsSchema(SearchRequestArgsSchema):
     locale = fields.Str()
     status = fields.Str()
     include_deleted = fields.Bool()
-
-
-class CommunityRecordsSearchRequestArgsSchema(SearchRequestArgsSchema):
-    """Extend schema with collection_id field."""
-
-    collection_id = fields.Integer()
diff --git a/invenio_rdm_records/resources/config.py b/invenio_rdm_records/resources/config.py
index 522372ee5..192ab61ad 100644
--- a/invenio_rdm_records/resources/config.py
+++ b/invenio_rdm_records/resources/config.py
@@ -46,7 +46,7 @@
     ReviewStateError,
     ValidationErrorWithMessageAsList,
 )
-from .args import CommunityRecordsSearchRequestArgsSchema, RDMSearchRequestArgsSchema
+from .args import RDMSearchRequestArgsSchema
 from .deserializers import ROCrateJSONDeserializer
 from .deserializers.errors import DeserializerError
 from .errors import HTTPJSONException, HTTPJSONValidationWithMessageAsListException
@@ -558,8 +558,6 @@ class RDMCommunityRecordsResourceConfig(RecordResourceConfig, ConfiguratorMixin)
         default=record_serializers,
     )
 
-    request_search_args = CommunityRecordsSearchRequestArgsSchema
-
 
 class RDMRecordCommunitiesResourceConfig(CommunityResourceConfig, ConfiguratorMixin):
     """Record communities resource config."""
diff --git a/invenio_rdm_records/services/community_records/service.py b/invenio_rdm_records/services/community_records/service.py
index 04be70ab3..7209f676b 100644
--- a/invenio_rdm_records/services/community_records/service.py
+++ b/invenio_rdm_records/services/community_records/service.py
@@ -67,14 +67,6 @@ def search(
         if extra_filter is not None:
             community_filter = community_filter & extra_filter
 
-        # Search in a specific collection
-        if "collection_id" in params:
-            collections_service = current_rdm_records.collections_service
-            collection = collections_service.read(
-                identity=identity, id_=params["collection_id"]
-            )
-            community_filter &= collection.query
-
         search = self._search(
             "search",
             identity,
diff --git a/invenio_rdm_records/views.py b/invenio_rdm_records/views.py
index 800d7a121..a9e4ab7f9 100644
--- a/invenio_rdm_records/views.py
+++ b/invenio_rdm_records/views.py
@@ -102,3 +102,9 @@ def create_iiif_bp(app):
     """Create IIIF blueprint."""
     ext = app.extensions["invenio-rdm-records"]
     return ext.iiif_resource.as_blueprint()
+
+
+def create_collections_bp(app):
+    """Create collections blueprint."""
+    ext = app.extensions["invenio-rdm-records"]
+    return ext.collections_resource.as_blueprint()
diff --git a/setup.cfg b/setup.cfg
index 0d73d84c0..2e1f9052c 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -97,6 +97,7 @@ invenio_base.api_blueprints =
     invenio_rdm_record_communities = invenio_rdm_records.views:create_record_communities_bp
     invenio_rdm_record_requests = invenio_rdm_records.views:create_record_requests_bp
     invenio_iiif = invenio_rdm_records.views:create_iiif_bp
+    invenio_rdm_records_collections = invenio_rdm_records.views:create_collections_bp
 invenio_base.api_finalize_app =
     invenio_rdm_records = invenio_rdm_records.ext:api_finalize_app
 invenio_base.blueprints =
diff --git a/tests/services/test_collections_service.py b/tests/services/test_collections_service.py
index e56a80d69..8915b02b7 100644
--- a/tests/services/test_collections_service.py
+++ b/tests/services/test_collections_service.py
@@ -170,6 +170,7 @@ def test_collections_results(
             "links": {
                 "search": "/api/communities/blr/records",
                 "self_html": "/communities/blr/collections/tree-1/collection-1",
+                "search": f"/api/collections/{c0.id}/records",
             },
             "num_records": 0,
             "order": c0.order,
@@ -183,6 +184,7 @@ def test_collections_results(
             "links": {
                 "search": "/api/communities/blr/records",
                 "self_html": "/communities/blr/collections/tree-1/collection-2",
+                "search": f"/api/collections/{c1.id}/records",
             },
             "num_records": 0,
             "order": c1.order,
@@ -215,6 +217,7 @@ def test_collections_results(
             "links": {
                 "search": "/api/communities/blr/records",
                 "self_html": "/communities/blr/collections/tree-1/collection-1",
+                "search": f"/api/collections/{c0.id}/records",
             },
             "num_records": 0,
             "order": c0.order,
@@ -228,6 +231,7 @@ def test_collections_results(
             "links": {
                 "search": "/api/communities/blr/records",
                 "self_html": "/communities/blr/collections/tree-1/collection-2",
+                "search": f"/api/collections/{c1.id}/records",
             },
             "num_records": 0,
             "order": c1.order,
@@ -241,6 +245,7 @@ def test_collections_results(
             "links": {
                 "search": "/api/communities/blr/records",
                 "self_html": "/communities/blr/collections/tree-1/collection-3",
+                "search": f"/api/collections/{c3.id}/records",
             },
             "num_records": 0,
             "order": c3.order,
diff --git a/tests/services/test_service_community_records.py b/tests/services/test_service_community_records.py
index 9c48c40ae..67885f4bb 100644
--- a/tests/services/test_service_community_records.py
+++ b/tests/services/test_service_community_records.py
@@ -159,38 +159,3 @@ def test_search_community_records(
             community_id=str(community.id),
         )
         assert results.to_dict()["hits"]["total"] == 3
-
-
-def test_search_community_records_in_collections(
-    community, record_community, service, uploader, minimal_record
-):
-    """Test search for records in a community collection."""
-    rec1 = deepcopy(minimal_record)
-    rec1["metadata"]["title"] = "Another record"
-    record_community.create_record(record_dict=rec1)
-    record_community.create_record(record_dict=minimal_record)
-    ctree = CollectionTree.create(
-        title="Tree 1",
-        order=10,
-        community_id=community.id,
-        slug="tree-1",
-    )
-    collection = Collection.create(
-        title="My Collection",
-        query='metadata.title:"Another record"',
-        slug="my-collection",
-        ctree=ctree,
-    )
-    all_results = service.search(
-        uploader.identity,
-        community_id=str(community.id),
-    )
-    assert all_results.total == 2
-
-    results = service.search(
-        uploader.identity,
-        community_id=str(community.id),
-        params={"collection_id": str(collection.id)},
-    )
-    assert results.total == 1
-    assert results.to_dict()["hits"]["hits"][0]["metadata"]["title"] == "Another record"

From 04f782ba48935425cabccda822edb348789f7d14 Mon Sep 17 00:00:00 2001
From: Carlin MacKenzie <carlin.mackenzie@gmail.com>
Date: Wed, 28 Aug 2024 14:50:57 +0200
Subject: [PATCH 65/76] UI: remove unnecessary divs to fix spacing bug

---
 .../CreatibutorsField/CreatibutorsModal.js    | 30 +++++++++----------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
index 45e08b727..2bb2b952e 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
@@ -559,7 +559,7 @@ export class CreatibutorsModal extends Component {
                     />
                   </Form.Group>
                   {_get(values, typeFieldPath, "") === CREATIBUTOR_TYPE.PERSON ? (
-                    <div>
+                    <>
                       {autocompleteNames !== NamesAutocompleteOptions.OFF && (
                         <RemoteSelectField
                           selectOnBlur={false}
@@ -587,7 +587,7 @@ export class CreatibutorsModal extends Component {
                         />
                       )}
                       {showPersonForm && (
-                        <div>
+                        <>
                           <Form.Group widths="equal">
                             <TextField
                               label={i18next.t("Family name")}
@@ -617,9 +617,9 @@ export class CreatibutorsModal extends Component {
                             fieldPath={affiliationsFieldPath}
                             selectRef={this.affiliationsRef}
                           />
-                        </div>
+                        </>
                       )}
-                    </div>
+                    </>
                   ) : (
                     <>
                       {autocompleteNames !== NamesAutocompleteOptions.OFF && (
@@ -678,18 +678,16 @@ export class CreatibutorsModal extends Component {
                   {(_get(values, typeFieldPath) === CREATIBUTOR_TYPE.ORGANIZATION ||
                     (showPersonForm &&
                       _get(values, typeFieldPath) === CREATIBUTOR_TYPE.PERSON)) && (
-                    <div>
-                      <SelectField
-                        fieldPath={roleFieldPath}
-                        label={i18next.t("Role")}
-                        options={roleOptions}
-                        placeholder={i18next.t("Select role")}
-                        {...(this.isCreator() && { clearable: true })}
-                        required={!this.isCreator()}
-                        optimized
-                        scrolling
-                      />
-                    </div>
+                    <SelectField
+                      fieldPath={roleFieldPath}
+                      label={i18next.t("Role")}
+                      options={roleOptions}
+                      placeholder={i18next.t("Select role")}
+                      {...(this.isCreator() && { clearable: true })}
+                      required={!this.isCreator()}
+                      optimized
+                      scrolling
+                    />
                   )}
                 </Form>
               </Modal.Content>

From 84bbf8eedf290c31eae0efd27cc31869e98b5c76 Mon Sep 17 00:00:00 2001
From: Carlin MacKenzie <carlin.mackenzie@gmail.com>
Date: Thu, 29 Aug 2024 10:35:35 +0200
Subject: [PATCH 66/76] refactor: create makeSubheader function

---
 .../CreatibutorsField/CreatibutorsModal.js    | 45 ++++++++++---------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
index 2bb2b952e..05c0b3f10 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
@@ -246,10 +246,9 @@ export class CreatibutorsModal extends Component {
       icon = "/static/images/gnd-icon.svg";
       link = "https://d-nb.info/gnd/" + identifier.identifier;
     } else if (identifier.scheme === "ror") {
-      icon = "/static/images/ror-icon.svg";
-      link = "https://ror.org/" + identifier.identifier;
+      return; // ROR doesn't recommend displaying ROR IDs
     } else if (identifier.scheme === "isni" || identifier.scheme === "grid") {
-      return null;
+      return;
     } else {
       return (
         <>
@@ -261,38 +260,42 @@ export class CreatibutorsModal extends Component {
     return (
       <span key={identifier.identifier}>
         <a href={link} target="_blank" rel="noopener noreferrer">
-          <Image
-            src={icon}
-            className="inline-id-icon ml-5 mr-5"
-            verticalAlign="middle"
-          />
+          <Image src={icon} className="inline-id-icon mr-5" verticalAlign="middle" />
           {identifier.scheme === "orcid" ? identifier.identifier : null}
         </a>
       </span>
     );
   };
 
+  makeSubheader = (creatibutor) => {
+    const { isOrganization } = this.state;
+
+    if (isOrganization) {
+      return creatibutor.acronym;
+    } else {
+      let affNames = "";
+      if ("affiliations" in creatibutor) {
+        affNames = creatibutor.affiliations
+          .map((affiliation) => affiliation.name)
+          .join(", ");
+      }
+      return affNames;
+    }
+  };
+
   serializeSuggestions = (creatibutors) => {
     const results = creatibutors.map((creatibutor) => {
       // ensure `affiliations` and `identifiers` are present
       creatibutor.affiliations = creatibutor.affiliations || [];
       creatibutor.identifiers = creatibutor.identifiers || [];
 
-      let affNames = "";
-      if ("affiliations" in creatibutor) {
-        creatibutor.affiliations.forEach((affiliation, idx) => {
-          affNames += affiliation.name;
-          if (idx < creatibutor.affiliations.length - 1) {
-            affNames += ", ";
-          }
-        });
-      }
+      const subheader = this.makeSubheader(creatibutor);
 
       const idString = [];
       creatibutor.identifiers?.forEach((i) => {
-        idString.push(this.makeIdEntry(i));
+        const idEntry = this.makeIdEntry(i);
+        if (idEntry) idString.push(idEntry);
       });
-      const { isOrganization } = this.state;
 
       return {
         text: creatibutor.name,
@@ -302,9 +305,7 @@ export class CreatibutorsModal extends Component {
         content: (
           <Header>
             {creatibutor.name} {idString.length ? <>({idString})</> : null}
-            <Header.Subheader>
-              {isOrganization ? creatibutor.acronym : affNames}
-            </Header.Subheader>
+            <Header.Subheader>{subheader}</Header.Subheader>
           </Header>
         ),
       };

From b6a47a040e941e68d0b9c778d9c133de293b4d85 Mon Sep 17 00:00:00 2001
From: Carlin MacKenzie <carlin.mackenzie@gmail.com>
Date: Thu, 29 Aug 2024 15:00:02 +0200
Subject: [PATCH 67/76] UI/UX: improve display of ROR information

---
 .../CreatibutorsField/CreatibutorsModal.js    | 27 ++++++++++++-------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
index 05c0b3f10..2e698ff88 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
@@ -271,15 +271,22 @@ export class CreatibutorsModal extends Component {
     const { isOrganization } = this.state;
 
     if (isOrganization) {
-      return creatibutor.acronym;
+      const locationPart = [creatibutor?.location_name, creatibutor?.country_name]
+        .filter(Boolean)
+        .join(", ");
+
+      const typesPart = creatibutor?.types
+        .map((type) => type.charAt(0).toUpperCase() + type.slice(1))
+        .join(", ");
+
+      return `${locationPart}${
+        locationPart && typesPart ? " — " : ""
+      }${typesPart}`.trim();
     } else {
-      let affNames = "";
-      if ("affiliations" in creatibutor) {
-        affNames = creatibutor.affiliations
-          .map((affiliation) => affiliation.name)
-          .join(", ");
-      }
-      return affNames;
+      return (
+        creatibutor?.affiliations?.map((affiliation) => affiliation.name)?.join(", ") ||
+        ""
+      );
     }
   };
 
@@ -290,6 +297,8 @@ export class CreatibutorsModal extends Component {
       creatibutor.identifiers = creatibutor.identifiers || [];
 
       const subheader = this.makeSubheader(creatibutor);
+      let name = creatibutor.name;
+      if (creatibutor.acronym) name += " (" + creatibutor.acronym + ")";
 
       const idString = [];
       creatibutor.identifiers?.forEach((i) => {
@@ -304,7 +313,7 @@ export class CreatibutorsModal extends Component {
         key: creatibutor.id,
         content: (
           <Header>
-            {creatibutor.name} {idString.length ? <>({idString})</> : null}
+            {name} {idString.length ? <>({idString})</> : null}
             <Header.Subheader>{subheader}</Header.Subheader>
           </Header>
         ),

From 52cce93d87e9cd6930ab2ce5c174a34ce3f2f7d7 Mon Sep 17 00:00:00 2001
From: Carlin MacKenzie <carlin.mackenzie@gmail.com>
Date: Fri, 30 Aug 2024 17:12:28 +0200
Subject: [PATCH 68/76] refactor: extract logic to react-invenio-forms

---
 .../CreatibutorsField/CreatibutorsModal.js    | 126 ++----------------
 1 file changed, 12 insertions(+), 114 deletions(-)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
index 2e698ff88..c73389250 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
@@ -15,13 +15,12 @@ import _isEmpty from "lodash/isEmpty";
 import _map from "lodash/map";
 import PropTypes from "prop-types";
 import React, { Component, createRef } from "react";
-import { Trans } from "react-i18next";
 import {
-  Image,
   RadioField,
   RemoteSelectField,
   SelectField,
   TextField,
+  AffiliationsSuggestions,
 } from "react-invenio-forms";
 import { Button, Form, Header, Modal } from "semantic-ui-react";
 import * as Yup from "yup";
@@ -235,118 +234,17 @@ export class CreatibutorsModal extends Component {
     }
   };
 
-  makeIdEntry = (identifier) => {
-    let icon = null;
-    let link = null;
-
-    if (identifier.scheme === "orcid") {
-      icon = "/static/images/orcid.svg";
-      link = "https://orcid.org/" + identifier.identifier;
-    } else if (identifier.scheme === "gnd") {
-      icon = "/static/images/gnd-icon.svg";
-      link = "https://d-nb.info/gnd/" + identifier.identifier;
-    } else if (identifier.scheme === "ror") {
-      return; // ROR doesn't recommend displaying ROR IDs
-    } else if (identifier.scheme === "isni" || identifier.scheme === "grid") {
-      return;
-    } else {
-      return (
-        <>
-          {identifier.scheme}: {identifier.identifier}
-        </>
-      );
-    }
-
-    return (
-      <span key={identifier.identifier}>
-        <a href={link} target="_blank" rel="noopener noreferrer">
-          <Image src={icon} className="inline-id-icon mr-5" verticalAlign="middle" />
-          {identifier.scheme === "orcid" ? identifier.identifier : null}
-        </a>
-      </span>
-    );
-  };
-
-  makeSubheader = (creatibutor) => {
-    const { isOrganization } = this.state;
-
-    if (isOrganization) {
-      const locationPart = [creatibutor?.location_name, creatibutor?.country_name]
-        .filter(Boolean)
-        .join(", ");
-
-      const typesPart = creatibutor?.types
-        .map((type) => type.charAt(0).toUpperCase() + type.slice(1))
-        .join(", ");
-
-      return `${locationPart}${
-        locationPart && typesPart ? " — " : ""
-      }${typesPart}`.trim();
-    } else {
-      return (
-        creatibutor?.affiliations?.map((affiliation) => affiliation.name)?.join(", ") ||
-        ""
-      );
-    }
-  };
-
-  serializeSuggestions = (creatibutors) => {
-    const results = creatibutors.map((creatibutor) => {
-      // ensure `affiliations` and `identifiers` are present
-      creatibutor.affiliations = creatibutor.affiliations || [];
-      creatibutor.identifiers = creatibutor.identifiers || [];
-
-      const subheader = this.makeSubheader(creatibutor);
-      let name = creatibutor.name;
-      if (creatibutor.acronym) name += " (" + creatibutor.acronym + ")";
-
-      const idString = [];
-      creatibutor.identifiers?.forEach((i) => {
-        const idEntry = this.makeIdEntry(i);
-        if (idEntry) idString.push(idEntry);
-      });
-
-      return {
-        text: creatibutor.name,
-        value: creatibutor.id,
-        extra: creatibutor,
-        key: creatibutor.id,
-        content: (
-          <Header>
-            {name} {idString.length ? <>({idString})</> : null}
-            <Header.Subheader>{subheader}</Header.Subheader>
-          </Header>
-        ),
-      };
-    });
-
-    const { showPersonForm } = this.state;
+  SuggestionsWrapper = (creatibutors) => {
+    const { isOrganization, showPersonForm } = this.state;
     const { autocompleteNames } = this.props;
 
-    const showManualEntry =
-      autocompleteNames === NamesAutocompleteOptions.SEARCH_ONLY && !showPersonForm;
-
-    if (showManualEntry) {
-      results.push({
-        text: "Manual entry",
-        value: "Manual entry",
-        extra: "Manual entry",
-        key: "manual-entry",
-        content: (
-          <Header textAlign="center">
-            <Header.Content>
-              <p>
-                <Trans>
-                  {/* eslint-disable-next-line jsx-a11y/anchor-is-valid*/}
-                  Couldn't find your person? You can <a>create a new entry</a>.
-                </Trans>
-              </p>
-            </Header.Content>
-          </Header>
-        ),
-      });
-    }
-    return results;
+    return AffiliationsSuggestions(
+      creatibutors,
+      isOrganization,
+      showPersonForm,
+      autocompleteNames,
+      NamesAutocompleteOptions
+    );
   };
 
   updateIdentifiersAndAffiliations(
@@ -591,7 +489,7 @@ export class CreatibutorsModal extends Component {
                           // Disable UI-side filtering of search results
                           search={(options) => options}
                           suggestionAPIUrl="/api/names"
-                          serializeSuggestions={this.serializeSuggestions}
+                          serializeSuggestions={this.SuggestionsWrapper}
                           onValueChange={this.onPersonSearchChange}
                           ref={this.namesAutocompleteRef}
                         />
@@ -653,7 +551,7 @@ export class CreatibutorsModal extends Component {
                           // Disable UI-side filtering of search results
                           search={(options) => options}
                           suggestionAPIUrl="/api/affiliations"
-                          serializeSuggestions={this.serializeSuggestions}
+                          serializeSuggestions={this.SuggestionsWrapper}
                           onValueChange={this.onOrganizationSearchChange}
                         />
                       )}

From 626b1d853c9e0ac921520ce6cd2dda1b8d683e5d Mon Sep 17 00:00:00 2001
From: Carlin MacKenzie <carlin.mackenzie@gmail.com>
Date: Fri, 30 Aug 2024 17:43:17 +0200
Subject: [PATCH 69/76] UI/UX: add consistent suggestions display to
 affiliations

---
 .../AffiliationsField/AffiliationsField.js    | 21 ++++++++-----------
 .../CreatibutorsField/CreatibutorsModal.js    | 11 ++++------
 2 files changed, 13 insertions(+), 19 deletions(-)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AffiliationsField/AffiliationsField.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AffiliationsField/AffiliationsField.js
index 3ac77ca87..575cb3b6a 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AffiliationsField/AffiliationsField.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AffiliationsField/AffiliationsField.js
@@ -8,22 +8,19 @@
 
 import PropTypes from "prop-types";
 import React, { Component } from "react";
-import { FieldLabel, RemoteSelectField } from "react-invenio-forms";
+import {
+  FieldLabel,
+  RemoteSelectField,
+  AffiliationsSuggestions,
+} from "react-invenio-forms";
 import { Field, getIn } from "formik";
 import { i18next } from "@translations/invenio_rdm_records/i18next";
 
 /**Affiliation input component */
 export class AffiliationsField extends Component {
-  serializeAffiliations = (affiliations) =>
-    affiliations.map((affiliation) => ({
-      text: affiliation.acronym
-        ? `${affiliation.name} (${affiliation.acronym})`
-        : affiliation.name,
-      value: affiliation.id || affiliation.name,
-      key: affiliation.id,
-      id: affiliation.id,
-      name: affiliation.name,
-    }));
+  SuggestionsWrapper = (affiliations) => {
+    return AffiliationsSuggestions(affiliations, true, false);
+  };
 
   render() {
     const { fieldPath, selectRef } = this.props;
@@ -38,7 +35,7 @@ export class AffiliationsField extends Component {
                 Accept: "application/vnd.inveniordm.v1+json",
               }}
               initialSuggestions={getIn(values, fieldPath, [])}
-              serializeSuggestions={this.serializeAffiliations}
+              serializeSuggestions={this.SuggestionsWrapper}
               placeholder={i18next.t("Search or create affiliation")}
               label={
                 <FieldLabel
diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
index c73389250..08e432f66 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
@@ -238,13 +238,10 @@ export class CreatibutorsModal extends Component {
     const { isOrganization, showPersonForm } = this.state;
     const { autocompleteNames } = this.props;
 
-    return AffiliationsSuggestions(
-      creatibutors,
-      isOrganization,
-      showPersonForm,
-      autocompleteNames,
-      NamesAutocompleteOptions
-    );
+    const showManualEntry =
+      autocompleteNames === NamesAutocompleteOptions.SEARCH_ONLY && !showPersonForm;
+
+    return AffiliationsSuggestions(creatibutors, isOrganization, showManualEntry);
   };
 
   updateIdentifiersAndAffiliations(

From 81579d730b3bcfe7e90425d7c005919bee6a8802 Mon Sep 17 00:00:00 2001
From: Carlin MacKenzie <carlin.mackenzie@gmail.com>
Date: Mon, 2 Sep 2024 17:36:17 +0200
Subject: [PATCH 70/76] refactor: naming

---
 .../deposit/fields/AffiliationsField/AffiliationsField.js | 4 ++--
 .../deposit/fields/CreatibutorsField/CreatibutorsModal.js | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AffiliationsField/AffiliationsField.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AffiliationsField/AffiliationsField.js
index 575cb3b6a..9264068e1 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AffiliationsField/AffiliationsField.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AffiliationsField/AffiliationsField.js
@@ -18,7 +18,7 @@ import { i18next } from "@translations/invenio_rdm_records/i18next";
 
 /**Affiliation input component */
 export class AffiliationsField extends Component {
-  SuggestionsWrapper = (affiliations) => {
+  serializeAffiliations = (affiliations) => {
     return AffiliationsSuggestions(affiliations, true, false);
   };
 
@@ -35,7 +35,7 @@ export class AffiliationsField extends Component {
                 Accept: "application/vnd.inveniordm.v1+json",
               }}
               initialSuggestions={getIn(values, fieldPath, [])}
-              serializeSuggestions={this.SuggestionsWrapper}
+              serializeSuggestions={this.serializeAffiliations}
               placeholder={i18next.t("Search or create affiliation")}
               label={
                 <FieldLabel
diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
index 08e432f66..a544343ed 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
@@ -22,7 +22,7 @@ import {
   TextField,
   AffiliationsSuggestions,
 } from "react-invenio-forms";
-import { Button, Form, Header, Modal } from "semantic-ui-react";
+import { Button, Form, Modal } from "semantic-ui-react";
 import * as Yup from "yup";
 import { AffiliationsField } from "./../AffiliationsField";
 import { CreatibutorsIdentifiers } from "./CreatibutorsIdentifiers";
@@ -234,7 +234,7 @@ export class CreatibutorsModal extends Component {
     }
   };
 
-  SuggestionsWrapper = (creatibutors) => {
+  serializeAffiliations = (creatibutors) => {
     const { isOrganization, showPersonForm } = this.state;
     const { autocompleteNames } = this.props;
 
@@ -486,7 +486,7 @@ export class CreatibutorsModal extends Component {
                           // Disable UI-side filtering of search results
                           search={(options) => options}
                           suggestionAPIUrl="/api/names"
-                          serializeSuggestions={this.SuggestionsWrapper}
+                          serializeSuggestions={this.serializeAffiliations}
                           onValueChange={this.onPersonSearchChange}
                           ref={this.namesAutocompleteRef}
                         />
@@ -548,7 +548,7 @@ export class CreatibutorsModal extends Component {
                           // Disable UI-side filtering of search results
                           search={(options) => options}
                           suggestionAPIUrl="/api/affiliations"
-                          serializeSuggestions={this.SuggestionsWrapper}
+                          serializeSuggestions={this.serializeAffiliations}
                           onValueChange={this.onOrganizationSearchChange}
                         />
                       )}

From a40c52d80214d9a2cac19f21f3a8d4e56009b9c8 Mon Sep 17 00:00:00 2001
From: Carlin MacKenzie <carlin.mackenzie@gmail.com>
Date: Tue, 10 Sep 2024 14:30:03 +0200
Subject: [PATCH 71/76] refactor: remove showManualEntry argument

---
 .../deposit/fields/AffiliationsField/AffiliationsField.js | 8 +++-----
 .../deposit/fields/CreatibutorsField/CreatibutorsModal.js | 8 ++------
 2 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AffiliationsField/AffiliationsField.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AffiliationsField/AffiliationsField.js
index 9264068e1..d94de8231 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AffiliationsField/AffiliationsField.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AffiliationsField/AffiliationsField.js
@@ -18,10 +18,6 @@ import { i18next } from "@translations/invenio_rdm_records/i18next";
 
 /**Affiliation input component */
 export class AffiliationsField extends Component {
-  serializeAffiliations = (affiliations) => {
-    return AffiliationsSuggestions(affiliations, true, false);
-  };
-
   render() {
     const { fieldPath, selectRef } = this.props;
     return (
@@ -35,7 +31,9 @@ export class AffiliationsField extends Component {
                 Accept: "application/vnd.inveniordm.v1+json",
               }}
               initialSuggestions={getIn(values, fieldPath, [])}
-              serializeSuggestions={this.serializeAffiliations}
+              serializeSuggestions={(affiliations) => {
+                return AffiliationsSuggestions(affiliations, true);
+              }}
               placeholder={i18next.t("Search or create affiliation")}
               label={
                 <FieldLabel
diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
index a544343ed..3905ee4cb 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
@@ -235,13 +235,9 @@ export class CreatibutorsModal extends Component {
   };
 
   serializeAffiliations = (creatibutors) => {
-    const { isOrganization, showPersonForm } = this.state;
-    const { autocompleteNames } = this.props;
-
-    const showManualEntry =
-      autocompleteNames === NamesAutocompleteOptions.SEARCH_ONLY && !showPersonForm;
+    const { isOrganization } = this.state;
 
-    return AffiliationsSuggestions(creatibutors, isOrganization, showManualEntry);
+    return AffiliationsSuggestions(creatibutors, isOrganization);
   };
 
   updateIdentifiersAndAffiliations(

From d20c082feca29876013f635dbb306d2ecd73d123 Mon Sep 17 00:00:00 2001
From: Alex Ioannidis <a.ioannidis@cern.ch>
Date: Wed, 4 Sep 2024 16:45:07 +0200
Subject: [PATCH 72/76] user_moderation: use search for faster actions

* Use search results to determine the user's list of records.
* Use a TaskOp and Unit of Work to avoid sending Celery tasks immediately.
* Add a cleanup task that will perform a more thorough check using the
  DB to lookup the user's records.
---
 .../requests/user_moderation/actions.py       | 64 +++++++++----------
 .../requests/user_moderation/tasks.py         | 40 ++++++++++++
 .../requests/user_moderation/utils.py         | 48 ++++++++++++++
 .../services/communities/components.py        |  3 +-
 .../services/components/verified.py           |  4 +-
 .../requests/test_user_moderation_actions.py  |  6 +-
 6 files changed, 128 insertions(+), 37 deletions(-)
 create mode 100644 invenio_rdm_records/requests/user_moderation/utils.py

diff --git a/invenio_rdm_records/requests/user_moderation/actions.py b/invenio_rdm_records/requests/user_moderation/actions.py
index 43a1442c4..c443af72c 100644
--- a/invenio_rdm_records/requests/user_moderation/actions.py
+++ b/invenio_rdm_records/requests/user_moderation/actions.py
@@ -8,38 +8,18 @@
 """RDM user moderation action."""
 
 from invenio_access.permissions import system_identity
-from invenio_db import db
 from invenio_pidstore.errors import PIDDoesNotExistError
+from invenio_records_resources.services.uow import TaskOp
 from invenio_vocabularies.proxies import current_service
 
 from ...proxies import current_rdm_records_service
-from .tasks import delete_record, restore_record
-
-
-def _get_records_for_user(user_id):
-    """Helper function for getting all the records of the user.
-
-    Note: This function performs DB queries yielding all records for a given
-    user (which is not hard-limited in the system) and performs service calls
-    on each of them. Thus, this function has the potential of being a very
-    heavy operation, and should not be called as part of the handling of an
-    HTTP request!
-    """
-    record_cls = current_rdm_records_service.record_cls
-    model_cls = record_cls.model_cls
-    parent_cls = record_cls.parent_record_cls
-    parent_model_cls = parent_cls.model_cls
-
-    records = (
-        db.session.query(model_cls.json["id"].as_string())
-        .join(parent_model_cls)
-        .filter(
-            parent_model_cls.json["access"]["owned_by"]["user"].as_string()
-            == str(user_id),
-        )
-    ).yield_per(1000)
-
-    return records
+from .tasks import (
+    delete_record,
+    restore_record,
+    user_block_cleanup,
+    user_restore_cleanup,
+)
+from .utils import get_user_records
 
 
 def on_block(user_id, uow=None, **kwargs):
@@ -63,8 +43,18 @@ def on_block(user_id, uow=None, **kwargs):
         pass
 
     # soft-delete all the published records of that user
-    for (recid,) in _get_records_for_user(user_id):
-        delete_record.delay(recid, tombstone_data)
+    for recid in get_user_records(user_id):
+        uow.register(TaskOp(delete_record, recid=recid, tombstone_data=tombstone_data))
+
+    # Send cleanup task to make sure all records are deleted
+    uow.register(
+        TaskOp.for_async_apply(
+            user_block_cleanup,
+            kwargs=dict(user_id=user_id, tombstone_data=tombstone_data),
+            # wait for 10 minutes before starting the cleanup
+            countdown=10 * 60,
+        )
+    )
 
 
 def on_restore(user_id, uow=None, **kwargs):
@@ -77,8 +67,18 @@ def on_restore(user_id, uow=None, **kwargs):
     user_id = str(user_id)
 
     # restore all the deleted records of that user
-    for (recid,) in _get_records_for_user(user_id):
-        restore_record.delay(recid)
+    for recid in get_user_records(user_id):
+        uow.register(TaskOp(restore_record, recid=recid))
+
+    # Send cleanup task to make sure all records are restored
+    uow.register(
+        TaskOp.for_async_apply(
+            user_restore_cleanup,
+            kwargs=dict(user_id=user_id),
+            # wait for 10 minutes before starting the cleanup
+            countdown=10 * 60,
+        )
+    )
 
 
 def on_approve(user_id, uow=None, **kwargs):
diff --git a/invenio_rdm_records/requests/user_moderation/tasks.py b/invenio_rdm_records/requests/user_moderation/tasks.py
index a5c4fde7b..20921cdbb 100644
--- a/invenio_rdm_records/requests/user_moderation/tasks.py
+++ b/invenio_rdm_records/requests/user_moderation/tasks.py
@@ -9,10 +9,50 @@
 
 from celery import shared_task
 from invenio_access.permissions import system_identity
+from invenio_users_resources.records.api import UserAggregate
 
 from invenio_rdm_records.proxies import current_rdm_records_service
+from invenio_rdm_records.records.systemfields.deletion_status import (
+    RecordDeletionStatusEnum,
+)
 from invenio_rdm_records.services.errors import DeletionStatusException
 
+from .utils import get_user_records
+
+
+@shared_task(ignore_result=True)
+def user_block_cleanup(user_id, tombstone_data):
+    """User block action cleanup."""
+    user = UserAggregate.get_record(user_id)
+    # Bail out if the user is not blocked (i.e. we restored him before the task ran)
+    if not user.blocked:
+        return
+
+    for recid in get_user_records(
+        user_id,
+        from_db=True,
+        # Only fetch published records that might have not been deleted yet.
+        status=[RecordDeletionStatusEnum.PUBLISHED],
+    ):
+        delete_record.delay(recid, tombstone_data)
+
+
+@shared_task(ignore_result=True)
+def user_restore_cleanup(user_id):
+    """User restore action cleanup."""
+    user = UserAggregate.get_record(user_id)
+    # Bail out if the user is blocked (i.e. we blocked him before the task ran)
+    if user.blocked:
+        return
+
+    for recid in get_user_records(
+        user_id,
+        from_db=True,
+        # Only fetch deleted records that might have not been restored yet.
+        status=[RecordDeletionStatusEnum.DELETED],
+    ):
+        restore_record.delay(recid)
+
 
 @shared_task(ignore_result=True)
 def delete_record(recid, tombstone_data):
diff --git a/invenio_rdm_records/requests/user_moderation/utils.py b/invenio_rdm_records/requests/user_moderation/utils.py
new file mode 100644
index 000000000..319eb9ba6
--- /dev/null
+++ b/invenio_rdm_records/requests/user_moderation/utils.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+# Copyright (C) 2023 TU Wien.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""RDM user moderation utilities."""
+
+from invenio_db import db
+from invenio_search.api import RecordsSearchV2
+
+from ...proxies import current_rdm_records_service
+
+
+def get_user_records(user_id, from_db=False, status=None):
+    """Helper function for getting all the records of the user."""
+    record_cls = current_rdm_records_service.record_cls
+    model_cls = record_cls.model_cls
+    parent_cls = record_cls.parent_record_cls
+    parent_model_cls = parent_cls.model_cls
+
+    if from_db:
+        query = (
+            db.session.query(model_cls.json["id"].as_string())
+            .join(parent_model_cls)
+            .filter(
+                parent_model_cls.json["access"]["owned_by"]["user"].as_string()
+                == str(user_id),
+            )
+        )
+        if status:
+            query = query.filter(model_cls.deletion_status.in_(status))
+
+        return (row[0] for row in query.yield_per(1000))
+    else:
+        search = (
+            RecordsSearchV2(index=record_cls.index._name)
+            .filter("term", **{"parent.access.owned_by.user": user_id})
+            .source(["id"])
+        )
+        if status:
+            if not isinstance(status, (tuple, list)):
+                status = [status]
+            status = [s.value for s in status]
+            search = search.filter("terms", deletion_status=status)
+        return (hit["id"] for hit in search.scan())
diff --git a/invenio_rdm_records/services/communities/components.py b/invenio_rdm_records/services/communities/components.py
index d1e450ffc..0c9922258 100644
--- a/invenio_rdm_records/services/communities/components.py
+++ b/invenio_rdm_records/services/communities/components.py
@@ -30,6 +30,7 @@
     MetadataComponent,
     RelationsComponent,
 )
+from invenio_records_resources.services.uow import TaskOp
 from invenio_requests.tasks import request_moderation
 from invenio_search.engine import dsl
 
@@ -81,7 +82,7 @@ def create(self, identity, data=None, record=None, **kwargs):
 
             if not is_verified:
                 # Spawn a task to request moderation.
-                request_moderation.delay(identity.id)
+                self.uow.register(TaskOp(request_moderation, user_id=identity.id))
 
 
 CommunityServiceComponents = [
diff --git a/invenio_rdm_records/services/components/verified.py b/invenio_rdm_records/services/components/verified.py
index 34e848875..65e123bae 100644
--- a/invenio_rdm_records/services/components/verified.py
+++ b/invenio_rdm_records/services/components/verified.py
@@ -9,6 +9,7 @@
 from flask import current_app
 from invenio_access.permissions import system_identity
 from invenio_drafts_resources.services.records.components import ServiceComponent
+from invenio_records_resources.services.uow import TaskOp
 from invenio_requests.tasks import request_moderation
 
 
@@ -27,4 +28,5 @@ def publish(self, identity, draft=None, record=None):
 
             if not is_verified:
                 # Spawn a task to request moderation.
-                request_moderation.delay(record.parent.access.owner.owner_id)
+                owner_id = record.parent.access.owner.owner_id
+                self.uow.register(TaskOp(request_moderation, user_id=owner_id))
diff --git a/tests/requests/test_user_moderation_actions.py b/tests/requests/test_user_moderation_actions.py
index 984b0d85b..cb024fb0c 100644
--- a/tests/requests/test_user_moderation_actions.py
+++ b/tests/requests/test_user_moderation_actions.py
@@ -21,8 +21,8 @@
 class MockRequestModerationTask(Task):
     """Mock celery task for moderation request."""
 
-    def delay(*args):
-        user_id = args[0]
+    def apply_async(self, args=None, kwargs=None, **kwargs_):
+        user_id = kwargs["user_id"]
         with db.session.begin_nested():
             try:
                 current_user_moderation_service.request_moderation(
@@ -55,7 +55,7 @@ def test_user_moderation_approve(
     # This is a patch for tests only.
     mocker.patch(
         "invenio_rdm_records.services.components.verified.request_moderation",
-        MockRequestModerationTask,
+        MockRequestModerationTask(),
     )
     new_record = records_service.publish(
         identity=unverified_user.identity, id_=new_version.id

From 24527790a4a6d60d2df3645a4e5fe7ca2d54abb7 Mon Sep 17 00:00:00 2001
From: David Eckhard <dabombeeer@gmail.com>
Date: Fri, 4 Oct 2024 09:21:52 +0200
Subject: [PATCH 73/76] deposit: add missing fields to record deserializer

---
 .../src/deposit/api/DepositRecordSerializer.js                 | 3 +++
 .../src/deposit/api/DepositRecordSerializer.test.js            | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/api/DepositRecordSerializer.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/api/DepositRecordSerializer.js
index 48145f1cd..735441ee4 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/api/DepositRecordSerializer.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/api/DepositRecordSerializer.js
@@ -323,6 +323,9 @@ export class RDMDepositRecordSerializer extends DepositRecordSerializer {
       "pids",
       "ui",
       "custom_fields",
+      "created",
+      "updated",
+      "revision_id",
     ]);
 
     // FIXME: move logic in a more sophisticated PIDField that allows empty values
diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/api/DepositRecordSerializer.test.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/api/DepositRecordSerializer.test.js
index 879ca4d03..ac6d32f37 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/api/DepositRecordSerializer.test.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/api/DepositRecordSerializer.test.js
@@ -368,6 +368,7 @@ describe("RDMDepositRecordSerializer tests", () => {
           files: false,
           metadata: false,
         },
+        created: "2020-10-28 18:35:58.113520",
         expanded: {},
         id: "wk205-00878",
         links: {
@@ -496,9 +497,11 @@ describe("RDMDepositRecordSerializer tests", () => {
           ],
           version: "v2.0.0",
         },
+        revision_id: 1,
         ui: {
           publication_date_l10n: "Sep 28, 2020",
         },
+        updated: "2020-10-28 18:35:58.125222",
       };
       expect(deserializedRecord).toEqual(expectedRecord);
     });

From b5fbfd4cb48c5f92c0f38d3f837fef4ab033352f Mon Sep 17 00:00:00 2001
From: Alex Ioannidis <a.ioannidis@cern.ch>
Date: Mon, 4 Nov 2024 09:31:59 +0100
Subject: [PATCH 74/76] components: make content moderation configurable

- Closes #1861.
- Adds a new `RRM_CONTENT_MODERATION_HANDLERS` config variable to allow
  for configuring multiple handlers for the different write actions.
---
 invenio_rdm_records/config.py                 | 13 +++
 .../services/communities/components.py        | 27 +-----
 .../services/communities/moderation.py        | 85 +++++++++++++++++++
 .../services/components/verified.py           | 83 ++++++++++++++++--
 tests/requests/conftest.py                    |  7 ++
 5 files changed, 182 insertions(+), 33 deletions(-)
 create mode 100644 invenio_rdm_records/services/communities/moderation.py

diff --git a/invenio_rdm_records/config.py b/invenio_rdm_records/config.py
index 53d634f22..80ee5d106 100644
--- a/invenio_rdm_records/config.py
+++ b/invenio_rdm_records/config.py
@@ -15,6 +15,9 @@
 import idutils
 from invenio_i18n import lazy_gettext as _
 
+import invenio_rdm_records.services.communities.moderation as communities_moderation
+from invenio_rdm_records.services.components.verified import UserModerationHandler
+
 from . import tokens
 from .resources.serializers import DataCite43JSONSerializer
 from .services import facets
@@ -555,6 +558,16 @@ def make_doi(prefix, record):
    def lock_edit_published_files(service, identity, record=None, draft=None):
 """
 
+RDM_CONTENT_MODERATION_HANDLERS = [
+    UserModerationHandler(),
+]
+"""Records content moderation handlers."""
+
+RDM_COMMUNITY_CONTENT_MODERATION_HANDLERS = [
+    communities_moderation.UserModerationHandler(),
+]
+"""Community content moderation handlers."""
+
 # Feature flag to enable/disable user moderation
 RDM_USER_MODERATION_ENABLED = False
 """Flag to enable creation of user moderation requests on specific user actions."""
diff --git a/invenio_rdm_records/services/communities/components.py b/invenio_rdm_records/services/communities/components.py
index 0c9922258..8c2f4c5a9 100644
--- a/invenio_rdm_records/services/communities/components.py
+++ b/invenio_rdm_records/services/communities/components.py
@@ -7,8 +7,6 @@
 
 """Record communities service components."""
 
-from flask import current_app
-from invenio_access.permissions import system_identity
 from invenio_communities.communities.records.systemfields.access import VisibilityEnum
 from invenio_communities.communities.services.components import ChildrenComponent
 from invenio_communities.communities.services.components import (
@@ -24,18 +22,16 @@
     OwnershipComponent,
     PIDComponent,
 )
-from invenio_drafts_resources.services.records.components import ServiceComponent
 from invenio_i18n import lazy_gettext as _
 from invenio_records_resources.services.records.components import (
     MetadataComponent,
     RelationsComponent,
 )
-from invenio_records_resources.services.uow import TaskOp
-from invenio_requests.tasks import request_moderation
 from invenio_search.engine import dsl
 
 from ...proxies import current_community_records_service
 from ..errors import InvalidCommunityVisibility
+from .moderation import ContentModerationComponent
 
 
 class CommunityAccessComponent(BaseAccessComponent):
@@ -66,25 +62,6 @@ def update(self, identity, data=None, record=None, **kwargs):
             self._check_visibility(identity, record)
 
 
-class ContentModerationComponent(ServiceComponent):
-    """Service component for content moderation."""
-
-    def create(self, identity, data=None, record=None, **kwargs):
-        """Create a moderation request if the user is not verified."""
-        if current_app.config["RDM_USER_MODERATION_ENABLED"]:
-            # If the publisher is the system process, we don't want to create a moderation request.
-            # Even if the record being published is owned by a user that is not system
-            if identity == system_identity:
-                return
-
-            # resolve current user and check if they are verified
-            is_verified = identity.user.verified_at is not None
-
-            if not is_verified:
-                # Spawn a task to request moderation.
-                self.uow.register(TaskOp(request_moderation, user_id=identity.id))
-
-
 CommunityServiceComponents = [
     MetadataComponent,
     CommunityThemeComponent,
@@ -95,8 +72,8 @@ def create(self, identity, data=None, record=None, **kwargs):
     OwnershipComponent,
     FeaturedCommunityComponent,
     OAISetComponent,
-    ContentModerationComponent,
     CommunityDeletionComponent,
     ChildrenComponent,
     CommunityParentComponent,
+    ContentModerationComponent,
 ]
diff --git a/invenio_rdm_records/services/communities/moderation.py b/invenio_rdm_records/services/communities/moderation.py
new file mode 100644
index 000000000..52e7e4342
--- /dev/null
+++ b/invenio_rdm_records/services/communities/moderation.py
@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2024 CERN.
+#
+# Invenio-RDM-Records is free software; you can redistribute it and/or modify
+# it under the terms of the MIT License; see LICENSE file for more details.
+
+"""Content moderation for communities."""
+
+from flask import current_app
+from invenio_access.permissions import system_identity
+from invenio_records_resources.services.records.components import ServiceComponent
+from invenio_records_resources.services.uow import TaskOp
+from invenio_requests.tasks import request_moderation
+
+
+class BaseHandler:
+    """Base class for content moderation handlers."""
+
+    def create(self, identity, record=None, data=None, uow=None, **kwargs):
+        """Create handler."""
+        pass
+
+    def update(self, identity, record=None, data=None, uow=None, **kwargs):
+        """Update handler."""
+        pass
+
+    def delete(self, identity, data=None, record=None, uow=None, **kwargs):
+        """Delete handler."""
+        pass
+
+
+class UserModerationHandler(BaseHandler):
+    """Creates user moderation request if the user publishing is not verified."""
+
+    @property
+    def enabled(self):
+        """Check if user moderation is enabled."""
+        return current_app.config["RDM_USER_MODERATION_ENABLED"]
+
+    def run(self, identity, record=None, uow=None):
+        """Calculate the moderation score for a given record or draft."""
+        if self.enabled:
+            # If the publisher is the system process, we don't want to create a moderation request.
+            # Even if the record being published is owned by a user that is not system
+            if identity == system_identity:
+                return
+
+            # resolve current user and check if they are verified
+            is_verified = identity.user.verified_at is not None
+            if not is_verified:
+                # Spawn a task to request moderation.
+                self.uow.register(TaskOp(request_moderation, user_id=identity.id))
+
+    def create(self, identity, record=None, data=None, uow=None, **kwargs):
+        """Handle create."""
+        self.run(identity, record=record, uow=uow)
+
+    def update(self, identity, record=None, data=None, uow=None, **kwargs):
+        """Handle update."""
+        self.run(identity, record=record, uow=uow)
+
+
+class ContentModerationComponent(ServiceComponent):
+    """Service component for content moderation."""
+
+    def handler_for(action):
+        """Get the handlers for an action."""
+
+        def _handler_method(self, *args, **kwargs):
+            handlers = current_app.config.get(
+                "RDM_COMMUNITY_CONTENT_MODERATION_HANDLERS", []
+            )
+            for handler in handlers:
+                action_method = getattr(handler, action, None)
+                if action_method:
+                    action_method(*args, **kwargs, uow=self.uow)
+
+        return _handler_method
+
+    create = handler_for("create")
+    update = handler_for("update")
+    delete = handler_for("delete")
+
+    del handler_for
diff --git a/invenio_rdm_records/services/components/verified.py b/invenio_rdm_records/services/components/verified.py
index 65e123bae..ce7ae19eb 100644
--- a/invenio_rdm_records/services/components/verified.py
+++ b/invenio_rdm_records/services/components/verified.py
@@ -13,20 +13,87 @@
 from invenio_requests.tasks import request_moderation
 
 
-class ContentModerationComponent(ServiceComponent):
-    """Service component for content moderation."""
+class BaseHandler:
+    """Base class for content moderation handlers."""
+
+    def update_draft(
+        self, identity, data=None, record=None, errors=None, uow=None, **kwargs
+    ):
+        """Update draft handler."""
+        pass
+
+    def delete_draft(
+        self, identity, draft=None, record=None, force=False, uow=None, **kwargs
+    ):
+        """Delete draft handler."""
+        pass
+
+    def edit(self, identity, draft=None, record=None, uow=None, **kwargs):
+        """Edit a record handler."""
+        pass
+
+    def new_version(self, identity, draft=None, record=None, uow=None, **kwargs):
+        """New version handler."""
+        pass
+
+    def publish(self, identity, draft=None, record=None, uow=None, **kwargs):
+        """Publish handler."""
+        pass
+
+    def post_publish(
+        self, identity, record=None, is_published=False, uow=None, **kwargs
+    ):
+        """Post publish handler."""
+        pass
+
+
+class UserModerationHandler(BaseHandler):
+    """Creates user moderation request if the user publishing is not verified."""
+
+    @property
+    def enabled(self):
+        """Check if user moderation is enabled."""
+        return current_app.config["RDM_USER_MODERATION_ENABLED"]
 
-    def publish(self, identity, draft=None, record=None):
-        """Create a moderation request if the user is not verified."""
-        if current_app.config["RDM_USER_MODERATION_ENABLED"]:
+    def run(self, identity, record=None, uow=None):
+        """Calculate the moderation score for a given record or draft."""
+        if self.enabled:
             # If the publisher is the system process, we don't want to create a moderation request.
             # Even if the record being published is owned by a user that is not system
             if identity == system_identity:
                 return
 
             is_verified = record.parent.is_verified
-
             if not is_verified:
                 # Spawn a task to request moderation.
-                owner_id = record.parent.access.owner.owner_id
-                self.uow.register(TaskOp(request_moderation, user_id=owner_id))
+                user_id = record.parent.access.owner.owner_id
+                uow.register(TaskOp(request_moderation, user_id=user_id))
+
+    def publish(self, identity, draft=None, record=None, uow=None, **kwargs):
+        """Handle publish."""
+        self.run(identity, record=record, uow=uow)
+
+
+class ContentModerationComponent(ServiceComponent):
+    """Service component for content moderation."""
+
+    def handler_for(action):
+        """Get the handlers for an action."""
+
+        def _handler_method(self, *args, **kwargs):
+            handlers = current_app.config.get("RDM_CONTENT_MODERATION_HANDLERS", [])
+            for handler in handlers:
+                action_method = getattr(handler, action, None)
+                if action_method:
+                    action_method(*args, **kwargs, uow=self.uow)
+
+        return _handler_method
+
+    update_draft = handler_for("update_draft")
+    delete_draft = handler_for("delete_draft")
+    edit = handler_for("edit")
+    publish = handler_for("publish")
+    post_publish = handler_for("post_publish")
+    new_version = handler_for("new_version")
+
+    del handler_for
diff --git a/tests/requests/conftest.py b/tests/requests/conftest.py
index 6a1248a63..b6638f452 100644
--- a/tests/requests/conftest.py
+++ b/tests/requests/conftest.py
@@ -15,6 +15,8 @@
 import pytest
 from invenio_records_permissions.generators import AuthenticatedUser, SystemProcess
 
+import invenio_rdm_records.services.communities.moderation as communities_moderation
+from invenio_rdm_records.services.components.verified import UserModerationHandler
 from invenio_rdm_records.services.permissions import RDMRecordPermissionPolicy
 
 
@@ -35,4 +37,9 @@ def app_config(app_config):
     app_config["RDM_PERMISSION_POLICY"] = CustomRDMRecordPermissionPolicy
     # Enable user moderation
     app_config["RDM_USER_MODERATION_ENABLED"] = True
+    # Enable content moderation handlers
+    app_config["RDM_CONTENT_MODERATION_HANDLERS"] = [UserModerationHandler()]
+    app_config["RDM_COMMUNITY_CONTENT_MODERATION_HANDLERS"] = [
+        communities_moderation.UserModerationHandler(),
+    ]
     return app_config

From c707cad1676ea3bd4e52366891ed786a1f3aa805 Mon Sep 17 00:00:00 2001
From: Alex Ioannidis <a.ioannidis@cern.ch>
Date: Mon, 4 Nov 2024 09:48:47 +0100
Subject: [PATCH 75/76] resources: make record error handlers configurable

---
 invenio_rdm_records/resources/config.py | 159 ++++++++++++------------
 1 file changed, 82 insertions(+), 77 deletions(-)

diff --git a/invenio_rdm_records/resources/config.py b/invenio_rdm_records/resources/config.py
index 192ab61ad..c34623f4e 100644
--- a/invenio_rdm_records/resources/config.py
+++ b/invenio_rdm_records/resources/config.py
@@ -135,6 +135,84 @@ def _bibliography_headers(obj_or_list, code, many=False):
     "application/linkset+json": ResponseHandler(FAIRSignpostingProfileLvl2Serializer()),
 }
 
+error_handlers = {
+    **ErrorHandlersMixin.error_handlers,
+    DeserializerError: create_error_handler(
+        lambda exc: HTTPJSONException(
+            code=400,
+            description=exc.args[0],
+        )
+    ),
+    StyleNotFoundError: create_error_handler(
+        HTTPJSONException(
+            code=400,
+            description=_("Citation string style not found."),
+        )
+    ),
+    ReviewNotFoundError: create_error_handler(
+        HTTPJSONException(
+            code=404,
+            description=_("Review for draft not found"),
+        )
+    ),
+    ReviewStateError: create_error_handler(
+        lambda e: HTTPJSONException(
+            code=400,
+            description=str(e),
+        )
+    ),
+    ReviewExistsError: create_error_handler(
+        lambda e: HTTPJSONException(
+            code=400,
+            description=str(e),
+        )
+    ),
+    InvalidRelationValue: create_error_handler(
+        lambda exc: HTTPJSONException(
+            code=400,
+            description=exc.args[0],
+        )
+    ),
+    InvalidAccessRestrictions: create_error_handler(
+        lambda exc: HTTPJSONException(
+            code=400,
+            description=exc.args[0],
+        )
+    ),
+    ValidationErrorWithMessageAsList: create_error_handler(
+        lambda e: HTTPJSONValidationWithMessageAsListException(e)
+    ),
+    AccessRequestExistsError: create_error_handler(
+        lambda e: HTTPJSONException(
+            code=400,
+            description=e.description,
+        )
+    ),
+    RecordDeletedException: create_error_handler(
+        lambda e: (
+            HTTPJSONException(code=404, description=_("Record not found"))
+            if not e.record.tombstone.is_visible
+            else HTTPJSONException(
+                code=410,
+                description=_("Record deleted"),
+                tombstone=e.record.tombstone.dump(),
+            )
+        )
+    ),
+    RecordSubmissionClosedCommunityError: create_error_handler(
+        lambda e: HTTPJSONException(
+            code=403,
+            description=e.description,
+        )
+    ),
+    CommunityRequiredError: create_error_handler(
+        HTTPJSONException(
+            code=400,
+            description=_("Cannot publish without selecting a community."),
+        )
+    ),
+}
+
 
 #
 # Records and record versions
@@ -187,83 +265,10 @@ class RDMRecordResourceConfig(RecordResourceConfig, ConfiguratorMixin):
         default=record_serializers,
     )
 
-    error_handlers = {
-        **ErrorHandlersMixin.error_handlers,
-        DeserializerError: create_error_handler(
-            lambda exc: HTTPJSONException(
-                code=400,
-                description=exc.args[0],
-            )
-        ),
-        StyleNotFoundError: create_error_handler(
-            HTTPJSONException(
-                code=400,
-                description=_("Citation string style not found."),
-            )
-        ),
-        ReviewNotFoundError: create_error_handler(
-            HTTPJSONException(
-                code=404,
-                description=_("Review for draft not found"),
-            )
-        ),
-        ReviewStateError: create_error_handler(
-            lambda e: HTTPJSONException(
-                code=400,
-                description=str(e),
-            )
-        ),
-        ReviewExistsError: create_error_handler(
-            lambda e: HTTPJSONException(
-                code=400,
-                description=str(e),
-            )
-        ),
-        InvalidRelationValue: create_error_handler(
-            lambda exc: HTTPJSONException(
-                code=400,
-                description=exc.args[0],
-            )
-        ),
-        InvalidAccessRestrictions: create_error_handler(
-            lambda exc: HTTPJSONException(
-                code=400,
-                description=exc.args[0],
-            )
-        ),
-        ValidationErrorWithMessageAsList: create_error_handler(
-            lambda e: HTTPJSONValidationWithMessageAsListException(e)
-        ),
-        AccessRequestExistsError: create_error_handler(
-            lambda e: HTTPJSONException(
-                code=400,
-                description=e.description,
-            )
-        ),
-        RecordDeletedException: create_error_handler(
-            lambda e: (
-                HTTPJSONException(code=404, description=_("Record not found"))
-                if not e.record.tombstone.is_visible
-                else HTTPJSONException(
-                    code=410,
-                    description=_("Record deleted"),
-                    tombstone=e.record.tombstone.dump(),
-                )
-            )
-        ),
-        RecordSubmissionClosedCommunityError: create_error_handler(
-            lambda e: HTTPJSONException(
-                code=403,
-                description=e.description,
-            )
-        ),
-        CommunityRequiredError: create_error_handler(
-            HTTPJSONException(
-                code=400,
-                description=_("Cannot publish without selecting a community."),
-            )
-        ),
-    }
+    error_handlers = FromConfig(
+        "RDM_RECORDS_ERROR_HANDLERS",
+        default=error_handlers,
+    )
 
 
 #

From 2a4a7afc70a4033bacc379f3d4b56d7ee1cf4295 Mon Sep 17 00:00:00 2001
From: Alex Ioannidis <a.ioannidis@cern.ch>
Date: Mon, 4 Nov 2024 12:30:31 +0100
Subject: [PATCH 76/76] =?UTF-8?q?=F0=9F=93=A6=20release:=20v15.7.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 CHANGES.rst                                   | 22 +++++++++++++++++++
 invenio_rdm_records/__init__.py               |  2 +-
 .../deposit/api/DepositRecordSerializer.js    |  2 +-
 .../api/DepositRecordSerializer.test.js       |  2 +-
 .../AffiliationsField/AffiliationsField.js    |  2 +-
 .../CreatibutorsField/CreatibutorsModal.js    |  2 +-
 .../services/components/verified.py           |  2 +-
 invenio_rdm_records/views.py                  |  2 +-
 tests/requests/conftest.py                    |  2 +-
 9 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/CHANGES.rst b/CHANGES.rst
index fcc2cc46f..2c2a84bc1 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -12,6 +12,28 @@
 Changes
 =======
 
+Version v15.7.0 (released 2024-11-04)
+
+- resources: make record error handlers configurable
+    * Possible via the new `RDM_RECORDS_ERROR_HANDLERS` config variable.
+- components: make content moderation configurable
+    * Closes #1861.
+    * Adds a new `RRM_CONTENT_MODERATION_HANDLERS` config variable to allow
+      for configuring multiple handlers for the different write actions.
+- user_moderation: use search for faster actions
+    * Use search results to determine the user's list of records.
+    * Use a TaskOp and Unit of Work to avoid sending Celery tasks immediately.
+    * Add a cleanup task that will perform a more thorough check using the
+      DB to lookup the user's records.
+- deposit: add missing fields to record deserializer
+- UI/UX: add consistent suggestions display to affiliations
+- UI/UX: improve display of ROR information
+- collections: move records search into service
+- collections: added task to compute number of records for each collection
+- services: make file-service components configurable
+- access notification: provide correct draft preview link
+    * Closes inveniosoftware/invenio-app-rdm#2827
+
 Version v15.6.0 (released 2024-10-18)
 
 - community: added myCommunitiesEnabled prop to CommunitySelectionSearch
diff --git a/invenio_rdm_records/__init__.py b/invenio_rdm_records/__init__.py
index eb38213ac..c45ae601c 100644
--- a/invenio_rdm_records/__init__.py
+++ b/invenio_rdm_records/__init__.py
@@ -11,6 +11,6 @@
 
 from .ext import InvenioRDMRecords
 
-__version__ = "15.6.0"
+__version__ = "15.7.0"
 
 __all__ = ("__version__", "InvenioRDMRecords")
diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/api/DepositRecordSerializer.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/api/DepositRecordSerializer.js
index 735441ee4..0c0d59a19 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/api/DepositRecordSerializer.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/api/DepositRecordSerializer.js
@@ -1,5 +1,5 @@
 // This file is part of Invenio-RDM-Records
-// Copyright (C) 2020-2023 CERN.
+// Copyright (C) 2020-2024 CERN.
 // Copyright (C) 2020-2022 Northwestern University.
 //
 // Invenio-RDM-Records is free software; you can redistribute it and/or modify it
diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/api/DepositRecordSerializer.test.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/api/DepositRecordSerializer.test.js
index ac6d32f37..fea57a5e3 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/api/DepositRecordSerializer.test.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/api/DepositRecordSerializer.test.js
@@ -1,5 +1,5 @@
 // This file is part of Invenio-RDM-Records
-// Copyright (C) 2020-2023 CERN.
+// Copyright (C) 2020-2024 CERN.
 // Copyright (C) 2020-2022 Northwestern University.
 //
 // Invenio-RDM-Records is free software; you can redistribute it and/or modify it
diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AffiliationsField/AffiliationsField.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AffiliationsField/AffiliationsField.js
index d94de8231..eac305cc2 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AffiliationsField/AffiliationsField.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/AffiliationsField/AffiliationsField.js
@@ -1,5 +1,5 @@
 // This file is part of Invenio-RDM-Records
-// Copyright (C) 2020-2023 CERN.
+// Copyright (C) 2020-2024 CERN.
 // Copyright (C) 2020-2022 Northwestern University.
 // Copyright (C) 2021 Graz University of Technology.
 //
diff --git a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
index 3905ee4cb..0d3f896d5 100644
--- a/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
+++ b/invenio_rdm_records/assets/semantic-ui/js/invenio_rdm_records/src/deposit/fields/CreatibutorsField/CreatibutorsModal.js
@@ -1,5 +1,5 @@
 // This file is part of Invenio-RDM-Records
-// Copyright (C) 2020-2023 CERN.
+// Copyright (C) 2020-2024 CERN.
 // Copyright (C) 2020-2022 Northwestern University.
 // Copyright (C) 2021 Graz University of Technology.
 // Copyright (C) 2022 data-futures.org.
diff --git a/invenio_rdm_records/services/components/verified.py b/invenio_rdm_records/services/components/verified.py
index ce7ae19eb..7b700f992 100644
--- a/invenio_rdm_records/services/components/verified.py
+++ b/invenio_rdm_records/services/components/verified.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2023 CERN.
+# Copyright (C) 2023-2024 CERN.
 #
 # Invenio-RDM-records is free software; you can redistribute it and/or modify
 # it under the terms of the MIT License; see LICENSE file for more details.
diff --git a/invenio_rdm_records/views.py b/invenio_rdm_records/views.py
index a9e4ab7f9..0956c770a 100644
--- a/invenio_rdm_records/views.py
+++ b/invenio_rdm_records/views.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2020-2022 CERN.
+# Copyright (C) 2020-2024 CERN.
 # Copyright (C) 2021 TU Wien.
 # Copyright (C) 2022 Universität Hamburg.
 # Copyright (C) 2024 Graz University of Technology.
diff --git a/tests/requests/conftest.py b/tests/requests/conftest.py
index b6638f452..27d1d8d29 100644
--- a/tests/requests/conftest.py
+++ b/tests/requests/conftest.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2019-2021 CERN.
+# Copyright (C) 2019-2024 CERN.
 # Copyright (C) 2019-2021 Northwestern University.
 #
 # Invenio-RDM-Records is free software; you can redistribute it and/or modify