Changes for oarepo-vocabularies and related fields (#50)

* Working on changes for vocabularies:

1. library-mode properties - used when a library is built, ignored when a library is used
2. read-only and dump-only schema properties

* Extensions & changes for oarepo-vocabularies package

* different loading of included schemas, changes for vocabularies

* nr fixes

* vocabulary changes

* tests fixed, upgrade to importlib

* 1.0.0dev8

.gitignore

@@ -80,4 +80,7 @@ example/data
 test-model
 
 # Testing
-sample/
+sample/
+
+tests/test-sample-app
+tests/test-sample-site

docs/model.md

@@ -202,6 +202,9 @@ model:
       oarepo:use: ./common.yaml#person
 ```
 
+**Note:** If you reference multiple files via ``oarepo:use: ["a", "b"]`` and both
+include the same property, the value from ``a`` will be used.
+
 ## Built-in extensions
 
 ### `oarepo:mapping` - elasticsearch definition
@@ -253,6 +256,16 @@ This customizes how the marshmallow field will be generated. The section might c
   generate `from <import> import <classname> as <alias?`
 * `generate` - if the current element is a jsontype object ("type": "object"), tell the builder to generate this class
   as well (if not set it is expected that the class already exists in sources)
+* `read`, `write` - defaults to True. If set to false, make the attribute load-only or dump-only. If both are
+  set to false, do not generate the field. 
+
+#### Rules for object nested properties:  
+
+ * `generate` is not set to False -> generate a nested schema class, otherwise do not generate the class
+ * `field` is set -> use this field directly as is
+ * `class` is set -> generate/use class with this name. If not set, infer the class name from the context
+ * `read=false, write=false` - do not generate the field at all
+
 
 Example:
 

oarepo_model_builder/builtin_models/invenio.json

@@ -1,29 +1,46 @@
 {
   "model": {
+    "oarepo:marshmallow": {
+      "base-classes": [
+        "invenio_records_resources.services.records.schema.BaseRecordSchema"
+      ]
+    },
     "properties": {
       "id": {
         "type": "keyword",
         "oarepo:sample": {
           "skip": true
+        },
+        "oarepo:marshmallow": {
+          "read": false,
+          "write": false
         }
       },
       "created": {
         "type": "date",
         "oarepo:sample": {
           "skip": true
+        },
+        "oarepo:marshmallow": {
+          "write": false,
+          "read": true
         }
       },
       "updated": {
         "type": "date",
         "oarepo:sample": {
           "skip": true
+        },
+        "oarepo:marshmallow": {
+          "write": false,
+          "read": true
         }
       },
       "$schema": {
         "type": "keyword",
         "oarepo:marshmallow": {
-          "field_name": "_schema",
-          "field_args": "data_key='$schema'"
+          "read": false,
+          "write": false
         },
         "oarepo:sample": {
           "skip": true

oarepo_model_builder/entrypoints.py

@@ -1,10 +1,13 @@
 import sys
+from functools import reduce
+from importlib import import_module
 from pathlib import Path
 
-import pkg_resources
+import importlib.metadata
+import importlib.resources
 
 from oarepo_model_builder.builder import ModelBuilder
-from oarepo_model_builder.schema import ModelSchema
+from oarepo_model_builder.schema import ModelSchema, remove_star_keys
 from oarepo_model_builder.utils.hyphen_munch import HyphenMunch
 
 
@@ -31,40 +34,50 @@ def create_builder_from_entrypoints(**kwargs):
 
 
 def load_entry_points_dict(name):
-    return {ep.name: ep.load() for ep in pkg_resources.iter_entry_points(group=name)}
+    return {ep.name: ep.load() for ep in importlib.metadata.entry_points().select(group=name)}
 
 
 def load_entry_points_list(name):
-    ret = [(ep.name, ep.load()) for ep in pkg_resources.iter_entry_points(group=name)]
+    ret = [(ep.name, ep.load()) for ep in importlib.metadata.entry_points().select(group=name)]
     ret.sort()
     return [x[1] for x in ret]
 
 
-def load_model_from_entrypoint(ep: pkg_resources.EntryPoint):
+def load_model_from_entrypoint(ep: importlib.metadata.EntryPoint):
     def load(schema):
-        filename = ".".join(ep.attrs)
-        data = pkg_resources.resource_string(ep.module_name, filename)
-        return schema._load(filename, content=data)
+        try:
+            loaded_schema = ep.load()
+        except:
+            module = import_module(ep.module)
+            split_attr = ep.attr.split('.')
+            fn = f'{split_attr[-2]}.{split_attr[-1]}'
+            if len(split_attr) > 2:
+                fn = reduce(lambda x, y: Path(x) / Path(y), split_attr[:-2]) / fn
+            content = importlib.resources.open_text(module, fn, encoding='utf-8').read()
+            loaded_schema = schema._load(fn, content=content)
+
+        remove_star_keys(loaded_schema)
+        return loaded_schema
 
     return load
 
 
 def load_included_models_from_entry_points():
     ret = {}
-    for ep in pkg_resources.iter_entry_points(group="oarepo.models"):
+    for ep in importlib.metadata.entry_points().select(group="oarepo.models"):
         ret[ep.name] = load_model_from_entrypoint(ep)
     return ret
 
 
 def load_model(
-    model_filename,
-    package=None,
-    configs=(),
-    black=True,
-    isort=True,
-    sets=(),
-    model_content=None,
-    extra_included=None,
+        model_filename,
+        package=None,
+        configs=(),
+        black=True,
+        isort=True,
+        sets=(),
+        model_content=None,
+        extra_included=None,
 ):
     loaders = load_entry_points_dict("oarepo_model_builder.loaders")
     included_models = load_included_models_from_entry_points()
@@ -114,11 +127,11 @@ def check_plugin_packages(schema):
     unknown_packages = [rp for rp in required_packages if rp not in known_packages]
     if unknown_packages:
         if (
-            input(
-                f'Required packages {", ".join(unknown_packages)} are missing. '
-                f"Should I install them for you via pip install? (y/n) "
-            )
-            == "y"
+                input(
+                    f'Required packages {", ".join(unknown_packages)} are missing. '
+                    f"Should I install them for you via pip install? (y/n) "
+                )
+                == "y"
         ):
             if subprocess.call(["pip", "install", *unknown_packages]):
                 sys.exit(1)

oarepo_model_builder/invenio/invenio_record.py

@@ -1,7 +1,25 @@
 from .invenio_base import InvenioBaseClassPythonBuilder
+from ..builders import process
 
 
 class InvenioRecordBuilder(InvenioBaseClassPythonBuilder):
     TYPE = "invenio_record"
     class_config = "record-class"
     template = "record"
+
+    def begin(self, schema, settings):
+        super().begin(schema, settings)
+        self.relations = []
+
+    @process("/model/**", condition=lambda current, stack: stack.schema_valid)
+    def enter_model_element(self):
+        self.build_children()
+        data = self.stack.top.data
+        if isinstance(data, dict) and 'invenio:relation' in data:
+            self.relations.append(data['invenio:relation'])
+
+    def process_template(self, python_path, template, **extra_kwargs):
+        return super().process_template(python_path, template, **{
+            **extra_kwargs,
+            'invenio_relations': self.relations
+        })

oarepo_model_builder/invenio/invenio_record_schema.py

@@ -50,6 +50,12 @@ def enter_model_element(self):
 
         definition = None
         recurse = True
+        if isinstance(self.stack.top.data, dict):
+            definition = self.stack.top.data.get(OAREPO_MARSHMALLOW_PROPERTY, {})
+            generate_key = definition.get('read', True) or definition.get('write', True)
+            if not generate_key:
+                return
+
         if schema_element_type == "properties":
             parent = self.stack[-2].data
             definition = parent.get(OAREPO_MARSHMALLOW_PROPERTY, {})
@@ -58,12 +64,18 @@ def enter_model_element(self):
             if "nested" not in definition:
                 definition["nested"] = True
 
-            generate_schema_class = definition.get("generate", True)
+            generate_schema_class = definition.get('generate')
             schema_class = None  # to make pycharm happy
             schema_class_base_classes = None
+
             if generate_schema_class:
                 if "class" not in definition:
-                    definition["class"] = self.stack.top.key.title()
+                    for se in reversed(self.stack.stack):
+                        if se.schema_element_type == 'property':
+                            definition["class"] = se.key.title()
+                            break
+                    else:
+                        definition["class"] = self.stack.top.key.title()
             if "class" in definition:
                 schema_class = definition["class"]
                 if "." not in schema_class:
@@ -194,12 +206,18 @@ def create_field(field_type, options=(), validators=(), definition=None):
     validators = [*validators, *definition.get("validators", [])]
     nested = definition.get("nested", False)
     required = definition.get('required', False)
+    read = definition.get('read', True)
+    write = definition.get('write', True)
 
     list_nested = definition.get("list_nested", False)
     if validators:
         opts.append(f'validate=[{",".join(validators)}]')
     if required:
         opts.append(f'required=' + str(required))
+    if not read and write:
+        opts.append('load_only=True')
+    if not write and read:
+        opts.append('dump_only=True')
     kwargs = definition.get("field_args", "")
     if kwargs and opts:
         kwargs = ", " + kwargs
@@ -208,10 +226,20 @@ def create_field(field_type, options=(), validators=(), definition=None):
     else:
         ret = f"{field_type}()"
     if nested:
-        if opts or kwargs:
-            ret = f'ma_fields.Nested(lambda: {ret}, {", ".join(opts)}{kwargs})'
+        if ret.endswith('()'):
+            ret = ret[:-2]
+        else:
+            ret = f'lamda: {ret}'
+        if isinstance(nested, str):
+            if opts or kwargs:
+                ret = f'{nested}({ret}, {", ".join(opts)}{kwargs})'
+            else:
+                ret = f"{nested}({ret})"
         else:
-            ret = f"ma_fields.Nested(lambda: {ret})"
+            if opts or kwargs:
+                ret = f'ma_fields.Nested({ret}, {", ".join(opts)}{kwargs})'
+            else:
+                ret = f"ma_fields.Nested({ret})"
     if list_nested:
         if opts or kwargs:
             ret = f'ma_fields.List(ma_fields.Nested(lambda: {ret}, {", ".join(opts)}{kwargs}))'
@@ -240,10 +268,12 @@ def marshmallow_boolean_generator(data, definition, schema, imports):
     validators = []
     return create_field("ma_fields.Boolean", [], validators, definition)
 
+
 def marshmallow_raw_generator(data, definition, schema, imports):
     validators = []
     return create_field("ma_fields.Raw", [], validators, definition)
 
+
 def marshmallow_generic_number_generator(datatype, data, definition, schema, imports):
     validators = definition.get('validators', [])
     if validators != []:

oarepo_model_builder/invenio/templates/invenio_record.py.jinja2

@@ -1,5 +1,6 @@
-from invenio_records.systemfields import ConstantField
-from invenio_records_resources.records.systemfields import IndexField
+from invenio_records.systemfields import ConstantField, RelationsField
+from invenio_records_resources.records.systemfields import IndexField,
+
 from invenio_records_resources.records.systemfields.pid import PIDField, PIDFieldContext
 from invenio_pidstore.providers.recordid_v2 import RecordIdProviderV2
 from invenio_records_resources.records.api import Record as InvenioBaseRecord
@@ -10,16 +11,41 @@ from {{ b|package_name }} import {{ b|base_name }}
 from {{ python.record_metadata_class|package_name }} import {{ python.record_metadata_class|base_name }}
 from {{ python.record_dumper_class|package_name }} import {{ python.record_dumper_class|base_name }}
 
+{% for rel in invenio_relations %}
+{% for imp in rel.imports or [] %}
+import {{ imp }}
+{% endfor %}
+{% endfor %}
+{% if invenio_relations %}
+from invenio_records.dumpers.relations import RelationDumperExt
+{% endif %}
 
 class {{ python.record_class|base_name }}({% for b in python.record_bases %}{{ b|base_name }}, {% endfor %}InvenioBaseRecord):
     model_cls = {{ python.record_metadata_class|base_name }}
     schema = ConstantField("$schema", "{{ settings.schema_server }}{{ settings.schema_name }}")
     index = IndexField("{{ settings.index_name }}")
+{% if python.generate_record_pid_field %}
     pid = PIDField(
         create=True,
         provider=RecordIdProviderV2,
         context_cls = PIDFieldContext
     )
-    dumper_extensions = []
+{% endif %}
+    dumper_extensions = [
+        {%- for ext in python.record_dumper_extensions %}{{ ext }}{% if not loop.last %}, {% endif %}{% endfor -%}
+        {% if invenio_relations %}
+            RelationDumperExt("relations"),
+        {% endif %}
+    ]
     dumper = {{ python.record_dumper_class|base_name }}(extensions=dumper_extensions)
 
+    {% if invenio_relations %}
+    relations = RelationsField(
+        {% for rel in invenio_relations %}
+        {{ rel.name }}={{ rel.type }}(
+            {% for param in rel.params %}{{ param }},
+            {% endfor %}
+        ),
+        {% endfor %}
+    )
+    {% endif %}

oarepo_model_builder/invenio/templates/invenio_record_schema.py.jinja2

@@ -2,8 +2,8 @@ from invenio_records_resources.services.records.schema import BaseRecordSchema a
 from marshmallow import ValidationError
 from marshmallow import validates as ma_validates
 
-{% for b in python.record_schema_bases %}
-    from {{ b|package_name }} import {{ b|base_name }}
+{% for b in schema_bases %}
+from {{ b|package_name }} import {{ b|base_name }}
 {% endfor %}
 
 {% include "imports" %}

oarepo_model_builder/invenio/templates/invenio_record_search_options.py.jinja2

@@ -20,18 +20,11 @@ class {{ python.record_search_options_class|base_name }}({% for b in python.reco
 {% endfor %}
     }
     sort_options = {
-            "bestmatch": dict(
-                title=_('Best match'),
-                fields=['_score'],  # ES defaults to desc on `_score` field
-            ),
-            "newest": dict(
-                title=_('Newest'),
-                fields=['-created'],
-            ),
-            "oldest": dict(
-                title=_('Oldest'),
-                fields=['created'],
-            ),
+        {% if python.record_search_options_bases %}
+        **{{ python.record_search_options_bases[0]|base_name }}.sort_options,
+        {% else %}
+        **InvenioSearchOptions.sort_options,
+        {% endif %}
 {% for dict in sort_definition %}
 {% for key, value in dict.items()%}
     '{{ key }}': {{ value }},