From e19b497fd8fb74c932e5356f2567cf8406d2c40a Mon Sep 17 00:00:00 2001 From: Alzpetka Date: Mon, 2 Oct 2023 15:50:55 +0200 Subject: [PATCH 1/7] field validation --- .../datatypes/components/facets/field.py | 1 + tests/test_facets.py | 38 +++++++++++++++++++ tests/test_search_options.py | 4 ++ 3 files changed, 43 insertions(+) diff --git a/oarepo_model_builder/datatypes/components/facets/field.py b/oarepo_model_builder/datatypes/components/facets/field.py index f2cc5029..ad5d6c61 100644 --- a/oarepo_model_builder/datatypes/components/facets/field.py +++ b/oarepo_model_builder/datatypes/components/facets/field.py @@ -23,6 +23,7 @@ class Meta: imports = fields.List(fields.Nested(ImportSchema), required=False) path = fields.String(required=False) keyword = fields.String(required=False) + facet_groups = fields.List(fields.String(), required=False) class RegularFacetsComponent(DataTypeComponent): diff --git a/tests/test_facets.py b/tests/test_facets.py index 91878fa7..e4a7b2c0 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -1122,3 +1122,41 @@ def test_customizations_field(): """, ) + +def test_facets_group(): + schema = load_model( + "test.yaml", + model_content={ + "record": { + "use": "invenio", + "module": {"qualified": "test"}, + "properties": { + "b": { + "type": "keyword", + "facets": { + "facet_groups" : ["curator"] + + }, + }, + "c": "fulltext", + }, + }, + }, + isort=False, + black=False, + autoflake=False, + ) + + filesystem = InMemoryFileSystem() + builder = create_builder_from_entrypoints(filesystem=filesystem) + builder.build(schema, "record", ["record"], "") + + data = ( + builder.filesystem.open( + os.path.join("test", "services", "records", "facets.py") + ) + .read() + .replace("'", '"') + ) + print(data) + diff --git a/tests/test_search_options.py b/tests/test_search_options.py index ac00170d..f598a13a 100644 --- a/tests/test_search_options.py +++ b/tests/test_search_options.py @@ -47,6 +47,10 @@ def test_sort(): os.path.join("test", "services", "records", "search.py") ).read() print(data) + data2 = builder.filesystem.open( + os.path.join("test", "services", "records", "facets.py") + ).read() + print(data2) assert re.sub(r"\s", "", data) == re.sub( r"\s", "", From ad61308348d5efb7f0cdacd8115d8f49f7c3fff9 Mon Sep 17 00:00:00 2001 From: Alzpetka Date: Tue, 3 Oct 2023 13:55:41 +0200 Subject: [PATCH 2/7] search groups --- .../datatypes/components/facets/__init__.py | 1 + .../datatypes/components/facets/field.py | 1 + .../invenio/invenio_record_search_options.py | 8 ++++++-- .../invenio_record_search_options.py.jinja2 | 15 ++++++++++----- tests/test_facets.py | 2 +- tests/test_search_options.py | 2 +- 6 files changed, 20 insertions(+), 9 deletions(-) diff --git a/oarepo_model_builder/datatypes/components/facets/__init__.py b/oarepo_model_builder/datatypes/components/facets/__init__.py index 8f155bcd..66d49498 100644 --- a/oarepo_model_builder/datatypes/components/facets/__init__.py +++ b/oarepo_model_builder/datatypes/components/facets/__init__.py @@ -8,6 +8,7 @@ class FacetDefinition: dot_path: str searchable: bool imports: List[Dict[str, str]] + facet_groups: List[str] field: Optional[str] = None def update(self, facet_section): diff --git a/oarepo_model_builder/datatypes/components/facets/field.py b/oarepo_model_builder/datatypes/components/facets/field.py index ad5d6c61..7b3b90d3 100644 --- a/oarepo_model_builder/datatypes/components/facets/field.py +++ b/oarepo_model_builder/datatypes/components/facets/field.py @@ -53,6 +53,7 @@ def process_facets(self, datatype, section, **__kwargs): dot_path=datatype.path, searchable=facet_section.get("searchable"), imports=facet_section.get("imports", []), + facet_groups= facet_section.get("facet_groups", ["default"]) ) # set the field on the definition diff --git a/oarepo_model_builder/invenio/invenio_record_search_options.py b/oarepo_model_builder/invenio/invenio_record_search_options.py index ec0bdacc..a4b771a4 100644 --- a/oarepo_model_builder/invenio/invenio_record_search_options.py +++ b/oarepo_model_builder/invenio/invenio_record_search_options.py @@ -12,14 +12,18 @@ class InvenioRecordSearchOptionsBuilder(InvenioBaseClassPythonBuilder): def finish(self, **extra_kwargs): facets: List[FacetDefinition] = get_distinct_facets(self.current_model) - + facet_groups = {} search_data = [] for f in facets: + for group in f.facet_groups: + if group not in facet_groups.keys(): + facet_groups[group] = {} + facet_groups[group][f.path] = "facets." + f.path search_data.append({f.path: "facets." + f.path}) if "sortable" in self.current_model.definition: sort_options = self.current_model.definition["sortable"] else: sort_options = {} - extra_kwargs["search_data"] = search_data + extra_kwargs["facet_groups"] = facet_groups extra_kwargs["sort_definition"] = sort_options super().finish(**extra_kwargs) diff --git a/oarepo_model_builder/invenio/templates/invenio_record_search_options.py.jinja2 b/oarepo_model_builder/invenio/templates/invenio_record_search_options.py.jinja2 index 0c29cc89..d50693e7 100644 --- a/oarepo_model_builder/invenio/templates/invenio_record_search_options.py.jinja2 +++ b/oarepo_model_builder/invenio/templates/invenio_record_search_options.py.jinja2 @@ -7,12 +7,17 @@ from . import facets class {{ vars.search_options| class_header }}: """{{ vars.record.class|base_name }} search options.""" + facet_groups ={ + {% for group in facet_groups %} + '{{ group }}': { + {% for dict in facet_groups[group].keys() %} + '{{ dict}}' : {{ facet_groups[group][dict] }}, + {% endfor %} + }, + {% endfor %} +} + facets = { -{% for dict in search_data %} -{% for key, value in dict.items()%} - '{{ key }}': {{ value }}, -{% endfor %} -{% endfor %} {% for base_class in vars.search_options.base_classes %} **getattr({{ base_class|base_name }}, 'facets', {}) {% endfor %} diff --git a/tests/test_facets.py b/tests/test_facets.py index e4a7b2c0..ef4263d2 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -1159,4 +1159,4 @@ def test_facets_group(): .replace("'", '"') ) print(data) - + diff --git a/tests/test_search_options.py b/tests/test_search_options.py index f598a13a..5ed66e6f 100644 --- a/tests/test_search_options.py +++ b/tests/test_search_options.py @@ -22,7 +22,7 @@ def test_sort(): }, "b": { "type": "keyword", - "facets": {"field": 'TermsFacet(field="cosi")'}, + "facets": {"field": 'TermsFacet(field="cosi")', "facet_groups": ["curator"]}, "sortable": {"key": "b_test", "order": "desc"}, }, "c": { From cb7e5e4bed23fbe456fd417e3aab96b8cc78eb39 Mon Sep 17 00:00:00 2001 From: Alzpetka Date: Tue, 3 Oct 2023 14:43:30 +0200 Subject: [PATCH 3/7] test + refactor --- .../datatypes/components/facets/field.py | 6 +- .../invenio_record_search_options.py.jinja2 | 2 +- tests/test_search_options.py | 79 ++++++++++++++++++- 3 files changed, 82 insertions(+), 5 deletions(-) diff --git a/oarepo_model_builder/datatypes/components/facets/field.py b/oarepo_model_builder/datatypes/components/facets/field.py index 7b3b90d3..9d417ee9 100644 --- a/oarepo_model_builder/datatypes/components/facets/field.py +++ b/oarepo_model_builder/datatypes/components/facets/field.py @@ -23,8 +23,8 @@ class Meta: imports = fields.List(fields.Nested(ImportSchema), required=False) path = fields.String(required=False) keyword = fields.String(required=False) - facet_groups = fields.List(fields.String(), required=False) - + facet_groups = fields.List(fields.String(), required=False, data_key="facet-groups", attribute="facet-groups") + facet = ma.fields.Bool(required=False) class RegularFacetsComponent(DataTypeComponent): eligible_datatypes = [] @@ -53,7 +53,7 @@ def process_facets(self, datatype, section, **__kwargs): dot_path=datatype.path, searchable=facet_section.get("searchable"), imports=facet_section.get("imports", []), - facet_groups= facet_section.get("facet_groups", ["default"]) + facet_groups= facet_section.get("facet-groups", ["default"]) ) # set the field on the definition diff --git a/oarepo_model_builder/invenio/templates/invenio_record_search_options.py.jinja2 b/oarepo_model_builder/invenio/templates/invenio_record_search_options.py.jinja2 index d50693e7..ea4e1970 100644 --- a/oarepo_model_builder/invenio/templates/invenio_record_search_options.py.jinja2 +++ b/oarepo_model_builder/invenio/templates/invenio_record_search_options.py.jinja2 @@ -15,7 +15,7 @@ class {{ vars.search_options| class_header }}: {% endfor %} }, {% endfor %} -} + } facets = { {% for base_class in vars.search_options.base_classes %} diff --git a/tests/test_search_options.py b/tests/test_search_options.py index 5ed66e6f..63bffef3 100644 --- a/tests/test_search_options.py +++ b/tests/test_search_options.py @@ -22,7 +22,7 @@ def test_sort(): }, "b": { "type": "keyword", - "facets": {"field": 'TermsFacet(field="cosi")', "facet_groups": ["curator"]}, + "facets": {"field": 'TermsFacet(field="cosi")', "facet-groups": ["curator"]}, "sortable": {"key": "b_test", "order": "desc"}, }, "c": { @@ -186,3 +186,80 @@ class TestSearchOptions(BaseSearchOptions): } """, ) + + +def test_facet_groups(): + schema = load_model( + DUMMY_YAML, + model_content={ + "record": { + "use": "invenio", + "module": {"qualified": "test"}, + "search-options": { + "base-classes": ["BaseSearchOptions"], + "imports": [{"import": "blah.BaseSearchOptions"}], + }, + "properties": { + "a": { + "type": "keyword", + "facets": { + "facet-groups": ["curator", "user"] + } + }, + "b": { + "type": "keyword", + "facets": { + "searchable": True, + "facet-groups": [] + } + }, + "c": { + "type": "keyword", + "facets": { + "searchable": True, + "facet": False + + }, + } + } + }, + }, + isort=False, + black=False, + autoflake=False, + ) + + filesystem = InMemoryFileSystem() + builder = create_builder_from_entrypoints(filesystem=filesystem) + + builder.build(schema, "record", ["record"], "") + + data = builder.filesystem.open( + os.path.join("test", "services", "records", "search.py") + ).read() + print(data) + data2 = builder.filesystem.open( + os.path.join("test", "services", "records", "facets.py") + ).read() + print(data2) + assert strip_whitespaces(data) == strip_whitespaces( + """ +from blah import BaseSearchOptions +from flask_babelex import lazy_gettext as _ +from . import facets + +class TestSearchOptions(BaseSearchOptions): + \"""TestRecord search options.\""" + + facets = { + '_schema': facets._schema, + 'created': facets.created, + '_id': facets._id, + 'updated': facets.updated, + **getattr(BaseSearchOptions, 'facets', {}) + } + sort_options = { + **BaseSearchOptions.sort_options, + } + """, + ) \ No newline at end of file From 004b729b6ac21ace0b980225e335029e241320f2 Mon Sep 17 00:00:00 2001 From: Alzpetka Date: Tue, 3 Oct 2023 15:04:53 +0200 Subject: [PATCH 4/7] facet option --- oarepo_model_builder/datatypes/components/facets/__init__.py | 1 + oarepo_model_builder/datatypes/components/facets/field.py | 1 + oarepo_model_builder/datatypes/components/model/facets.py | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/oarepo_model_builder/datatypes/components/facets/__init__.py b/oarepo_model_builder/datatypes/components/facets/__init__.py index 66d49498..21989777 100644 --- a/oarepo_model_builder/datatypes/components/facets/__init__.py +++ b/oarepo_model_builder/datatypes/components/facets/__init__.py @@ -9,6 +9,7 @@ class FacetDefinition: searchable: bool imports: List[Dict[str, str]] facet_groups: List[str] + facet: Optional[bool] field: Optional[str] = None def update(self, facet_section): diff --git a/oarepo_model_builder/datatypes/components/facets/field.py b/oarepo_model_builder/datatypes/components/facets/field.py index 9d417ee9..1998d1bd 100644 --- a/oarepo_model_builder/datatypes/components/facets/field.py +++ b/oarepo_model_builder/datatypes/components/facets/field.py @@ -53,6 +53,7 @@ def process_facets(self, datatype, section, **__kwargs): dot_path=datatype.path, searchable=facet_section.get("searchable"), imports=facet_section.get("imports", []), + facet=facet_section.get("facet", None), facet_groups= facet_section.get("facet-groups", ["default"]) ) diff --git a/oarepo_model_builder/datatypes/components/model/facets.py b/oarepo_model_builder/datatypes/components/model/facets.py index 64f30214..b061e6e4 100644 --- a/oarepo_model_builder/datatypes/components/model/facets.py +++ b/oarepo_model_builder/datatypes/components/model/facets.py @@ -53,7 +53,7 @@ def build_facet_definition( ): if facet_definition.searchable is None: facet_definition.searchable = datatype.definition.get("searchable", True) - if facet_definition.searchable is not False: + if facet_definition.searchable is not False and facet_definition.facet is not False: return [facet_definition] else: # facet will not be generated From 4fbac2b8948ddb2ebc47a1c1729eda1bbed96937 Mon Sep 17 00:00:00 2001 From: Alzpetka Date: Tue, 3 Oct 2023 15:25:34 +0200 Subject: [PATCH 5/7] facets --- tests/test_facets.py | 29 ++++++++++++++++++++++++- tests/test_search_options.py | 41 +++++++++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/tests/test_facets.py b/tests/test_facets.py index ef4263d2..6e895cfa 100644 --- a/tests/test_facets.py +++ b/tests/test_facets.py @@ -1134,7 +1134,7 @@ def test_facets_group(): "b": { "type": "keyword", "facets": { - "facet_groups" : ["curator"] + "facet-groups" : ["curator"] }, }, @@ -1159,4 +1159,31 @@ def test_facets_group(): .replace("'", '"') ) print(data) + assert re.sub(r"\s", "", data) == re.sub( + r"\s", + "", + """ +\"""Facet definitions.\""" + +from invenio_search.engine import dsl +from flask_babelex import lazy_gettext as _ + +from invenio_records_resources.services.records.facets import TermsFacet +from oarepo_runtime.facets.date import DateTimeFacet + + +_schema = TermsFacet(field="$schema", label =_("$schema.label")) + +b = TermsFacet(field="b", label =_("b.label")) + +created = DateTimeFacet(field="created", label =_("created.label")) + +_id = TermsFacet(field="id", label =_("id.label")) + +updated = DateTimeFacet(field="updated", label =_("updated.label")) + + + + """, + ) diff --git a/tests/test_search_options.py b/tests/test_search_options.py index 63bffef3..9d70e333 100644 --- a/tests/test_search_options.py +++ b/tests/test_search_options.py @@ -220,7 +220,40 @@ def test_facet_groups(): "facet": False }, - } + }, + "d": { + "type" : "keyword", + "facets": { + "searchable": False + } + }, + "g": { + "type": "array", + + "items": { + "type": "keyword", + "facets": { + "facet-groups": ["curator"] + } + } + + }, + "arr": { + "type": "array", + "facets": {"searchable": True}, + "items": { + "type": "nested", + "properties": { + "d": {"type": "keyword", "facets" : {"facet": False}}, + "e": { + "type": "object", + "properties": { + "f": "keyword", + }, + }, + }, + }, + }, } }, }, @@ -242,6 +275,12 @@ def test_facet_groups(): os.path.join("test", "services", "records", "facets.py") ).read() print(data2) + data3 = builder.filesystem.read( + os.path.join("test", "records", "mappings", "os-v2", "test", "test-1.0.0.json") + ) + import json + data3 = json.loads(data3) + print(data3) assert strip_whitespaces(data) == strip_whitespaces( """ from blah import BaseSearchOptions From 7910c5eccdd14d79cd02b1396da2794b4f54d80f Mon Sep 17 00:00:00 2001 From: Alzpetka Date: Wed, 4 Oct 2023 13:09:55 +0200 Subject: [PATCH 6/7] default facet group --- .../invenio/invenio_record_search_options.py | 11 ++++++++--- .../templates/invenio_record_search_options.py.jinja2 | 9 +++++++++ setup.cfg | 2 +- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/oarepo_model_builder/invenio/invenio_record_search_options.py b/oarepo_model_builder/invenio/invenio_record_search_options.py index a4b771a4..61a0b836 100644 --- a/oarepo_model_builder/invenio/invenio_record_search_options.py +++ b/oarepo_model_builder/invenio/invenio_record_search_options.py @@ -13,17 +13,22 @@ class InvenioRecordSearchOptionsBuilder(InvenioBaseClassPythonBuilder): def finish(self, **extra_kwargs): facets: List[FacetDefinition] = get_distinct_facets(self.current_model) facet_groups = {} + default_group = [] search_data = [] for f in facets: for group in f.facet_groups: - if group not in facet_groups.keys(): - facet_groups[group] = {} - facet_groups[group][f.path] = "facets." + f.path + if group != 'default': + if group not in facet_groups.keys(): + facet_groups[group] = {} + facet_groups[group][f.path] = "facets." + f.path + if group == 'default': + default_group.append({f.path: "facets." + f.path}) search_data.append({f.path: "facets." + f.path}) if "sortable" in self.current_model.definition: sort_options = self.current_model.definition["sortable"] else: sort_options = {} extra_kwargs["facet_groups"] = facet_groups + extra_kwargs["default_group"] = default_group extra_kwargs["sort_definition"] = sort_options super().finish(**extra_kwargs) diff --git a/oarepo_model_builder/invenio/templates/invenio_record_search_options.py.jinja2 b/oarepo_model_builder/invenio/templates/invenio_record_search_options.py.jinja2 index ea4e1970..f3fa2021 100644 --- a/oarepo_model_builder/invenio/templates/invenio_record_search_options.py.jinja2 +++ b/oarepo_model_builder/invenio/templates/invenio_record_search_options.py.jinja2 @@ -12,12 +12,21 @@ class {{ vars.search_options| class_header }}: '{{ group }}': { {% for dict in facet_groups[group].keys() %} '{{ dict}}' : {{ facet_groups[group][dict] }}, + {% endfor %} + {% for base_class in vars.search_options.base_classes %} + **getattr(({{ base_class|base_name }}, 'facet_groups', {}).get('{{group }}', {})) + {% endfor %} }, {% endfor %} } facets = { +{% for dict in default_group %} +{% for key, value in dict.items()%} + '{{ key }}': {{ value }}, +{% endfor %} +{% endfor %} {% for base_class in vars.search_options.base_classes %} **getattr({{ base_class|base_name }}, 'facets', {}) {% endfor %} diff --git a/setup.cfg b/setup.cfg index 9c2a8a41..4d4fe819 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = oarepo-model-builder -version = 4.0.50 +version = 4.0.51 description = A utility library that generates OARepo required data model files from a JSON specification file authors = Miroslav Bauer , Miroslav Simek readme = README.md From f57948c984e56086a8d2ea5f49088a3310cacb25 Mon Sep 17 00:00:00 2001 From: Alzpetka Date: Wed, 4 Oct 2023 13:21:39 +0200 Subject: [PATCH 7/7] tests --- tests/test_search_options.py | 210 ++++++++++++++++++++++++++++++----- 1 file changed, 183 insertions(+), 27 deletions(-) diff --git a/tests/test_search_options.py b/tests/test_search_options.py index 9d70e333..264e5db5 100644 --- a/tests/test_search_options.py +++ b/tests/test_search_options.py @@ -46,11 +46,6 @@ def test_sort(): data = builder.filesystem.open( os.path.join("test", "services", "records", "search.py") ).read() - print(data) - data2 = builder.filesystem.open( - os.path.join("test", "services", "records", "facets.py") - ).read() - print(data2) assert re.sub(r"\s", "", data) == re.sub( r"\s", "", @@ -59,24 +54,74 @@ def test_sort(): from flask_babelex import lazy_gettext as _ from . import facets + class TestSearchOptions(InvenioSearchOptions): \"""TestRecord search options.\""" + facet_groups ={ + + 'curator': { + + 'b' : facets.b, + + + + **getattr((InvenioSearchOptions, 'facet_groups', {}).get('curator', {})) + + }, + + } + facets = { + + '_schema': facets._schema, + + + 'a': facets.a, - 'b': facets.b, + + + 'c_d': facets.c_d, + + + 'created': facets.created, + + + '_id': facets._id, + + + 'updated': facets.updated, + + + **getattr(InvenioSearchOptions, 'facets', {}) + } sort_options = { - **InvenioSearchOptions.sort_options, + + + **InvenioSearchOptions.sort_options, + + + + + 'a_test': {'fields': ['a']}, + + + 'b_test': {'fields': ['-b']}, + + + 'c_d': {'fields': ['-c.d']}, + + } """, ) @@ -111,6 +156,7 @@ def test_search_class(): data = builder.filesystem.open( os.path.join("test", "services", "records", "search.py") ).read() + print(data) assert strip_whitespaces(data) == strip_whitespaces( """ from invenio_records_resources.services import SearchOptions as InvenioSearchOptions @@ -121,17 +167,48 @@ def test_search_class(): class TestSearchOptions(InvenioSearchOptions): \"""TestRecord search options.\""" + facet_groups ={ + + } + facets = { - '_schema': facets._schema, - 'a': facets.a, - 'b': facets.b, - 'created': facets.created, - '_id': facets._id, - 'updated': facets.updated, - **getattr(InvenioSearchOptions,'facets',{}) + + + '_schema': facets._schema, + + + + 'a': facets.a, + + + + 'b': facets.b, + + + + 'created': facets.created, + + + + '_id': facets._id, + + + + 'updated': facets.updated, + + + + **getattr(InvenioSearchOptions, 'facets', {}) + } sort_options = { - **InvenioSearchOptions.sort_options, + + + **InvenioSearchOptions.sort_options, + + + + } """, ) @@ -171,18 +248,44 @@ def test_search_options_base_class(): from flask_babelex import lazy_gettext as _ from . import facets + class TestSearchOptions(BaseSearchOptions): \"""TestRecord search options.\""" + facet_groups ={ + + } + facets = { - '_schema': facets._schema, - 'created': facets.created, - '_id': facets._id, - 'updated': facets.updated, - **getattr(BaseSearchOptions, 'facets', {}) + + + '_schema': facets._schema, + + + + 'created': facets.created, + + + + '_id': facets._id, + + + + 'updated': facets.updated, + + + + **getattr(BaseSearchOptions, 'facets', {}) + } sort_options = { - **BaseSearchOptions.sort_options, + + + **BaseSearchOptions.sort_options, + + + + } """, ) @@ -287,18 +390,71 @@ def test_facet_groups(): from flask_babelex import lazy_gettext as _ from . import facets + class TestSearchOptions(BaseSearchOptions): \"""TestRecord search options.\""" + facet_groups ={ + + 'curator': { + + 'a' : facets.a, + + + 'g' : facets.g, + + + + **getattr((BaseSearchOptions, 'facet_groups', {}).get('curator', {})) + + }, + + 'user': { + + 'a' : facets.a, + + + + **getattr((BaseSearchOptions, 'facet_groups', {}).get('user', {})) + + }, + + } + facets = { - '_schema': facets._schema, - 'created': facets.created, - '_id': facets._id, - 'updated': facets.updated, - **getattr(BaseSearchOptions, 'facets', {}) + + + '_schema': facets._schema, + + + + 'arr_e_f': facets.arr_e_f, + + + + 'created': facets.created, + + + + '_id': facets._id, + + + + 'updated': facets.updated, + + + + **getattr(BaseSearchOptions, 'facets', {}) + } sort_options = { - **BaseSearchOptions.sort_options, + + + **BaseSearchOptions.sort_options, + + + + } """, ) \ No newline at end of file