Miroslavsimek/be 184 better builder import handling (#234)

Fixed EDTF
Automatic imports for class names & fields & extra code

.github/workflows/build.yaml

@@ -0,0 +1,79 @@
+name: Build and test
+
+on:
+  workflow_call:
+    inputs:
+      oarepo:
+        description: OARepo version (11, 12, ...)
+        required: true
+        default: 11
+        type: string
+
+env:
+  OAREPO_VERSION: ${{ github.event.inputs.oarepo }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ "3.9", "3.10" ]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          # This path is specific to Ubuntu
+          path: ~/.cache/pip
+          # Look to see if there is a cache hit for the corresponding requirements file
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+
+      - name: Configure sysctl limits
+        run: |
+          sudo swapoff -a
+          sudo sysctl -w vm.swappiness=1
+          sudo sysctl -w fs.file-max=262144
+          sudo sysctl -w vm.max_map_count=262144
+
+      - name: Runs Opensearch
+        uses: ankane/setup-opensearch@v1
+        with:
+          plugins: analysis-icu
+
+      - name: Start Redis
+        uses: supercharge/[email protected]
+        with:
+          redis-version: ${{ matrix.redis-version }}
+
+      - name: Run tests
+        run: |
+          ./run-tests.sh
+
+      - name: Build package to publish
+        run: |
+          .venv/bin/python setup.py sdist bdist_wheel
+
+      - name: Freeze packages
+        run: |
+          .venv/bin/pip freeze > requirements.txt
+          .venv-server/bin/pip freeze >>requirements.txt
+          .venv-tests/bin/pip freeze >>requirements.txt
+
+      - name: Archive production artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: dist
+          path: dist
+
+      - name: Archive production artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: requirements.txt
+          path: requirements.txt

.github/workflows/manual.yaml

@@ -0,0 +1,15 @@
+name: Dispatch
+
+on:
+  workflow_dispatch:
+    inputs:
+      oarepo:
+        description: OARepo version (11, 12, ...)
+        required: true
+        default: 11
+
+jobs:
+  build:
+    uses: ./.github/workflows/build.yaml
+    with:
+      oarepo: ${{ github.event.inputs.oarepo }}

.github/workflows/push.yaml

@@ -0,0 +1,36 @@
+name: Build, test and publish
+
+on: push
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  build:
+    uses: ./.github/workflows/build.yaml
+    with:
+      oarepo: 11
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Use built artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: dist
+          path: dist
+
+      - name: List files
+        run: |
+          ls -la
+          ls -la dist
+
+      - name: Publish package
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          skip_existing: true
+          user: __token__
+          password: ${{ secrets.PYPI_PASSWORD }}

.gitignore

@@ -90,3 +90,8 @@ schema-dump.json5
 schema-dump.json5
 vocabulary-postprocessed-schema.json
 .model_venv
+
+complex-model
+.venv*
+
+.token

oarepo_model_builder/builders/setup_cfg.py

@@ -34,11 +34,7 @@ def finish(self):
         output.setdefault("metadata", "authors", "")
 
         output.setdefault("options", "python", ">=3.9")
-        try:
-            ov = int(self.settings["oarepo-version"])
-            output.add_dependency("oarepo", f">={ov},<{ov+1}")
-        except ValueError:
-            output.add_dependency("oarepo", self.settings["oarepo-version"])
+
         output.add_dependency("oarepo-runtime", ">=1.0.0")
 
         output.setdefault("options", "packages", "find:")

oarepo_model_builder/builtin_models/invenio.json

@@ -1,20 +1,12 @@
 {
   "marshmallow": {
-    "base-classes": ["BaseRecordSchema"],
-      "imports": [
-        {
-          "import": "oarepo_runtime.marshmallow.BaseRecordSchema"
-        }
-      ]
+    "base-classes": ["oarepo_runtime.marshmallow.BaseRecordSchema"],
+      "imports": []
   },
   "ui": {
     "marshmallow": {
-      "base-classes": ["InvenioUISchema"],
-      "imports": [
-        {
-          "import": "oarepo_runtime.ui.marshmallow.InvenioUISchema"
-        }
-      ]
+      "base-classes": ["oarepo_runtime.services.schema.ui.InvenioUISchema"],
+      "imports": []
     }
   },
   "properties": {

oarepo_model_builder/datatypes/__init__.py

@@ -27,7 +27,7 @@
     NestedDataType,
     ObjectDataType,
 )
-from .datatypes import DataType, DataTypeComponent, Import, Section, datatypes  # noqa
+from .datatypes import DataType, DataTypeComponent, Section, datatypes  # noqa
 from .dates import (  # noqa
     DateDataType,
     DateTimeDataType,

oarepo_model_builder/datatypes/components/__init__.py

@@ -17,6 +17,7 @@
     AppModelComponent,
     BlueprintsModelComponent,
     DefaultsModelComponent,
+    EDTFIntervalDumperModelComponent,
     ExtResourceModelComponent,
     FacetsModelComponent,
     JSONSchemaModelComponent,
@@ -81,4 +82,5 @@
     FieldSortComponent,
     StringMarshmallowComponent,
     NumberMarshmallowComponent,
+    EDTFIntervalDumperModelComponent,
 ]

oarepo_model_builder/datatypes/components/enum.py

@@ -13,12 +13,12 @@ def process_marshmallow(self, datatype, section, **kwargs):
         enum = datatype.definition.get("enum")
         if enum:
             section.config.setdefault("validators", []).append(
-                f"ma_validate.OneOf({repr(enum)})"
+                f"{{{{marshmallow.validate.OneOf}}}}({repr(enum)})"
             )
 
     def process_ui(self, datatype, section, **kwargs):
         enum = datatype.definition.get("enum")
         if enum:
             section.config.setdefault("marshmallow", {}).setdefault(
                 "validators", []
-            ).append(f"ma_validate.OneOf({repr(enum)})")
+            ).append(f"{{{{marshmallow.validate.OneOf}}}}({repr(enum)})")

oarepo_model_builder/datatypes/components/facets/__init__.py

@@ -1,6 +1,12 @@
 import dataclasses
 from typing import Dict, List, Optional
 
+from oarepo_model_builder.utils.jinja import (
+    extract_extra_code_imports,
+    generate_extra_code,
+)
+from oarepo_model_builder.utils.python_name import PythonQualifiedName
+
 
 @dataclasses.dataclass
 class FacetDefinition:
@@ -20,10 +26,17 @@ def update(self, facet_section):
 
     def set_field(self, facet_section, arguments, field_class=None):
         field = facet_section.get("field")
+        extra_imports = None
         if field:
-            self.field = field
+            extra_imports = extract_extra_code_imports(field)
+            self.field = generate_extra_code(field)
         else:
             if not field_class:
                 field_class = facet_section.get("facet-class")
             if field_class:
-                self.field = f"{field_class}({', '.join(arguments)})"
+                field_class = PythonQualifiedName(field_class)
+                self.field = f"{field_class.local_name}({', '.join(arguments)})"
+                extra_imports = field_class.imports
+
+        if extra_imports:
+            self.imports = [*self.imports, *extra_imports]

oarepo_model_builder/datatypes/components/marshmallow/field.py

@@ -5,10 +5,18 @@
 from marshmallow import fields
 
 from oarepo_model_builder.datatypes import DataTypeComponent
-from oarepo_model_builder.utils.python_name import convert_name_to_python
+from oarepo_model_builder.utils.jinja import (
+    extract_extra_code_imports,
+    generate_extra_code,
+)
+from oarepo_model_builder.utils.python_name import (
+    Import,
+    PythonQualifiedName,
+    convert_name_to_python,
+)
 from oarepo_model_builder.validation.utils import ImportSchema, StrictSchema
 
-from ...datatypes import DataType, Import
+from ...datatypes import DataType
 from .graph import MarshmallowField
 
 
@@ -52,8 +60,11 @@ def _create_marshmallow_field(
             field_class = marshmallow.get("field-class")
             if not field_class:
                 return
+
+            field_class = PythonQualifiedName(field_class)
+            imports.extend(field_class.imports)
             field_decl = [
-                field_class,
+                field_class.local_name,
                 "(",
                 ", ".join(
                     self._marshmallow_field_arguments(
@@ -64,6 +75,11 @@ def _create_marshmallow_field(
             ]
             field = "".join(field_decl)
 
+            validators = marshmallow.get("validators", [])
+            if validators:
+                for validator in validators:
+                    imports.extend(extract_extra_code_imports(validator))
+
         fields.append(MarshmallowField(field_name, field, imports))
 
     def _marshmallow_field_arguments(
@@ -88,6 +104,7 @@ def _marshmallow_field_arguments(
 
         validators = marshmallow.get("validators", [])
         if validators:
+            validators = [generate_extra_code(x) for x in validators]
             arguments.append(f"validate=[{', '.join(validators)}]")
 
         return arguments