Merge branch 'main' into cwa/close-195-minimize-soft-model

.github/utils/end2end_test.py

@@ -54,7 +54,7 @@ def main(oteapi_url: str) -> None:
     from otelib import OTEClient
     from pydantic import ValidationError
 
-    from oteapi_optimade.models import OPTIMADEResourceSession
+    from oteapi_optimade.models import OPTIMADEResourceResult
 
     client = OTEClient(oteapi_url)
 
@@ -66,24 +66,25 @@ def main(oteapi_url: str) -> None:
     }
 
     source = client.create_dataresource(
+        resourceType="OPTIMADE/structures",
         accessService="OPTIMADE",
         accessUrl=OPTIMADE_URL,
         configuration=config,
     )
 
-    session = source.get()
+    output = source.get()
 
     error_message = "Could not parse returned session as an OPTIMADEResourceStrategy."
 
     try:
-        session = OPTIMADEResourceSession(**json.loads(session))
+        output = OPTIMADEResourceResult(**json.loads(output))
     except ValidationError as exc_:
         raise RuntimeError(error_message) from exc_
 
-    assert session.optimade_resource_model == f"{Structure.__module__}:Structure"
-    assert len(session.optimade_resources) == 2
+    assert output.optimade_resource_model == f"{Structure.__module__}:Structure"
+    assert len(output.optimade_resources) == 2
 
-    for resource in tuple(session.optimade_resources):
+    for resource in tuple(output.optimade_resources):
         parsed_resource = Structure(resource)
         assert parsed_resource.id in ["mpf_1", "mpf_110"]
 
@@ -96,18 +97,18 @@ def main(oteapi_url: str) -> None:
     )
 
     pipeline = query >> source
-    session = pipeline.get()
+    output = pipeline.get()
 
     try:
-        # Should be an OPTIMADEResourceSession because `source` is last in the pipeline
-        session = OPTIMADEResourceSession(**json.loads(session))
+        # Should be an OPTIMADEResourceResult because `source` is last in the pipeline
+        output = OPTIMADEResourceResult(**json.loads(output))
     except ValidationError as exc_:
         raise RuntimeError(error_message) from exc_
 
-    assert session.optimade_resource_model == f"{Structure.__module__}:Structure"
-    assert len(session.optimade_resources) == 4
+    assert output.optimade_resource_model == f"{Structure.__module__}:Structure"
+    assert len(output.optimade_resources) == 4
 
-    for resource in tuple(session.optimade_resources):
+    for resource in tuple(output.optimade_resources):
         parsed_resource = Structure(resource)
         assert parsed_resource.id in [
             "mpf_1",
@@ -132,11 +133,11 @@ def main(oteapi_url: str) -> None:
     # Configuration
     PORT = os.getenv("OTEAPI_PORT", "8080")
     OTEAPI_SERVICE_URL = f"http://localhost:{PORT}"
-    OTEAPI_PREFIX = os.getenv("OTEAPI_prefix", "/api/v1")  # noqa: SIM112
+    OTEAPI_PREFIX = os.getenv("OTEAPI_PREFIX", "/api/v1")
     OPTIMADE_URL = f"http://{os.getenv('OPTIMADE_HOST', 'localhost')}:{os.getenv('OPTIMADE_PORT', '5000')}/"
-    if "OTEAPI_prefix" not in os.environ:
+    if "OTEAPI_PREFIX" not in os.environ:
         # Set environment variables
-        os.environ["OTEAPI_prefix"] = OTEAPI_PREFIX  # noqa: SIM112
+        os.environ["OTEAPI_PREFIX"] = OTEAPI_PREFIX
 
     try:
         _check_service_availability(service_url=OTEAPI_SERVICE_URL)

.github/utils/requirements_upload.txt

@@ -0,0 +1 @@
+entities-service[cli] @ git+https://github.com/SINTEF/[email protected]

.github/workflows/cd_upload_entities.yml

@@ -0,0 +1,65 @@
+name: CD - Upload Entities
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  update-public-entities:
+    name: Update public entities
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'SINTEF'
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -U setuptools wheel
+          pip install -r .github/utils/requirements_upload.txt
+
+      - name: Check 'entities-service' CLI version
+        run: entities-service --version
+
+      - name: Gather Entities
+        id: gather_entities
+        run: |
+          if [ "${{ github.event_name }}" == "push" ]; then
+            SHA_BEFORE="${{ github.event.before }}"
+          else
+            SHA_BEFORE="${{ github.event.pull_request.base.sha }}"
+          fi
+
+          git diff --name-only ${SHA_BEFORE} | grep -E '^oteapi_optimade/dlite/entities/.*\.ya?ml$' > entities.txt ||:
+
+          if [ -s entities.txt ]; then
+            echo "relevant_entities=true" >> $GITHUB_OUTPUT
+
+            echo "Relevant Entities:"
+            cat entities.txt
+          else
+            echo "relevant_entities=false" >> $GITHUB_OUTPUT
+
+            echo "No entities to validate (and upload)."
+            exit 0
+          fi
+
+      - name: Validate Entities
+        if: steps.gather_entities.outputs.relevant_entities == 'true'
+        run: cat entities.txt | entities-service validate --strict --format=yaml -
+
+      - name: Upload Entities
+        if: steps.gather_entities.outputs.relevant_entities == 'true' && github.event_name == 'push' && github.ref_name == 'main'
+        run: cat entities.txt | entities-service upload --auto-confirm --strict --format=yaml -
+        env:
+          ENTITIES_SERVICE_ACCESS_TOKEN: ${{ secrets.ENTITIES_SERVICE_ACCESS_TOKEN }}

.github/workflows/ci_tests.yml

@@ -47,7 +47,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
         os:
         - ["ubuntu-latest", "linux"]
         - ["windows-latest", "windows"]
@@ -90,7 +90,7 @@ jobs:
     env:
       OTEAPI_PORT: 8080
       OPTIMADE_PORT: 5000
-      OTEAPI_prefix: "/api/v1"
+      OTEAPI_PREFIX: "/api/v1"
 
     services:
       redis:
@@ -129,7 +129,7 @@ jobs:
           --env "OTEAPI_REDIS_TYPE=redis" \
           --env "OTEAPI_REDIS_HOST=localhost" \
           --env "OTEAPI_REDIS_PORT=6379" \
-          --env "OTEAPI_prefix=${OTEAPI_prefix}" \
+          --env "OTEAPI_PREFIX=${OTEAPI_PREFIX}" \
           --network "host" \
           --volume "${PWD}:/oteapi-optimade" \
           --entrypoint "" \
@@ -139,11 +139,7 @@ jobs:
         .github/utils/wait_for_it.sh localhost:${OTEAPI_PORT} -t 240
         sleep 5
       env:
-        # Use version 1.20240228.345 until
-        # https://github.com/SINTEF/oteapi-optimade/issues/213 has been resolved.
-        # See also
-        # https://github.com/EMMC-ASBL/oteapi-services/tree/8306d7212419764fb87e5cefdb5a869db9c68ef7?tab=readme-ov-file#open-translation-environment-ote-api
-        DOCKER_OTEAPI_VERSION: '1.20240228.345'
+        DOCKER_OTEAPI_VERSION: 'latest'
 
     - name: Run end-2-end tests
       run: python .github/utils/end2end_test.py

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ ci:
   autoupdate_branch: 'main'
   autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate'
   autoupdate_schedule: 'weekly'
-  skip: []
+  skip: [validate-entities]  # doesn't seem to work. Also, this is checked with dedicated CD workflow
   submodules: false
 
 # hooks
@@ -26,6 +26,7 @@ repos:
     - id: debug-statements
     - id: end-of-file-fixer
     - id: mixed-line-ending
+      exclude: ^CHANGELOG.md$
     - id: name-tests-test
       args: ["--pytest-test-first"]
     - id: trailing-whitespace
@@ -78,3 +79,13 @@ repos:
       - "--package-dir=oteapi_optimade"
       - "--full-docs-folder=models"
     - id: docs-landing-page
+
+  # entities-service can validate SOFT/DLite entities
+  # More information can be found in the repository README:
+  # https://github.com/SINTEF/entities-service?tab=readme-ov-file#readme
+  - repo: https://github.com/SINTEF/entities-service
+    rev: v0.7.0
+    hooks:
+    - id: validate-entities
+      additional_dependencies: [".[cli]"]
+      files: ^oteapi_optimade/dlite/entities/.*\.ya?ml$

compose.yaml

@@ -0,0 +1,44 @@
+services:
+  oteapi:
+    image: ghcr.io/emmc-asbl/oteapi:${DOCKER_OTEAPI_VERSION:-latest}
+    ports:
+      - "${OTEAPI_PORT:-8080}:8080"
+    environment:
+      OTEAPI_REDIS_TYPE: redis
+      OTEAPI_REDIS_HOST: redis
+      OTEAPI_REDIS_PORT: 6379
+      OTEAPI_PREFIX: "${OTEAPI_PREFIX:-/api/v1}"
+      PATH_TO_OTEAPI_CORE:
+      OTEAPI_PLUGIN_PACKAGES: "-v -e /oteapi-optimade"
+    depends_on:
+      - redis
+    networks:
+      - otenet
+    volumes:
+      - "${PATH_TO_OTEAPI_CORE:-/dev/null}:/oteapi-core"
+      - "${PWD}:/oteapi-optimade"
+    entrypoint: |
+      /bin/bash -c "if [ \"${PATH_TO_OTEAPI_CORE}\" != \"/dev/null\" ] && [ -n \"${PATH_TO_OTEAPI_CORE}\" ]; then \
+      pip install -U --force-reinstall -e /oteapi-core; fi && ./entrypoint.sh --reload --debug --log-level debug"
+    stop_grace_period: 1s
+
+  redis:
+    image: redis:latest
+    volumes:
+      - redis-persist:/data
+    networks:
+      - otenet
+
+  optimade:
+    image: ghcr.io/materials-consortia/optimade:develop
+    ports:
+      - "${OPTIMADE_PORT:-5000}:5000"
+    networks:
+      - otenet
+    stop_grace_period: 1s
+
+volumes:
+  redis-persist:
+
+networks:
+  otenet:

mkdocs.yml

@@ -93,6 +93,7 @@ plugins:
       allow_errors: false
       execute_ignore:
         - "**/examples/otelib.ipynb"
+        - "**/examples/dlite.ipynb"
       kernel_name: python3
 
 nav:

oteapi_optimade/dlite/entities/JSONAPIResourceLinks.yaml

@@ -1,7 +1,7 @@
 uri: http://onto-ns.com/meta/1.0/JSONAPIResourceLinks
 meta: http://onto-ns.com/meta/0.3/EntitySchema
 description: A Resource Links object.
-dimensions: []
+dimensions: {}
 properties:
   self:
     type: string

oteapi_optimade/dlite/entities/OPTIMADERelationships.yaml

@@ -1,7 +1,7 @@
 uri: http://onto-ns.com/meta/1.0/OPTIMADERelationships
 meta: http://onto-ns.com/meta/0.3/EntitySchema
 description: This model wraps the JSON API Relationships to include type-specific top level keys.
-dimensions: []
+dimensions: {}
 properties:
   references:
     type: ref

oteapi_optimade/dlite/entities/OPTIMADEStructure.yaml

@@ -1,14 +1,14 @@
-uri: http://onto-ns.com/meta/1.0/OPTIMADEStructure
+uri: http://onto-ns.com/meta/1.0.1/OPTIMADEStructure
 meta: http://onto-ns.com/meta/0.3/EntitySchema
 description: An OPTIMADE structure.
-dimensions: []
+dimensions: {}
 properties:
   type:
     type: string
     description: The name of the type of an entry. Must always be 'structures'.
   attributes:
     type: ref
-    $ref: http://onto-ns.com/meta/1.0/OPTIMADEStructureAttributes
+    $ref: http://onto-ns.com/meta/1.0.1/OPTIMADEStructureAttributes
     description: The attributes used to represent a structure, e.g. unit cell, atoms, positions.
   id:
     type: str

oteapi_optimade/dlite/entities/OPTIMADEStructureAttributes.yaml

@@ -1,4 +1,4 @@
-uri: http://onto-ns.com/meta/1.0/OPTIMADEStructureAttributes
+uri: http://onto-ns.com/meta/1.0.1/OPTIMADEStructureAttributes
 meta: http://onto-ns.com/meta/0.3/EntitySchema
 description: The attributes used to represent a structure, e.g. unit cell, atoms, positions.
 dimensions:
@@ -46,7 +46,7 @@ properties:
     description: Cartesian positions of each site in the structure. A site is usually used to describe positions of atoms; what atoms can be encountered at a given site is conveyed by the `species_at_sites` property, and the species themselves are described in the `species` property.
   species:
     type: ref
-    $ref: http://onto-ns.com/meta/1.0/OPTIMADEStructureSpecies
+    $ref: http://onto-ns.com/meta/1.0.1/OPTIMADEStructureSpecies
     shape: [nspecies]
     description: A list describing the species of the sites of this structure. Species can represent pure chemical elements, virtual-crystal atoms representing a statistical occupation of a given site by multiple chemical elements, and/or a location to which there are attached atoms, i.e., atoms whose precise location are unknown beyond that they are attached to that position (frequently used to indicate hydrogen atoms attached to another element, e.g., a carbon with three attached hydrogens might represent a methyl group, -CH3).
   species_at_sites:

oteapi_optimade/dlite/entities/OPTIMADEStructureSpecies.yaml

@@ -1,4 +1,4 @@
-uri: http://onto-ns.com/meta/1.0/OPTIMADEStructureSpecies
+uri: http://onto-ns.com/meta/1.0.1/OPTIMADEStructureSpecies
 meta: http://onto-ns.com/meta/0.3/EntitySchema
 description: Species can represent pure chemical elements, virtual-crystal atoms representing a statistical occupation of a given site by multiple chemical elements, and/or a location to which there are attached atoms, i.e., atoms whose precise location are unknown beyond that they are attached to that position (frequently used to indicate hydrogen atoms attached to another element, e.g., a carbon with three attached hydrogens might represent a methyl group, -CH3).
 dimensions: