Merge branch 'main' into cwa/close-153-semantic-mapping-to-cif-ontology

.github/dependabot.yml

@@ -5,7 +5,24 @@ updates:
   schedule:
     interval: daily
     time: "05:24"
-  target-branch: ci/dependency-updates
+  target-branch: main
   labels:
     - CI/CD
     - skip_changelog
+- package-ecosystem: pip
+  directory: "/"
+  schedule:
+    interval: weekly
+    day: monday
+    time: "05:18"
+  target-branch: main
+  labels:
+    - dependencies
+    - skip_changelog
+  groups:
+    packages:
+      applies-to: version-updates
+      patterns: ["*"]
+    security:
+      applies-to: security-updates
+      patterns: ["*"]

.github/utils/end2end_test.py

@@ -54,7 +54,7 @@ def main(oteapi_url: str) -> None:
     from otelib import OTEClient
     from pydantic import ValidationError
 
-    from oteapi_optimade.models import OPTIMADEResourceSession
+    from oteapi_optimade.models import OPTIMADEResourceResult
 
     client = OTEClient(oteapi_url)
 
@@ -66,24 +66,25 @@ def main(oteapi_url: str) -> None:
     }
 
     source = client.create_dataresource(
+        resourceType="OPTIMADE/structures",
         accessService="OPTIMADE",
         accessUrl=OPTIMADE_URL,
         configuration=config,
     )
 
-    session = source.get()
+    output = source.get()
 
     error_message = "Could not parse returned session as an OPTIMADEResourceStrategy."
 
     try:
-        session = OPTIMADEResourceSession(**json.loads(session))
+        output = OPTIMADEResourceResult(**json.loads(output))
     except ValidationError as exc_:
         raise RuntimeError(error_message) from exc_
 
-    assert session.optimade_resource_model == f"{Structure.__module__}:Structure"
-    assert len(session.optimade_resources) == 2
+    assert output.optimade_resource_model == f"{Structure.__module__}:Structure"
+    assert len(output.optimade_resources) == 2
 
-    for resource in tuple(session.optimade_resources):
+    for resource in tuple(output.optimade_resources):
         parsed_resource = Structure(resource)
         assert parsed_resource.id in ["mpf_1", "mpf_110"]
 
@@ -96,18 +97,18 @@ def main(oteapi_url: str) -> None:
     )
 
     pipeline = query >> source
-    session = pipeline.get()
+    output = pipeline.get()
 
     try:
-        # Should be an OPTIMADEResourceSession because `source` is last in the pipeline
-        session = OPTIMADEResourceSession(**json.loads(session))
+        # Should be an OPTIMADEResourceResult because `source` is last in the pipeline
+        output = OPTIMADEResourceResult(**json.loads(output))
     except ValidationError as exc_:
         raise RuntimeError(error_message) from exc_
 
-    assert session.optimade_resource_model == f"{Structure.__module__}:Structure"
-    assert len(session.optimade_resources) == 4
+    assert output.optimade_resource_model == f"{Structure.__module__}:Structure"
+    assert len(output.optimade_resources) == 4
 
-    for resource in tuple(session.optimade_resources):
+    for resource in tuple(output.optimade_resources):
         parsed_resource = Structure(resource)
         assert parsed_resource.id in [
             "mpf_1",
@@ -132,11 +133,11 @@ def main(oteapi_url: str) -> None:
     # Configuration
     PORT = os.getenv("OTEAPI_PORT", "8080")
     OTEAPI_SERVICE_URL = f"http://localhost:{PORT}"
-    OTEAPI_PREFIX = os.getenv("OTEAPI_prefix", "/api/v1")  # noqa: SIM112
+    OTEAPI_PREFIX = os.getenv("OTEAPI_PREFIX", "/api/v1")
     OPTIMADE_URL = f"http://{os.getenv('OPTIMADE_HOST', 'localhost')}:{os.getenv('OPTIMADE_PORT', '5000')}/"
-    if "OTEAPI_prefix" not in os.environ:
+    if "OTEAPI_PREFIX" not in os.environ:
         # Set environment variables
-        os.environ["OTEAPI_prefix"] = OTEAPI_PREFIX  # noqa: SIM112
+        os.environ["OTEAPI_PREFIX"] = OTEAPI_PREFIX
 
     try:
         _check_service_availability(service_url=OTEAPI_SERVICE_URL)

.github/utils/requirements_upload.txt

@@ -0,0 +1 @@
+entities-service[cli] @ git+https://github.com/SINTEF/[email protected]

.github/workflows/cd_release.yml

@@ -6,9 +6,9 @@ on:
     - published
 
 jobs:
-  publish-package-and-docs:
+  build:
     name: External
-    uses: SINTEF/ci-cd/.github/workflows/cd_release.yml@v2.7.4
+    uses: SINTEF/ci-cd/.github/workflows/cd_release.yml@v2.8.2
     if: github.repository == 'SINTEF/oteapi-optimade' && startsWith(github.ref, 'refs/tags/v')
     with:
       # General
@@ -21,8 +21,11 @@ jobs:
       release_branch: main
       install_extras: "[dev]"
       python_version_build: "3.9"
-      build_cmd: "pip install -U flit && flit build"
-      publish_on_pypi: true
+      build_libs: flit
+      build_cmd: "flit build"
+      build_dir: "dist"
+      publish_on_pypi: false
+      upload_distribution: true
 
       # Documentation
       update_docs: true
@@ -31,5 +34,29 @@ jobs:
       changelog_exclude_labels: "skip_changelog,duplicate,question,invalid,wontfix"
 
     secrets:
-      PyPI_token: ${{ secrets.PYPI_TOKEN }}
       PAT: ${{ secrets.RELEASE_PAT }}
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+
+    # Using environments is recommended by PyPI when using Trusted Publishers
+    environment:
+      name: pypi
+      url: https://pypi.org/project/oteapi-optimade
+
+    # The id-token:write permission is required by the PyPI upload action for
+    # Trusted Publishers
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download distribution
+        uses: actions/download-artifact@v4
+        with:
+          name: dist  # The artifact will always be called 'dist'
+          path: dist
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/cd_upload_entities.yml

@@ -0,0 +1,65 @@
+name: CD - Upload Entities
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  update-public-entities:
+    name: Update public entities
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'SINTEF'
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -U setuptools wheel
+          pip install -r .github/utils/requirements_upload.txt
+
+      - name: Check 'entities-service' CLI version
+        run: entities-service --version
+
+      - name: Gather Entities
+        id: gather_entities
+        run: |
+          if [ "${{ github.event_name }}" == "push" ]; then
+            SHA_BEFORE="${{ github.event.before }}"
+          else
+            SHA_BEFORE="${{ github.event.pull_request.base.sha }}"
+          fi
+
+          git diff --name-only ${SHA_BEFORE} | grep -E '^entities/.*\.ya?ml$' > entities.txt ||:
+
+          if [ -s entities.txt ]; then
+            echo "relevant_entities=true" >> $GITHUB_OUTPUT
+
+            echo "Relevant Entities:"
+            cat entities.txt
+          else
+            echo "relevant_entities=false" >> $GITHUB_OUTPUT
+
+            echo "No entities to validate (and upload)."
+            exit 0
+          fi
+
+      - name: Validate Entities
+        if: steps.gather_entities.outputs.relevant_entities == 'true'
+        run: cat entities.txt | entities-service validate --strict --format=yaml -
+
+      - name: Upload Entities
+        if: steps.gather_entities.outputs.relevant_entities == 'true' && github.event_name == 'push' && github.ref_name == 'main'
+        run: cat entities.txt | entities-service upload --auto-confirm --strict --format=yaml -
+        env:
+          ENTITIES_SERVICE_ACCESS_TOKEN: ${{ secrets.ENTITIES_SERVICE_ACCESS_TOKEN }}

.github/workflows/ci_automerge_dependency_prs.yml → .github/workflows/ci_automerge_automated_prs.yml

@@ -1,13 +1,13 @@
-name: CI - Activate auto-merging for dependencies PRs
+name: CI - Activate auto-merging for automated PRs
 
 on:
   pull_request_target:
-    branches: [ci/dependency-updates]
+    branches: [main]
 
 jobs:
   update-dependencies-branch:
     name: External
-    uses: SINTEF/ci-cd/.github/workflows/ci_automerge_prs.yml@v2.7.4
-    if: github.repository_owner == 'SINTEF' && ( ( startsWith(github.event.pull_request.head.ref, 'dependabot/') && github.actor == 'dependabot[bot]' ) || ( github.event.pull_request.head.ref == 'ci/update-pyproject' && github.actor == 'TEAM4-0' ) )
+    uses: SINTEF/ci-cd/.github/workflows/ci_automerge_prs.yml@v2.8.2
+    if: github.repository_owner == 'SINTEF' && ( ( startsWith(github.event.pull_request.head.ref, 'dependabot/') && github.actor == 'dependabot[bot]' ) || ( github.event.pull_request.head.ref == 'pre-commit-ci-update-config' && github.actor == 'pre-commit-ci[bot]' ) )
     secrets:
       PAT: ${{ secrets.RELEASE_PAT }}

.github/workflows/ci_cd_updated_main.yml

@@ -7,16 +7,15 @@ on:
 jobs:
   update-deps-branch-and-docs:
     name: External
-    uses: SINTEF/ci-cd/.github/workflows/ci_cd_updated_default_branch.yml@v2.7.4
+    uses: SINTEF/ci-cd/.github/workflows/ci_cd_updated_default_branch.yml@v2.8.2
     if: github.repository_owner == 'SINTEF'
     with:
       # General
       git_username: "TEAM 4.0[bot]"
       git_email: "[email protected]"
 
       # Update dependency branch
-      permanent_dependencies_branch: "ci/dependency-updates"
-      default_repo_branch: main
+      update_dependencies_branch: false
 
       # Update documentation
       update_docs: true

.github/workflows/ci_tests.yml

@@ -10,18 +10,25 @@ on:
 jobs:
   basic-tests:
     name: External
-    uses: SINTEF/ci-cd/.github/workflows/ci_tests.yml@v2.7.4
+    uses: SINTEF/ci-cd/.github/workflows/ci_tests.yml@v2.8.2
     with:
       # General setup
       install_extras: "[dev]"
 
       # pre-commit
-      python_version_pre-commit: "3.9"
+      run_pre-commit: false
 
       # pylint & safety
       python_version_pylint_safety: "3.9"
       run_pylint: false
+
       run_safety: true
+      # ID: 70612
+      #   Package: Jinja2
+      #   Has been disputed by the maintainer and multiple third parties.
+      #   For more information see: https://github.com/advisories/GHSA-f6pv-j8mr-w6rr
+      safety_options: |
+        --ignore=70612
 
       # Build dist
       python_version_package: "3.9"
@@ -40,7 +47,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
         os:
         - ["ubuntu-latest", "linux"]
         - ["windows-latest", "windows"]
@@ -83,7 +90,7 @@ jobs:
     env:
       OTEAPI_PORT: 8080
       OPTIMADE_PORT: 5000
-      OTEAPI_prefix: "/api/v1"
+      OTEAPI_PREFIX: "/api/v1"
 
     services:
       redis:
@@ -122,7 +129,7 @@ jobs:
           --env "OTEAPI_REDIS_TYPE=redis" \
           --env "OTEAPI_REDIS_HOST=localhost" \
           --env "OTEAPI_REDIS_PORT=6379" \
-          --env "OTEAPI_prefix=${OTEAPI_prefix}" \
+          --env "OTEAPI_PREFIX=${OTEAPI_PREFIX}" \
           --network "host" \
           --volume "${PWD}:/oteapi-optimade" \
           --entrypoint "" \
@@ -132,11 +139,7 @@ jobs:
         .github/utils/wait_for_it.sh localhost:${OTEAPI_PORT} -t 240
         sleep 5
       env:
-        # Use version 1.20240228.345 until
-        # https://github.com/SINTEF/oteapi-optimade/issues/213 has been resolved.
-        # See also
-        # https://github.com/EMMC-ASBL/oteapi-services/tree/8306d7212419764fb87e5cefdb5a869db9c68ef7?tab=readme-ov-file#open-translation-environment-ote-api
-        DOCKER_OTEAPI_VERSION: '1.20240228.345'
+        DOCKER_OTEAPI_VERSION: 'latest'
 
     - name: Run end-2-end tests
       run: python .github/utils/end2end_test.py