Merge pull request #82 from Materials-Consortia/issue_66_pagination

Minor re-design. Remove the gateway-as-an-OPTIMADE-DB concept. This concept introduced a lot of edge-cases for the base code and in general didn't provide much benefit, other than being able to validate. Responses can still be returned as proper OPTIMADE responses by using the `as_optimade=True` query parameter in the `GET /search` endpoint. Furthermore, this PR represents updates to the code as a result of updates in OPTIMADE Python tools and its public Python API, as well as updated ways of extending the various functional classes. Finally, a more modularized query processing has been introduced. The queries here being the actual gateway queries; preparation, handling, and processing.
Materials-Consortia · Sep 7, 2021 · cb7430c · cb7430c
2 parents f6d12a9 + 784c5af
commit cb7430c
Show file tree

Hide file tree

Showing 52 changed files with 1,211 additions and 1,990 deletions.
diff --git a/.ci/test_gateways.json b/.ci/test_gateways.json
@@ -4,7 +4,7 @@
     "last_modified": "2021-01-28T15:49:58Z",
     "databases": [
       {
-        "id": "2dstructures",
+        "id": "mcloud/2dstructures",
         "type": "links",
         "attributes": {
           "name": "2D Structures",
@@ -16,7 +16,7 @@
         }
       },
       {
-        "id": "optimade-sample",
+        "id": "mcloud/optimade-sample",
         "type": "links",
         "attributes": {
           "name": "OPTIMADE Sample Database",
@@ -28,7 +28,7 @@
         }
       },
       {
-        "id": "scdm",
+        "id": "mcloud/scdm",
         "type": "links",
         "attributes": {
           "name": "Automated high-throughput Wannierisation",
@@ -40,7 +40,7 @@
         }
       },
       {
-        "id": "mcloud",
+        "id": "mcloud/index",
         "type": "links",
         "attributes": {
           "name": "Materials Cloud",
@@ -57,7 +57,7 @@
     "last_modified": "2021-01-28T16:35:37Z",
     "databases": [
       {
-        "id": "2dstructures",
+        "id": "mcloud/2dstructures",
         "type": "links",
         "attributes": {
           "name": "2D Structures",
@@ -69,7 +69,7 @@
         }
       },
       {
-        "id": "optimade-sample",
+        "id": "mcloud/optimade-sample",
         "type": "links",
         "attributes": {
           "name": "OPTIMADE Sample Database",
@@ -81,7 +81,7 @@
         }
       },
       {
-        "id": "scdm",
+        "id": "mcloud/scdm",
         "type": "links",
         "attributes": {
           "name": "Automated high-throughput Wannierisation",
@@ -99,7 +99,7 @@
     "last_modified": "2021-01-28T16:36:19Z",
     "databases": [
       {
-        "id": "optimade-sample",
+        "id": "mcloud/optimade-sample",
         "type": "links",
         "attributes": {
           "name": "OPTIMADE Sample Database",
@@ -117,7 +117,7 @@
     "last_modified": "2021-01-28T16:36:55Z",
     "databases": [
       {
-        "id": "2dstructures",
+        "id": "mcloud/2dstructures",
         "type": "links",
         "attributes": {
           "name": "2D Structures",
@@ -129,7 +129,7 @@
         }
       },
       {
-        "id": "optimade-sample",
+        "id": "mcloud/optimade-sample",
         "type": "links",
         "attributes": {
           "name": "OPTIMADE Sample Database",
@@ -165,7 +165,7 @@
     "last_modified": "2021-04-06T16:44:35Z",
     "databases": [
       {
-        "id": "optimade-sample",
+        "id": "mcloud/optimade-sample",
         "type": "links",
         "attributes": {
           "name": "OPTIMADE Sample Database",
@@ -177,7 +177,7 @@
         }
       },
       {
-        "id": "mcloud",
+        "id": "mcloud/index",
         "type": "links",
         "attributes": {
           "name": "Materials Cloud",
@@ -212,7 +212,7 @@
     "last_modified": "2021-04-29T14:44:48Z",
     "databases": [
       {
-        "id": "2dstructures",
+        "id": "mcloud/2dstructures",
         "type": "links",
         "attributes": {
           "name": "2D Structures",
@@ -224,7 +224,7 @@
         }
       },
       {
-        "id": "optimade-sample_single",
+        "id": "mcloud/optimade-sample_single",
         "type": "links",
         "attributes": {
           "name": "OPTIMADE Sample Database",

diff --git a/.ci/test_queries.json b/.ci/test_queries.json
@@ -7,8 +7,7 @@
     "query_parameters": {
       "filter": "elements HAS \"Si\""
     },
-    "endpoint": "structures",
-    "endpoint_model": ["optimade.models.responses", "StructureResponseMany"]
+    "endpoint": "structures"
   },
   {
     "id": "twodbs_query_2",
@@ -18,8 +17,7 @@
     "query_parameters": {
       "filter": "elements HAS \"Si\""
     },
-    "endpoint": "structures",
-    "endpoint_model": ["optimade.models.responses", "StructureResponseMany"]
+    "endpoint": "structures"
   },
   {
     "id": "singledb_query_1",
@@ -30,7 +28,6 @@
       "filter": "elements HAS \"Si\"",
       "page_limit": 5
     },
-    "endpoint": "structures",
-    "endpoint_model": ["optimade.models.responses", "StructureResponseMany"]
+    "endpoint": "structures"
   }
 ]
diff --git a/.github/docker/docker_config.json b/.github/docker/docker_config.json
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -102,29 +102,16 @@ jobs:
       with:
         fetch-depth: 2
 
-    - name: Setup and run OPTIMADE Python tools server
-      run: |
-        git clone --recurse-submodules https://github.com/Materials-Consortia/optimade-python-tools
-        docker-compose -f ./optimade-python-tools/docker-compose.yml up optimade &
-        .github/utils/wait_for_it.sh localhost:3213 -t 120
-
     - name: Build the Docker app
-      run: docker-compose build --build-arg CONFIG_FILE=".github/docker/docker_config.json"
+      run: docker-compose build
+      env:
+        OPTIMADE_BASE_URL: "http://gh_actions_host:5000"
 
-    - name: Start the Docker app (adding OPT server to 'gateway' network)
+    - name: Start the Docker app
       run: |
         docker-compose up &
         .github/utils/wait_for_it.sh localhost:5000 -t 120
         sleep 5
-        docker network connect --alias optimade-sample-server optimade-gateway_gateway optimade-python-tools_optimade_1
-
-    - name: OPTIMADE Validator for gateway
-      uses: Materials-Consortia/optimade-validator-action@v2
-      with:
-        port: 5000
-        path: /gateways/docker_ci
-        all versioned paths: True
-        validate unversioned path: True
 
   docs:
     name: Documentation
@@ -141,7 +128,7 @@ jobs:
 
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
+        python -m pip install -U pip
         pip install -U setuptools
         pip install -e .[docs]
 

diff --git a/mkdocs.yml b/mkdocs.yml
@@ -48,8 +48,6 @@ markdown_extensions:
 plugins:
   - search:
       lang: en
-  - minify:
-      minify_html: true
   - mkdocstrings:
       default_handler: python
       handlers:

diff --git a/optimade_gateway/common/utils.py b/optimade_gateway/common/utils.py
@@ -33,7 +33,7 @@ async def clean_python_types(data: Any) -> Any:
 
 
 def get_resource_attribute(
-    resource: Union[BaseModel, Dict[str, Any]],
+    resource: Union[BaseModel, Dict[str, Any], None],
     field: str,
     default: Any = None,
     disambiguate: bool = True,
@@ -53,7 +53,7 @@ def get_resource_attribute(
 
     Parameters:
         resource: The resource, from which to get the field value.
-        field: The resource field. This can be a comma-separated nested field, e.g.,
+        field: The resource field. This can be a dot-separated nested field, e.g.,
             `"attributes.base_url"`.
         default: The default value to return if `field` does not exist.
         disambiguate: Whether or not to "shortcut" a field value.
@@ -71,6 +71,9 @@ def get_resource_attribute(
         def _get_attr(mapping: dict, key: str, default: Any) -> Any:
             return mapping.get(key, default)
 
+    elif resource is None:
+        # Allow passing `None`, but simply return `default`
+        return default
     else:
         raise TypeError(
             "resource must be either a pydantic model or a Python dictionary, it was of type "

diff --git a/optimade_gateway/main.py b/optimade_gateway/main.py
@@ -19,13 +19,6 @@
     queries,
     search,
 )
-from optimade_gateway.routers.gateway import (
-    info as gateway_info,
-    links as gateway_links,
-    queries as gateway_queries,
-    structures,
-    versions,
-)
 
 APP = FastAPI(
     title="OPTIMADE Gateway",
@@ -62,16 +55,10 @@ async def get_root(request: Request) -> RedirectResponse:
 
 # Add the special /versions endpoint(s)
 APP.include_router(versions_router)
-APP.include_router(versions.ROUTER)
 
 # Add endpoints to / and /vMAJOR
 for prefix in list(BASE_URL_PREFIXES.values()) + [""]:
-    for router in (databases, gateways, info, links, queries, search) + (
-        gateway_info,
-        gateway_links,
-        gateway_queries,
-        structures,
-    ):
+    for router in (databases, gateways, info, links, queries, search):
         APP.include_router(router.ROUTER, prefix=prefix, include_in_schema=prefix == "")
 
 for event, func in EVENTS:

diff --git a/optimade_gateway/mappers/base.py b/optimade_gateway/mappers/base.py
@@ -0,0 +1,45 @@
+from typing import Iterable, List, Union
+
+from optimade.models import EntryResource
+from optimade.server.mappers.entries import (
+    BaseResourceMapper as OptimadeBaseResourceMapper,
+)
+
+
+class BaseResourceMapper(OptimadeBaseResourceMapper):
+    """
+    Generic Resource Mapper that defines and performs the mapping
+    between objects in the database and the resource objects defined by
+    the specification.
+
+    Note:
+        This is a "wrapped" sub-class to make certain methods asynchronous.
+
+    Attributes:
+        ALIASES: a tuple of aliases between
+            OPTIMADE field names and the field names in the database ,
+            e.g. `(("elements", "custom_elements_field"))`.
+        LENGTH_ALIASES: a tuple of aliases between
+            a field name and another field that defines its length, to be used
+            when querying, e.g. `(("elements", "nelements"))`.
+            e.g. `(("elements", "custom_elements_field"))`.
+        ENTRY_RESOURCE_CLASS: The entry type that this mapper corresponds to.
+        PROVIDER_FIELDS: a tuple of extra field names that this
+            mapper should support when querying with the database prefix.
+        TOP_LEVEL_NON_ATTRIBUTES_FIELDS: the set of top-level
+            field names common to all endpoints.
+        SUPPORTED_PREFIXES: The set of prefixes registered by this mapper.
+        ALL_ATTRIBUTES: The set of attributes defined across the entry
+            resource class and the server configuration.
+        ENTRY_RESOURCE_ATTRIBUTES: A dictionary of attributes and their definitions
+            defined by the schema of the entry resource class.
+        ENDPOINT: The expected endpoint name for this resource, as defined by
+            the `type` in the schema of the entry resource class.
+
+    """
+
+    @classmethod
+    async def deserialize(
+        cls, results: Union[dict, Iterable[dict]]
+    ) -> Union[List[EntryResource], EntryResource]:
+        return super(BaseResourceMapper, cls).deserialize(results)
diff --git a/optimade_gateway/mappers/databases.py b/optimade_gateway/mappers/databases.py
@@ -1,16 +1,15 @@
-from optimade.models import LinksResource
-from optimade.server.mappers.links import LinksMapper
 from pydantic import AnyUrl  # pylint: disable=no-name-in-module
 
 from optimade_gateway.common.config import CONFIG
 
+from optimade_gateway.mappers.links import LinksMapper
+
 __all__ = ("DatabasesMapper",)
 
 
 class DatabasesMapper(LinksMapper):
 
     ENDPOINT = "databases"
-    ENTRY_RESOURCE_CLASS = LinksResource
 
     @classmethod
     def map_back(cls, doc: dict) -> dict:
@@ -30,7 +29,4 @@ def map_back(cls, doc: dict) -> dict:
         }
 
         # Ensure the type does not change to "databases"
-        # The `LinksMapper.map_back()` method ensures the value for doc["type"] is kept.
-        doc["type"] = "links"
-
         return super().map_back(doc)
diff --git a/optimade_gateway/mappers/gateways.py b/optimade_gateway/mappers/gateways.py
@@ -1,9 +1,10 @@
-from optimade.server.mappers.entries import BaseResourceMapper
 from pydantic import AnyUrl  # pylint: disable=no-name-in-module
 
 from optimade_gateway.common.config import CONFIG
 from optimade_gateway.models import GatewayResource
 
+from optimade_gateway.mappers.base import BaseResourceMapper
+
 __all__ = ("GatewaysMapper",)
 
 

diff --git a/optimade_gateway/mappers/links.py b/optimade_gateway/mappers/links.py
@@ -0,0 +1,18 @@
+from optimade.models.links import LinksResource
+
+from optimade_gateway.mappers.base import BaseResourceMapper
+
+__all__ = ("LinksMapper",)
+
+
+class LinksMapper(BaseResourceMapper):
+
+    ENDPOINT = "links"
+    ENTRY_RESOURCE_CLASS = LinksResource
+
+    @classmethod
+    def map_back(cls, doc: dict) -> dict:
+        type_ = doc.get("type", None) or "links"
+        newdoc = super().map_back(doc)
+        newdoc["type"] = type_
+        return newdoc