From 7a14a1017bb6b3a6267be6476a901da00af48c5a Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Fri, 7 May 2021 14:01:42 +0200 Subject: [PATCH 01/13] Extend documentation for `db_get_all_resources` --- optimade_gateway/queries/perform.py | 7 +++++++ optimade_gateway/queries/utils.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/optimade_gateway/queries/perform.py b/optimade_gateway/queries/perform.py index 94028dfb..d5a9633d 100644 --- a/optimade_gateway/queries/perform.py +++ b/optimade_gateway/queries/perform.py @@ -282,6 +282,13 @@ async def db_get_all_resources( ]: """Recursively retrieve all resources from an entry-listing endpoint + This function keeps pulling the `links.next` link if `meta.more_data_available` is `True` to + ultimately retrieve *all* entries for `endpoint`. + + !!! warning + This function can be dangerous if an endpoint with hundreds or thousands of entries is + requested. + Parameters: database: The OPTIMADE implementation to be queried. It **must** have a valid base URL and id. diff --git a/optimade_gateway/queries/utils.py b/optimade_gateway/queries/utils.py index f83083a7..405b2d02 100644 --- a/optimade_gateway/queries/utils.py +++ b/optimade_gateway/queries/utils.py @@ -10,7 +10,7 @@ async def update_query(query: QueryResource, field: str, value: Any) -> None: """Update a query's `field` attribute with `value`. !!! note - This can _only_ update a field for a query's `attributes`, i.e., this function cannot + This can *only* update a field for a query's `attributes`, i.e., this function cannot update `id`, `type` or any other top-level resource field. Parameters: From ba4d5a1b33407c6cc75391fb99c0e584b9c12b0d Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 8 Jun 2021 17:09:26 +0200 Subject: [PATCH 02/13] Modularize processing a DB response A new data structure has been added to represent a gateway query response: `GatewayQueryResponse`. This new response changes the structure for the value of the top-level `data` field by always returning a dictionary/JSON object. The keys of the dictionary are the database `id`s and the value is a list of the returned resources from each database. In this way there is no need to alter/add anything to the returned resources, which has been the case up to now, where the `id` for each resource was prepended with the database `id`. This is however still in place, somewhat, if a `QueryResource` is not used for the query, but a straight-through result is desired. Then an OPTIMADE-compliant response is produced, where the database `id` is added as a `meta` entry to each returned resource. To handle this new approach a new utility function for processing an OPTIMADE database response has been written and implemented. An Enum for OPTIMADE endpoint entry types has been implemented with methods for returning the correct pydantic model for either the resources or response (both many and single), removing the need to pass the import path for these models. Make entry ID ambiguity a warning instead of an exception. --- .ci/test_queries.json | 9 +- optimade_gateway/common/utils.py | 7 +- optimade_gateway/models/__init__.py | 3 +- optimade_gateway/models/queries.py | 92 ++++--- optimade_gateway/queries/perform.py | 230 ++++++++++-------- optimade_gateway/queries/prepare.py | 58 ++--- optimade_gateway/queries/process.py | 133 ++++++++++ optimade_gateway/queries/utils.py | 52 +++- .../routers/gateway/structures.py | 162 +++++++----- optimade_gateway/routers/info.py | 2 + optimade_gateway/routers/queries.py | 8 +- optimade_gateway/routers/search.py | 8 - optimade_gateway/routers/utils.py | 12 +- .../gateway/test_gateway_structures.py | 6 +- tests/routers/test_info.py | 3 +- tests/routers/test_queries.py | 9 +- 16 files changed, 512 insertions(+), 282 deletions(-) create mode 100644 optimade_gateway/queries/process.py diff --git a/.ci/test_queries.json b/.ci/test_queries.json index 204f223e..b1214391 100644 --- a/.ci/test_queries.json +++ b/.ci/test_queries.json @@ -7,8 +7,7 @@ "query_parameters": { "filter": "elements HAS \"Si\"" }, - "endpoint": "structures", - "endpoint_model": ["optimade.models.responses", "StructureResponseMany"] + "endpoint": "structures" }, { "id": "twodbs_query_2", @@ -18,8 +17,7 @@ "query_parameters": { "filter": "elements HAS \"Si\"" }, - "endpoint": "structures", - "endpoint_model": ["optimade.models.responses", "StructureResponseMany"] + "endpoint": "structures" }, { "id": "singledb_query_1", @@ -30,7 +28,6 @@ "filter": "elements HAS \"Si\"", "page_limit": 5 }, - "endpoint": "structures", - "endpoint_model": ["optimade.models.responses", "StructureResponseMany"] + "endpoint": "structures" } ] diff --git a/optimade_gateway/common/utils.py b/optimade_gateway/common/utils.py index 621272a8..299f64dd 100644 --- a/optimade_gateway/common/utils.py +++ b/optimade_gateway/common/utils.py @@ -33,7 +33,7 @@ async def clean_python_types(data: Any) -> Any: def get_resource_attribute( - resource: Union[BaseModel, Dict[str, Any]], + resource: Union[BaseModel, Dict[str, Any], None], field: str, default: Any = None, disambiguate: bool = True, @@ -53,7 +53,7 @@ def get_resource_attribute( Parameters: resource: The resource, from which to get the field value. - field: The resource field. This can be a comma-separated nested field, e.g., + field: The resource field. This can be a dot-separated nested field, e.g., `"attributes.base_url"`. default: The default value to return if `field` does not exist. disambiguate: Whether or not to "shortcut" a field value. @@ -71,6 +71,9 @@ def get_resource_attribute( def _get_attr(mapping: dict, key: str, default: Any) -> Any: return mapping.get(key, default) + elif resource is None: + # Allow passing `None`, but simply return `default` + return default else: raise TypeError( "resource must be either a pydantic model or a Python dictionary, it was of type " diff --git a/optimade_gateway/models/__init__.py b/optimade_gateway/models/__init__.py index 87be0471..d62d9b32 100644 --- a/optimade_gateway/models/__init__.py +++ b/optimade_gateway/models/__init__.py @@ -6,7 +6,7 @@ from .databases import DatabaseCreate from .gateways import GatewayCreate, GatewayResource, GatewayResourceAttributes -from .queries import QueryCreate, QueryResource, QueryState +from .queries import GatewayQueryResponse, QueryCreate, QueryResource, QueryState from .resources import EntryResourceCreate from .responses import ( DatabasesResponse, @@ -25,6 +25,7 @@ "DatabasesResponseSingle", "EntryResourceCreate", "GatewayCreate", + "GatewayQueryResponse", "GatewayResource", "GatewayResourceAttributes", "GatewaysResponse", diff --git a/optimade_gateway/models/queries.py b/optimade_gateway/models/queries.py index 4c9017d7..28895c48 100644 --- a/optimade_gateway/models/queries.py +++ b/optimade_gateway/models/queries.py @@ -1,6 +1,6 @@ """Pydantic models/schemas for the Queries resource""" from enum import Enum -from typing import Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Union import warnings from optimade.models import ( @@ -8,6 +8,12 @@ EntryResourceAttributes, EntryResponseMany, ErrorResponse, + ReferenceResource, + ReferenceResponseMany, + ReferenceResponseOne, + StructureResource, + StructureResponseMany, + StructureResponseOne, ) from optimade.server.query_params import EntryListingQueryParams from pydantic import BaseModel, EmailStr, Field, validator @@ -16,6 +22,39 @@ from optimade_gateway.warnings import SortNotSupported +class EndpointEntryType(Enum): + """Entry endpoint resource types, mapping to their pydantic models from the `optimade` package.""" + + REFERENCES = "references" + STRUCTURES = "structures" + + def get_resource_model(self) -> Union[ReferenceResource, StructureResource]: + """Get the matching pydantic model for a resource.""" + return { + "references": ReferenceResource, + "structures": StructureResource, + }[self.value] + + def get_response_model( + self, single: bool = False + ) -> Union[ + ReferenceResponseMany, + ReferenceResponseOne, + StructureResponseMany, + StructureResponseOne, + ]: + """Get the matching pydantic model for a successful response.""" + if single: + return { + "references": ReferenceResponseOne, + "structures": StructureResponseOne, + }[self.value] + return { + "references": ReferenceResponseMany, + "structures": StructureResponseMany, + }[self.value] + + QUERY_PARAMETERS = EntryListingQueryParams() """Entry listing URL query parameters from the `optimade` package ([`EntryListingQueryParams`](https://www.optimade.org/optimade-python-tools/api_reference/server/query_params/#optimade.server.query_params.EntryListingQueryParams)).""" @@ -99,8 +138,17 @@ class QueryState(Enum): FINISHED = "finished" +class GatewayQueryResponse(EntryResponseMany): + """Response from a Gateway Query.""" + + data: Dict[str, Union[List[EntryResource], List[Dict[str, Any]]]] = Field( + ..., + uniqueItems=True, + ) + + class QueryResourceAttributes(EntryResourceAttributes): - """Attributes for an OPTIMADE gateway query""" + """Attributes for an OPTIMADE gateway query.""" gateway_id: str = Field( ..., @@ -117,45 +165,26 @@ class QueryResourceAttributes(EntryResourceAttributes): title="State", type="enum", ) - response: Optional[Union[EntryResponseMany, ErrorResponse]] = Field( + response: Optional[Union[GatewayQueryResponse, ErrorResponse]] = Field( None, description="Response from gateway query.", type="object", ) - endpoint: str = Field( - ..., description="The entry endpoint queried, e.g., 'structures'." - ) - endpoint_model: Tuple[str, str] = Field( - ..., - description=( - "The full importable path to the pydantic response model class (not an instance of " - "the class). It should be a tuple of the Python module and the Class name." - ), + endpoint: EndpointEntryType = Field( + EndpointEntryType.STRUCTURES, + description="The entry endpoint queried, e.g., 'structures'.", + title="Endpoint", + type="enum", ) @validator("endpoint") - def remove_endpoints_slashes(cls, value: str) -> str: - """Remove prep-/appended slashes (`/`)""" - org_value = value - value = value.strip() - while value.startswith("/"): - value = value[1:] - while value.endswith("/"): - value = value[:-1] - if not value: - raise ValueError( - "endpoint must not be an empty string or be prep-/appended with slashes (`/`). " - f"Original value: {org_value!r}. Final value (after removing prep-/appended " - f"slashes): {value!r}" - ) - - # Temporarily only allow queries to "structures" endpoints. - if value != "structures": + def only_allow_structures(cls, value: EndpointEntryType) -> EndpointEntryType: + """Temporarily only allow queries to "structures" endpoints.""" + if value != EndpointEntryType.STRUCTURES: raise NotImplementedError( 'OPTIMADE Gateway temporarily only supports queries to "structures" endpoints, ' 'i.e.: endpoint="structures"' ) - return value @@ -175,8 +204,7 @@ class QueryCreate(EntryResourceCreate, QueryResourceAttributes): """Model for creating new Query resources in the MongoDB""" state: Optional[QueryState] - endpoint: Optional[str] - endpoint_model: Optional[Tuple[str, str]] + endpoint: Optional[EndpointEntryType] @validator("query_parameters") def sort_not_supported( diff --git a/optimade_gateway/queries/perform.py b/optimade_gateway/queries/perform.py index d5a9633d..ee1a18d2 100644 --- a/optimade_gateway/queries/perform.py +++ b/optimade_gateway/queries/perform.py @@ -1,3 +1,4 @@ +"""Perform OPTIMADE queries""" import asyncio from concurrent.futures import ThreadPoolExecutor import functools @@ -22,17 +23,22 @@ from optimade_gateway.common.logger import LOGGER from optimade_gateway.common.utils import get_resource_attribute -from optimade_gateway.models import GatewayResource, QueryResource, QueryState -from optimade_gateway.queries.prepare import get_query_params, prepare_query +from optimade_gateway.models import ( + GatewayQueryResponse, + GatewayResource, + QueryResource, + QueryState, +) +from optimade_gateway.queries.prepare import get_query_params, prepare_query_filter +from optimade_gateway.queries.process import process_db_response from optimade_gateway.queries.utils import update_query -from optimade_gateway.warnings import OptimadeGatewayWarning async def perform_query( url: URL, query: QueryResource, use_query_resource: bool = True, -) -> Union[EntryResponseMany, ErrorResponse]: +) -> Union[EntryResponseMany, ErrorResponse, GatewayQueryResponse]: """Perform OPTIMADE query with gateway. Parameters: @@ -44,28 +50,57 @@ async def perform_query( through the `/queries` endpoint. Returns: - This function returns the final response; either an `ErrorResponse` or a subclass of - `EntryResponseMany`. + This function returns the final response; either an `ErrorResponse`, a subclass of + `EntryResponseMany` or if `use_query_resource` is true, then a + [`GatewayQueryResponse`][optimade_gateway.models.queries.GatewayQueryResponse]. """ from optimade_gateway.routers.gateways import GATEWAYS_COLLECTION from optimade_gateway.routers.utils import get_valid_resource - data_available = data_returned = 0 - errors = [] - more_data_available = False - results = [] + response = None + + if use_query_resource: + await update_query(query, "state", QueryState.STARTED) gateway: GatewayResource = await get_valid_resource( GATEWAYS_COLLECTION, query.attributes.gateway_id ) - (filter_queries, response_model) = await prepare_query( + filter_queries = await prepare_query_filter( database_ids=[_.id for _ in gateway.attributes.databases], - endpoint_model=query.attributes.endpoint_model, filter_query=query.attributes.query_parameters.filter, ) + if use_query_resource: + url = url.replace(path=f"{url.path.rstrip('/')}/{query.id}") + await update_query( + query, + "response", + GatewayQueryResponse( + data={}, + links=ToplevelLinks(next=None), + meta=meta_values( + url=url, + data_available=0, + data_returned=0, + more_data_available=False, + ), + ), + **{"$set": {"state": QueryState.IN_PROGRESS}}, + ) + else: + response = EntryResponseMany( + data=[], + links=ToplevelLinks(next=None), + meta=meta_values( + url=url, + data_available=0, + data_returned=0, + more_data_available=False, + ), + ) + loop = asyncio.get_running_loop() with ThreadPoolExecutor( max_workers=min(32, os.cpu_count() + 4, len(gateway.attributes.databases)) @@ -85,105 +120,106 @@ async def perform_query( func=functools.partial( db_find, database=database, - endpoint=query.attributes.endpoint, - response_model=response_model, + endpoint=query.attributes.endpoint.value, + response_model=query.attributes.endpoint.get_response_model(), query_params=query_params, ), ) ) - if use_query_resource: - await update_query(query, "state", QueryState.STARTED) - for query_task in query_tasks: - (response, db_id) = await query_task - - if use_query_resource and query.attributes.state != QueryState.IN_PROGRESS: - await update_query(query, "state", QueryState.IN_PROGRESS) - - if isinstance(response, ErrorResponse): - for error in response.errors: - if isinstance(error.id, str) and error.id.startswith( - "OPTIMADE_GATEWAY" - ): - import warnings + (db_response, db_id) = await query_task + + results, errors, response_meta = await process_db_response( + response=db_response, + database_id=db_id, + query=query, + gateway=gateway, + use_query_resource=use_query_resource, + ) - warnings.warn(error.detail, OptimadeGatewayWarning) + if not use_query_resource: + # Create a standard OPTIMADE response, adding the database ID to each returned + # resource's meta field. + database_id_meta = {"_optimade_gateway_": {"source_database_id": db_id}} + if errors or isinstance(response, ErrorResponse): + # Error response + if isinstance(response, ErrorResponse): + response.errors.append(errors) else: - meta = {} - if error.meta: - meta = error.meta.dict() - meta.update( - { - "optimade_gateway": { - "gateway": gateway, - "source_database_id": db_id, - } - } - ) - error.meta = Meta(**meta) - errors.append(error) - else: - for datum in response.data: - if isinstance(datum, dict) and datum.get("id") is not None: - datum["id"] = f"{db_id}/{datum['id']}" + response = ErrorResponse(errors=errors, meta=response.meta) + response.meta.data_returned = 0 + response.meta.more_data_available = False + elif isinstance(results, list): + # "Many" response + for resource in results: + if hasattr(resource, "meta") and resource.meta: + meta = resource.meta.dict() + meta.update(database_id_meta) + resource.meta = Meta(**meta) + elif isinstance(resource, dict) and resource.get("meta", {}): + resource["meta"] = resource["meta"].update(database_id_meta) + elif isinstance(resource, dict): + resource["meta"] = database_id_meta + else: + resource.meta = Meta(**database_id_meta) + response.data.append(results) + response.meta.data_returned += response_meta["data_returned"] + if not response.meta.more_data_available: + # Keep it True, if set to True once. + response.meta.more_data_available = response_meta[ + "more_data_available" + ] + else: + # "One" response + if hasattr(results, "meta") and results.meta: + meta = results.meta.dict() + meta.update(database_id_meta) + results.meta = Meta(**meta) + elif isinstance(results, dict) and results.get("meta", {}): + results["meta"] = results["meta"].update(database_id_meta) + elif isinstance(results, dict): + results["meta"] = database_id_meta else: - datum.id = f"{db_id}/{datum.id}" - results.extend(response.data) - - data_available += response.meta.data_available or 0 - data_returned += response.meta.data_returned or len(response.data) - - if not more_data_available: - # Keep it True, if set to True once. - more_data_available = response.meta.more_data_available - - if use_query_resource: - url = url.replace(path=f"{url.path.rstrip('/')}/{query.id}") - - meta = meta_values( - url=url, - data_available=data_available, - data_returned=data_returned, - more_data_available=more_data_available, - ) + results.meta = Meta(**database_id_meta) + response.data.append(results) + response.meta.data_returned += response_meta["data_returned"] + if not response.meta.more_data_available: + # Keep it True, if set to True once. + response.meta.more_data_available = response_meta[ + "more_data_available" + ] + response.meta.data_available += response_meta["data_available"] + + if get_resource_attribute( + query, + "attributes.response.meta.more_data_available", + False, + disambiguate=False, # Extremely minor speed-up + ) or get_resource_attribute( + response, + "meta.more_data_available", + False, + disambiguate=False, + ): + # Deduce the `next` link from the current request + query_string = urllib.parse.parse_qs(url.query) + query_string["page_offset"] = int( + query_string.get("page_offset", [0])[0] + ) + len(results[: query.attributes.query_parameters.page_limit]) + urlencoded = urllib.parse.urlencode(query_string, doseq=True) + base_url = get_base_url(url) + + links = ToplevelLinks(next=f"{base_url}{url.path}?{urlencoded}") - if errors: - response = ErrorResponse(errors=errors, meta=meta) - else: - # Sort results over two steps, first by database id, - # and then by (original) "id", since "id" MUST always be present. - # NOTE: Possbly remove this sorting? - results.sort(key=lambda data: data["id"] if "id" in data else data.id) - results.sort( - key=lambda data: "/".join(data["id"].split("/")[1:]) - if "id" in data - else "/".join(data.id.split("/")[1:]) - ) - - if more_data_available: - # Deduce the `next` link from the current request - query_string = urllib.parse.parse_qs(url.query) - query_string["page_offset"] = int( - query_string.get("page_offset", [0])[0] - ) + len(results[: query.attributes.query_parameters.page_limit]) - urlencoded = urllib.parse.urlencode(query_string, doseq=True) - base_url = get_base_url(url) - - links = ToplevelLinks(next=f"{base_url}{url.path}?{urlencoded}") + if use_query_resource: + await update_query(query, "response.links", links) else: - links = ToplevelLinks(next=None) - - response = response_model( - links=links, - data=results, - meta=meta, - ) + response.links = links if use_query_resource: - await update_query(query, "response", response) await update_query(query, "state", QueryState.FINISHED) - + return query.attributes.response return response diff --git a/optimade_gateway/queries/prepare.py b/optimade_gateway/queries/prepare.py index ee1d9073..99e09fc3 100644 --- a/optimade_gateway/queries/prepare.py +++ b/optimade_gateway/queries/prepare.py @@ -1,27 +1,12 @@ -import importlib import re -from typing import Dict, List, Tuple, Union +from typing import Dict, List, Union import urllib.parse -from optimade.models.responses import EntryResponseMany, EntryResponseOne -from optimade.server.exceptions import BadRequest - from optimade_gateway.models.queries import OptimadeQueryParameters +from optimade_gateway.warnings import OptimadeGatewayWarning -async def prepare_query( - database_ids: List[str], - endpoint_model: Tuple[str, str], - filter_query: Union[str, None], -) -> Tuple[Dict[str, Union[str, None]], Union[EntryResponseMany, EntryResponseOne]]: - """Prepare a query by returning necessary variables.""" - return ( - await update_query_filter(database_ids, filter_query), - await get_response_model(endpoint_model), - ) - - -async def update_query_filter( +async def prepare_query_filter( database_ids: List[str], filter_query: Union[str, None] ) -> Dict[str, Union[str, None]]: """Update the query parameter `filter` value to be database-specific @@ -60,12 +45,18 @@ async def update_query_filter( break # TODO: Remove `id="value"` sections here for queries to databases that doesn't match the id value! else: - raise BadRequest( - detail=( - f"Structures entry not found. To get a specific structures " - "entry one needs to prepend the ID with a database ID belonging to the gateway," - f" e.g., '{database_ids[0]}/'. Available" - f"databases for this gateway: {database_ids}" + from warnings import warn + + warn( + OptimadeGatewayWarning( + title="Non-Unique Entry ID", + detail=( + f"The passed entry ID may be ambiguous! To get a " + "specific structures entry, one can prepend the ID with a database ID " + "belonging to the gateway, followed by a forward slash, e.g., " + f"'{database_ids[0]}/'. Available databases for this " + f"gateway: {database_ids}" + ), ) ) return updated_filter @@ -83,22 +74,3 @@ async def get_query_params( if filter_mapping[database_id]: query_params.update({"filter": filter_mapping[database_id]}) return urllib.parse.urlencode(query_params) - - -async def get_response_model( - endpoint_model: Tuple[str, str] -) -> Union[EntryResponseMany, EntryResponseOne]: - """Import and return response model based on `endpoint_model`. - - Parameters: - endpoint_model: The - [`endpoint_model`][optimade_gateway.models.queries.QueryResourceAttributes.endpoint_model] - from the - [`QueryResource` attributes][optimade_gateway.models.queries.QueryResourceAttributes]. - - Returns: - The imported response model class, e.g., `StructureResponseMany`. - - """ - module, name = endpoint_model - return getattr(importlib.import_module(module), name) diff --git a/optimade_gateway/queries/process.py b/optimade_gateway/queries/process.py new file mode 100644 index 00000000..e91764c8 --- /dev/null +++ b/optimade_gateway/queries/process.py @@ -0,0 +1,133 @@ +"""Process performed OPTIMADE queries""" +from typing import Any, Dict, List, Tuple, Union + +from optimade.models import ( + EntryResource, + EntryResponseMany, + EntryResponseOne, + ErrorResponse, + Meta, + OptimadeError, +) + +from optimade_gateway.common.utils import get_resource_attribute +from optimade_gateway.models import GatewayResource, QueryResource +from optimade_gateway.queries.utils import update_query +from optimade_gateway.warnings import OptimadeGatewayWarning + + +async def process_db_response( + response: Union[ErrorResponse, EntryResponseMany, EntryResponseOne], + database_id: str, + query: QueryResource, + gateway: GatewayResource, + use_query_resource: bool = True, +) -> Tuple[ + Union[ + List[EntryResource], List[Dict[str, Any]], EntryResource, Dict[str, Any], None + ], + List[OptimadeError], + Dict[str, Union[bool, int]], +]: + """Process an OPTIMADE database response. + + The passed `query` will be updated with the top-level `meta` information: `data_available`, + `data_returned`, and `more_data_available`. + + Since, only either `data` or `errors` should ever be present, one or the other will be either + an empty list or `None`. + `meta` will only be a non-empty dictionary when not using a + [`QueryResource`][optimade_gateway.models.queries.QueryResource], i.e., if `use_query_resource` + is `False`. + + Parameters: + response: The OPTIMADE database response to be processed. + database_id: The database's `id` under which the returned resources or errors will be + delivered. + query: A resource representing the performed query. + gateway: A resource representing the gateway that was queried. + use_query_resource: Whether or not to update the passed + [`QueryResource`][optimade_gateway.models.queries.QueryResource]. + + Returns: + A tuple of the response's `data`, `errors`, and `meta`. + + """ + results = [] + errors = [] + meta = {} + + if isinstance(response, ErrorResponse): + for error in response.errors: + if isinstance(error.id, str) and error.id.startswith("OPTIMADE_GATEWAY"): + import warnings + + warnings.warn(error.detail, OptimadeGatewayWarning) + else: + # The model `ErrorResponse` does not allow the objects in the top-level `errors` + # list to be parsed as dictionaries - they must be a pydantic model. + meta_error = {} + if error.meta: + meta_error = error.meta.dict() + meta_error.update( + { + "optimade_gateway": { + "gateway": gateway, + "source_database_id": database_id, + } + } + ) + error.meta = Meta(**meta_error) + errors.append(error) + data_returned = 0 + more_data_available = False + else: + results = response.data + + if isinstance(results, list): + data_returned = response.meta.data_returned or len(results) + else: + data_returned = response.meta.data_returned or (0 if not results else 1) + + more_data_available = response.meta.more_data_available or False + + data_available = response.meta.data_available or 0 + + if use_query_resource: + extra_updates = { + "$inc": { + "response.meta.data_available": data_available, + "response.meta.data_returned": data_returned, + } + } + if not get_resource_attribute( + query, + "attributes.response.meta.more_data_available", + False, + disambiguate=False, # Extremely minor speed-up + ): + # Keep it True, if set to True once. + extra_updates.update( + {"$set": {"response.meta.more_data_available": more_data_available}} + ) + + # This ensures an empty list under `response.data.{database_id}` is returned if the case is + # simply that there is no results to return. + # It also ensures that only `response.errors.{database_id}` is created if there are any + # errors. + await update_query( + query, + f"response.errors.{database_id}" + if errors + else f"response.data.{database_id}", + errors or results, + **extra_updates, + ) + else: + meta = { + "data_returned": data_returned, + "data_available": data_available, + "more_data_available": more_data_available, + } + + return results, errors, meta diff --git a/optimade_gateway/queries/utils.py b/optimade_gateway/queries/utils.py index 405b2d02..21656b31 100644 --- a/optimade_gateway/queries/utils.py +++ b/optimade_gateway/queries/utils.py @@ -6,17 +6,30 @@ from optimade_gateway.models import QueryResource -async def update_query(query: QueryResource, field: str, value: Any) -> None: +async def update_query( + query: QueryResource, field: str, value: Any, operator: str = None, **mongo_kwargs +) -> None: """Update a query's `field` attribute with `value`. + If `field` is a dot-separated value, then only the last field part may be a non-pre-existing + field. Otherwise a `KeyError` or `AttributeError` will be raised. + !!! note This can *only* update a field for a query's `attributes`, i.e., this function cannot update `id`, `type` or any other top-level resource field. + !!! important + `mongo_kwargs` will not be considered for updating the pydantic model instance. + Parameters: query: The query to be updated. field: The `attributes` field (key) to be set. + This can be a dot-separated key value to signify embedded fields. + + **Example**: `response.meta`. value: The (possibly) new value for `field`. + operator: A MongoDB operator to be used for updating `field` with `value`. + mongo_kwargs (dict): Further MongoDB update filters. """ from datetime import datetime @@ -24,17 +37,27 @@ async def update_query(query: QueryResource, field: str, value: Any) -> None: from optimade_gateway.common.utils import clean_python_types from optimade_gateway.routers.queries import QUERIES_COLLECTION + operator = operator or "$set" + + if operator and not operator.startswith("$"): + operator = f"${operator}" + update_time = datetime.utcnow() + update_kwargs = {"$set": {"last_modified": update_time}} + + if mongo_kwargs: + update_kwargs.update(await clean_python_types(mongo_kwargs)) + + if operator and operator == "$set": + update_kwargs["$set"].update({field: await clean_python_types(value)}) + elif operator: + update_kwargs.update({operator: {field: await clean_python_types(value)}}) + # MongoDB result: UpdateResult = await QUERIES_COLLECTION.collection.update_one( filter={"id": {"$eq": query.id}}, - update={ - "$set": { - "last_modified": update_time, - field: await clean_python_types(value), - } - }, + update=update_kwargs, ) if result.matched_count != 1: LOGGER.error( @@ -45,4 +68,17 @@ async def update_query(query: QueryResource, field: str, value: Any) -> None: # Pydantic model instance query.attributes.last_modified = update_time - setattr(query.attributes, field, value) + if "." in field: + field_list = field.split(".") + field = getattr(query.attributes, field_list[0]) + for field_part in field_list[1:-1]: + if isinstance(field, dict): + field = field.get(field_part) + else: + field = getattr(field, field_part) + if isinstance(field, dict): + field[field_list[-1]] = value + else: + setattr(field, field_list[-1], value) + else: + setattr(query.attributes, field, value) diff --git a/optimade_gateway/routers/gateway/structures.py b/optimade_gateway/routers/gateway/structures.py index 0086673a..c36f18e1 100644 --- a/optimade_gateway/routers/gateway/structures.py +++ b/optimade_gateway/routers/gateway/structures.py @@ -21,11 +21,11 @@ StructureResponseOne, ToplevelLinks, ) -from optimade.server.exceptions import BadRequest from optimade.server.query_params import EntryListingQueryParams, SingleEntryQueryParams from optimade.server.routers.utils import meta_values from optimade_gateway.models import QueryResource +from optimade_gateway.queries import perform_query from optimade_gateway.routers.gateway.utils import validate_version from optimade_gateway.routers.gateways import GATEWAYS_COLLECTION from optimade_gateway.warnings import OptimadeGatewayWarning, SortNotSupported @@ -53,7 +53,6 @@ async def get_structures( Return a regular `/structures` response for an OPTIMADE implementation, including responses from all the gateway's databases. """ - from optimade_gateway.queries import perform_query from optimade_gateway.routers.utils import validate_resource await validate_resource(GATEWAYS_COLLECTION, gateway_id) @@ -76,10 +75,6 @@ async def get_structures( key: value for key, value in params.__dict__.items() if value }, "endpoint": "structures", - "endpoint_model": ( - StructureResponseMany.__module__, - StructureResponseMany.__name__, - ), }, } ), @@ -115,7 +110,21 @@ async def get_single_structure( """`GET /gateways/{gateway_id}/structures/{structure_id}` Return a regular `/structures/{id}` response for an OPTIMADE implementation. - The `structure_id` must be of the type `{database ID}/{id}`. + + !!! important + There are two options for the `structure_id`. + + Either one supplies an entry ID similar to what is given in the local databases. + This will result in this endpoint returning the first entry it finds from any database that + matches the given ID. + + Example: `GET /gateway/some_gateway/structures/some_structure`. + + Otherwise, one can supply the database ID (call `GET /databases` to see all available + databases and their IDs), and then the local entry ID, separated by a forward slash. + + Example: `GET /gateways/some_gateway/structures/some_database/some_structure`. + """ from optimade_gateway.models import GatewayResource from optimade_gateway.queries import db_find @@ -124,83 +133,110 @@ async def get_single_structure( gateway: GatewayResource = await get_valid_resource(GATEWAYS_COLLECTION, gateway_id) local_structure_id = None + database_id = None + for database in gateway.attributes.databases: if structure_id.startswith(f"{database.id}/"): # Database found - local_structure_id = structure_id[len(f"{database.id}/") :] + database_id = database.id + local_structure_id = structure_id[len(f"{database_id}/") :] break else: - raise BadRequest( - detail=( - f"Structures entry not found. To get a specific structures entry " - "one needs to prepend the ID with a database ID belonging to the gateway, e.g., " - f"'{gateway.attributes.databases[0].id}/'. Available databases for " - f"gateway {gateway_id!r}: {[_.id for _ in gateway.attributes.databases]}" - ) - ) + # Assume the given ID is already a local database ID - find and return the first one + # available. + local_structure_id = structure_id errors = [] - result = None parsed_params = urllib.parse.urlencode( {param: value for param, value in params.__dict__.items() if value} ) - (response, _) = await asyncio.get_running_loop().run_in_executor( - executor=None, # Run in thread with the event loop - func=functools.partial( - db_find, - database=database, - endpoint=f"structures/{local_structure_id}", - response_model=StructureResponseOne, - query_params=parsed_params, - ), - ) - - if isinstance(response, ErrorResponse): - for error in response.errors: - if isinstance(error.id, str) and error.id.startswith("OPTIMADE_GATEWAY"): - warnings.warn(error.detail, OptimadeGatewayWarning) - else: - meta = {} - if error.meta: - meta = error.meta.dict() - meta.update( - { - "optimade_gateway": { - "gateway": gateway, - "source_database_id": database.id, + if database_id: + (gateway_response, _) = await asyncio.get_running_loop().run_in_executor( + executor=None, # Run in thread with the event loop + func=functools.partial( + db_find, + database=database, + endpoint=f"structures/{local_structure_id}", + response_model=StructureResponseOne, + query_params=parsed_params, + ), + ) + if isinstance(gateway_response, ErrorResponse): + for error in gateway_response.errors: + if isinstance(error.id, str) and error.id.startswith( + "OPTIMADE_GATEWAY" + ): + warnings.warn(error.detail, OptimadeGatewayWarning) + else: + meta = {} + if error.meta: + meta = error.meta.dict() + meta.update( + { + "optimade_gateway": { + "gateway": gateway, + "source_database_id": database.id, + } } - } - ) - error.meta = Meta(**meta) - errors.append(error) - else: - result = response.data - if isinstance(result, dict) and result.get("id") is not None: - result["id"] = f"{database.id}/{result['id']}" - elif result is None: - pass + ) + error.meta = Meta(**meta) + errors.append(error) + + meta = meta_values( + url=request.url, + data_returned=gateway_response.meta.data_returned, + data_available=None, # Don't set this, as we'd have to request ALL gateway databases + more_data_available=gateway_response.meta.more_data_available, + ) + del meta.data_available + + if errors: + gateway_response = ErrorResponse(errors=errors, meta=meta) else: - result.id = f"{database.id}/{result.id}" + gateway_response = StructureResponseOne( + links=ToplevelLinks(next=None), data=gateway_response.data, meta=meta + ) - meta = meta_values( - url=request.url, - data_returned=response.meta.data_returned, - data_available=None, # Don't set this, as we'd have to request ALL gateway databases - more_data_available=response.meta.more_data_available, - ) - del meta.data_available + else: + params.filter = f'id="{local_structure_id}"' + gateway_response = await perform_query( + url=request.url, + query=QueryResource( + **{ + "id": "temp", + "type": "queries", + "attributes": { + "last_modified": datetime.utcnow(), + "gateway_id": gateway_id, + "state": "created", + "query_parameters": { + key: value + for key, value in params.__dict__.items() + if value + }, + "endpoint": "structures", + }, + } + ), + use_query_resource=False, + ) + if isinstance(gateway_response, StructureResponseMany): + gateway_response = gateway_response.dict(exclude_unset=True) + gateway_response["data"] = ( + gateway_response["data"][0] if gateway_response["data"] else None + ) + gateway_response = StructureResponseOne(**gateway_response) - if errors: - for error in errors: + if isinstance(gateway_response, ErrorResponse): + for error in errors or gateway_response.errors: if error.status: response.status_code = int(error.status) break else: response.status_code = 500 - return ErrorResponse(errors=errors, meta=meta) - return StructureResponseOne(links=ToplevelLinks(next=None), data=result, meta=meta) + return gateway_response @ROUTER.get( diff --git a/optimade_gateway/routers/info.py b/optimade_gateway/routers/info.py index 386f14de..68259864 100644 --- a/optimade_gateway/routers/info.py +++ b/optimade_gateway/routers/info.py @@ -16,6 +16,7 @@ EntryInfoResponse, ErrorResponse, InfoResponse, + LinksResource, ) from optimade.server.routers.utils import get_base_url, meta_values @@ -24,6 +25,7 @@ ROUTER = APIRouter(redirect_slashes=True) ENTRY_INFO_SCHEMAS = { + "databases": LinksResource.schema, "gateways": GatewayResource.schema, "queries": QueryResource.schema, } diff --git a/optimade_gateway/routers/queries.py b/optimade_gateway/routers/queries.py index ba43bc96..da70e7f7 100644 --- a/optimade_gateway/routers/queries.py +++ b/optimade_gateway/routers/queries.py @@ -21,10 +21,10 @@ from optimade.server.query_params import EntryListingQueryParams from optimade.server.routers.utils import meta_values -from optimade_gateway.common.logger import LOGGER from optimade_gateway.common.config import CONFIG from optimade_gateway.mappers import QueryMapper from optimade_gateway.models import ( + GatewayQueryResponse, QueryCreate, QueryResource, QueryState, @@ -114,7 +114,7 @@ async def post_queries( @ROUTER.get( "/queries/{query_id:path}", - response_model=Union[EntryResponseMany, ErrorResponse], + response_model=Union[EntryResponseMany, ErrorResponse, GatewayQueryResponse], response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, @@ -124,7 +124,7 @@ async def get_query( request: Request, query_id: str, response: Response, -) -> Union[EntryResponseMany, ErrorResponse]: +) -> Union[EntryResponseMany, ErrorResponse, GatewayQueryResponse]: """`GET /queries/{query_id}` Return the response from a query @@ -132,9 +132,7 @@ async def get_query( """ from optimade_gateway.routers.utils import get_valid_resource - LOGGER.debug("At /queries/ with id=%s", query_id) query: QueryResource = await get_valid_resource(QUERIES_COLLECTION, query_id) - LOGGER.debug("Found query (in /queries/): %s", query) if query.attributes.state != QueryState.FINISHED: return EntryResponseMany( diff --git a/optimade_gateway/routers/search.py b/optimade_gateway/routers/search.py index 3996a632..362e1f1e 100644 --- a/optimade_gateway/routers/search.py +++ b/optimade_gateway/routers/search.py @@ -26,7 +26,6 @@ ToplevelLinks, ) from optimade.models.links import LinkType -from optimade.models import StructureResponseMany, ReferenceResponseMany, LinksResponse from optimade.server.query_params import EntryListingQueryParams from optimade.server.routers.utils import meta_values from pydantic import AnyUrl, ValidationError @@ -47,12 +46,6 @@ ROUTER = APIRouter(redirect_slashes=True) -ENDPOINT_MODELS = { - "structures": (StructureResponseMany.__module__, StructureResponseMany.__name__), - "references": (ReferenceResponseMany.__module__, ReferenceResponseMany.__name__), - "links": (LinksResponse.__module__, LinksResponse.__name__), -} - @ROUTER.post( "/search", @@ -151,7 +144,6 @@ async def post_search(request: Request, search: Search) -> QueriesResponseSingle query = QueryCreate( endpoint=search.endpoint, - endpoint_model=ENDPOINT_MODELS.get(search.endpoint), gateway_id=gateway.id, query_parameters=search.query_parameters, ) diff --git a/optimade_gateway/routers/utils.py b/optimade_gateway/routers/utils.py index f746d12a..92a2bf93 100644 --- a/optimade_gateway/routers/utils.py +++ b/optimade_gateway/routers/utils.py @@ -151,10 +151,9 @@ async def resource_factory( !!! attention Only the `/structures` entry endpoint can be queried with multiple expected responses. - This means the `endpoint` field defaults to `"structures"` and `endpoint_model` - defaults to `("optimade.models.responses", "StructureResponseMany")`, i.e., the - [`StructureResponseMany`](https://www.optimade.org/optimade-python-tools/api_reference/models/responses/#optimade.models.responses.StructureResponseMany) - response model. + This means the `endpoint` field defaults to `"structures"`, i.e., the + [`StructureResource`](https://www.optimade.org/optimade-python-tools/all_models/#optimade.models.structures.StructureResource) + resource model. Parameters: create_resource: The resource to be retrieved or created anew. @@ -208,11 +207,6 @@ async def resource_factory( create_resource.endpoint = ( create_resource.endpoint if create_resource.endpoint else "structures" ) - create_resource.endpoint_model = ( - create_resource.endpoint_model - if create_resource.endpoint_model - else ("optimade.models.responses", "StructureResponseMany") - ) mongo_query = { "gateway_id": {"$eq": create_resource.gateway_id}, diff --git a/tests/routers/gateway/test_gateway_structures.py b/tests/routers/gateway/test_gateway_structures.py index 7a66f4c0..84755053 100644 --- a/tests/routers/gateway/test_gateway_structures.py +++ b/tests/routers/gateway/test_gateway_structures.py @@ -105,7 +105,7 @@ async def test_get_single_structure( database = [_ for _ in gateway["databases"] if "_single" in _["id"]][0] - assert response.data is not None + assert response.data is not None, f"Response:\n{response.json(indent=2)}" url = f"{database['attributes']['base_url']}/structures/{structure_id[len(database['id']) + 1:]}" db_response = httpx.get(url) @@ -115,7 +115,7 @@ async def test_get_single_structure( db_response = db_response.json() assert db_response["data"] is not None - db_response["data"]["id"] = f"{database['id']}/{db_response['data']['id']}" + assert db_response["data"] == json.loads(response.json(exclude_unset=True))["data"] assert db_response["meta"]["data_returned"] == response.meta.data_returned assert response.meta.data_available is None @@ -123,8 +123,6 @@ async def test_get_single_structure( db_response["meta"]["more_data_available"] == response.meta.more_data_available ) - assert db_response["data"] == json.loads(response.json(exclude_unset=True))["data"] - async def test_sort_no_effect( client: Callable[ diff --git a/tests/routers/test_info.py b/tests/routers/test_info.py index 597687bd..2f7512e4 100644 --- a/tests/routers/test_info.py +++ b/tests/routers/test_info.py @@ -33,9 +33,10 @@ async def test_get_info( "redoc", "search", "queries", + "databases", ] ) - entry_types_by_format = {"json": ["gateways", "queries"]} + entry_types_by_format = {"json": ["databases", "gateways", "queries"]} response = await client("/info") diff --git a/tests/routers/test_queries.py b/tests/routers/test_queries.py index 57d55b5b..d3b15366 100644 --- a/tests/routers/test_queries.py +++ b/tests/routers/test_queries.py @@ -156,10 +156,13 @@ async def test_query_results( """Test POST /queries and GET /queries/{id}""" import asyncio from optimade.models import EntryResponseMany - from optimade.models import StructureResponseMany from optimade_gateway.common.config import CONFIG - from optimade_gateway.models.queries import QueryState, QueryResource + from optimade_gateway.models.queries import ( + GatewayQueryResponse, + QueryState, + QueryResource, + ) data = { "id": "test", @@ -193,7 +196,7 @@ async def test_query_results( response = await client(f"/queries/{data['id']}") assert response.status_code == 200, f"Request failed: {response.json()}" - response = StructureResponseMany(**response.json()) + response = GatewayQueryResponse(**response.json()) assert response.data assert ( getattr(response.meta, f"_{CONFIG.provider.prefix}_query", "NOT FOUND") From 96c13025fdf770cabc97b06fd77449c252af0667 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Wed, 9 Jun 2021 15:42:03 +0200 Subject: [PATCH 03/13] Use responses property for routes Only supply the pydantic model to `response_model` that represents the successful response. Add a dictionary of erroneous status codes with the ErrorResponse pydantic model to `responses` for all routes. Fix bugs caught by the tests introduced in the latest commit, modularizing out processing a database response. Use new specific exception classes from `optimade`. --- optimade_gateway/models/queries.py | 25 ++++++++---- optimade_gateway/queries/perform.py | 4 +- optimade_gateway/queries/process.py | 14 ++++--- optimade_gateway/queries/utils.py | 11 +++-- optimade_gateway/routers/databases.py | 14 ++++--- optimade_gateway/routers/gateway/info.py | 16 ++++---- optimade_gateway/routers/gateway/links.py | 10 ++--- optimade_gateway/routers/gateway/queries.py | 13 +++--- .../routers/gateway/structures.py | 22 +++++++--- optimade_gateway/routers/gateway/versions.py | 2 + optimade_gateway/routers/gateways.py | 14 ++++--- optimade_gateway/routers/info.py | 10 ++--- optimade_gateway/routers/links.py | 9 ++--- optimade_gateway/routers/queries.py | 12 ++++-- optimade_gateway/routers/search.py | 19 +++++---- optimade_gateway/routers/utils.py | 14 +++---- tests/conftest.py | 12 ++++++ .../gateway/test_gateway_structures.py | 10 +++-- tests/routers/test_queries.py | 10 ++--- tests/routers/test_search.py | 40 ++++++++++--------- 20 files changed, 169 insertions(+), 112 deletions(-) diff --git a/optimade_gateway/models/queries.py b/optimade_gateway/models/queries.py index 28895c48..49fd14d2 100644 --- a/optimade_gateway/models/queries.py +++ b/optimade_gateway/models/queries.py @@ -6,15 +6,17 @@ from optimade.models import ( EntryResource, EntryResourceAttributes, - EntryResponseMany, - ErrorResponse, + OptimadeError, ReferenceResource, ReferenceResponseMany, ReferenceResponseOne, + Response, + ResponseMeta, StructureResource, StructureResponseMany, StructureResponseOne, ) +from optimade.models.utils import StrictField from optimade.server.query_params import EntryListingQueryParams from pydantic import BaseModel, EmailStr, Field, validator @@ -138,13 +140,23 @@ class QueryState(Enum): FINISHED = "finished" -class GatewayQueryResponse(EntryResponseMany): +class GatewayQueryResponse(Response): """Response from a Gateway Query.""" - data: Dict[str, Union[List[EntryResource], List[Dict[str, Any]]]] = Field( - ..., + data: Dict[str, Union[List[EntryResource], List[Dict[str, Any]]]] = StrictField( + ..., uniqueItems=True, description="Outputted Data" + ) + meta: ResponseMeta = StrictField( + ..., description="A meta object containing non-standard information" + ) + errors: Optional[List[OptimadeError]] = StrictField( + [], + description="A list of OPTIMADE-specific JSON API error objects, where the field detail MUST be present.", uniqueItems=True, ) + included: Optional[Union[List[EntryResource], List[Dict[str, Any]]]] = Field( + None, uniqueItems=True + ) class QueryResourceAttributes(EntryResourceAttributes): @@ -165,10 +177,9 @@ class QueryResourceAttributes(EntryResourceAttributes): title="State", type="enum", ) - response: Optional[Union[GatewayQueryResponse, ErrorResponse]] = Field( + response: Optional[GatewayQueryResponse] = Field( None, description="Response from gateway query.", - type="object", ) endpoint: EndpointEntryType = Field( EndpointEntryType.STRUCTURES, diff --git a/optimade_gateway/queries/perform.py b/optimade_gateway/queries/perform.py index ee1a18d2..98f7ca23 100644 --- a/optimade_gateway/queries/perform.py +++ b/optimade_gateway/queries/perform.py @@ -90,7 +90,7 @@ async def perform_query( **{"$set": {"state": QueryState.IN_PROGRESS}}, ) else: - response = EntryResponseMany( + response = query.attributes.endpoint.get_response_model()( data=[], links=ToplevelLinks(next=None), meta=meta_values( @@ -163,7 +163,7 @@ async def perform_query( resource["meta"] = database_id_meta else: resource.meta = Meta(**database_id_meta) - response.data.append(results) + response.data.extend(results) response.meta.data_returned += response_meta["data_returned"] if not response.meta.more_data_available: # Keep it True, if set to True once. diff --git a/optimade_gateway/queries/process.py b/optimade_gateway/queries/process.py index e91764c8..753dc658 100644 --- a/optimade_gateway/queries/process.py +++ b/optimade_gateway/queries/process.py @@ -57,6 +57,10 @@ async def process_db_response( errors = [] meta = {} + from optimade_gateway.common.logger import LOGGER + + LOGGER.debug("database_id: %s", database_id) + if isinstance(response, ErrorResponse): for error in response.errors: if isinstance(error.id, str) and error.id.startswith("OPTIMADE_GATEWAY"): @@ -113,14 +117,12 @@ async def process_db_response( # This ensures an empty list under `response.data.{database_id}` is returned if the case is # simply that there is no results to return. - # It also ensures that only `response.errors.{database_id}` is created if there are any - # errors. + if errors: + extra_updates.update({"$addToSet": {"response.errors": {"$each": errors}}}) await update_query( query, - f"response.errors.{database_id}" - if errors - else f"response.data.{database_id}", - errors or results, + f"response.data.{database_id}", + results, **extra_updates, ) else: diff --git a/optimade_gateway/queries/utils.py b/optimade_gateway/queries/utils.py index 21656b31..f0f22a15 100644 --- a/optimade_gateway/queries/utils.py +++ b/optimade_gateway/queries/utils.py @@ -47,17 +47,20 @@ async def update_query( update_kwargs = {"$set": {"last_modified": update_time}} if mongo_kwargs: - update_kwargs.update(await clean_python_types(mongo_kwargs)) + update_kwargs.update(mongo_kwargs) if operator and operator == "$set": - update_kwargs["$set"].update({field: await clean_python_types(value)}) + update_kwargs["$set"].update({field: value}) elif operator: - update_kwargs.update({operator: {field: await clean_python_types(value)}}) + if operator in update_kwargs: + update_kwargs[operator].update({field: value}) + else: + update_kwargs.update({operator: {field: value}}) # MongoDB result: UpdateResult = await QUERIES_COLLECTION.collection.update_one( filter={"id": {"$eq": query.id}}, - update=update_kwargs, + update=await clean_python_types(update_kwargs), ) if result.matched_count != 1: LOGGER.error( diff --git a/optimade_gateway/routers/databases.py b/optimade_gateway/routers/databases.py index c5d3dbfe..dd8d1fdf 100644 --- a/optimade_gateway/routers/databases.py +++ b/optimade_gateway/routers/databases.py @@ -11,12 +11,11 @@ One can register a new database (by using `POST /databases`) or look through the available databases (by using `GET /databases`) using standard OPTIMADE filtering. """ -from typing import Union - from fastapi import APIRouter, Depends, Request -from optimade.models import ErrorResponse, LinksResource, ToplevelLinks +from optimade.models import LinksResource, ToplevelLinks from optimade.server.query_params import EntryListingQueryParams, SingleEntryQueryParams from optimade.server.routers.utils import handle_response_fields, meta_values +from optimade.server.schemas import ERROR_RESPONSES from optimade_gateway.common.config import CONFIG from optimade_gateway.mappers import DatabasesMapper @@ -39,11 +38,12 @@ @ROUTER.get( "/databases", - response_model=Union[DatabasesResponse, ErrorResponse], + response_model=DatabasesResponse, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Databases"], + responses=ERROR_RESPONSES, ) async def get_databases( request: Request, @@ -65,11 +65,12 @@ async def get_databases( @ROUTER.post( "/databases", - response_model=Union[DatabasesResponseSingle, ErrorResponse], + response_model=DatabasesResponseSingle, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Databases"], + responses=ERROR_RESPONSES, ) async def post_databases( request: Request, database: DatabaseCreate @@ -99,11 +100,12 @@ async def post_databases( @ROUTER.get( "/databases/{database_id:path}", - response_model=Union[DatabasesResponseSingle, ErrorResponse], + response_model=DatabasesResponseSingle, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Databases"], + responses=ERROR_RESPONSES, ) async def get_database( request: Request, diff --git a/optimade_gateway/routers/gateway/info.py b/optimade_gateway/routers/gateway/info.py index df1a2802..809d2ba4 100644 --- a/optimade_gateway/routers/gateway/info.py +++ b/optimade_gateway/routers/gateway/info.py @@ -6,19 +6,17 @@ where `version` and `entry` may be left out. """ -from typing import Union - from fastapi import APIRouter, Request from optimade import __api_version__ from optimade.models import ( BaseInfoAttributes, BaseInfoResource, EntryInfoResponse, - ErrorResponse, InfoResponse, StructureResource, ) from optimade.server.routers.utils import get_base_url, meta_values +from optimade.server.schemas import ERROR_RESPONSES ROUTER = APIRouter(redirect_slashes=True) @@ -27,11 +25,12 @@ @ROUTER.get( "/gateways/{gateway_id}/info", - response_model=Union[InfoResponse, ErrorResponse], + response_model=InfoResponse, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Info"], + responses=ERROR_RESPONSES, ) async def get_gateways_info( request: Request, @@ -88,11 +87,12 @@ async def get_gateways_info( @ROUTER.get( "/gateways/{gateway_id}/info/{entry}", - response_model=Union[EntryInfoResponse, ErrorResponse], + response_model=EntryInfoResponse, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Info"], + responses=ERROR_RESPONSES, ) async def get_gateways_entry_info( request: Request, gateway_id: str, entry: str @@ -145,11 +145,12 @@ async def get_gateways_entry_info( @ROUTER.get( "/gateways/{gateway_id}/{version}/info", - response_model=Union[InfoResponse, ErrorResponse], + response_model=InfoResponse, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Info"], + responses=ERROR_RESPONSES, ) async def get_versioned_gateways_info( request: Request, @@ -168,11 +169,12 @@ async def get_versioned_gateways_info( @ROUTER.get( "/gateways/{gateway_id}/{version}/info/{entry}", - response_model=Union[EntryInfoResponse, ErrorResponse], + response_model=EntryInfoResponse, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Info"], + responses=ERROR_RESPONSES, ) async def get_versioned_gateways_entry_info( request: Request, diff --git a/optimade_gateway/routers/gateway/links.py b/optimade_gateway/routers/gateway/links.py index e75344fe..1abd15d6 100644 --- a/optimade_gateway/routers/gateway/links.py +++ b/optimade_gateway/routers/gateway/links.py @@ -6,25 +6,24 @@ where `version` may be left out. """ -from typing import Union - from fastapi import APIRouter, Depends, Request from optimade.models import ( - ErrorResponse, LinksResponse, ) from optimade.server.query_params import EntryListingQueryParams +from optimade.server.schemas import ERROR_RESPONSES ROUTER = APIRouter(redirect_slashes=True) @ROUTER.get( "/gateways/{gateway_id}/links", - response_model=Union[LinksResponse, ErrorResponse], + response_model=LinksResponse, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Links"], + responses=ERROR_RESPONSES, ) async def get_gateways_links( request: Request, @@ -45,11 +44,12 @@ async def get_gateways_links( @ROUTER.get( "/gateways/{gateway_id}/{version}/links", - response_model=Union[LinksResponse, ErrorResponse], + response_model=LinksResponse, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Links"], + responses=ERROR_RESPONSES, ) async def get_versioned_gateways_links( request: Request, diff --git a/optimade_gateway/routers/gateway/queries.py b/optimade_gateway/routers/gateway/queries.py index 62a4b943..a014e7c6 100644 --- a/optimade_gateway/routers/gateway/queries.py +++ b/optimade_gateway/routers/gateway/queries.py @@ -6,13 +6,11 @@ where `version` and the last `id` may be left out. """ -from typing import Union - from fastapi import APIRouter, Depends, Request, status -from optimade.models import ErrorResponse from optimade.models.responses import EntryResponseMany from optimade.server.exceptions import Forbidden from optimade.server.query_params import EntryListingQueryParams +from optimade.server.schemas import ERROR_RESPONSES from optimade_gateway.models import ( QueryCreate, @@ -27,11 +25,12 @@ @ROUTER.get( "/gateways/{gateway_id}/queries", - response_model=Union[QueriesResponse, ErrorResponse], + response_model=QueriesResponse, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Gateways", "Queries"], + responses=ERROR_RESPONSES, ) async def get_gateway_queries( request: Request, @@ -57,12 +56,13 @@ async def get_gateway_queries( @ROUTER.post( "/gateways/{gateway_id}/queries", - response_model=Union[QueriesResponseSingle, ErrorResponse], + response_model=QueriesResponseSingle, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Gateways", "Queries"], status_code=status.HTTP_202_ACCEPTED, + responses=ERROR_RESPONSES, ) async def post_gateway_queries( request: Request, @@ -91,11 +91,12 @@ async def post_gateway_queries( @ROUTER.get( "/gateways/{gateway_id}/queries/{query_id}", - response_model=Union[EntryResponseMany, ErrorResponse], + response_model=EntryResponseMany, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Gateways", "Queries"], + responses=ERROR_RESPONSES, ) async def get_gateway_query( request: Request, gateway_id: str, query_id: str diff --git a/optimade_gateway/routers/gateway/structures.py b/optimade_gateway/routers/gateway/structures.py index c36f18e1..99972e14 100644 --- a/optimade_gateway/routers/gateway/structures.py +++ b/optimade_gateway/routers/gateway/structures.py @@ -23,6 +23,7 @@ ) from optimade.server.query_params import EntryListingQueryParams, SingleEntryQueryParams from optimade.server.routers.utils import meta_values +from optimade.server.schemas import ERROR_RESPONSES from optimade_gateway.models import QueryResource from optimade_gateway.queries import perform_query @@ -36,11 +37,12 @@ @ROUTER.get( "/gateways/{gateway_id}/structures", - response_model=Union[StructureResponseMany, ErrorResponse], + response_model=StructureResponseMany, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Structures"], + responses=ERROR_RESPONSES, ) async def get_structures( request: Request, @@ -88,17 +90,23 @@ async def get_structures( break else: response.status_code = 500 - - return gateway_response + return gateway_response + elif isinstance(gateway_response, StructureResponseMany): + return gateway_response + else: + raise TypeError( + "The response should be either StructureResponseMany or ErrorResponse." + ) @ROUTER.get( "/gateways/{gateway_id}/structures/{structure_id:path}", - response_model=Union[StructureResponseOne, ErrorResponse], + response_model=StructureResponseOne, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Structures"], + responses=ERROR_RESPONSES, ) async def get_single_structure( request: Request, @@ -241,12 +249,13 @@ async def get_single_structure( @ROUTER.get( "/gateways/{gateway_id}/{version}/structures", - response_model=Union[StructureResponseMany, ErrorResponse], + response_model=StructureResponseMany, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Structures"], include_in_schema=False, + responses=ERROR_RESPONSES, ) async def get_versioned_structures( request: Request, @@ -265,12 +274,13 @@ async def get_versioned_structures( @ROUTER.get( "/gateways/{gateway_id}/{version}/structures/{structure_id:path}", - response_model=Union[StructureResponseOne, ErrorResponse], + response_model=StructureResponseOne, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Structures"], include_in_schema=False, + responses=ERROR_RESPONSES, ) async def get_versioned_single_structure( request: Request, diff --git a/optimade_gateway/routers/gateway/versions.py b/optimade_gateway/routers/gateway/versions.py index 94ca58ea..cbf162e4 100644 --- a/optimade_gateway/routers/gateway/versions.py +++ b/optimade_gateway/routers/gateway/versions.py @@ -7,6 +7,7 @@ """ from fastapi import APIRouter, Request from optimade.server.routers.versions import CsvResponse +from optimade.server.schemas import ERROR_RESPONSES ROUTER = APIRouter(redirect_slashes=True) @@ -15,6 +16,7 @@ "/gateways/{gateway_id}/versions", response_class=CsvResponse, tags=["Versions"], + responses=ERROR_RESPONSES, ) async def get_gateway_versions(request: Request, gateway_id: str) -> CsvResponse: """`GET /gateways/{gateway_id}/versions` diff --git a/optimade_gateway/routers/gateways.py b/optimade_gateway/routers/gateways.py index 533e777d..122e55ee 100644 --- a/optimade_gateway/routers/gateways.py +++ b/optimade_gateway/routers/gateways.py @@ -6,13 +6,12 @@ where, `id` may be left out. """ -from typing import Union - from fastapi import APIRouter, Depends, Request from fastapi.responses import RedirectResponse -from optimade.models import ErrorResponse, ToplevelLinks +from optimade.models import ToplevelLinks from optimade.server.query_params import EntryListingQueryParams from optimade.server.routers.utils import meta_values +from optimade.server.schemas import ERROR_RESPONSES from optimade_gateway.common.config import CONFIG from optimade_gateway.mappers import GatewaysMapper @@ -36,11 +35,12 @@ @ROUTER.get( "/gateways", - response_model=Union[GatewaysResponse, ErrorResponse], + response_model=GatewaysResponse, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Gateways"], + responses=ERROR_RESPONSES, ) async def get_gateways( request: Request, @@ -62,11 +62,12 @@ async def get_gateways( @ROUTER.post( "/gateways", - response_model=Union[GatewaysResponseSingle, ErrorResponse], + response_model=GatewaysResponseSingle, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Gateways"], + responses=ERROR_RESPONSES, ) async def post_gateways( request: Request, gateway: GatewayCreate @@ -110,11 +111,12 @@ async def post_gateways( @ROUTER.get( "/gateways/{gateway_id}", - response_model=Union[GatewaysResponseSingle, ErrorResponse], + response_model=GatewaysResponseSingle, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Gateways"], + responses=ERROR_RESPONSES, ) async def get_gateway(request: Request, gateway_id: str) -> GatewaysResponseSingle: """`GET /gateways/{gateway ID}` diff --git a/optimade_gateway/routers/info.py b/optimade_gateway/routers/info.py index 68259864..78d1df1a 100644 --- a/optimade_gateway/routers/info.py +++ b/optimade_gateway/routers/info.py @@ -6,19 +6,17 @@ where, `entry` may be left out. """ -from typing import Union - from fastapi import APIRouter, Request from optimade import __api_version__ from optimade.models import ( BaseInfoAttributes, BaseInfoResource, EntryInfoResponse, - ErrorResponse, InfoResponse, LinksResource, ) from optimade.server.routers.utils import get_base_url, meta_values +from optimade.server.schemas import ERROR_RESPONSES from optimade_gateway.models import GatewayResource, QueryResource @@ -33,11 +31,12 @@ @ROUTER.get( "/info", - response_model=Union[InfoResponse, ErrorResponse], + response_model=InfoResponse, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Info"], + responses=ERROR_RESPONSES, ) async def get_info(request: Request) -> InfoResponse: """`GET /info` @@ -83,11 +82,12 @@ async def get_info(request: Request) -> InfoResponse: @ROUTER.get( "/info/{entry}", - response_model=Union[EntryInfoResponse, ErrorResponse], + response_model=EntryInfoResponse, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Info"], + responses=ERROR_RESPONSES, ) async def get_entry_info(request: Request, entry: str) -> EntryInfoResponse: """`GET /info/{entry}` diff --git a/optimade_gateway/routers/links.py b/optimade_gateway/routers/links.py index 3be28677..d66996ed 100644 --- a/optimade_gateway/routers/links.py +++ b/optimade_gateway/routers/links.py @@ -5,17 +5,15 @@ /links """ -from typing import Union - from fastapi import APIRouter, Depends, Request -from optimade.models import ErrorResponse, LinksResponse, LinksResource +from optimade.models import LinksResponse, LinksResource from optimade.server.mappers import LinksMapper from optimade.server.query_params import EntryListingQueryParams +from optimade.server.schemas import ERROR_RESPONSES from optimade_gateway.common.config import CONFIG from optimade_gateway.mongo.collection import AsyncMongoCollection - from optimade_gateway.routers.utils import get_entries ROUTER = APIRouter(redirect_slashes=True) @@ -29,11 +27,12 @@ @ROUTER.get( "/links", - response_model=Union[LinksResponse, ErrorResponse], + response_model=LinksResponse, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Links"], + responses=ERROR_RESPONSES, ) async def get_links( request: Request, params: EntryListingQueryParams = Depends() diff --git a/optimade_gateway/routers/queries.py b/optimade_gateway/routers/queries.py index da70e7f7..87d910a6 100644 --- a/optimade_gateway/routers/queries.py +++ b/optimade_gateway/routers/queries.py @@ -20,6 +20,7 @@ from optimade.models.responses import EntryResponseMany from optimade.server.query_params import EntryListingQueryParams from optimade.server.routers.utils import meta_values +from optimade.server.schemas import ERROR_RESPONSES from optimade_gateway.common.config import CONFIG from optimade_gateway.mappers import QueryMapper @@ -45,11 +46,12 @@ @ROUTER.get( "/queries", - response_model=Union[QueriesResponse, ErrorResponse], + response_model=QueriesResponse, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Queries"], + responses=ERROR_RESPONSES, ) async def get_queries( request: Request, @@ -71,12 +73,13 @@ async def get_queries( @ROUTER.post( "/queries", - response_model=Union[QueriesResponseSingle, ErrorResponse], + response_model=QueriesResponseSingle, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Queries"], status_code=status.HTTP_202_ACCEPTED, + responses=ERROR_RESPONSES, ) async def post_queries( request: Request, @@ -114,17 +117,18 @@ async def post_queries( @ROUTER.get( "/queries/{query_id:path}", - response_model=Union[EntryResponseMany, ErrorResponse, GatewayQueryResponse], + response_model=GatewayQueryResponse, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Queries"], + responses=ERROR_RESPONSES, ) async def get_query( request: Request, query_id: str, response: Response, -) -> Union[EntryResponseMany, ErrorResponse, GatewayQueryResponse]: +) -> Union[ErrorResponse, GatewayQueryResponse]: """`GET /queries/{query_id}` Return the response from a query diff --git a/optimade_gateway/routers/search.py b/optimade_gateway/routers/search.py index 362e1f1e..a5d196ed 100644 --- a/optimade_gateway/routers/search.py +++ b/optimade_gateway/routers/search.py @@ -19,8 +19,6 @@ from fastapi.responses import RedirectResponse from optimade.server.exceptions import BadRequest from optimade.models import ( - EntryResponseMany, - ErrorResponse, LinksResource, LinksResourceAttributes, ToplevelLinks, @@ -28,6 +26,7 @@ from optimade.models.links import LinkType from optimade.server.query_params import EntryListingQueryParams from optimade.server.routers.utils import meta_values +from optimade.server.schemas import ERROR_RESPONSES from pydantic import AnyUrl, ValidationError from optimade_gateway.common.config import CONFIG @@ -40,7 +39,11 @@ QueryResource, Search, ) -from optimade_gateway.models.queries import OptimadeQueryParameters, QueryState +from optimade_gateway.models.queries import ( + GatewayQueryResponse, + OptimadeQueryParameters, + QueryState, +) from optimade_gateway.queries import perform_query, SearchQueryParams @@ -49,12 +52,13 @@ @ROUTER.post( "/search", - response_model=Union[QueriesResponseSingle, ErrorResponse], + response_model=QueriesResponseSingle, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Search"], status_code=status.HTTP_202_ACCEPTED, + responses=ERROR_RESPONSES, ) async def post_search(request: Request, search: Search) -> QueriesResponseSingle: """`POST /search` @@ -118,7 +122,7 @@ async def post_search(request: Request, search: Search) -> QueriesResponseSingle f"{url.user + '@' if url.user else ''}{url.host}" f"{':' + url.port if url.port else ''}" f"{url.path.rstrip('/') if url.path else ''}" - ), + ).replace(".", "__"), type="links", attributes=LinksResourceAttributes( name=( @@ -169,18 +173,19 @@ async def post_search(request: Request, search: Search) -> QueriesResponseSingle @ROUTER.get( "/search", - response_model=Union[EntryResponseMany, ErrorResponse], + response_model=GatewayQueryResponse, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, tags=["Search"], + responses=ERROR_RESPONSES, ) async def get_search( request: Request, response: Response, search_params: SearchQueryParams = Depends(), entry_params: EntryListingQueryParams = Depends(), -) -> Union[EntryResponseMany, ErrorResponse, RedirectResponse]: +) -> Union[GatewayQueryResponse, RedirectResponse]: """`GET /search` Coordinate a new OPTIMADE query in multiple databases through a gateway: diff --git a/optimade_gateway/routers/utils.py b/optimade_gateway/routers/utils.py index 92a2bf93..bcaa10a6 100644 --- a/optimade_gateway/routers/utils.py +++ b/optimade_gateway/routers/utils.py @@ -180,8 +180,8 @@ async def resource_factory( mongo_query = { "$or": [ - {"base_url": {"$eq": await clean_python_types(base_url)}}, - {"base_url.href": {"$eq": await clean_python_types(base_url)}}, + {"base_url": {"$eq": base_url}}, + {"base_url.href": {"$eq": base_url}}, ] } elif isinstance(create_resource, GatewayCreate): @@ -192,9 +192,7 @@ async def resource_factory( mongo_query = { "databases": {"$size": len(create_resource.databases)}, "databases.attributes.base_url": { - "$all": await clean_python_types( - [_.attributes.base_url for _ in create_resource.databases] - ) + "$all": [_.attributes.base_url for _ in create_resource.databases] }, } elif isinstance(create_resource, QueryCreate): @@ -210,9 +208,7 @@ async def resource_factory( mongo_query = { "gateway_id": {"$eq": create_resource.gateway_id}, - "query_parameters": { - "$eq": await clean_python_types(create_resource.query_parameters), - }, + "query_parameters": {"$eq": create_resource.query_parameters}, "endpoint": {"$eq": create_resource.endpoint}, } else: @@ -222,7 +218,7 @@ async def resource_factory( ) result, more_data_available, _ = await RESOURCE_COLLECTION.find( - criteria={"filter": mongo_query} + criteria={"filter": await clean_python_types(mongo_query)} ) if more_data_available: diff --git a/tests/conftest.py b/tests/conftest.py index 1a935de7..4e94982d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -173,9 +173,21 @@ def _mock_response(gateway: dict) -> None: top_dir / f"tests/static/db_responses/{database['id']}.json" ) as handle: data = json.load(handle) + + if data.get("errors", []): + for error in data.get("errors", []): + if "status" in error: + status_code = int(error["status"]) + break + else: + status_code = 500 + else: + status_code = 200 + httpx_mock.add_response( url=re.compile(fr"{database['attributes']['base_url']}.*"), json=data, + status_code=status_code, ) def sleep_response(request: httpx.Request, extensions: dict) -> MockResponse: diff --git a/tests/routers/gateway/test_gateway_structures.py b/tests/routers/gateway/test_gateway_structures.py index 84755053..d74a69f3 100644 --- a/tests/routers/gateway/test_gateway_structures.py +++ b/tests/routers/gateway/test_gateway_structures.py @@ -63,15 +63,19 @@ async def test_get_structures( more_data_available = db_response["meta"]["more_data_available"] for datum in db_response["data"]: - datum["id"] = f"{database['id']}/{datum['id']}" + database_id_meta = { + "_optimade_gateway_": {"source_database_id": database["id"]} + } + if "meta" in datum: + datum["meta"].update(database_id_meta) + else: + datum["meta"] = database_id_meta data.append(datum) assert data_returned == response.meta.data_returned assert data_available == response.meta.data_available assert more_data_available == response.meta.more_data_available - data.sort(key=lambda datum: datum["id"]) - data.sort(key=lambda datum: "/".join(datum["id"].split("/")[1:])) assert data == json.loads(response.json(exclude_unset=True))["data"], ( f"IDs in test not in response: {set([_['id'] for _ in data]) - set([_['id'] for _ in json.loads(response.json(exclude_unset=True))['data']])}\n\n" f"IDs in response not in test: {set([_['id'] for _ in json.loads(response.json(exclude_unset=True))['data']]) - set([_['id'] for _ in data])}\n\n" diff --git a/tests/routers/test_queries.py b/tests/routers/test_queries.py index d3b15366..db7e1628 100644 --- a/tests/routers/test_queries.py +++ b/tests/routers/test_queries.py @@ -155,7 +155,6 @@ async def test_query_results( ): """Test POST /queries and GET /queries/{id}""" import asyncio - from optimade.models import EntryResponseMany from optimade_gateway.common.config import CONFIG from optimade_gateway.models.queries import ( @@ -182,8 +181,8 @@ async def test_query_results( response = await client(f"/queries/{data['id']}") assert response.status_code == 200, f"Request failed: {response.json()}" - response = EntryResponseMany(**response.json()) - assert response.data == [] + response = GatewayQueryResponse(**response.json()) + assert response.data == {} query: QueryResource = QueryResource( **getattr(response.meta, f"_{CONFIG.provider.prefix}_query") @@ -215,8 +214,8 @@ async def test_errored_query_results( ): """Test POST /queries and GET /queries/{id} with an erroneous response""" import asyncio - from optimade.models import ErrorResponse + from optimade_gateway.models.queries import GatewayQueryResponse from optimade_gateway.models.responses import QueriesResponseSingle data = { @@ -239,7 +238,8 @@ async def test_errored_query_results( response.status_code == 404 ), f"Request succeeded, where it should have failed:\n{json.dumps(response.json(), indent=2)}" - response = ErrorResponse(**response.json()) + response = GatewayQueryResponse(**response.json()) + assert response.errors @pytest.mark.usefixtures("reset_db_after") diff --git a/tests/routers/test_search.py b/tests/routers/test_search.py index f26ad30b..1da2733d 100644 --- a/tests/routers/test_search.py +++ b/tests/routers/test_search.py @@ -30,9 +30,8 @@ async def test_get_search( this should ensure a new gateway is created, specifically for use with these versioned base URLs, but we can reuse the mock_gateway_responses for the "twodbs" gateway. """ - from optimade.models import StructureResponseMany - from optimade_gateway.common.config import CONFIG + from optimade_gateway.models import GatewayQueryResponse gateway_id = "twodbs" gateway: dict = await get_gateway(gateway_id) @@ -52,7 +51,7 @@ async def test_get_search( assert response.status_code == 200, f"Request failed: {response.json()}" - response = StructureResponseMany(**response.json()) + response = GatewayQueryResponse(**response.json()) assert response.data assert ( getattr(response.meta, f"_{CONFIG.provider.prefix}_query", "NOT FOUND") @@ -72,9 +71,8 @@ async def test_get_search_existing_gateway( caplog: pytest.LogCaptureFixture, ): """Test GET /search for base URLs matching an existing gateway""" - from optimade.models import StructureResponseMany - from optimade_gateway.common.config import CONFIG + from optimade_gateway.models import GatewayQueryResponse gateway_id = "twodbs" gateway: dict = await get_gateway(gateway_id) @@ -115,7 +113,7 @@ async def test_get_search_existing_gateway( assert response.status_code == 200, f"Request failed: {response.json()}" - response = StructureResponseMany(**response.json()) + response = GatewayQueryResponse(**response.json()) assert response.data, f"No data: {response.json(indent=2)}" assert ( getattr(response.meta, f"_{CONFIG.provider.prefix}_query", "NOT FOUND") @@ -135,10 +133,12 @@ async def test_get_search_not_finishing( caplog: pytest.LogCaptureFixture, ): """Test GET /search for unfinished query (redirect to query URL)""" - from optimade.models import EntryResponseMany - from optimade_gateway.common.config import CONFIG - from optimade_gateway.models.queries import QueryResource, QueryState + from optimade_gateway.models.queries import ( + GatewayQueryResponse, + QueryResource, + QueryState, + ) gateway_id = "slow-query" gateway: dict = await get_gateway(gateway_id) @@ -160,12 +160,13 @@ async def test_get_search_not_finishing( assert "A gateway was found and reused for a query" in caplog.text, caplog.text - response = EntryResponseMany(**response.json()) - assert response.data == [], f"Data was found in response: {response.json(indent=2)}" + response = GatewayQueryResponse(**response.json()) + assert response.data == {}, f"Data was found in response: {response.json(indent=2)}" assert getattr( response.meta, f"_{CONFIG.provider.prefix}_query", False ), f"Special __query field not found in meta. Response: {response.json(indent=2)}" + query: QueryResource = QueryResource( **getattr(response.meta, f"_{CONFIG.provider.prefix}_query") ) @@ -175,9 +176,9 @@ async def test_get_search_not_finishing( assert ( query.attributes.query_parameters.page_limit == query_params["page_limit"] ), query - assert ( - query.attributes.response == query.attributes.__fields__["response"].default - ), query + assert isinstance(query.attributes.response, GatewayQueryResponse) + assert query.attributes.response.data == {} + assert query.attributes.response.errors == [] assert query.attributes.gateway_id == gateway_id, query @@ -239,7 +240,7 @@ async def test_post_search( == OptimadeQueryParameters(**data["query_parameters"]).dict() ), f"Response: {datum.attributes.query_parameters!r}\n\nTest data: {OptimadeQueryParameters(**data['query_parameters'])!r}" - assert datum.attributes.state == QueryState.CREATED + assert datum.attributes.state in [QueryState.CREATED, QueryState.STARTED] assert datum.attributes.response is None with open(top_dir / "tests/static/test_gateways.json") as handle: @@ -321,7 +322,7 @@ async def test_post_search_existing_gateway( == OptimadeQueryParameters(**gateway_create_data["query_parameters"]).dict() ), f"Response: {datum.attributes.query_parameters!r}\n\nTest data: {OptimadeQueryParameters(**gateway_create_data['query_parameters'])!r}" - assert datum.attributes.state == QueryState.CREATED + assert datum.attributes.state in [QueryState.CREATED, QueryState.STARTED] assert datum.attributes.response is None assert datum.attributes.gateway_id == gateway_id @@ -345,8 +346,9 @@ async def test_sort_no_effect( This means if the `sort` parameter is used, the response should not change - it should be ignored. """ - from optimade.models import StructureResponseMany, Warnings + from optimade.models import Warnings + from optimade_gateway.models import GatewayQueryResponse from optimade_gateway.models.responses import QueriesResponseSingle from optimade_gateway.warnings import SortNotSupported @@ -373,9 +375,9 @@ async def test_sort_no_effect( assert response_asc.status_code == 200, f"Request failed: {response_asc.json()}" assert response_desc.status_code == 200, f"Request failed: {response_desc.json()}" - response_asc = StructureResponseMany(**response_asc.json()) + response_asc = GatewayQueryResponse(**response_asc.json()) assert response_asc - response_desc = StructureResponseMany(**response_desc.json()) + response_desc = GatewayQueryResponse(**response_desc.json()) assert response_desc assert response_asc.data == response_desc.data From 759f63c80653bc4e00c614ec975f7da344c23d71 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 15 Jul 2021 01:44:23 +0200 Subject: [PATCH 04/13] Update to new Collection design --- optimade_gateway/mappers/base.py | 45 ++++ optimade_gateway/mappers/databases.py | 12 +- optimade_gateway/mappers/gateways.py | 3 +- optimade_gateway/mappers/links.py | 26 +++ optimade_gateway/mappers/queries.py | 3 +- optimade_gateway/mongo/collection.py | 317 ++++++++++++++------------ optimade_gateway/routers/databases.py | 16 +- optimade_gateway/routers/gateways.py | 2 +- optimade_gateway/routers/links.py | 2 +- optimade_gateway/routers/utils.py | 21 +- 10 files changed, 284 insertions(+), 163 deletions(-) create mode 100644 optimade_gateway/mappers/base.py create mode 100644 optimade_gateway/mappers/links.py diff --git a/optimade_gateway/mappers/base.py b/optimade_gateway/mappers/base.py new file mode 100644 index 00000000..65c50d22 --- /dev/null +++ b/optimade_gateway/mappers/base.py @@ -0,0 +1,45 @@ +from typing import Iterable, List, Union + +from optimade.models import EntryResource +from optimade.server.mappers.entries import ( + BaseResourceMapper as OptimadeBaseResourceMapper, +) + + +class BaseResourceMapper(OptimadeBaseResourceMapper): + """ + Generic Resource Mapper that defines and performs the mapping + between objects in the database and the resource objects defined by + the specification. + + Note: + This is a "wrapped" sub-class to make certain methods asynchronous. + + Attributes: + ALIASES: a tuple of aliases between + OPTIMADE field names and the field names in the database , + e.g. `(("elements", "custom_elements_field"))`. + LENGTH_ALIASES: a tuple of aliases between + a field name and another field that defines its length, to be used + when querying, e.g. `(("elements", "nelements"))`. + e.g. `(("elements", "custom_elements_field"))`. + ENTRY_RESOURCE_CLASS: The entry type that this mapper corresponds to. + PROVIDER_FIELDS: a tuple of extra field names that this + mapper should support when querying with the database prefix. + TOP_LEVEL_NON_ATTRIBUTES_FIELDS: the set of top-level + field names common to all endpoints. + SUPPORTED_PREFIXES: The set of prefixes registered by this mapper. + ALL_ATTRIBUTES: The set of attributes defined across the entry + resource class and the server configuration. + ENTRY_RESOURCE_ATTRIBUTES: A dictionary of attributes and their definitions + defined by the schema of the entry resource class. + ENDPOINT: The expected endpoint name for this resource, as defined by + the `type` in the schema of the entry resource class. + + """ + + @classmethod + async def deserialize( + cls, results: Union[dict, Iterable[dict]] + ) -> Union[List[EntryResource], EntryResource]: + return super(BaseResourceMapper, cls).deserialize(results) diff --git a/optimade_gateway/mappers/databases.py b/optimade_gateway/mappers/databases.py index 6e6b97b2..4b733276 100644 --- a/optimade_gateway/mappers/databases.py +++ b/optimade_gateway/mappers/databases.py @@ -1,13 +1,14 @@ from optimade.models import LinksResource -from optimade.server.mappers.links import LinksMapper from pydantic import AnyUrl # pylint: disable=no-name-in-module from optimade_gateway.common.config import CONFIG +from optimade_gateway.mappers.base import BaseResourceMapper + __all__ = ("DatabasesMapper",) -class DatabasesMapper(LinksMapper): +class DatabasesMapper(BaseResourceMapper): ENDPOINT = "databases" ENTRY_RESOURCE_CLASS = LinksResource @@ -30,7 +31,6 @@ def map_back(cls, doc: dict) -> dict: } # Ensure the type does not change to "databases" - # The `LinksMapper.map_back()` method ensures the value for doc["type"] is kept. - doc["type"] = "links" - - return super().map_back(doc) + newdoc = super().map_back(doc) + newdoc["type"] = "links" + return newdoc diff --git a/optimade_gateway/mappers/gateways.py b/optimade_gateway/mappers/gateways.py index 9cffa71e..cdc5c6b6 100644 --- a/optimade_gateway/mappers/gateways.py +++ b/optimade_gateway/mappers/gateways.py @@ -1,9 +1,10 @@ -from optimade.server.mappers.entries import BaseResourceMapper from pydantic import AnyUrl # pylint: disable=no-name-in-module from optimade_gateway.common.config import CONFIG from optimade_gateway.models import GatewayResource +from optimade_gateway.mappers.base import BaseResourceMapper + __all__ = ("GatewaysMapper",) diff --git a/optimade_gateway/mappers/links.py b/optimade_gateway/mappers/links.py new file mode 100644 index 00000000..9b07fa6c --- /dev/null +++ b/optimade_gateway/mappers/links.py @@ -0,0 +1,26 @@ +from optimade.models.links import LinksResource + +from optimade_gateway.mappers.base import BaseResourceMapper + +__all__ = ("LinksMapper",) + + +class LinksMapper(BaseResourceMapper): + + ENDPOINT = "links" + ENTRY_RESOURCE_CLASS = LinksResource + + @classmethod + def map_back(cls, doc: dict) -> dict: + """Map properties from MongoDB to OPTIMADE + + :param doc: A resource object in MongoDB format + :type doc: dict + + :return: A resource object in OPTIMADE format + :rtype: dict + """ + type_ = doc["type"] + newdoc = super().map_back(doc) + newdoc["type"] = type_ + return newdoc diff --git a/optimade_gateway/mappers/queries.py b/optimade_gateway/mappers/queries.py index 60374b37..287236a5 100644 --- a/optimade_gateway/mappers/queries.py +++ b/optimade_gateway/mappers/queries.py @@ -1,9 +1,10 @@ -from optimade.server.mappers.entries import BaseResourceMapper from pydantic import AnyUrl # pylint: disable=no-name-in-module from optimade_gateway.common.config import CONFIG from optimade_gateway.models import QueryResource +from optimade_gateway.mappers.base import BaseResourceMapper + __all__ = ("QueryMapper",) diff --git a/optimade_gateway/mongo/collection.py b/optimade_gateway/mongo/collection.py index ed87cdd8..e9f838ed 100644 --- a/optimade_gateway/mongo/collection.py +++ b/optimade_gateway/mongo/collection.py @@ -1,13 +1,15 @@ from datetime import datetime -from typing import Any, Dict, List, Tuple, Union +from typing import Any, Dict, List, Set, Tuple, Union +import warnings -from fastapi import HTTPException from optimade.filterparser import LarkParser from optimade.filtertransformers.mongo import MongoTransformer from optimade.models import EntryResource from optimade.server.entry_collections.entry_collections import EntryCollection +from optimade.server.exceptions import BadRequest, NotFound from optimade.server.mappers.entries import BaseResourceMapper from optimade.server.query_params import EntryListingQueryParams, SingleEntryQueryParams +from optimade.server.warnings import UnknownProviderProperty from pymongo.collection import Collection as MongoCollection from optimade_gateway.common.logger import LOGGER @@ -55,73 +57,25 @@ def __init__( self._check_aliases(self.resource_mapper.all_length_aliases()) def __str__(self) -> str: - """Standard printing result for an instance""" + """Standard printing result for an instance.""" return f"<{self.__class__.__name__}: resource={self.resource_cls.__name__} endpoint(mapper)={self.resource_mapper.ENDPOINT} DB_collection={self.collection.name}>" def __repr__(self) -> str: - """Representation of instance""" - return f"{self.__class__.__name__}(name={self.collection.name!r}, resource_cls={self.resource_cls!r}, resource_mapper={self.resource_mapper!r}" + """Representation of instance.""" + return f"{self.__class__.__name__}(name={self.collection.name!r}, resource_cls={self.resource_cls!r}, resource_mapper={self.resource_mapper!r})" def __len__(self) -> int: - """Length of collection""" import warnings warnings.warn( - "Cannot calculate length of collection using `len()`. Use `count()` instead.", - OptimadeGatewayWarning, + OptimadeGatewayWarning( + detail="Cannot calculate length of collection using `len()`. Use `count()` instead." + ) ) return 0 - def _check_aliases(self, aliases: Tuple[Tuple[str, str]]) -> None: - """Check that aliases do not clash with mongo keywords. - - Parameters: - aliases: Tuple of tuple of aliases to be checked. - - Raises: - RuntimeError: If any alias starts with the dollar (`$`) character. - - """ - if any( - alias[0].startswith("$") or alias[1].startswith("$") for alias in aliases - ): - raise RuntimeError(f"Cannot define an alias starting with a '$': {aliases}") - - @staticmethod - def _valid_find_keys(**kwargs) -> Dict[str, Any]: - """Return valid MongoDB find() keys with values from kwargs - - Note, not including deprecated flags - (see https://pymongo.readthedocs.io/en/3.11.0/api/pymongo/collection.html#pymongo.collection.Collection.find). - """ - valid_method_keys = ( - "filter", - "projection", - "session", - "skip", - "limit", - "no_cursor_timeout", - "cursor_type", - "sort", - "allow_partial_results", - "batch_size", - "collation", - "return_key", - "show_record_id", - "hint", - "max_time_ms", - "min", - "max", - "comment", - "allow_disk_use", - ) - criteria = {key: kwargs[key] for key in valid_method_keys if key in kwargs} - - if criteria.get("filter") is None: - # Ensure documents are included in the result set - criteria["filter"] = {} - - return criteria + async def insert(self, data: List[EntryResource]) -> None: + await self.collection.insert_many(await clean_python_types(data)) async def count( self, @@ -165,10 +119,140 @@ async def count( return await self.collection.count_documents(**criteria) + async def find( + self, + params: Union[EntryListingQueryParams, SingleEntryQueryParams] = None, + criteria: Dict[str, Any] = None, + ) -> Tuple[ + Union[List[EntryResource], EntryResource, None], bool, Set[str], Set[str] + ]: + """Perform the query on the underlying MongoDB Collection, handling projection + and pagination of the output. + + Either provide `params` or `criteria`. Not both, but at least one. + + Parameters: + params: URL query parameters, either from a general entry endpoint or a single-entry endpoint. + criteria: Already handled/parsed URL query parameters. + + Returns: + A list of entry resource objects, whether more data is available with pagination, and fields + (excluded, included). + + """ + if (params is None and criteria is None) or ( + params is not None and criteria is not None + ): + raise ValueError( + "Exacly one of either `params` and `criteria` must be specified." + ) + + # Set single_entry to False, this is done since if criteria is defined, + # this is an unknown factor - better to then get a list of results. + single_entry = False + if criteria is None: + criteria = await self.handle_query_params(params) + else: + single_entry = isinstance(params, SingleEntryQueryParams) + + response_fields = criteria.pop("fields", self.all_fields) + + results, data_returned, more_data_available = await self._run_db_query( + criteria=criteria, + single_entry=single_entry, + ) + + if single_entry: + results = results[0] if results else None + + if data_returned > 1: + raise NotFound( + detail=f"Instead of a single entry, {data_returned} entries were found", + ) + + include_fields = ( + response_fields - self.resource_mapper.TOP_LEVEL_NON_ATTRIBUTES_FIELDS + ) + bad_optimade_fields = set() + bad_provider_fields = set() + for field in include_fields: + if field not in self.resource_mapper.ALL_ATTRIBUTES: + if field.startswith("_"): + if any( + field.startswith(f"_{prefix}_") + for prefix in self.resource_mapper.SUPPORTED_PREFIXES + ): + bad_provider_fields.add(field) + else: + bad_optimade_fields.add(field) + + if bad_provider_fields: + warnings.warn( + UnknownProviderProperty( + detail=f"Unrecognised field(s) for this provider requested in `response_fields`: {bad_provider_fields}." + ) + ) + + if bad_optimade_fields: + raise BadRequest( + detail=f"Unrecognised OPTIMADE field(s) in requested `response_fields`: {bad_optimade_fields}." + ) + + if results: + results = await self.resource_mapper.deserialize(results) + + return ( + results, + data_returned, + more_data_available, + self.all_fields - response_fields, + include_fields, + ) + + async def handle_query_params( + self, params: Union[EntryListingQueryParams, SingleEntryQueryParams] + ) -> Dict[str, Any]: + return super().handle_query_params(params) + + async def _run_db_query( + self, criteria: Dict[str, Any], single_entry: bool + ) -> Tuple[List[Dict[str, Any]], int, bool]: + results = [] + async for document in self.collection.find(**self._valid_find_keys(**criteria)): + if criteria.get("projection", {}).get("_id"): + document["_id"] = str(document["_id"]) + results.append(document) + + if single_entry: + data_returned = len(results) + more_data_available = False + else: + criteria_nolimit = criteria.copy() + criteria_nolimit.pop("limit", None) + data_returned = await self.count(params=None, **criteria_nolimit) + more_data_available = len(results) < data_returned + + return results, data_returned, more_data_available + + def _check_aliases(self, aliases: Tuple[Tuple[str, str]]) -> None: + """Check that aliases do not clash with mongo keywords. + + Parameters: + aliases: Tuple of tuple of aliases to be checked. + + Raises: + RuntimeError: If any alias starts with the dollar (`$`) character. + + """ + if any( + alias[0].startswith("$") or alias[1].startswith("$") for alias in aliases + ): + raise RuntimeError(f"Cannot define an alias starting with a '$': {aliases}") + async def get_one(self, **criteria: Dict[str, Any]) -> EntryResource: """Get one resource based on criteria - !!! warning + Warning: This is not to be used for creating a REST API response, but is rather a utility function to easily retrieve a single resource. @@ -190,7 +274,7 @@ async def get_one(self, **criteria: Dict[str, Any]) -> EntryResource: async def get_multiple(self, **criteria: Dict[str, Any]) -> List[EntryResource]: """Get a list of resources based on criteria - !!! warning + Warning: This is not to be used for creating a REST API response, but is rather a utility function to easily retrieve a list of resources. @@ -209,73 +293,6 @@ async def get_multiple(self, **criteria: Dict[str, Any]) -> List[EntryResource]: return results - async def find( - self, - params: Union[EntryListingQueryParams, SingleEntryQueryParams] = None, - criteria: Dict[str, Any] = None, - ) -> Tuple[Union[List[EntryResource], EntryResource], bool, set]: - """Perform the query on the underlying MongoDB Collection, handling projection - and pagination of the output. - - Either provide `params` or `criteria`. Not both, but at least one. - - Parameters: - params: URL query parameters, either from a general entry endpoint or a single-entry endpoint. - criteria: Already handled/parsed URL query parameters. - - Returns: - A list of entry resource objects, whether more data is available with pagination, and fields. - - """ - if (params is None and criteria is None) or ( - params is not None and criteria is not None - ): - raise ValueError( - "Exacly one of either `params` and `criteria` must be specified." - ) - - if criteria is None: - criteria = await self.handle_query_params(params) - - fields = criteria.get("fields", self.all_fields) - - results = [] - async for document in self.collection.find(**self._valid_find_keys(**criteria)): - if criteria.get("projection", {}).get("_id"): - document["_id"] = str(document["_id"]) - results.append(self.resource_cls(**self.resource_mapper.map_back(document))) - - if params is None or isinstance(params, EntryListingQueryParams): - criteria_nolimit = criteria.copy() - criteria_nolimit.pop("limit", None) - more_data_available = len(results) < await self.count(**criteria_nolimit) - else: - # SingleEntryQueryParams, e.g., /structures/{entry_id} - more_data_available = False - if len(results) > 1: - raise HTTPException( - status_code=404, - detail=f"Instead of a single entry, {len(results)} entries were found", - ) - results = results[0] if results else None - - return results, more_data_available, self.all_fields - fields - - async def handle_query_params( - self, params: Union[EntryListingQueryParams, SingleEntryQueryParams] - ) -> dict: - cursor_kwargs = super().handle_query_params(params) - - if getattr(params, "response_fields", False): - fields = set(params.response_fields.split(",")) - fields |= self.resource_mapper.get_required_fields() - cursor_kwargs["fields"] = fields - else: - # cursor_kwargs["fields"] is already set to self.all_fields - pass - - return cursor_kwargs - async def create_one(self, resource: EntryResourceCreate) -> EntryResource: """Create a new document in the MongoDB collection based on query parameters. @@ -322,20 +339,38 @@ async def exists(self, entry_id: str) -> bool: """ return bool(await self.collection.count_documents({"id": entry_id})) - async def insert(self, data: List[EntryResource]) -> None: - """Add the given entries to the underlying database. - - Warning: - No validation is performed on the incoming data. - - Parameters: - data: The entry resource objects to add to the database. + @staticmethod + def _valid_find_keys(**kwargs) -> Dict[str, Any]: + """Return valid MongoDB find() keys with values from kwargs + Note, not including deprecated flags + (see https://pymongo.readthedocs.io/en/3.11.0/api/pymongo/collection.html#pymongo.collection.Collection.find). """ - await self.collection.insert_many(await clean_python_types(data)) + valid_method_keys = ( + "filter", + "projection", + "session", + "skip", + "limit", + "no_cursor_timeout", + "cursor_type", + "sort", + "allow_partial_results", + "batch_size", + "collation", + "return_key", + "show_record_id", + "hint", + "max_time_ms", + "min", + "max", + "comment", + "allow_disk_use", + ) + criteria = {key: kwargs[key] for key in valid_method_keys if key in kwargs} - def _run_db_query( - self, criteria: Dict[str, Any], single_entry: bool - ) -> Tuple[List[Dict[str, Any]], int, bool]: - """Abstract class - not implemented""" - return super()._run_db_query(criteria, single_entry=single_entry) + if criteria.get("filter") is None: + # Ensure documents are included in the result set + criteria["filter"] = {} + + return criteria diff --git a/optimade_gateway/routers/databases.py b/optimade_gateway/routers/databases.py index dd8d1fdf..a00e9cd9 100644 --- a/optimade_gateway/routers/databases.py +++ b/optimade_gateway/routers/databases.py @@ -119,18 +119,24 @@ async def get_database( representing the database resource object with `id={database ID}`. """ params.filter = f'id="{database_id}"' - result, _, fields = await DATABASES_COLLECTION.find(params=params) + ( + result, + data_returned, + more_data_available, + fields, + include_fields, + ) = await DATABASES_COLLECTION.find(params=params) - if fields and result is not None: - result = handle_response_fields(result, fields, set())[0] + if fields or include_fields and result is not None: + result = handle_response_fields(result, fields, include_fields)[0] return DatabasesResponseSingle( links=ToplevelLinks(next=None), data=result, meta=meta_values( url=request.url, - data_returned=0 if result is None else 1, + data_returned=data_returned, data_available=await DATABASES_COLLECTION.count(), - more_data_available=False, + more_data_available=more_data_available, ), ) diff --git a/optimade_gateway/routers/gateways.py b/optimade_gateway/routers/gateways.py index 122e55ee..518c8a1e 100644 --- a/optimade_gateway/routers/gateways.py +++ b/optimade_gateway/routers/gateways.py @@ -10,7 +10,6 @@ from fastapi.responses import RedirectResponse from optimade.models import ToplevelLinks from optimade.server.query_params import EntryListingQueryParams -from optimade.server.routers.utils import meta_values from optimade.server.schemas import ERROR_RESPONSES from optimade_gateway.common.config import CONFIG @@ -76,6 +75,7 @@ async def post_gateways( Create or return existing gateway according to `gateway`. """ + from optimade.server.routers.utils import meta_values from optimade_gateway.common.utils import clean_python_types from optimade_gateway.routers.utils import resource_factory diff --git a/optimade_gateway/routers/links.py b/optimade_gateway/routers/links.py index d66996ed..36c93e36 100644 --- a/optimade_gateway/routers/links.py +++ b/optimade_gateway/routers/links.py @@ -8,11 +8,11 @@ from fastapi import APIRouter, Depends, Request from optimade.models import LinksResponse, LinksResource -from optimade.server.mappers import LinksMapper from optimade.server.query_params import EntryListingQueryParams from optimade.server.schemas import ERROR_RESPONSES from optimade_gateway.common.config import CONFIG +from optimade_gateway.mappers.links import LinksMapper from optimade_gateway.mongo.collection import AsyncMongoCollection from optimade_gateway.routers.utils import get_entries diff --git a/optimade_gateway/routers/utils.py b/optimade_gateway/routers/utils.py index bcaa10a6..7518e8f0 100644 --- a/optimade_gateway/routers/utils.py +++ b/optimade_gateway/routers/utils.py @@ -37,7 +37,13 @@ async def get_entries( params: EntryListingQueryParams, ) -> EntryResponseMany: """Generalized `/{entries}` endpoint getter""" - results, more_data_available, fields = await collection.find(params=params) + ( + results, + data_returned, + more_data_available, + fields, + include_fields, + ) = await collection.find(params=params) if more_data_available: # Deduce the `next` link from the current request @@ -50,15 +56,15 @@ async def get_entries( else: links = ToplevelLinks(next=None) - if fields: - results = handle_response_fields(results, fields, set()) + if fields or include_fields: + results = handle_response_fields(results, fields, include_fields) return response_cls( links=links, data=results, meta=meta_values( url=request.url, - data_returned=await collection.count(params=params), + data_returned=data_returned, data_available=await collection.count(), more_data_available=more_data_available, ), @@ -217,7 +223,7 @@ async def resource_factory( f"{type(create_resource)!r}" ) - result, more_data_available, _ = await RESOURCE_COLLECTION.find( + result, data_returned, more_data_available, _, _ = await RESOURCE_COLLECTION.find( criteria={"filter": await clean_python_types(mongo_query)} ) @@ -228,12 +234,13 @@ async def resource_factory( ) if result: - if len(result) > 1: + if data_returned > 1: raise OptimadeGatewayError( f"More than one {result[0].type} were found. IDs of found {result[0].type}: " f"{[_.id for _ in result]}" ) - result = result[0] + if isinstance(result, list): + result = result[0] else: if isinstance(create_resource, DatabaseCreate): # Set required `LinksResourceAttributes` values if not set From c486c6ec1cacc840d872e466a5be2f846464966b Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 9 Aug 2021 10:56:06 +0200 Subject: [PATCH 05/13] Various optimizations - Minor renaming of routers in main.py. - Fix validation errors when returning error responses. - Minimize special meta fields for query response. - Utilize local `LinksMapper` in `DatabaseMapper`. --- optimade_gateway/main.py | 8 ++++---- optimade_gateway/mappers/databases.py | 10 +++------- optimade_gateway/mappers/links.py | 10 +--------- optimade_gateway/queries/perform.py | 5 ++++- optimade_gateway/routers/gateway/structures.py | 8 ++++---- tests/routers/gateway/test_gateway_structures.py | 2 +- 6 files changed, 17 insertions(+), 26 deletions(-) diff --git a/optimade_gateway/main.py b/optimade_gateway/main.py index db7940e8..2f850e15 100644 --- a/optimade_gateway/main.py +++ b/optimade_gateway/main.py @@ -23,8 +23,8 @@ info as gateway_info, links as gateway_links, queries as gateway_queries, - structures, - versions, + structures as gateway_structures, + versions as gateway_versions, ) APP = FastAPI( @@ -62,7 +62,7 @@ async def get_root(request: Request) -> RedirectResponse: # Add the special /versions endpoint(s) APP.include_router(versions_router) -APP.include_router(versions.ROUTER) +APP.include_router(gateway_versions.ROUTER) # Add endpoints to / and /vMAJOR for prefix in list(BASE_URL_PREFIXES.values()) + [""]: @@ -70,7 +70,7 @@ async def get_root(request: Request) -> RedirectResponse: gateway_info, gateway_links, gateway_queries, - structures, + gateway_structures, ): APP.include_router(router.ROUTER, prefix=prefix, include_in_schema=prefix == "") diff --git a/optimade_gateway/mappers/databases.py b/optimade_gateway/mappers/databases.py index 4b733276..e712b511 100644 --- a/optimade_gateway/mappers/databases.py +++ b/optimade_gateway/mappers/databases.py @@ -1,17 +1,15 @@ -from optimade.models import LinksResource from pydantic import AnyUrl # pylint: disable=no-name-in-module from optimade_gateway.common.config import CONFIG -from optimade_gateway.mappers.base import BaseResourceMapper +from optimade_gateway.mappers.links import LinksMapper __all__ = ("DatabasesMapper",) -class DatabasesMapper(BaseResourceMapper): +class DatabasesMapper(LinksMapper): ENDPOINT = "databases" - ENTRY_RESOURCE_CLASS = LinksResource @classmethod def map_back(cls, doc: dict) -> dict: @@ -31,6 +29,4 @@ def map_back(cls, doc: dict) -> dict: } # Ensure the type does not change to "databases" - newdoc = super().map_back(doc) - newdoc["type"] = "links" - return newdoc + return super().map_back(doc) diff --git a/optimade_gateway/mappers/links.py b/optimade_gateway/mappers/links.py index 9b07fa6c..3017dc95 100644 --- a/optimade_gateway/mappers/links.py +++ b/optimade_gateway/mappers/links.py @@ -12,15 +12,7 @@ class LinksMapper(BaseResourceMapper): @classmethod def map_back(cls, doc: dict) -> dict: - """Map properties from MongoDB to OPTIMADE - - :param doc: A resource object in MongoDB format - :type doc: dict - - :return: A resource object in OPTIMADE format - :rtype: dict - """ - type_ = doc["type"] + type_ = doc.get("type", None) or "links" newdoc = super().map_back(doc) newdoc["type"] = type_ return newdoc diff --git a/optimade_gateway/queries/perform.py b/optimade_gateway/queries/perform.py index 98f7ca23..81b8b259 100644 --- a/optimade_gateway/queries/perform.py +++ b/optimade_gateway/queries/perform.py @@ -21,6 +21,7 @@ from pydantic import ValidationError from starlette.datastructures import URL +from optimade_gateway.common.config import CONFIG from optimade_gateway.common.logger import LOGGER from optimade_gateway.common.utils import get_resource_attribute from optimade_gateway.models import ( @@ -141,7 +142,9 @@ async def perform_query( if not use_query_resource: # Create a standard OPTIMADE response, adding the database ID to each returned # resource's meta field. - database_id_meta = {"_optimade_gateway_": {"source_database_id": db_id}} + database_id_meta = { + f"_{CONFIG.provider.prefix}_source_database_id": db_id + } if errors or isinstance(response, ErrorResponse): # Error response if isinstance(response, ErrorResponse): diff --git a/optimade_gateway/routers/gateway/structures.py b/optimade_gateway/routers/gateway/structures.py index 99972e14..6e6d8b85 100644 --- a/optimade_gateway/routers/gateway/structures.py +++ b/optimade_gateway/routers/gateway/structures.py @@ -37,7 +37,7 @@ @ROUTER.get( "/gateways/{gateway_id}/structures", - response_model=StructureResponseMany, + response_model=Union[StructureResponseMany, ErrorResponse], response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, @@ -101,7 +101,7 @@ async def get_structures( @ROUTER.get( "/gateways/{gateway_id}/structures/{structure_id:path}", - response_model=StructureResponseOne, + response_model=Union[StructureResponseOne, ErrorResponse], response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, @@ -249,7 +249,7 @@ async def get_single_structure( @ROUTER.get( "/gateways/{gateway_id}/{version}/structures", - response_model=StructureResponseMany, + response_model=Union[StructureResponseMany, ErrorResponse], response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, @@ -274,7 +274,7 @@ async def get_versioned_structures( @ROUTER.get( "/gateways/{gateway_id}/{version}/structures/{structure_id:path}", - response_model=StructureResponseOne, + response_model=Union[StructureResponseOne, ErrorResponse], response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, diff --git a/tests/routers/gateway/test_gateway_structures.py b/tests/routers/gateway/test_gateway_structures.py index d74a69f3..ffdcbe55 100644 --- a/tests/routers/gateway/test_gateway_structures.py +++ b/tests/routers/gateway/test_gateway_structures.py @@ -64,7 +64,7 @@ async def test_get_structures( for datum in db_response["data"]: database_id_meta = { - "_optimade_gateway_": {"source_database_id": database["id"]} + f"_{CONFIG.provider.prefix}_source_database_id": database["id"] } if "meta" in datum: datum["meta"].update(database_id_meta) From 33a7c83717b74bf05b9b4c54fc61bef5145c4267 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 9 Aug 2021 17:46:37 +0200 Subject: [PATCH 06/13] Always return latest found results in queries --- optimade_gateway/routers/queries.py | 36 ++++++++++++++++++----------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/optimade_gateway/routers/queries.py b/optimade_gateway/routers/queries.py index 87d910a6..dbeffdf9 100644 --- a/optimade_gateway/routers/queries.py +++ b/optimade_gateway/routers/queries.py @@ -7,7 +7,6 @@ where, `id` may be left out. """ import asyncio -from typing import Union from fastapi import ( APIRouter, @@ -16,8 +15,7 @@ Response, status, ) -from optimade.models import ErrorResponse, ToplevelLinks -from optimade.models.responses import EntryResponseMany +from optimade.models import ToplevelLinks from optimade.server.query_params import EntryListingQueryParams from optimade.server.routers.utils import meta_values from optimade.server.schemas import ERROR_RESPONSES @@ -128,7 +126,7 @@ async def get_query( request: Request, query_id: str, response: Response, -) -> Union[ErrorResponse, GatewayQueryResponse]: +) -> GatewayQueryResponse: """`GET /queries/{query_id}` Return the response from a query @@ -139,15 +137,27 @@ async def get_query( query: QueryResource = await get_valid_resource(QUERIES_COLLECTION, query_id) if query.attributes.state != QueryState.FINISHED: - return EntryResponseMany( - data=[], - meta=meta_values( - url=request.url, - data_returned=0, - data_available=None, # It is at this point unknown - more_data_available=False, - **{f"_{CONFIG.provider.prefix}_query": query}, - ), + unfinished_meta = {f"_{CONFIG.provider.prefix}_query": query} + if hasattr(query.attributes.response, "meta"): + unfinished_meta.update( + { + "data_available": query.attributes.response.meta.data_available, + "data_returned": query.attributes.response.meta.data_returned, + "more_data_available": query.attributes.response.meta.more_data_available, + } + ) + else: + unfinished_meta.update( + { + "data_available": 0, + "data_returned": 0, + "more_data_available": False, + } + ) + return GatewayQueryResponse( + data=getattr(query.attributes.response, "data", {}), + links=getattr(query.attributes.response, "links", ToplevelLinks(next=None)), + meta=meta_values(url=request.url, **unfinished_meta), ) if query.attributes.response.errors: From e496a1be1e82acf045e2f2b981fcf0058c21a6ac Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 10 Aug 2021 14:48:51 +0200 Subject: [PATCH 07/13] New query endpoint response model Instead of returning a `GatewayQueryResponse` for the single `QueryResource` endpoints as well as the `GET /search` endpoint, a `QueriesResponseSingle` is now returned. This is done to ensure all necessary information is communicated along with the actual data response, like the used query parameters and such. The example/test gateways have been updated to use "correct" database `id`s, where the provider is prepended, separating the provider `id` and the database `id` with a slash. --- .ci/test_gateways.json | 28 ++++---- optimade_gateway/queries/perform.py | 6 +- optimade_gateway/queries/process.py | 2 +- optimade_gateway/routers/gateway/queries.py | 9 ++- optimade_gateway/routers/queries.py | 46 ++++-------- optimade_gateway/routers/search.py | 36 +++++++--- tests/conftest.py | 3 +- .../gateway/test_gateway_structures.py | 2 +- tests/routers/test_gateways.py | 4 +- tests/routers/test_queries.py | 30 +++----- tests/routers/test_search.py | 70 ++++++++----------- .../db_responses/{mcloud.json => index.json} | 0 12 files changed, 106 insertions(+), 130 deletions(-) rename tests/static/db_responses/{mcloud.json => index.json} (100%) diff --git a/.ci/test_gateways.json b/.ci/test_gateways.json index 24154ece..e7c563b6 100644 --- a/.ci/test_gateways.json +++ b/.ci/test_gateways.json @@ -4,7 +4,7 @@ "last_modified": "2021-01-28T15:49:58Z", "databases": [ { - "id": "2dstructures", + "id": "mcloud/2dstructures", "type": "links", "attributes": { "name": "2D Structures", @@ -16,7 +16,7 @@ } }, { - "id": "optimade-sample", + "id": "mcloud/optimade-sample", "type": "links", "attributes": { "name": "OPTIMADE Sample Database", @@ -28,7 +28,7 @@ } }, { - "id": "scdm", + "id": "mcloud/scdm", "type": "links", "attributes": { "name": "Automated high-throughput Wannierisation", @@ -40,7 +40,7 @@ } }, { - "id": "mcloud", + "id": "mcloud/index", "type": "links", "attributes": { "name": "Materials Cloud", @@ -57,7 +57,7 @@ "last_modified": "2021-01-28T16:35:37Z", "databases": [ { - "id": "2dstructures", + "id": "mcloud/2dstructures", "type": "links", "attributes": { "name": "2D Structures", @@ -69,7 +69,7 @@ } }, { - "id": "optimade-sample", + "id": "mcloud/optimade-sample", "type": "links", "attributes": { "name": "OPTIMADE Sample Database", @@ -81,7 +81,7 @@ } }, { - "id": "scdm", + "id": "mcloud/scdm", "type": "links", "attributes": { "name": "Automated high-throughput Wannierisation", @@ -99,7 +99,7 @@ "last_modified": "2021-01-28T16:36:19Z", "databases": [ { - "id": "optimade-sample", + "id": "mcloud/optimade-sample", "type": "links", "attributes": { "name": "OPTIMADE Sample Database", @@ -117,7 +117,7 @@ "last_modified": "2021-01-28T16:36:55Z", "databases": [ { - "id": "2dstructures", + "id": "mcloud/2dstructures", "type": "links", "attributes": { "name": "2D Structures", @@ -129,7 +129,7 @@ } }, { - "id": "optimade-sample", + "id": "mcloud/optimade-sample", "type": "links", "attributes": { "name": "OPTIMADE Sample Database", @@ -165,7 +165,7 @@ "last_modified": "2021-04-06T16:44:35Z", "databases": [ { - "id": "optimade-sample", + "id": "mcloud/optimade-sample", "type": "links", "attributes": { "name": "OPTIMADE Sample Database", @@ -177,7 +177,7 @@ } }, { - "id": "mcloud", + "id": "mcloud/index", "type": "links", "attributes": { "name": "Materials Cloud", @@ -212,7 +212,7 @@ "last_modified": "2021-04-29T14:44:48Z", "databases": [ { - "id": "2dstructures", + "id": "mcloud/2dstructures", "type": "links", "attributes": { "name": "2D Structures", @@ -224,7 +224,7 @@ } }, { - "id": "optimade-sample_single", + "id": "mcloud/optimade-sample_single", "type": "links", "attributes": { "name": "OPTIMADE Sample Database", diff --git a/optimade_gateway/queries/perform.py b/optimade_gateway/queries/perform.py index 81b8b259..3176fc55 100644 --- a/optimade_gateway/queries/perform.py +++ b/optimade_gateway/queries/perform.py @@ -292,7 +292,11 @@ def db_find( ErrorResponse( errors=[ { - "detail": f"Could not pass response from {url} as either a {response_model.__name__!r} or 'ErrorResponse'. ValidationError: {exc}", + "detail": ( + f"Could not pass response from {url} as either a " + f"{response_model.__name__!r} or 'ErrorResponse'. " + f"ValidationError: {exc}" + ), "id": "OPTIMADE_GATEWAY_DB_FIND_MANY_VALIDATIONERROR", } ], diff --git a/optimade_gateway/queries/process.py b/optimade_gateway/queries/process.py index 753dc658..03eac1ce 100644 --- a/optimade_gateway/queries/process.py +++ b/optimade_gateway/queries/process.py @@ -116,7 +116,7 @@ async def process_db_response( ) # This ensures an empty list under `response.data.{database_id}` is returned if the case is - # simply that there is no results to return. + # simply that there are no results to return. if errors: extra_updates.update({"$addToSet": {"response.errors": {"$each": errors}}}) await update_query( diff --git a/optimade_gateway/routers/gateway/queries.py b/optimade_gateway/routers/gateway/queries.py index a014e7c6..69e90159 100644 --- a/optimade_gateway/routers/gateway/queries.py +++ b/optimade_gateway/routers/gateway/queries.py @@ -7,7 +7,6 @@ where `version` and the last `id` may be left out. """ from fastapi import APIRouter, Depends, Request, status -from optimade.models.responses import EntryResponseMany from optimade.server.exceptions import Forbidden from optimade.server.query_params import EntryListingQueryParams from optimade.server.schemas import ERROR_RESPONSES @@ -47,9 +46,9 @@ async def get_gateway_queries( await validate_resource(GATEWAYS_COLLECTION, gateway_id) params.filter = ( - f'( {params.filter} ) AND ( gateway="{gateway_id}" )' + f'( {params.filter} ) AND ( gateway_id="{gateway_id}" )' if params.filter - else f'gateway="{gateway_id}"' + else f'gateway_id="{gateway_id}"' ) return await get_queries(request, params) @@ -91,7 +90,7 @@ async def post_gateway_queries( @ROUTER.get( "/gateways/{gateway_id}/queries/{query_id}", - response_model=EntryResponseMany, + response_model=QueriesResponseSingle, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, @@ -100,7 +99,7 @@ async def post_gateway_queries( ) async def get_gateway_query( request: Request, gateway_id: str, query_id: str -) -> EntryResponseMany: +) -> QueriesResponseSingle: """`GET /gateways/{gateway_id}/queries/{query_id}` Return the response from a gateway query diff --git a/optimade_gateway/routers/queries.py b/optimade_gateway/routers/queries.py index dbeffdf9..9cbd7cfa 100644 --- a/optimade_gateway/routers/queries.py +++ b/optimade_gateway/routers/queries.py @@ -23,10 +23,8 @@ from optimade_gateway.common.config import CONFIG from optimade_gateway.mappers import QueryMapper from optimade_gateway.models import ( - GatewayQueryResponse, QueryCreate, QueryResource, - QueryState, QueriesResponse, QueriesResponseSingle, ) @@ -114,8 +112,8 @@ async def post_queries( @ROUTER.get( - "/queries/{query_id:path}", - response_model=GatewayQueryResponse, + "/queries/{query_id}", + response_model=QueriesResponseSingle, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, @@ -126,40 +124,15 @@ async def get_query( request: Request, query_id: str, response: Response, -) -> GatewayQueryResponse: +) -> QueriesResponseSingle: """`GET /queries/{query_id}` - Return the response from a query - ([`QueryResource.attributes.response`][optimade_gateway.models.queries.QueryResourceAttributes.response]). + Return a single [`QueryResource`][optimade_gateway.models.queries.QueryResource]. """ from optimade_gateway.routers.utils import get_valid_resource query: QueryResource = await get_valid_resource(QUERIES_COLLECTION, query_id) - if query.attributes.state != QueryState.FINISHED: - unfinished_meta = {f"_{CONFIG.provider.prefix}_query": query} - if hasattr(query.attributes.response, "meta"): - unfinished_meta.update( - { - "data_available": query.attributes.response.meta.data_available, - "data_returned": query.attributes.response.meta.data_returned, - "more_data_available": query.attributes.response.meta.more_data_available, - } - ) - else: - unfinished_meta.update( - { - "data_available": 0, - "data_returned": 0, - "more_data_available": False, - } - ) - return GatewayQueryResponse( - data=getattr(query.attributes.response, "data", {}), - links=getattr(query.attributes.response, "links", ToplevelLinks(next=None)), - meta=meta_values(url=request.url, **unfinished_meta), - ) - if query.attributes.response.errors: for error in query.attributes.response.errors: if error.status: @@ -168,4 +141,13 @@ async def get_query( else: response.status_code = 500 - return query.attributes.response + return QueriesResponseSingle( + links=ToplevelLinks(next=None), + data=query, + meta=meta_values( + url=request.url, + data_returned=1, + data_available=await QUERIES_COLLECTION.count(), + more_data_available=False, + ), + ) diff --git a/optimade_gateway/routers/search.py b/optimade_gateway/routers/search.py index a5d196ed..8fc97e8d 100644 --- a/optimade_gateway/routers/search.py +++ b/optimade_gateway/routers/search.py @@ -40,7 +40,6 @@ Search, ) from optimade_gateway.models.queries import ( - GatewayQueryResponse, OptimadeQueryParameters, QueryState, ) @@ -173,7 +172,7 @@ async def post_search(request: Request, search: Search) -> QueriesResponseSingle @ROUTER.get( "/search", - response_model=GatewayQueryResponse, + response_model=QueriesResponseSingle, response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, @@ -185,19 +184,25 @@ async def get_search( response: Response, search_params: SearchQueryParams = Depends(), entry_params: EntryListingQueryParams = Depends(), -) -> Union[GatewayQueryResponse, RedirectResponse]: +) -> Union[QueriesResponseSingle, RedirectResponse]: """`GET /search` Coordinate a new OPTIMADE query in multiple databases through a gateway: - 1. Create a [`Search`][optimade_gateway.models.search.Search] `POST` data - calling `POST /search` + 1. Create a [`Search`][optimade_gateway.models.search.Search] `POST` data - calling `POST /search`. 1. Wait [`search_params.timeout`][optimade_gateway.queries.params.SearchQueryParams] - seconds until the query has finished - 1. Return successful response + seconds before returning the query, if it has not finished before. + 1. Return query - similar to `GET /queries/{query_id}`. + + This endpoint works similarly to `GET /queries/{query_id}`, where one passes the query + parameters directly in the URL, instead of first POSTing a query and then going to its URL. + Hence, a [`QueryResponseSingle`][optimade_gateway.models.responses.QueriesResponseSingle] is + the standard response model for this endpoint. + + If the timeout time is reached and the query has not yet finished, the user is redirected to the + specific URL for the query. - !!! attention "Contingency" - If the query has not finished within the set timeout period, the client will be redirected - to the query's URL instead. + In the future, this might introduce a mode to return a response as a standard OPTIMADE response. """ from time import time @@ -220,7 +225,7 @@ async def get_search( raise BadRequest( detail=( "A Search object could not be created from the given URL query parameters. " - f"Error(s): {[exc.errors]}" + f"Error(s): {exc.errors}" ) ) @@ -245,7 +250,16 @@ async def get_search( else: response.status_code = 500 - return query.attributes.response + return QueriesResponseSingle( + links=ToplevelLinks(next=None), + data=query, + meta=meta_values( + url=request.url, + data_returned=1, + data_available=await QUERIES_COLLECTION.count(), + more_data_available=False, + ), + ) await asyncio.sleep(0.1) diff --git a/tests/conftest.py b/tests/conftest.py index 4e94982d..6519243d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -170,7 +170,8 @@ def _mock_response(gateway: dict) -> None: pass else: with open( - top_dir / f"tests/static/db_responses/{database['id']}.json" + top_dir + / f"tests/static/db_responses/{''.join(database['id'].split('/')[1:])}.json" ) as handle: data = json.load(handle) diff --git a/tests/routers/gateway/test_gateway_structures.py b/tests/routers/gateway/test_gateway_structures.py index ffdcbe55..adf46334 100644 --- a/tests/routers/gateway/test_gateway_structures.py +++ b/tests/routers/gateway/test_gateway_structures.py @@ -94,7 +94,7 @@ async def test_get_single_structure( from optimade.models import StructureResponseOne gateway_id = "single-structure_optimade-sample" - structure_id = "optimade-sample_single/1" + structure_id = "mcloud/optimade-sample_single/1" gateway = await get_gateway(gateway_id) mock_gateway_responses(gateway) diff --git a/tests/routers/test_gateways.py b/tests/routers/test_gateways.py index e4033119..69fb2787 100644 --- a/tests/routers/test_gateways.py +++ b/tests/routers/test_gateways.py @@ -173,7 +173,7 @@ async def test_post_gateways_database_ids( assert datum.id == "twodbs" for database in datum.attributes.databases: - assert database.id in [_.split("/")[-1] for _ in data["database_ids"]] + assert database.id in data["database_ids"] assert datum.links.dict() == { "self": AnyUrl( @@ -187,7 +187,7 @@ async def test_post_gateways_database_ids( assert await MONGO_DB["gateways"].count_documents(mongo_filter) == 1 db_datum = await MONGO_DB["gateways"].find_one(mongo_filter) for db in db_datum["databases"]: - assert db["id"] in [_.split("/")[-1] for _ in data["database_ids"]] + assert db["id"] in data["database_ids"] async def test_post_gateways_create_with_db_ids( diff --git a/tests/routers/test_queries.py b/tests/routers/test_queries.py index db7e1628..79f782b0 100644 --- a/tests/routers/test_queries.py +++ b/tests/routers/test_queries.py @@ -156,12 +156,8 @@ async def test_query_results( """Test POST /queries and GET /queries/{id}""" import asyncio - from optimade_gateway.common.config import CONFIG - from optimade_gateway.models.queries import ( - GatewayQueryResponse, - QueryState, - QueryResource, - ) + from optimade_gateway.models.queries import QueryState + from optimade_gateway.models.responses import QueriesResponseSingle data = { "id": "test", @@ -181,12 +177,10 @@ async def test_query_results( response = await client(f"/queries/{data['id']}") assert response.status_code == 200, f"Request failed: {response.json()}" - response = GatewayQueryResponse(**response.json()) - assert response.data == {} + response = QueriesResponseSingle(**response.json()) + assert response.data.attributes.response.data == {} - query: QueryResource = QueryResource( - **getattr(response.meta, f"_{CONFIG.provider.prefix}_query") - ) + query = response.data assert query assert query.attributes.state in (QueryState.STARTED, QueryState.IN_PROGRESS) @@ -195,12 +189,9 @@ async def test_query_results( response = await client(f"/queries/{data['id']}") assert response.status_code == 200, f"Request failed: {response.json()}" - response = GatewayQueryResponse(**response.json()) - assert response.data - assert ( - getattr(response.meta, f"_{CONFIG.provider.prefix}_query", "NOT FOUND") - == "NOT FOUND" - ) + response = QueriesResponseSingle(**response.json()) + assert response.data.attributes.response.data + assert response.data.attributes.state == QueryState.FINISHED @pytest.mark.usefixtures("reset_db_after") @@ -215,7 +206,6 @@ async def test_errored_query_results( """Test POST /queries and GET /queries/{id} with an erroneous response""" import asyncio - from optimade_gateway.models.queries import GatewayQueryResponse from optimade_gateway.models.responses import QueriesResponseSingle data = { @@ -238,8 +228,8 @@ async def test_errored_query_results( response.status_code == 404 ), f"Request succeeded, where it should have failed:\n{json.dumps(response.json(), indent=2)}" - response = GatewayQueryResponse(**response.json()) - assert response.errors + response = QueriesResponseSingle(**response.json()) + assert response.data.attributes.response.errors @pytest.mark.usefixtures("reset_db_after") diff --git a/tests/routers/test_search.py b/tests/routers/test_search.py index 1da2733d..d92a3ca5 100644 --- a/tests/routers/test_search.py +++ b/tests/routers/test_search.py @@ -30,8 +30,7 @@ async def test_get_search( this should ensure a new gateway is created, specifically for use with these versioned base URLs, but we can reuse the mock_gateway_responses for the "twodbs" gateway. """ - from optimade_gateway.common.config import CONFIG - from optimade_gateway.models import GatewayQueryResponse + from optimade_gateway.models import QueriesResponseSingle gateway_id = "twodbs" gateway: dict = await get_gateway(gateway_id) @@ -51,12 +50,9 @@ async def test_get_search( assert response.status_code == 200, f"Request failed: {response.json()}" - response = GatewayQueryResponse(**response.json()) - assert response.data - assert ( - getattr(response.meta, f"_{CONFIG.provider.prefix}_query", "NOT FOUND") - == "NOT FOUND" - ) + response = QueriesResponseSingle(**response.json()) + assert response.data.attributes.response.data + assert response.data.attributes.state.value == "finished" assert "A new gateway was created for a query" in caplog.text, caplog.text @@ -71,8 +67,7 @@ async def test_get_search_existing_gateway( caplog: pytest.LogCaptureFixture, ): """Test GET /search for base URLs matching an existing gateway""" - from optimade_gateway.common.config import CONFIG - from optimade_gateway.models import GatewayQueryResponse + from optimade_gateway.models import QueriesResponseSingle gateway_id = "twodbs" gateway: dict = await get_gateway(gateway_id) @@ -91,9 +86,7 @@ async def test_get_search_existing_gateway( { "filter": 'elements HAS "Cu"', "page_limit": 15, - "database_ids": [ - f"mcloud/{_.get('id')}" for _ in gateway.get("databases", [{}]) - ], + "database_ids": [_.get("id") for _ in gateway.get("databases", [{}])], }, # Both optimade_urls & database_ids { @@ -102,7 +95,7 @@ async def test_get_search_existing_gateway( "optimade_urls": [ gateway.get("databases", [{}])[0].get("attributes", {}).get("base_url") ], - "database_ids": [f"mcloud/{gateway.get('databases', [{}])[-1].get('id')}"], + "database_ids": [gateway.get("databases", [{}])[-1].get("id")], }, ] @@ -113,12 +106,13 @@ async def test_get_search_existing_gateway( assert response.status_code == 200, f"Request failed: {response.json()}" - response = GatewayQueryResponse(**response.json()) - assert response.data, f"No data: {response.json(indent=2)}" + response = QueriesResponseSingle(**response.json()) + assert ( + response.data.attributes.response.data + ), f"No data: {response.json(indent=2)}" assert ( - getattr(response.meta, f"_{CONFIG.provider.prefix}_query", "NOT FOUND") - == "NOT FOUND" - ), f"Special __query field was found in meta. Response: {response.json(indent=2)}" + response.data.attributes.state.value == "finished" + ), f"Query never finished. Response: {response.json(indent=2)}" assert "A gateway was found and reused for a query" in caplog.text, caplog.text @@ -133,12 +127,8 @@ async def test_get_search_not_finishing( caplog: pytest.LogCaptureFixture, ): """Test GET /search for unfinished query (redirect to query URL)""" - from optimade_gateway.common.config import CONFIG - from optimade_gateway.models.queries import ( - GatewayQueryResponse, - QueryResource, - QueryState, - ) + from optimade_gateway.models.queries import GatewayQueryResponse, QueryState + from optimade_gateway.models.responses import QueriesResponseSingle gateway_id = "slow-query" gateway: dict = await get_gateway(gateway_id) @@ -160,16 +150,12 @@ async def test_get_search_not_finishing( assert "A gateway was found and reused for a query" in caplog.text, caplog.text - response = GatewayQueryResponse(**response.json()) - assert response.data == {}, f"Data was found in response: {response.json(indent=2)}" - - assert getattr( - response.meta, f"_{CONFIG.provider.prefix}_query", False - ), f"Special __query field not found in meta. Response: {response.json(indent=2)}" + response = QueriesResponseSingle(**response.json()) + assert ( + response.data.attributes.response.data == {} + ), f"Data was found in response: {response.json(indent=2)}" - query: QueryResource = QueryResource( - **getattr(response.meta, f"_{CONFIG.provider.prefix}_query") - ) + query = response.data assert query, query assert query.attributes.state in (QueryState.STARTED, QueryState.IN_PROGRESS), query assert query.attributes.query_parameters.filter == query_params["filter"], query @@ -284,14 +270,12 @@ async def test_post_search_existing_gateway( # database_ids { "query_parameters": {"filter": 'elements HAS "Cu"', "page_limit": 15}, - "database_ids": [ - f"mcloud/{_.get('id')}" for _ in gateway.get("databases", [{}]) - ], + "database_ids": [_.get("id") for _ in gateway.get("databases", [{}])], }, # Both optimade_urls & database_ids { "query_parameters": {"filter": 'elements HAS "Cu"', "page_limit": 15}, - "database_ids": [f"mcloud/{gateway.get('databases', [{}])[0].get('id')}"], + "database_ids": [gateway.get("databases", [{}])[0].get("id")], "optimade_urls": [ gateway.get("databases", [{}])[-1].get("attributes", {}).get("base_url") ], @@ -348,7 +332,6 @@ async def test_sort_no_effect( """ from optimade.models import Warnings - from optimade_gateway.models import GatewayQueryResponse from optimade_gateway.models.responses import QueriesResponseSingle from optimade_gateway.warnings import SortNotSupported @@ -375,12 +358,15 @@ async def test_sort_no_effect( assert response_asc.status_code == 200, f"Request failed: {response_asc.json()}" assert response_desc.status_code == 200, f"Request failed: {response_desc.json()}" - response_asc = GatewayQueryResponse(**response_asc.json()) + response_asc = QueriesResponseSingle(**response_asc.json()) assert response_asc - response_desc = GatewayQueryResponse(**response_desc.json()) + response_desc = QueriesResponseSingle(**response_desc.json()) assert response_desc - assert response_asc.data == response_desc.data + assert ( + response_asc.data.attributes.response.data + == response_desc.data.attributes.response.data + ) sort_warning = SortNotSupported() diff --git a/tests/static/db_responses/mcloud.json b/tests/static/db_responses/index.json similarity index 100% rename from tests/static/db_responses/mcloud.json rename to tests/static/db_responses/index.json From 4eef4de700b682d283734a560e9d9547528e4f7a Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 10 Aug 2021 15:29:22 +0200 Subject: [PATCH 08/13] Return gateway resource for /gateways/{id} This is instead of redirecting to /gateways/{id}/structures as this is now moving more towards *not* reproducing an actual OPTIMADE implementation. --- optimade_gateway/routers/gateways.py | 24 ++++++++++++++---------- tests/conftest.py | 13 +++++++++++++ tests/routers/test_gateways.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 10 deletions(-) diff --git a/optimade_gateway/routers/gateways.py b/optimade_gateway/routers/gateways.py index 518c8a1e..fbc8da92 100644 --- a/optimade_gateway/routers/gateways.py +++ b/optimade_gateway/routers/gateways.py @@ -7,7 +7,6 @@ where, `id` may be left out. """ from fastapi import APIRouter, Depends, Request -from fastapi.responses import RedirectResponse from optimade.models import ToplevelLinks from optimade.server.query_params import EntryListingQueryParams from optimade.server.schemas import ERROR_RESPONSES @@ -121,15 +120,20 @@ async def post_gateways( async def get_gateway(request: Request, gateway_id: str) -> GatewaysResponseSingle: """`GET /gateways/{gateway ID}` - Represent an OPTIMADE server. - - !!! note - For now, redirect to the gateway's `/structures` entry listing endpoint. - + Return a single [`GatewayResource`][optimade_gateway.models.gateways.GatewayResource]. """ - from optimade_gateway.routers.utils import validate_resource + from optimade.server.routers.utils import meta_values + from optimade_gateway.routers.utils import get_valid_resource + + result = await get_valid_resource(GATEWAYS_COLLECTION, gateway_id) - await validate_resource(GATEWAYS_COLLECTION, gateway_id) - return RedirectResponse( - request.url.replace(path=f"{request.url.path.rstrip('/')}/structures") + return GatewaysResponseSingle( + links=ToplevelLinks(next=None), + data=result, + meta=meta_values( + url=request.url, + data_returned=1, + data_available=await GATEWAYS_COLLECTION.count(), + more_data_available=False, + ), ) diff --git a/tests/conftest.py b/tests/conftest.py index 6519243d..1df9a105 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -132,6 +132,19 @@ async def _get_gateway(id: str) -> dict: return _get_gateway +@pytest.fixture +async def random_gateway() -> dict: + """Get a random gateway currently in the MongoDB""" + from optimade_gateway.mongo.database import MONGO_DB + + gateway_ids = set() + async for gateway in MONGO_DB["gateways"].find( + filter={}, projection={"id": True, "_id": False} + ): + gateway_ids.add(gateway["id"]) + return gateway_ids.pop() + + @pytest.fixture async def reset_db_after(top_dir: Path) -> None: """Reset MongoDB with original test data after the test has run""" diff --git a/tests/routers/test_gateways.py b/tests/routers/test_gateways.py index 69fb2787..ce25f427 100644 --- a/tests/routers/test_gateways.py +++ b/tests/routers/test_gateways.py @@ -190,6 +190,7 @@ async def test_post_gateways_database_ids( assert db["id"] in data["database_ids"] +@pytest.mark.usefixtures("reset_db_after") async def test_post_gateways_create_with_db_ids( client: Callable[ [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], @@ -252,3 +253,30 @@ async def test_post_gateways_create_with_db_ids( db_datum = await MONGO_DB["gateways"].find_one(mongo_filter) for db in db_datum["databases"]: assert db["id"] in [data["databases"][0]["id"], data["database_ids"][0]] + + +async def test_get_single_gateway( + client: Callable[ + [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], + Awaitable[httpx.Response], + ], + random_gateway: str, + top_dir: Path, +): + """Test GET /gateways/{gateway_id}""" + import json + + from optimade_gateway.models.responses import GatewaysResponseSingle + + response = await client(f"/gateways/{random_gateway}") + + assert response.status_code == 200, f"Request failed: {response.json()}" + response = GatewaysResponseSingle(**response.json()) + assert response + + with open(top_dir / "tests/static/test_gateways.json") as handle: + test_data = json.load(handle) + + assert response.meta.data_returned == 1 + assert response.meta.data_available == len(test_data) + assert not response.meta.more_data_available From b31aaa1fe800be2057db688971180eca0a4b69b9 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 16 Aug 2021 17:49:44 +0200 Subject: [PATCH 09/13] Return GET /search as OPTIMADE or not Add possibility to return responses from `GET /search` as an OPTIMADE entry-listing response or as the "standard" gateway query response. In doing this, some things have been optimized. Mainly the logic surrounding retrieving and naming databases in the `POST /search` endpoint. In order to return a query response as a valid OPTIMADE response, a method has been added to the `QueryResource` model, which works similarly to `GET /gateways//structures`, but working from the basis of the finished query. It updates the entry `id`s by prepending the `provider/database/` name, making the `id`s unique. Extra safety has been added to the creation of gateway resources, as the `resource_factory()` now expects a pre-treating `GatewayCreate` entity, where there are no "unknown" database `id`s in the `database_ids` attribute. Unknown meaning that all `database_ids` must be represented in the `databases` attribute. --- optimade_gateway/models/gateways.py | 4 +- optimade_gateway/models/queries.py | 117 +++++++++++++++++- optimade_gateway/queries/params.py | 15 +++ optimade_gateway/queries/perform.py | 30 ++++- optimade_gateway/queries/process.py | 20 ++- optimade_gateway/routers/databases.py | 4 +- .../routers/gateway/structures.py | 35 ++++-- optimade_gateway/routers/queries.py | 10 +- optimade_gateway/routers/search.py | 97 ++++++++++----- optimade_gateway/routers/utils.py | 22 +++- tests/routers/test_search.py | 87 ++++++++++++- 11 files changed, 391 insertions(+), 50 deletions(-) diff --git a/optimade_gateway/models/gateways.py b/optimade_gateway/models/gateways.py index d2412e05..ec3372d0 100644 --- a/optimade_gateway/models/gateways.py +++ b/optimade_gateway/models/gateways.py @@ -140,7 +140,9 @@ class GatewayCreate(EntryResourceCreate, GatewayResourceAttributes): @root_validator def specify_databases(cls, values: dict) -> dict: - """Either `database_ids` or `databases` must be non-empty""" + """Either `database_ids` or `databases` must be non-empty. + Both together is also fine. + """ if not any(values.get(field) for field in ("database_ids", "databases")): raise ValueError("Either 'database_ids' or 'databases' MUST be specified") return values diff --git a/optimade_gateway/models/queries.py b/optimade_gateway/models/queries.py index 49fd14d2..2406fb3b 100644 --- a/optimade_gateway/models/queries.py +++ b/optimade_gateway/models/queries.py @@ -1,11 +1,15 @@ """Pydantic models/schemas for the Queries resource""" +from datetime import timezone from enum import Enum from typing import Any, Dict, List, Optional, Union +import urllib.parse import warnings from optimade.models import ( - EntryResource, + EntryResource as OptimadeEntryResource, EntryResourceAttributes, + EntryResponseMany, + ErrorResponse, OptimadeError, ReferenceResource, ReferenceResponseMany, @@ -19,6 +23,7 @@ from optimade.models.utils import StrictField from optimade.server.query_params import EntryListingQueryParams from pydantic import BaseModel, EmailStr, Field, validator +from starlette.datastructures import URL as StarletteURL from optimade_gateway.models.resources import EntryResourceCreate from optimade_gateway.warnings import SortNotSupported @@ -140,6 +145,19 @@ class QueryState(Enum): FINISHED = "finished" +class EntryResource(OptimadeEntryResource): + """Entry Resource ensuring datetimes are not naive.""" + + @validator("attributes") + def ensure_non_naive_datetime( + cls, value: EntryResourceAttributes + ) -> EntryResourceAttributes: + """Set timezone to UTC if datetime is naive.""" + if value.last_modified and value.last_modified.tzinfo is None: + value.last_modified = value.last_modified.replace(tzinfo=timezone.utc) + return value + + class GatewayQueryResponse(Response): """Response from a Gateway Query.""" @@ -210,6 +228,103 @@ class QueryResource(EntryResource): ) attributes: QueryResourceAttributes + async def response_as_optimade( + self, + url: Optional[ + Union[urllib.parse.ParseResult, urllib.parse.SplitResult, StarletteURL, str] + ] = None, + ) -> Union[EntryResponseMany, ErrorResponse]: + """Return `attributes.response` as a valid OPTIMADE entry listing response. + + Note, this method disregards the state of the query and will simply return the query results + as they currently are (if there are any at all). + + Parameters: + url: Optionally, update the `meta.query.representation` value with this. + + Returns: + A valid OPTIMADE entry-listing response according to the + [OPTIMADE specification](https://github.com/Materials-Consortia/OPTIMADE/blob/master/optimade.rst#entry-listing-endpoints) + or an error response, if errors were returned or occurred during the query. + + """ + from copy import deepcopy + from optimade.server.routers.utils import meta_values + + async def _update_id( + entry_: Union[EntryResource, Dict[str, Any]], database_provider_: str + ) -> Union[EntryResource, Dict[str, Any]]: + """Internal utility function to prepend the entries' `id` with `provider/database/`. + + Parameters: + entry_: The entry as a model or a dictionary. + database_provider_: `provider/database` string. + + Returns: + The entry with an updated `id` value. + + """ + if isinstance(entry_, dict): + _entry = deepcopy(entry_) + _entry["id"] = f"{database_provider_}/{entry_['id']}" + else: + _entry = entry_.copy(deep=True) + _entry.id = f"{database_provider_}/{entry_.id}" + return _entry + + if not self.attributes.response: + # The query has not yet been initiated + return ErrorResponse( + errors=[ + { + "detail": ( + "Can not return as a valid OPTIMADE response as the query has not yet " + "been initialized." + ), + "id": "OPTIMADE_GATEWAY_QUERY_NOT_INITIALIZED", + } + ], + meta=meta_values( + url=url or f"/queries/{self.id}?", + data_returned=0, + data_available=0, + more_data_available=False, + ), + ) + + meta_ = self.attributes.response.meta + if url: + meta_ = meta_.dict(exclude_unset=True) + for repeated_key in ( + "query", + "api_version", + "time_stamp", + "provider", + "implementation", + ): + meta_.pop(repeated_key, None) + meta_ = meta_values(url=url, **meta_) + + # Error response + if self.attributes.response.errors: + return ErrorResponse( + errors=self.attributes.response.errors, + meta=meta_, + ) + + # Data response + results = [] + for database_provider, entries in self.attributes.response.data.items(): + results.extend( + [await _update_id(entry, database_provider) for entry in entries] + ) + + return self.attributes.endpoint.get_response_model()( + data=results, + meta=meta_, + links=self.attributes.response.links, + ) + class QueryCreate(EntryResourceCreate, QueryResourceAttributes): """Model for creating new Query resources in the MongoDB""" diff --git a/optimade_gateway/queries/params.py b/optimade_gateway/queries/params.py index 96abc6b1..3c4ee24f 100644 --- a/optimade_gateway/queries/params.py +++ b/optimade_gateway/queries/params.py @@ -35,6 +35,11 @@ class in `optimade´, which defines the standard entry listing endpoint query pa time, a redirection will still be performed, but to a zero-results page, which can be refreshed to get the finished query (once it has finished). + as_optimade (bool): Return the response as a standard OPTIMADE entry listing endpoint + response. Otherwise, the response will be based on the + [`QueriesResponseSingle`][optimade_gateway.models.responses.QueriesResponseSingle] + model. + """ def __init__( @@ -72,8 +77,18 @@ def __init__( "which can be refreshed to get the finished query (once it has finished)." ), ), + as_optimade: bool = Query( + False, + description=( + "Return the response as a standard OPTIMADE entry listing endpoint response. " + "Otherwise, the response will be based on the " + "[`QueriesResponseSingle`][optimade_gateway.models.responses.QueriesResponseSingle]" + " model." + ), + ) ) -> None: self.database_ids = database_ids self.optimade_urls = optimade_urls self.endpoint = endpoint self.timeout = timeout + self.as_optimade = as_optimade diff --git a/optimade_gateway/queries/perform.py b/optimade_gateway/queries/perform.py index 3176fc55..e4ff7c42 100644 --- a/optimade_gateway/queries/perform.py +++ b/optimade_gateway/queries/perform.py @@ -288,6 +288,34 @@ def db_find( try: response = ErrorResponse(**response) except ValidationError as exc: + # If it's an error and `meta` is missing, it is not a valid OPTIMADE response, + # but this happens a lot, and is therefore worth having an edge-case for. + if "errors" in response: + errors = list(response["errors"]) + errors.append( + { + "detail": ( + f"Could not pass response from {url} as either a " + f"{response_model.__name__!r} or 'ErrorResponse'. " + f"ValidationError: {exc}" + ), + "id": "OPTIMADE_GATEWAY_DB_FINDS_MANY_VALIDATIONERRORS", + } + ) + return ( + ErrorResponse( + errors=errors, + meta={ + "query": { + "representation": f"/{endpoint.strip('/')}?{query_params}" + }, + "api_version": __api_version__, + "more_data_available": False, + }, + ), + get_resource_attribute(database, "id"), + ) + return ( ErrorResponse( errors=[ @@ -297,7 +325,7 @@ def db_find( f"{response_model.__name__!r} or 'ErrorResponse'. " f"ValidationError: {exc}" ), - "id": "OPTIMADE_GATEWAY_DB_FIND_MANY_VALIDATIONERROR", + "id": "OPTIMADE_GATEWAY_DB_FINDS_MANY_VALIDATIONERRORS", } ], meta={ diff --git a/optimade_gateway/queries/process.py b/optimade_gateway/queries/process.py index 03eac1ce..6691afea 100644 --- a/optimade_gateway/queries/process.py +++ b/optimade_gateway/queries/process.py @@ -10,6 +10,7 @@ OptimadeError, ) +from optimade_gateway.common.config import CONFIG from optimade_gateway.common.utils import get_resource_attribute from optimade_gateway.models import GatewayResource, QueryResource from optimade_gateway.queries.utils import update_query @@ -75,10 +76,21 @@ async def process_db_response( meta_error = error.meta.dict() meta_error.update( { - "optimade_gateway": { - "gateway": gateway, - "source_database_id": database_id, - } + f"_{CONFIG.provider.prefix}_source_gateway": { + "id": gateway.id, + "type": gateway.type, + "links": {"self": gateway.links.self}, + }, + f"_{CONFIG.provider.prefix}_source_database": { + "id": database_id, + "type": "links", + "links": { + "self": ( + str(gateway.links.self).split("gateways")[0] + + f"databases/{database_id}" + ) + }, + }, } ) error.meta = Meta(**meta_error) diff --git a/optimade_gateway/routers/databases.py b/optimade_gateway/routers/databases.py index a00e9cd9..92afe273 100644 --- a/optimade_gateway/routers/databases.py +++ b/optimade_gateway/routers/databases.py @@ -128,7 +128,9 @@ async def get_database( ) = await DATABASES_COLLECTION.find(params=params) if fields or include_fields and result is not None: - result = handle_response_fields(result, fields, include_fields)[0] + result = handle_response_fields(result, fields, include_fields) + + result = result[0] if data_returned else None return DatabasesResponseSingle( links=ToplevelLinks(next=None), diff --git a/optimade_gateway/routers/gateway/structures.py b/optimade_gateway/routers/gateway/structures.py index 6e6d8b85..1e802184 100644 --- a/optimade_gateway/routers/gateway/structures.py +++ b/optimade_gateway/routers/gateway/structures.py @@ -86,8 +86,14 @@ async def get_structures( if isinstance(gateway_response, ErrorResponse): for error in gateway_response.errors: if error.status: - response.status_code = int(error.status) - break + for part in error.status.split(" "): + try: + response.status_code = int(part) + break + except ValueError: + pass + if response.status_code and response.status_code >= 300: + break else: response.status_code = 500 return gateway_response @@ -134,6 +140,7 @@ async def get_single_structure( Example: `GET /gateways/some_gateway/structures/some_database/some_structure`. """ + from optimade_gateway.common.config import CONFIG from optimade_gateway.models import GatewayResource from optimade_gateway.queries import db_find from optimade_gateway.routers.utils import get_valid_resource @@ -182,10 +189,16 @@ async def get_single_structure( meta = error.meta.dict() meta.update( { - "optimade_gateway": { - "gateway": gateway, - "source_database_id": database.id, - } + f"_{CONFIG.provider.prefix}_source_gateway": { + "id": gateway.id, + "type": gateway.type, + "links": {"self": gateway.links.self}, + }, + f"_{CONFIG.provider.prefix}_source_database": { + "id": database.id, + "type": database.type, + "links": {"self": database.links.self}, + }, } ) error.meta = Meta(**meta) @@ -239,8 +252,14 @@ async def get_single_structure( if isinstance(gateway_response, ErrorResponse): for error in errors or gateway_response.errors: if error.status: - response.status_code = int(error.status) - break + for part in error.status.split(" "): + try: + response.status_code = int(part) + break + except ValueError: + pass + if response.status_code and response.status_code >= 300: + break else: response.status_code = 500 diff --git a/optimade_gateway/routers/queries.py b/optimade_gateway/routers/queries.py index 9cbd7cfa..3d04e30c 100644 --- a/optimade_gateway/routers/queries.py +++ b/optimade_gateway/routers/queries.py @@ -136,8 +136,14 @@ async def get_query( if query.attributes.response.errors: for error in query.attributes.response.errors: if error.status: - response.status_code = int(error.status) - break + for part in error.status.split(" "): + try: + response.status_code = int(part) + break + except ValueError: + pass + if response.status_code and response.status_code >= 300: + break else: response.status_code = 500 diff --git a/optimade_gateway/routers/search.py b/optimade_gateway/routers/search.py index 8fc97e8d..d9888d27 100644 --- a/optimade_gateway/routers/search.py +++ b/optimade_gateway/routers/search.py @@ -11,13 +11,13 @@ from fastapi import ( APIRouter, Depends, - HTTPException, Request, Response, status, ) from fastapi.responses import RedirectResponse -from optimade.server.exceptions import BadRequest +from optimade.models.responses import EntryResponseMany, ErrorResponse +from optimade.server.exceptions import BadRequest, InternalServerError from optimade.models import ( LinksResource, LinksResourceAttributes, @@ -109,35 +109,60 @@ async def post_search(request: Request, search: Search) -> QueriesResponseSingle # Ensure all URLs are `pydantic.AnyUrl`s if not all([isinstance(_, AnyUrl) for _ in base_urls]): - raise HTTPException( - status_code=500, - detail="Could unexpectedly not get all base URLs as `pydantic.AnyUrl`s.", + raise InternalServerError( + "Could unexpectedly not validate all base URLs as proper URLs." ) - gateway = GatewayCreate( - databases=[ - LinksResource( - id=( - f"{url.user + '@' if url.user else ''}{url.host}" - f"{':' + url.port if url.port else ''}" - f"{url.path.rstrip('/') if url.path else ''}" - ).replace(".", "__"), - type="links", - attributes=LinksResourceAttributes( - name=( + databases = await DATABASES_COLLECTION.get_multiple( + filter={"base_url": {"$in": await clean_python_types(base_urls)}} + ) + if len(databases) == len(base_urls): + # At this point it is expected that the list of databases in `databases` + # is a complete set of databases requested. + gateway = GatewayCreate(databases=databases) + elif len(databases) < len(base_urls): + # There are unregistered databases + current_base_urls = set( + [ + get_resource_attribute(database, "attributes.base_url") + for database in databases + ] + ) + databases.extend( + [ + LinksResource( + id=( f"{url.user + '@' if url.user else ''}{url.host}" f"{':' + url.port if url.port else ''}" f"{url.path.rstrip('/') if url.path else ''}" + ).replace(".", "__"), + type="links", + attributes=LinksResourceAttributes( + name=( + f"{url.user + '@' if url.user else ''}{url.host}" + f"{':' + url.port if url.port else ''}" + f"{url.path.rstrip('/') if url.path else ''}" + ), + description="", + base_url=url, + link_type=LinkType.CHILD, + homepage=None, ), - description="", - base_url=url, - link_type=LinkType.CHILD, - homepage=None, - ), - ) - for url in base_urls - ] - ) + ) + for url in base_urls - current_base_urls + ] + ) + else: + LOGGER.error( + "Found more database entries in MongoDB than then number of passed base URLs. " + "This suggests ambiguity in the base URLs of databases stored in MongoDB.\n" + " base_urls: %s\n databases %s", + base_urls, + databases, + ) + raise InternalServerError("Unambiguous base URLs. See logs for more details.") + + gateway = GatewayCreate(databases=databases) gateway, created = await resource_factory(gateway) if created: @@ -172,7 +197,7 @@ async def post_search(request: Request, search: Search) -> QueriesResponseSingle @ROUTER.get( "/search", - response_model=QueriesResponseSingle, + response_model=Union[QueriesResponseSingle, EntryResponseMany, ErrorResponse], response_model_exclude_defaults=False, response_model_exclude_none=False, response_model_exclude_unset=True, @@ -184,7 +209,7 @@ async def get_search( response: Response, search_params: SearchQueryParams = Depends(), entry_params: EntryListingQueryParams = Depends(), -) -> Union[QueriesResponseSingle, RedirectResponse]: +) -> Union[QueriesResponseSingle, EntryResponseMany, ErrorResponse, RedirectResponse]: """`GET /search` Coordinate a new OPTIMADE query in multiple databases through a gateway: @@ -202,7 +227,10 @@ async def get_search( If the timeout time is reached and the query has not yet finished, the user is redirected to the specific URL for the query. - In the future, this might introduce a mode to return a response as a standard OPTIMADE response. + If the `as_optimade` query parameter is `True`, the response will be parseable as a standard + OPTIMADE entry listing endpoint like, e.g., `/structures`. + For more information see the + [OPTIMADE specification](https://github.com/Materials-Consortia/OPTIMADE/blob/master/optimade.rst#entry-listing-endpoints). """ from time import time @@ -245,11 +273,20 @@ async def get_search( if query.attributes.response.errors: for error in query.attributes.response.errors: if error.status: - response.status_code = int(error.status) - break + for part in error.status.split(" "): + try: + response.status_code = int(part) + break + except ValueError: + pass + if response.status_code and response.status_code >= 300: + break else: response.status_code = 500 + if search_params.as_optimade: + return await query.response_as_optimade(url=request.url) + return QueriesResponseSingle( links=ToplevelLinks(next=None), data=query, diff --git a/optimade_gateway/routers/utils.py b/optimade_gateway/routers/utils.py index 7518e8f0..81adfe63 100644 --- a/optimade_gateway/routers/utils.py +++ b/optimade_gateway/routers/utils.py @@ -149,6 +149,13 @@ async def resource_factory( [`databases.attributes.base_url`](https://www.optimade.org/optimade-python-tools/api_reference/models/links/#optimade.models.links.LinksResourceAttributes.base_url) element values, when compared with the `create_resource`. + !!! important + The `database_ids` attribute **must not** contain values that are not also included in the + `databases` attribute, in the form of the IDs for the individual databases. + If this should be the case an + [`OptimadeGatewayError`][optimade_gateway.common.exceptions.OptimadeGatewayError] will be + thrown. + === "Queries" The `gateway_id`, `query_parameters`, and `endpoint` fields are collectively considered to define uniqueness for a [`QueryResource`][optimade_gateway.models.queries.QueryResource] @@ -195,10 +202,23 @@ async def resource_factory( GATEWAYS_COLLECTION as RESOURCE_COLLECTION, ) + # One MUST have taken care of database_ids prior to calling `resource_factory()` + database_attr_ids = {_.id for _ in create_resource.databases or []} + unknown_ids = { + database_id + for database_id in create_resource.database_ids or [] + if database_id not in database_attr_ids + } + if unknown_ids: + raise OptimadeGatewayError( + "When using `resource_factory()` for `GatewayCreate`, `database_ids` MUST not " + f"include unknown IDs. Passed unknown IDs: {unknown_ids}" + ) + mongo_query = { "databases": {"$size": len(create_resource.databases)}, "databases.attributes.base_url": { - "$all": [_.attributes.base_url for _ in create_resource.databases] + "$all": [_.attributes.base_url for _ in create_resource.databases or []] }, } elif isinstance(create_resource, QueryCreate): diff --git a/tests/routers/test_search.py b/tests/routers/test_search.py index d92a3ca5..fdc52104 100644 --- a/tests/routers/test_search.py +++ b/tests/routers/test_search.py @@ -12,9 +12,10 @@ import pytest -pytestmark = [pytest.mark.asyncio, pytest.mark.usefixtures("reset_db_after")] +pytestmark = [pytest.mark.asyncio] +@pytest.mark.usefixtures("reset_db_after") async def test_get_search( client: Callable[ [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], @@ -55,6 +56,7 @@ async def test_get_search( assert response.data.attributes.state.value == "finished" assert "A new gateway was created for a query" in caplog.text, caplog.text + assert "A gateway was found and reused for a query" not in caplog.text, caplog.text async def test_get_search_existing_gateway( @@ -115,6 +117,7 @@ async def test_get_search_existing_gateway( ), f"Query never finished. Response: {response.json(indent=2)}" assert "A gateway was found and reused for a query" in caplog.text, caplog.text + assert "A new gateway was created for a query" not in caplog.text, caplog.text async def test_get_search_not_finishing( @@ -149,6 +152,7 @@ async def test_get_search_not_finishing( assert response.status_code == 200, f"Request failed: {response.json()}" assert "A gateway was found and reused for a query" in caplog.text, caplog.text + assert "A new gateway was created for a query" not in caplog.text, caplog.text response = QueriesResponseSingle(**response.json()) assert ( @@ -168,6 +172,85 @@ async def test_get_search_not_finishing( assert query.attributes.gateway_id == gateway_id, query +async def test_get_as_optimade( + client: Callable[ + [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], + Awaitable[httpx.Response], + ], + mock_gateway_responses: Callable[[dict], None], + get_gateway: Callable[[str], Awaitable[dict]], + caplog: pytest.LogCaptureFixture, +): + """Test GET /search with `as_optimade=True` + + This should be equivalent of `GET /gateways/{gateway_id}/structures`. + """ + from optimade.models import StructureResponseMany + + from optimade_gateway.common.config import CONFIG + + gateway_id = "twodbs" + gateway: dict = await get_gateway(gateway_id) + + query_params = { + "filter": 'elements HAS "Cu"', + "page_limit": CONFIG.page_limit, + "database_ids": [_.get("id") for _ in gateway.get("databases", [{}])], + "as_optimade": True, + } + + mock_gateway_responses(gateway) + + response = await client("/search", params=query_params) + + assert response.status_code == 200, f"Request failed: {response.json()}" + + response = StructureResponseMany(**response.json()) + assert response + + assert response.meta.more_data_available + + more_data_available = False + data_returned = 0 + data_available = 0 + data = [] + + assert len(response.data) == query_params["page_limit"] * len(gateway["databases"]) + + for database in gateway["databases"]: + url = f"{database['attributes']['base_url']}/structures?page_limit={query_params['page_limit']}" + db_response = httpx.get(url) + assert ( + db_response.status_code == 200 + ), f"Request to {url} failed: {db_response.json()}" + db_response = StructureResponseMany(**db_response.json()) + + data_returned += db_response.meta.data_returned + data_available += db_response.meta.data_available + if not more_data_available: + more_data_available = db_response.meta.more_data_available + + for datum in db_response.data: + datum = datum.dict(exclude_unset=True, exclude_none=True) + datum["id"] = f"{database['id']}/{datum['id']}" + data.append(datum) + + assert data_returned == response.meta.data_returned + assert data_available == response.meta.data_available + assert more_data_available == response.meta.more_data_available + + assert data == response.dict(exclude_unset=True, exclude_none=True)["data"], ( + f"IDs in test not in response: {set([_['id'] for _ in data]) - set([_['id'] for _ in response.dict(exclude_unset=True)['data']])}\n\n" + f"IDs in response not in test: {set([_['id'] for _ in response.dict(exclude_unset=True)['data']]) - set([_['id'] for _ in data])}\n\n" + f"A /search datum: {response.dict(exclude_unset=True)['data'][0]}\n\n" + f"A retrieved datum: {[_ for _ in data if _['id'] == response.dict(exclude_unset=True)['data'][0]['id']][0]}" + ) + + assert "A gateway was found and reused for a query" in caplog.text, caplog.text + assert "A new gateway was created for a query" not in caplog.text, caplog.text + + +@pytest.mark.usefixtures("reset_db_after") async def test_post_search( client: Callable[ [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], @@ -217,6 +300,7 @@ async def test_post_search( ), response.meta.dict() assert "A new gateway was created for a query" in caplog.text, caplog.text + assert "A gateway was found and reused for a query" not in caplog.text, caplog.text datum = response.data assert datum, response @@ -297,6 +381,7 @@ async def test_post_search_existing_gateway( ), response.meta.dict() assert "A gateway was found and reused for a query" in caplog.text, caplog.text + assert "A new gateway was created for a query" not in caplog.text, caplog.text datum = response.data assert datum, response From 4547d0d36e73f53544ff5a90a6edb1c31bd462c2 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 6 Sep 2021 17:30:43 +0200 Subject: [PATCH 10/13] Adapt to updated optimade Remove root validator from `Response` for `GatewayQueryResponse`. The `GatewayQueryResponse` is special, since it can allow `errors` to be present together with `data`. And `meta` is guaranteed to always be present, so there is no need for the validator. Also, update the test data to respect the new validators for `chemical_formula_anonymous`. Update optimade and pre-commit dependencies. --- optimade_gateway/models/queries.py | 23 +++++++++++++++++-- optimade_gateway/models/resources.py | 4 ++-- requirements.txt | 2 +- requirements_dev.txt | 2 +- .../static/db_responses/optimade-sample.json | 2 +- .../db_responses/optimade-sample_single.json | 2 +- 6 files changed, 27 insertions(+), 8 deletions(-) diff --git a/optimade_gateway/models/queries.py b/optimade_gateway/models/queries.py index 2406fb3b..73fba090 100644 --- a/optimade_gateway/models/queries.py +++ b/optimade_gateway/models/queries.py @@ -162,10 +162,10 @@ class GatewayQueryResponse(Response): """Response from a Gateway Query.""" data: Dict[str, Union[List[EntryResource], List[Dict[str, Any]]]] = StrictField( - ..., uniqueItems=True, description="Outputted Data" + ..., uniqueItems=True, description="Outputted Data." ) meta: ResponseMeta = StrictField( - ..., description="A meta object containing non-standard information" + ..., description="A meta object containing non-standard information." ) errors: Optional[List[OptimadeError]] = StrictField( [], @@ -176,6 +176,25 @@ class GatewayQueryResponse(Response): None, uniqueItems=True ) + @classmethod + def _remove_pre_root_validators(cls): + """Remove `either_data_meta_or_errors_must_be_set` pre root_validator. + This will always be available through `meta`, and more importantly, + `errors` should be allowed to be present always for this special response. + """ + pre_root_validators = [] + for validator_ in cls.__pre_root_validators__: + if not str(validator_).startswith( + " None: + """Remove root_validator `either_data_meta_or_errors_must_be_set`.""" + self._remove_pre_root_validators() + super().__init__(**data) + class QueryResourceAttributes(EntryResourceAttributes): """Attributes for an OPTIMADE gateway query.""" diff --git a/optimade_gateway/models/resources.py b/optimade_gateway/models/resources.py index f35b7481..b0672c23 100644 --- a/optimade_gateway/models/resources.py +++ b/optimade_gateway/models/resources.py @@ -16,7 +16,7 @@ class Config: @classmethod def _remove_pre_root_validators(cls): - """Remove `check_illegal_attributes_fields` pre root_validators""" + """Remove `check_illegal_attributes_fields` pre root_validators.""" pre_root_validators = [] for validator in cls.__pre_root_validators__: if not str(validator).startswith( @@ -26,6 +26,6 @@ def _remove_pre_root_validators(cls): cls.__pre_root_validators__ = pre_root_validators def __init__(self, **data: Any) -> None: - """Remove root_validator `check_illegal_attributes_fields`""" + """Remove root_validator `check_illegal_attributes_fields`.""" self._remove_pre_root_validators() super().__init__(**data) diff --git a/requirements.txt b/requirements.txt index f70314e1..546c2dac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ httpx~=0.19.0 motor~=2.5 -optimade[server]~=0.16.2 +optimade[server]~=0.16.3 diff --git a/requirements_dev.txt b/requirements_dev.txt index 19c7680a..07e5c2ee 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,5 +1,5 @@ invoke~=1.6 -pre-commit~=2.14 +pre-commit~=2.15 pylint~=2.10 pytest~=6.2 pytest-asyncio~=0.15.1 diff --git a/tests/static/db_responses/optimade-sample.json b/tests/static/db_responses/optimade-sample.json index 35b1f0b7..bd393f36 100644 --- a/tests/static/db_responses/optimade-sample.json +++ b/tests/static/db_responses/optimade-sample.json @@ -1 +1 @@ -{"data":[{"id":"1151","type":"structures","links":null,"attributes":{"immutable_id":"18a7b936-8fc9-4793-838e-0d49e89cbba1","last_modified":"2021-02-23T00:03:13Z","elements":["C"],"nelements":1,"elements_ratios":[1],"chemical_formula_descriptive":"C2","chemical_formula_reduced":"C","chemical_formula_hill":"C2","chemical_formula_anonymous":"A2","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[2.3116547558853364,0,2.8052588632054447],[1.0069144937119427,2.0808341867530236,2.8052588632054447],[0,0,3.635]],"cartesian_site_positions":[[0.5442453569339538,0.34125680662749586,1.5162649071313858],[2.7743238926633254,1.7395773801255277,7.729252819279503]],"nsites":2,"species":[{"name":"C","chemical_symbols":["C"],"concentration":[1],"mass":[12.0107],"original_name":"C"}],"species_at_sites":["C","C"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-01T10:07:29Z"},"relationships":null},{"id":"1138","type":"structures","links":null,"attributes":{"immutable_id":"1650d1ef-ccfa-4384-acbf-437bd57b034c","last_modified":"2021-02-23T00:03:12Z","elements":["C"],"nelements":1,"elements_ratios":[1],"chemical_formula_descriptive":"C2","chemical_formula_reduced":"C","chemical_formula_hill":"C2","chemical_formula_anonymous":"A2","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[2.3116547558853364,0,2.8052588632054447],[1.0069144937119427,2.0808341867530236,2.8052588632054447],[0,0,3.635]],"cartesian_site_positions":[[0.5442453569339538,0.34125680662749586,1.5162649071313858],[2.7743238926633254,1.7395773801255277,7.729252819279503]],"nsites":2,"species":[{"name":"C","chemical_symbols":["C"],"concentration":[1],"mass":[12.0107],"original_name":"C"}],"species_at_sites":["C","C"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-01T10:13:36Z"},"relationships":null},{"id":"1541","type":"structures","links":null,"attributes":{"immutable_id":"d5f353a1-79c7-40e5-9ce7-6e3c7db6f839","last_modified":"2021-02-23T00:03:52Z","elements":["C"],"nelements":1,"elements_ratios":[1],"chemical_formula_descriptive":"C2","chemical_formula_reduced":"C","chemical_formula_hill":"C2","chemical_formula_anonymous":"A2","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[2.3116547558853364,0,2.8052588632054447],[1.0069144937119427,2.0808341867530236,2.8052588632054447],[0,0,3.635]],"cartesian_site_positions":[[0.5442453569339538,0.34125680662749586,1.5162649071313858],[2.7743238926633254,1.7395773801255277,7.729252819279503]],"nsites":2,"species":[{"name":"C","chemical_symbols":["C"],"concentration":[1],"mass":[12.0107],"original_name":"C"}],"species_at_sites":["C","C"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-01T10:19:40Z"},"relationships":null},{"id":"580","type":"structures","links":null,"attributes":{"immutable_id":"e112d828-27d2-4129-932f-f18f205aa0b6","last_modified":"2021-02-23T00:02:10Z","elements":["Cl","Na"],"nelements":2,"elements_ratios":[0.5,0.5],"chemical_formula_descriptive":"ClNa","chemical_formula_reduced":"ClNa","chemical_formula_hill":"ClNa","chemical_formula_anonymous":"AB","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[2.86,0,0],[0,2.86,0],[0,0,2.86]],"cartesian_site_positions":[[0,0,0],[1.43,1.43,1.43]],"nsites":2,"species":[{"name":"Na","chemical_symbols":["Na"],"concentration":[1],"mass":[22.98977],"original_name":"Na"},{"name":"Cl","chemical_symbols":["Cl"],"concentration":[1],"mass":[35.453],"original_name":"Cl"}],"species_at_sites":["Na","Cl"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2019-11-19T17:44:10Z"},"relationships":null},{"id":"1095","type":"structures","links":null,"attributes":{"immutable_id":"01ded4dd-7bc9-4664-8288-cf86116cda45","last_modified":"2021-02-23T00:03:07Z","elements":["Br","Mg"],"nelements":2,"elements_ratios":[0.6666666666666666,0.3333333333333333],"chemical_formula_descriptive":"Br2Mg","chemical_formula_reduced":"Br2Mg","chemical_formula_hill":"Br2Mg","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[3.81,0,0],[-1.904999999999999,3.299556788418712,0],[0,0,6.26]],"cartesian_site_positions":[[0,0,0],[0,2.1997045256124745,1.5650000000000002],[1.9050000000000005,1.0998522628062373,4.695]],"nsites":3,"species":[{"name":"Mg","chemical_symbols":["Mg"],"concentration":[1],"mass":[24.305],"original_name":"Mg"},{"name":"Br","chemical_symbols":["Br"],"concentration":[1],"mass":[79.904],"original_name":"Br"}],"species_at_sites":["Mg","Br","Br"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-02T18:42:02Z"},"relationships":null},{"id":"1584","type":"structures","links":null,"attributes":{"immutable_id":"ebedd552-a2aa-4782-857a-b19a05c3bf11","last_modified":"2021-02-23T00:03:57Z","elements":["Nb","S"],"nelements":2,"elements_ratios":[0.3333333333333333,0.6666666666666666],"chemical_formula_descriptive":"NbS2","chemical_formula_reduced":"NbS2","chemical_formula_hill":"NbS2","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[3.42,0,0],[-1.709999999999999,2.9618068809427807,0],[0,0,5.938]],"cartesian_site_positions":[[0,0,0],[0,1.9745379206285203,1.4791558],[1.7100000000000004,0.9872689603142604,4.4588442]],"nsites":3,"species":[{"name":"Nb","chemical_symbols":["Nb"],"concentration":[1],"mass":[92.90638],"original_name":"Nb"},{"name":"S","chemical_symbols":["S"],"concentration":[1],"mass":[32.065],"original_name":"S"}],"species_at_sites":["Nb","S","S"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-02T21:59:28Z"},"relationships":null},{"id":"1390","type":"structures","links":null,"attributes":{"immutable_id":"8b22d26e-d1f8-443c-9736-96f5d5fb2c4d","last_modified":"2021-02-23T00:03:37Z","elements":["Se","Ti"],"nelements":2,"elements_ratios":[0.6666666666666666,0.3333333333333333],"chemical_formula_descriptive":"Se2Ti","chemical_formula_reduced":"Se2Ti","chemical_formula_hill":"Se2Ti","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[3.533,0,0],[-1.766499999999999,3.0596677515704225,0],[0,0,5.995]],"cartesian_site_positions":[[0,0,0],[0,2.039778501046948,1.4987500000000002],[1.7665000000000006,1.0198892505234742,4.49625]],"nsites":3,"species":[{"name":"Ti","chemical_symbols":["Ti"],"concentration":[1],"mass":[47.867],"original_name":"Ti"},{"name":"Se","chemical_symbols":["Se"],"concentration":[1],"mass":[78.96],"original_name":"Se"}],"species_at_sites":["Ti","Se","Se"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-02T09:01:13Z"},"relationships":null},{"id":"1317","type":"structures","links":null,"attributes":{"immutable_id":"60e3fc0b-2605-47ec-bd37-1a55770d3dda","last_modified":"2021-02-23T00:03:30Z","elements":["S","Sn"],"nelements":2,"elements_ratios":[0.6666666666666666,0.3333333333333333],"chemical_formula_descriptive":"S2Sn","chemical_formula_reduced":"S2Sn","chemical_formula_hill":"S2Sn","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[3.639,0,0],[-1.8194999999999988,3.1514664443715725,0],[0,0,5.868]],"cartesian_site_positions":[[0,0,0],[0,2.100977629581048,1.4670000000000003],[1.8195000000000001,1.050488814790524,4.401000000000001]],"nsites":3,"species":[{"name":"Sn","chemical_symbols":["Sn"],"concentration":[1],"mass":[118.71],"original_name":"Sn"},{"name":"S","chemical_symbols":["S"],"concentration":[1],"mass":[32.065],"original_name":"S"}],"species_at_sites":["Sn","S","S"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-02T19:33:42Z"},"relationships":null},{"id":"1576","type":"structures","links":null,"attributes":{"immutable_id":"e9b88297-8de9-49ad-91a1-185009731f42","last_modified":"2021-02-23T00:03:56Z","elements":["Br","Co"],"nelements":2,"elements_ratios":[0.6666666666666666,0.3333333333333333],"chemical_formula_descriptive":"Br2Co","chemical_formula_reduced":"Br2Co","chemical_formula_hill":"Br2Co","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[3.728,0,0],[-1.863999999999999,3.228542705308388,0],[0,0,6.169]],"cartesian_site_positions":[[0,0,0],[0,2.152361803538925,1.5422500000000001],[1.864,1.0761809017694626,4.62675]],"nsites":3,"species":[{"name":"Co","chemical_symbols":["Co"],"concentration":[1],"mass":[58.933195],"original_name":"Co"},{"name":"Br","chemical_symbols":["Br"],"concentration":[1],"mass":[79.904],"original_name":"Br"}],"species_at_sites":["Co","Br","Br"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-03T02:45:40Z"},"relationships":null},{"id":"1111","type":"structures","links":null,"attributes":{"immutable_id":"076be8d6-ea11-432e-a35f-85308f25c7a3","last_modified":"2021-02-23T00:03:09Z","elements":["Cl","V"],"nelements":2,"elements_ratios":[0.6666666666666666,0.3333333333333333],"chemical_formula_descriptive":"Cl2V","chemical_formula_reduced":"Cl2V","chemical_formula_hill":"Cl2V","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[3.6,0,0],[-1.799999999999999,3.11769145362398,0],[0,0,5.83]],"cartesian_site_positions":[[0,0,0],[0,2.078460969082653,1.4575000000000002],[1.8000000000000003,1.0392304845413265,4.3725000000000005]],"nsites":3,"species":[{"name":"V","chemical_symbols":["V"],"concentration":[1],"mass":[50.9415],"original_name":"V"},{"name":"Cl","chemical_symbols":["Cl"],"concentration":[1],"mass":[35.453],"original_name":"Cl"}],"species_at_sites":["V","Cl","Cl"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-03T11:38:56Z"},"relationships":null},{"id":"1590","type":"structures","links":null,"attributes":{"immutable_id":"f0c2cfab-9f53-4fb4-8c85-386c4198247b","last_modified":"2021-02-23T00:03:58Z","elements":["Cl","V"],"nelements":2,"elements_ratios":[0.6666666666666666,0.3333333333333333],"chemical_formula_descriptive":"Cl2V","chemical_formula_reduced":"Cl2V","chemical_formula_hill":"Cl2V","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[3.6,0,0],[-1.799999999999999,3.11769145362398,0],[0,0,5.83]],"cartesian_site_positions":[[0,0,0],[0,2.078460969082653,1.4575000000000002],[1.8000000000000003,1.0392304845413265,4.3725000000000005]],"nsites":3,"species":[{"name":"V","chemical_symbols":["V"],"concentration":[1],"mass":[50.9415],"original_name":"V"},{"name":"Cl","chemical_symbols":["Cl"],"concentration":[1],"mass":[35.453],"original_name":"Cl"}],"species_at_sites":["V","Cl","Cl"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-03T12:28:56Z"},"relationships":null},{"id":"1568","type":"structures","links":null,"attributes":{"immutable_id":"e66d5339-b127-4d9d-a167-900bc33cb579","last_modified":"2021-02-23T00:03:56Z","elements":["Cl","Ti"],"nelements":2,"elements_ratios":[0.6666666666666666,0.3333333333333333],"chemical_formula_descriptive":"Cl2Ti","chemical_formula_reduced":"Cl2Ti","chemical_formula_hill":"Cl2Ti","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[3.561,0,0],[-1.780499999999999,3.0839164628763864,0],[0,0,5.875]],"cartesian_site_positions":[[0,0,0],[0,2.0559443085842575,1.4687500000000002],[1.7805,1.0279721542921287,4.40625]],"nsites":3,"species":[{"name":"Ti","chemical_symbols":["Ti"],"concentration":[1],"mass":[47.867],"original_name":"Ti"},{"name":"Cl","chemical_symbols":["Cl"],"concentration":[1],"mass":[35.453],"original_name":"Cl"}],"species_at_sites":["Ti","Cl","Cl"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-02T18:45:32Z"},"relationships":null},{"id":"1599","type":"structures","links":null,"attributes":{"immutable_id":"f608478b-0bc7-44ee-931a-fcd12a8fe347","last_modified":"2021-02-23T00:03:59Z","elements":["Cl","Fe"],"nelements":2,"elements_ratios":[0.6666666666666666,0.3333333333333333],"chemical_formula_descriptive":"Cl2Fe","chemical_formula_reduced":"Cl2Fe","chemical_formula_hill":"Cl2Fe","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[3.585,0,0],[-1.792499999999999,3.104701072567213,0],[0,0,5.735]],"cartesian_site_positions":[[0,0,0],[0,2.0698007150448086,1.3706650000000002],[1.7925,1.0349003575224043,4.3643350000000005]],"nsites":3,"species":[{"name":"Fe","chemical_symbols":["Fe"],"concentration":[1],"mass":[55.845],"original_name":"Fe"},{"name":"Cl","chemical_symbols":["Cl"],"concentration":[1],"mass":[35.453],"original_name":"Cl"}],"species_at_sites":["Fe","Cl","Cl"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-03T02:36:51Z"},"relationships":null},{"id":"1606","type":"structures","links":null,"attributes":{"immutable_id":"fa289ffa-6ec4-4bff-b504-90908cf2aa9b","last_modified":"2021-02-23T00:04:00Z","elements":["Si","Te"],"nelements":2,"elements_ratios":[0.3333333333333333,0.6666666666666666],"chemical_formula_descriptive":"SiTe2","chemical_formula_reduced":"SiTe2","chemical_formula_hill":"SiTe2","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[4.28,0,0],[-2.139999999999999,3.7065887281973984,0],[0,0,6.71]],"cartesian_site_positions":[[0,0,0],[0,2.4710591521315988,1.7781500000000003],[2.1400000000000006,1.2355295760657994,4.93185]],"nsites":3,"species":[{"name":"Si","chemical_symbols":["Si"],"concentration":[1],"mass":[28.0855],"original_name":"Si"},{"name":"Te","chemical_symbols":["Te"],"concentration":[1],"mass":[127.6],"original_name":"Te"}],"species_at_sites":["Si","Te","Te"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-02T19:31:34Z"},"relationships":null},{"id":"1341","type":"structures","links":null,"attributes":{"immutable_id":"6f72f1b5-d7d9-4c6b-bfd9-a3eead7cc600","last_modified":"2021-02-23T00:03:32Z","elements":["I","Zn"],"nelements":2,"elements_ratios":[0.6666666666666666,0.3333333333333333],"chemical_formula_descriptive":"I2Zn","chemical_formula_reduced":"I2Zn","chemical_formula_hill":"I2Zn","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[4.25,0,0],[-2.1249999999999987,3.680607966083865,0],[0,0,6.54]],"cartesian_site_positions":[[0,0,0],[0,2.45373864405591,1.6350000000000002],[2.125,1.226869322027955,4.905]],"nsites":3,"species":[{"name":"Zn","chemical_symbols":["Zn"],"concentration":[1],"mass":[65.38],"original_name":"Zn"},{"name":"I","chemical_symbols":["I"],"concentration":[1],"mass":[126.90447],"original_name":"I"}],"species_at_sites":["Zn","I","I"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-02T19:19:11Z"},"relationships":null}],"meta":{"query":{"representation":"/structures?"},"api_version":"1.0.1","more_data_available":true,"time_stamp":"2021-04-01T23:04:19Z","data_returned":146,"provider":{"name":"Materials Cloud","description":"A platform for Open Science built for seamless sharing of resources in computational materials science","prefix":"mcloud","homepage":"https://materialscloud.org"},"data_available":1620,"implementation":{"name":"aiida-optimade","version":"0.16.3+mcloud","source_url":"https://github.com/aiidateam/aiida-optimade","maintainer":{"email":"casper.andersen@epfl.ch"}}},"links":{"next":"https://aiida.materialscloud.org/optimade-sample/optimade/structures?page_offset=15"}} \ No newline at end of file +{"data":[{"id":"1151","type":"structures","links":null,"attributes":{"immutable_id":"18a7b936-8fc9-4793-838e-0d49e89cbba1","last_modified":"2021-02-23T00:03:13Z","elements":["C"],"nelements":1,"elements_ratios":[1],"chemical_formula_descriptive":"C2","chemical_formula_reduced":"C","chemical_formula_hill":"C2","chemical_formula_anonymous":"A","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[2.3116547558853364,0,2.8052588632054447],[1.0069144937119427,2.0808341867530236,2.8052588632054447],[0,0,3.635]],"cartesian_site_positions":[[0.5442453569339538,0.34125680662749586,1.5162649071313858],[2.7743238926633254,1.7395773801255277,7.729252819279503]],"nsites":2,"species":[{"name":"C","chemical_symbols":["C"],"concentration":[1],"mass":[12.0107],"original_name":"C"}],"species_at_sites":["C","C"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-01T10:07:29Z"},"relationships":null},{"id":"1138","type":"structures","links":null,"attributes":{"immutable_id":"1650d1ef-ccfa-4384-acbf-437bd57b034c","last_modified":"2021-02-23T00:03:12Z","elements":["C"],"nelements":1,"elements_ratios":[1],"chemical_formula_descriptive":"C2","chemical_formula_reduced":"C","chemical_formula_hill":"C2","chemical_formula_anonymous":"A","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[2.3116547558853364,0,2.8052588632054447],[1.0069144937119427,2.0808341867530236,2.8052588632054447],[0,0,3.635]],"cartesian_site_positions":[[0.5442453569339538,0.34125680662749586,1.5162649071313858],[2.7743238926633254,1.7395773801255277,7.729252819279503]],"nsites":2,"species":[{"name":"C","chemical_symbols":["C"],"concentration":[1],"mass":[12.0107],"original_name":"C"}],"species_at_sites":["C","C"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-01T10:13:36Z"},"relationships":null},{"id":"1541","type":"structures","links":null,"attributes":{"immutable_id":"d5f353a1-79c7-40e5-9ce7-6e3c7db6f839","last_modified":"2021-02-23T00:03:52Z","elements":["C"],"nelements":1,"elements_ratios":[1],"chemical_formula_descriptive":"C2","chemical_formula_reduced":"C","chemical_formula_hill":"C2","chemical_formula_anonymous":"A","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[2.3116547558853364,0,2.8052588632054447],[1.0069144937119427,2.0808341867530236,2.8052588632054447],[0,0,3.635]],"cartesian_site_positions":[[0.5442453569339538,0.34125680662749586,1.5162649071313858],[2.7743238926633254,1.7395773801255277,7.729252819279503]],"nsites":2,"species":[{"name":"C","chemical_symbols":["C"],"concentration":[1],"mass":[12.0107],"original_name":"C"}],"species_at_sites":["C","C"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-01T10:19:40Z"},"relationships":null},{"id":"580","type":"structures","links":null,"attributes":{"immutable_id":"e112d828-27d2-4129-932f-f18f205aa0b6","last_modified":"2021-02-23T00:02:10Z","elements":["Cl","Na"],"nelements":2,"elements_ratios":[0.5,0.5],"chemical_formula_descriptive":"ClNa","chemical_formula_reduced":"ClNa","chemical_formula_hill":"ClNa","chemical_formula_anonymous":"AB","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[2.86,0,0],[0,2.86,0],[0,0,2.86]],"cartesian_site_positions":[[0,0,0],[1.43,1.43,1.43]],"nsites":2,"species":[{"name":"Na","chemical_symbols":["Na"],"concentration":[1],"mass":[22.98977],"original_name":"Na"},{"name":"Cl","chemical_symbols":["Cl"],"concentration":[1],"mass":[35.453],"original_name":"Cl"}],"species_at_sites":["Na","Cl"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2019-11-19T17:44:10Z"},"relationships":null},{"id":"1095","type":"structures","links":null,"attributes":{"immutable_id":"01ded4dd-7bc9-4664-8288-cf86116cda45","last_modified":"2021-02-23T00:03:07Z","elements":["Br","Mg"],"nelements":2,"elements_ratios":[0.6666666666666666,0.3333333333333333],"chemical_formula_descriptive":"Br2Mg","chemical_formula_reduced":"Br2Mg","chemical_formula_hill":"Br2Mg","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[3.81,0,0],[-1.904999999999999,3.299556788418712,0],[0,0,6.26]],"cartesian_site_positions":[[0,0,0],[0,2.1997045256124745,1.5650000000000002],[1.9050000000000005,1.0998522628062373,4.695]],"nsites":3,"species":[{"name":"Mg","chemical_symbols":["Mg"],"concentration":[1],"mass":[24.305],"original_name":"Mg"},{"name":"Br","chemical_symbols":["Br"],"concentration":[1],"mass":[79.904],"original_name":"Br"}],"species_at_sites":["Mg","Br","Br"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-02T18:42:02Z"},"relationships":null},{"id":"1584","type":"structures","links":null,"attributes":{"immutable_id":"ebedd552-a2aa-4782-857a-b19a05c3bf11","last_modified":"2021-02-23T00:03:57Z","elements":["Nb","S"],"nelements":2,"elements_ratios":[0.3333333333333333,0.6666666666666666],"chemical_formula_descriptive":"NbS2","chemical_formula_reduced":"NbS2","chemical_formula_hill":"NbS2","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[3.42,0,0],[-1.709999999999999,2.9618068809427807,0],[0,0,5.938]],"cartesian_site_positions":[[0,0,0],[0,1.9745379206285203,1.4791558],[1.7100000000000004,0.9872689603142604,4.4588442]],"nsites":3,"species":[{"name":"Nb","chemical_symbols":["Nb"],"concentration":[1],"mass":[92.90638],"original_name":"Nb"},{"name":"S","chemical_symbols":["S"],"concentration":[1],"mass":[32.065],"original_name":"S"}],"species_at_sites":["Nb","S","S"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-02T21:59:28Z"},"relationships":null},{"id":"1390","type":"structures","links":null,"attributes":{"immutable_id":"8b22d26e-d1f8-443c-9736-96f5d5fb2c4d","last_modified":"2021-02-23T00:03:37Z","elements":["Se","Ti"],"nelements":2,"elements_ratios":[0.6666666666666666,0.3333333333333333],"chemical_formula_descriptive":"Se2Ti","chemical_formula_reduced":"Se2Ti","chemical_formula_hill":"Se2Ti","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[3.533,0,0],[-1.766499999999999,3.0596677515704225,0],[0,0,5.995]],"cartesian_site_positions":[[0,0,0],[0,2.039778501046948,1.4987500000000002],[1.7665000000000006,1.0198892505234742,4.49625]],"nsites":3,"species":[{"name":"Ti","chemical_symbols":["Ti"],"concentration":[1],"mass":[47.867],"original_name":"Ti"},{"name":"Se","chemical_symbols":["Se"],"concentration":[1],"mass":[78.96],"original_name":"Se"}],"species_at_sites":["Ti","Se","Se"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-02T09:01:13Z"},"relationships":null},{"id":"1317","type":"structures","links":null,"attributes":{"immutable_id":"60e3fc0b-2605-47ec-bd37-1a55770d3dda","last_modified":"2021-02-23T00:03:30Z","elements":["S","Sn"],"nelements":2,"elements_ratios":[0.6666666666666666,0.3333333333333333],"chemical_formula_descriptive":"S2Sn","chemical_formula_reduced":"S2Sn","chemical_formula_hill":"S2Sn","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[3.639,0,0],[-1.8194999999999988,3.1514664443715725,0],[0,0,5.868]],"cartesian_site_positions":[[0,0,0],[0,2.100977629581048,1.4670000000000003],[1.8195000000000001,1.050488814790524,4.401000000000001]],"nsites":3,"species":[{"name":"Sn","chemical_symbols":["Sn"],"concentration":[1],"mass":[118.71],"original_name":"Sn"},{"name":"S","chemical_symbols":["S"],"concentration":[1],"mass":[32.065],"original_name":"S"}],"species_at_sites":["Sn","S","S"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-02T19:33:42Z"},"relationships":null},{"id":"1576","type":"structures","links":null,"attributes":{"immutable_id":"e9b88297-8de9-49ad-91a1-185009731f42","last_modified":"2021-02-23T00:03:56Z","elements":["Br","Co"],"nelements":2,"elements_ratios":[0.6666666666666666,0.3333333333333333],"chemical_formula_descriptive":"Br2Co","chemical_formula_reduced":"Br2Co","chemical_formula_hill":"Br2Co","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[3.728,0,0],[-1.863999999999999,3.228542705308388,0],[0,0,6.169]],"cartesian_site_positions":[[0,0,0],[0,2.152361803538925,1.5422500000000001],[1.864,1.0761809017694626,4.62675]],"nsites":3,"species":[{"name":"Co","chemical_symbols":["Co"],"concentration":[1],"mass":[58.933195],"original_name":"Co"},{"name":"Br","chemical_symbols":["Br"],"concentration":[1],"mass":[79.904],"original_name":"Br"}],"species_at_sites":["Co","Br","Br"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-03T02:45:40Z"},"relationships":null},{"id":"1111","type":"structures","links":null,"attributes":{"immutable_id":"076be8d6-ea11-432e-a35f-85308f25c7a3","last_modified":"2021-02-23T00:03:09Z","elements":["Cl","V"],"nelements":2,"elements_ratios":[0.6666666666666666,0.3333333333333333],"chemical_formula_descriptive":"Cl2V","chemical_formula_reduced":"Cl2V","chemical_formula_hill":"Cl2V","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[3.6,0,0],[-1.799999999999999,3.11769145362398,0],[0,0,5.83]],"cartesian_site_positions":[[0,0,0],[0,2.078460969082653,1.4575000000000002],[1.8000000000000003,1.0392304845413265,4.3725000000000005]],"nsites":3,"species":[{"name":"V","chemical_symbols":["V"],"concentration":[1],"mass":[50.9415],"original_name":"V"},{"name":"Cl","chemical_symbols":["Cl"],"concentration":[1],"mass":[35.453],"original_name":"Cl"}],"species_at_sites":["V","Cl","Cl"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-03T11:38:56Z"},"relationships":null},{"id":"1590","type":"structures","links":null,"attributes":{"immutable_id":"f0c2cfab-9f53-4fb4-8c85-386c4198247b","last_modified":"2021-02-23T00:03:58Z","elements":["Cl","V"],"nelements":2,"elements_ratios":[0.6666666666666666,0.3333333333333333],"chemical_formula_descriptive":"Cl2V","chemical_formula_reduced":"Cl2V","chemical_formula_hill":"Cl2V","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[3.6,0,0],[-1.799999999999999,3.11769145362398,0],[0,0,5.83]],"cartesian_site_positions":[[0,0,0],[0,2.078460969082653,1.4575000000000002],[1.8000000000000003,1.0392304845413265,4.3725000000000005]],"nsites":3,"species":[{"name":"V","chemical_symbols":["V"],"concentration":[1],"mass":[50.9415],"original_name":"V"},{"name":"Cl","chemical_symbols":["Cl"],"concentration":[1],"mass":[35.453],"original_name":"Cl"}],"species_at_sites":["V","Cl","Cl"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-03T12:28:56Z"},"relationships":null},{"id":"1568","type":"structures","links":null,"attributes":{"immutable_id":"e66d5339-b127-4d9d-a167-900bc33cb579","last_modified":"2021-02-23T00:03:56Z","elements":["Cl","Ti"],"nelements":2,"elements_ratios":[0.6666666666666666,0.3333333333333333],"chemical_formula_descriptive":"Cl2Ti","chemical_formula_reduced":"Cl2Ti","chemical_formula_hill":"Cl2Ti","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[3.561,0,0],[-1.780499999999999,3.0839164628763864,0],[0,0,5.875]],"cartesian_site_positions":[[0,0,0],[0,2.0559443085842575,1.4687500000000002],[1.7805,1.0279721542921287,4.40625]],"nsites":3,"species":[{"name":"Ti","chemical_symbols":["Ti"],"concentration":[1],"mass":[47.867],"original_name":"Ti"},{"name":"Cl","chemical_symbols":["Cl"],"concentration":[1],"mass":[35.453],"original_name":"Cl"}],"species_at_sites":["Ti","Cl","Cl"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-02T18:45:32Z"},"relationships":null},{"id":"1599","type":"structures","links":null,"attributes":{"immutable_id":"f608478b-0bc7-44ee-931a-fcd12a8fe347","last_modified":"2021-02-23T00:03:59Z","elements":["Cl","Fe"],"nelements":2,"elements_ratios":[0.6666666666666666,0.3333333333333333],"chemical_formula_descriptive":"Cl2Fe","chemical_formula_reduced":"Cl2Fe","chemical_formula_hill":"Cl2Fe","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[3.585,0,0],[-1.792499999999999,3.104701072567213,0],[0,0,5.735]],"cartesian_site_positions":[[0,0,0],[0,2.0698007150448086,1.3706650000000002],[1.7925,1.0349003575224043,4.3643350000000005]],"nsites":3,"species":[{"name":"Fe","chemical_symbols":["Fe"],"concentration":[1],"mass":[55.845],"original_name":"Fe"},{"name":"Cl","chemical_symbols":["Cl"],"concentration":[1],"mass":[35.453],"original_name":"Cl"}],"species_at_sites":["Fe","Cl","Cl"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-03T02:36:51Z"},"relationships":null},{"id":"1606","type":"structures","links":null,"attributes":{"immutable_id":"fa289ffa-6ec4-4bff-b504-90908cf2aa9b","last_modified":"2021-02-23T00:04:00Z","elements":["Si","Te"],"nelements":2,"elements_ratios":[0.3333333333333333,0.6666666666666666],"chemical_formula_descriptive":"SiTe2","chemical_formula_reduced":"SiTe2","chemical_formula_hill":"SiTe2","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[4.28,0,0],[-2.139999999999999,3.7065887281973984,0],[0,0,6.71]],"cartesian_site_positions":[[0,0,0],[0,2.4710591521315988,1.7781500000000003],[2.1400000000000006,1.2355295760657994,4.93185]],"nsites":3,"species":[{"name":"Si","chemical_symbols":["Si"],"concentration":[1],"mass":[28.0855],"original_name":"Si"},{"name":"Te","chemical_symbols":["Te"],"concentration":[1],"mass":[127.6],"original_name":"Te"}],"species_at_sites":["Si","Te","Te"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-02T19:31:34Z"},"relationships":null},{"id":"1341","type":"structures","links":null,"attributes":{"immutable_id":"6f72f1b5-d7d9-4c6b-bfd9-a3eead7cc600","last_modified":"2021-02-23T00:03:32Z","elements":["I","Zn"],"nelements":2,"elements_ratios":[0.6666666666666666,0.3333333333333333],"chemical_formula_descriptive":"I2Zn","chemical_formula_reduced":"I2Zn","chemical_formula_hill":"I2Zn","chemical_formula_anonymous":"A2B","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[4.25,0,0],[-2.1249999999999987,3.680607966083865,0],[0,0,6.54]],"cartesian_site_positions":[[0,0,0],[0,2.45373864405591,1.6350000000000002],[2.125,1.226869322027955,4.905]],"nsites":3,"species":[{"name":"Zn","chemical_symbols":["Zn"],"concentration":[1],"mass":[65.38],"original_name":"Zn"},{"name":"I","chemical_symbols":["I"],"concentration":[1],"mass":[126.90447],"original_name":"I"}],"species_at_sites":["Zn","I","I"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2015-10-02T19:19:11Z"},"relationships":null}],"meta":{"query":{"representation":"/structures?"},"api_version":"1.0.1","more_data_available":true,"time_stamp":"2021-04-01T23:04:19Z","data_returned":146,"provider":{"name":"Materials Cloud","description":"A platform for Open Science built for seamless sharing of resources in computational materials science","prefix":"mcloud","homepage":"https://materialscloud.org"},"data_available":1620,"implementation":{"name":"aiida-optimade","version":"0.16.3+mcloud","source_url":"https://github.com/aiidateam/aiida-optimade","maintainer":{"email":"casper.andersen@epfl.ch"}}},"links":{"next":"https://aiida.materialscloud.org/optimade-sample/optimade/structures?page_offset=15"}} \ No newline at end of file diff --git a/tests/static/db_responses/optimade-sample_single.json b/tests/static/db_responses/optimade-sample_single.json index 585a2c68..c6b2f257 100644 --- a/tests/static/db_responses/optimade-sample_single.json +++ b/tests/static/db_responses/optimade-sample_single.json @@ -1 +1 @@ -{"data":{"id":"1","type":"structures","links":null,"attributes":{"immutable_id":"509fff89-329d-494c-a73f-69e153c9dfdc","last_modified":"2021-02-22T22:02:01Z","elements":["Cl","Na"],"nelements":2,"elements_ratios":[0.5,0.5],"chemical_formula_descriptive":"Cl4Na4","chemical_formula_reduced":"ClNa","chemical_formula_hill":"Cl4Na4","chemical_formula_anonymous":"A4B4","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[5.7632,0,0],[0,5.7632,0],[0,0,5.7632]],"cartesian_site_positions":[[0,0,0],[0,2.8816,2.8816],[2.8816,0,2.8816],[2.8816,2.8816,0],[2.8816,2.8816,2.8816],[2.8816,0,0],[0,2.8816,0],[0,0,2.8816]],"nsites":8,"species":[{"name":"Na","chemical_symbols":["Na"],"concentration":[1],"mass":[22.98977],"original_name":"Na"},{"name":"Cl","chemical_symbols":["Cl"],"concentration":[1],"mass":[35.453],"original_name":"Cl"}],"species_at_sites":["Na","Na","Na","Na","Cl","Cl","Cl","Cl"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2019-11-19T17:52:30Z"},"relationships":null},"meta":{"query":{"representation":"/structures/1?"},"api_version":"1.0.1","more_data_available":false,"time_stamp":"2021-04-29T13:41:20Z","data_returned":1,"provider":{"name":"Materials Cloud","description":"A platform for Open Science built for seamless sharing of resources in computational materials science","prefix":"mcloud","homepage":"https://materialscloud.org"},"data_available":1620,"implementation":{"name":"aiida-optimade","version":"0.17.0+mcloud","source_url":"https://github.com/aiidateam/aiida-optimade","maintainer":{"email":"casper.andersen@epfl.ch"}}},"links":{"next":null}} \ No newline at end of file +{"data":{"id":"1","type":"structures","links":null,"attributes":{"immutable_id":"509fff89-329d-494c-a73f-69e153c9dfdc","last_modified":"2021-02-22T22:02:01Z","elements":["Cl","Na"],"nelements":2,"elements_ratios":[0.5,0.5],"chemical_formula_descriptive":"Cl4Na4","chemical_formula_reduced":"ClNa","chemical_formula_hill":"Cl4Na4","chemical_formula_anonymous":"AB","dimension_types":[1,1,1],"nperiodic_dimensions":3,"lattice_vectors":[[5.7632,0,0],[0,5.7632,0],[0,0,5.7632]],"cartesian_site_positions":[[0,0,0],[0,2.8816,2.8816],[2.8816,0,2.8816],[2.8816,2.8816,0],[2.8816,2.8816,2.8816],[2.8816,0,0],[0,2.8816,0],[0,0,2.8816]],"nsites":8,"species":[{"name":"Na","chemical_symbols":["Na"],"concentration":[1],"mass":[22.98977],"original_name":"Na"},{"name":"Cl","chemical_symbols":["Cl"],"concentration":[1],"mass":[35.453],"original_name":"Cl"}],"species_at_sites":["Na","Na","Na","Na","Cl","Cl","Cl","Cl"],"assemblies":null,"structure_features":[],"_mcloud_ctime":"2019-11-19T17:52:30Z"},"relationships":null},"meta":{"query":{"representation":"/structures/1?"},"api_version":"1.0.1","more_data_available":false,"time_stamp":"2021-04-29T13:41:20Z","data_returned":1,"provider":{"name":"Materials Cloud","description":"A platform for Open Science built for seamless sharing of resources in computational materials science","prefix":"mcloud","homepage":"https://materialscloud.org"},"data_available":1620,"implementation":{"name":"aiida-optimade","version":"0.17.0+mcloud","source_url":"https://github.com/aiidateam/aiida-optimade","maintainer":{"email":"casper.andersen@epfl.ch"}}},"links":{"next":null}} \ No newline at end of file From 2cd426689d07b6fbd44d6c7be9d81200cfd45c77 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 6 Sep 2021 18:30:18 +0200 Subject: [PATCH 11/13] Remove `/gateways//*` completely Remove the routers and tests. This also simplifies the query processing as a `QueryResource` will *always* be used, and hence all special cases where it is not used can be removed. --- optimade_gateway/main.py | 15 +- optimade_gateway/queries/perform.py | 119 +------ optimade_gateway/queries/process.py | 79 ++--- optimade_gateway/routers/gateway/__init__.py | 0 optimade_gateway/routers/gateway/info.py | 192 ----------- optimade_gateway/routers/gateway/links.py | 67 ---- optimade_gateway/routers/gateway/queries.py | 113 ------- .../routers/gateway/structures.py | 319 ------------------ optimade_gateway/routers/gateway/utils.py | 22 -- optimade_gateway/routers/gateway/versions.py | 32 -- optimade_gateway/routers/queries.py | 4 +- optimade_gateway/routers/search.py | 4 +- tests/routers/gateway/test_gateway_info.py | 291 ---------------- tests/routers/gateway/test_gateway_links.py | 124 ------- .../gateway/test_gateway_structures.py | 177 ---------- .../routers/gateway/test_gateway_versions.py | 33 -- tests/routers/test_versions.py | 39 --- 17 files changed, 48 insertions(+), 1582 deletions(-) delete mode 100644 optimade_gateway/routers/gateway/__init__.py delete mode 100644 optimade_gateway/routers/gateway/info.py delete mode 100644 optimade_gateway/routers/gateway/links.py delete mode 100644 optimade_gateway/routers/gateway/queries.py delete mode 100644 optimade_gateway/routers/gateway/structures.py delete mode 100644 optimade_gateway/routers/gateway/utils.py delete mode 100644 optimade_gateway/routers/gateway/versions.py delete mode 100644 tests/routers/gateway/test_gateway_info.py delete mode 100644 tests/routers/gateway/test_gateway_links.py delete mode 100644 tests/routers/gateway/test_gateway_structures.py delete mode 100644 tests/routers/gateway/test_gateway_versions.py delete mode 100644 tests/routers/test_versions.py diff --git a/optimade_gateway/main.py b/optimade_gateway/main.py index 2f850e15..08ae97e4 100644 --- a/optimade_gateway/main.py +++ b/optimade_gateway/main.py @@ -19,13 +19,6 @@ queries, search, ) -from optimade_gateway.routers.gateway import ( - info as gateway_info, - links as gateway_links, - queries as gateway_queries, - structures as gateway_structures, - versions as gateway_versions, -) APP = FastAPI( title="OPTIMADE Gateway", @@ -62,16 +55,10 @@ async def get_root(request: Request) -> RedirectResponse: # Add the special /versions endpoint(s) APP.include_router(versions_router) -APP.include_router(gateway_versions.ROUTER) # Add endpoints to / and /vMAJOR for prefix in list(BASE_URL_PREFIXES.values()) + [""]: - for router in (databases, gateways, info, links, queries, search) + ( - gateway_info, - gateway_links, - gateway_queries, - gateway_structures, - ): + for router in (databases, gateways, info, links, queries, search): APP.include_router(router.ROUTER, prefix=prefix, include_in_schema=prefix == "") for event, func in EVENTS: diff --git a/optimade_gateway/queries/perform.py b/optimade_gateway/queries/perform.py index e4ff7c42..6489e0b1 100644 --- a/optimade_gateway/queries/perform.py +++ b/optimade_gateway/queries/perform.py @@ -14,14 +14,12 @@ EntryResponseOne, ErrorResponse, LinksResource, - Meta, ToplevelLinks, ) from optimade.server.routers.utils import BASE_URL_PREFIXES, get_base_url, meta_values from pydantic import ValidationError from starlette.datastructures import URL -from optimade_gateway.common.config import CONFIG from optimade_gateway.common.logger import LOGGER from optimade_gateway.common.utils import get_resource_attribute from optimade_gateway.models import ( @@ -38,31 +36,22 @@ async def perform_query( url: URL, query: QueryResource, - use_query_resource: bool = True, ) -> Union[EntryResponseMany, ErrorResponse, GatewayQueryResponse]: """Perform OPTIMADE query with gateway. Parameters: url: Original request URL. query: The query to be performed. - use_query_resource: Whether or not to update the passed - [`QueryResource`][optimade_gateway.models.queries.QueryResource] or not. - The URL will be changed if this is `True`, to allow for requesting the query back - through the `/queries` endpoint. Returns: - This function returns the final response; either an `ErrorResponse`, a subclass of - `EntryResponseMany` or if `use_query_resource` is true, then a + This function returns the final response; a [`GatewayQueryResponse`][optimade_gateway.models.queries.GatewayQueryResponse]. """ from optimade_gateway.routers.gateways import GATEWAYS_COLLECTION from optimade_gateway.routers.utils import get_valid_resource - response = None - - if use_query_resource: - await update_query(query, "state", QueryState.STARTED) + await update_query(query, "state", QueryState.STARTED) gateway: GatewayResource = await get_valid_resource( GATEWAYS_COLLECTION, query.attributes.gateway_id @@ -73,26 +62,12 @@ async def perform_query( filter_query=query.attributes.query_parameters.filter, ) - if use_query_resource: - url = url.replace(path=f"{url.path.rstrip('/')}/{query.id}") - await update_query( - query, - "response", - GatewayQueryResponse( - data={}, - links=ToplevelLinks(next=None), - meta=meta_values( - url=url, - data_available=0, - data_returned=0, - more_data_available=False, - ), - ), - **{"$set": {"state": QueryState.IN_PROGRESS}}, - ) - else: - response = query.attributes.endpoint.get_response_model()( - data=[], + url = url.replace(path=f"{url.path.rstrip('/')}/{query.id}") + await update_query( + query, + "response", + GatewayQueryResponse( + data={}, links=ToplevelLinks(next=None), meta=meta_values( url=url, @@ -100,7 +75,9 @@ async def perform_query( data_returned=0, more_data_available=False, ), - ) + ), + **{"$set": {"state": QueryState.IN_PROGRESS}}, + ) loop = asyncio.get_running_loop() with ThreadPoolExecutor( @@ -131,79 +108,18 @@ async def perform_query( for query_task in query_tasks: (db_response, db_id) = await query_task - results, errors, response_meta = await process_db_response( + results = await process_db_response( response=db_response, database_id=db_id, query=query, gateway=gateway, - use_query_resource=use_query_resource, ) - if not use_query_resource: - # Create a standard OPTIMADE response, adding the database ID to each returned - # resource's meta field. - database_id_meta = { - f"_{CONFIG.provider.prefix}_source_database_id": db_id - } - if errors or isinstance(response, ErrorResponse): - # Error response - if isinstance(response, ErrorResponse): - response.errors.append(errors) - else: - response = ErrorResponse(errors=errors, meta=response.meta) - response.meta.data_returned = 0 - response.meta.more_data_available = False - elif isinstance(results, list): - # "Many" response - for resource in results: - if hasattr(resource, "meta") and resource.meta: - meta = resource.meta.dict() - meta.update(database_id_meta) - resource.meta = Meta(**meta) - elif isinstance(resource, dict) and resource.get("meta", {}): - resource["meta"] = resource["meta"].update(database_id_meta) - elif isinstance(resource, dict): - resource["meta"] = database_id_meta - else: - resource.meta = Meta(**database_id_meta) - response.data.extend(results) - response.meta.data_returned += response_meta["data_returned"] - if not response.meta.more_data_available: - # Keep it True, if set to True once. - response.meta.more_data_available = response_meta[ - "more_data_available" - ] - else: - # "One" response - if hasattr(results, "meta") and results.meta: - meta = results.meta.dict() - meta.update(database_id_meta) - results.meta = Meta(**meta) - elif isinstance(results, dict) and results.get("meta", {}): - results["meta"] = results["meta"].update(database_id_meta) - elif isinstance(results, dict): - results["meta"] = database_id_meta - else: - results.meta = Meta(**database_id_meta) - response.data.append(results) - response.meta.data_returned += response_meta["data_returned"] - if not response.meta.more_data_available: - # Keep it True, if set to True once. - response.meta.more_data_available = response_meta[ - "more_data_available" - ] - response.meta.data_available += response_meta["data_available"] - if get_resource_attribute( query, "attributes.response.meta.more_data_available", False, disambiguate=False, # Extremely minor speed-up - ) or get_resource_attribute( - response, - "meta.more_data_available", - False, - disambiguate=False, ): # Deduce the `next` link from the current request query_string = urllib.parse.parse_qs(url.query) @@ -215,15 +131,10 @@ async def perform_query( links = ToplevelLinks(next=f"{base_url}{url.path}?{urlencoded}") - if use_query_resource: - await update_query(query, "response.links", links) - else: - response.links = links + await update_query(query, "response.links", links) - if use_query_resource: - await update_query(query, "state", QueryState.FINISHED) - return query.attributes.response - return response + await update_query(query, "state", QueryState.FINISHED) + return query.attributes.response def db_find( diff --git a/optimade_gateway/queries/process.py b/optimade_gateway/queries/process.py index 6691afea..6c02db0f 100644 --- a/optimade_gateway/queries/process.py +++ b/optimade_gateway/queries/process.py @@ -1,5 +1,5 @@ """Process performed OPTIMADE queries""" -from typing import Any, Dict, List, Tuple, Union +from typing import Any, Dict, List, Union from optimade.models import ( EntryResource, @@ -7,7 +7,6 @@ EntryResponseOne, ErrorResponse, Meta, - OptimadeError, ) from optimade_gateway.common.config import CONFIG @@ -22,13 +21,8 @@ async def process_db_response( database_id: str, query: QueryResource, gateway: GatewayResource, - use_query_resource: bool = True, -) -> Tuple[ - Union[ - List[EntryResource], List[Dict[str, Any]], EntryResource, Dict[str, Any], None - ], - List[OptimadeError], - Dict[str, Union[bool, int]], +) -> Union[ + List[EntryResource], List[Dict[str, Any]], EntryResource, Dict[str, Any], None ]: """Process an OPTIMADE database response. @@ -37,9 +31,6 @@ async def process_db_response( Since, only either `data` or `errors` should ever be present, one or the other will be either an empty list or `None`. - `meta` will only be a non-empty dictionary when not using a - [`QueryResource`][optimade_gateway.models.queries.QueryResource], i.e., if `use_query_resource` - is `False`. Parameters: response: The OPTIMADE database response to be processed. @@ -47,16 +38,13 @@ async def process_db_response( delivered. query: A resource representing the performed query. gateway: A resource representing the gateway that was queried. - use_query_resource: Whether or not to update the passed - [`QueryResource`][optimade_gateway.models.queries.QueryResource]. Returns: - A tuple of the response's `data`, `errors`, and `meta`. + The response's `data`. """ results = [] errors = [] - meta = {} from optimade_gateway.common.logger import LOGGER @@ -109,39 +97,32 @@ async def process_db_response( data_available = response.meta.data_available or 0 - if use_query_resource: - extra_updates = { - "$inc": { - "response.meta.data_available": data_available, - "response.meta.data_returned": data_returned, - } + extra_updates = { + "$inc": { + "response.meta.data_available": data_available, + "response.meta.data_returned": data_returned, } - if not get_resource_attribute( - query, - "attributes.response.meta.more_data_available", - False, - disambiguate=False, # Extremely minor speed-up - ): - # Keep it True, if set to True once. - extra_updates.update( - {"$set": {"response.meta.more_data_available": more_data_available}} - ) - - # This ensures an empty list under `response.data.{database_id}` is returned if the case is - # simply that there are no results to return. - if errors: - extra_updates.update({"$addToSet": {"response.errors": {"$each": errors}}}) - await update_query( - query, - f"response.data.{database_id}", - results, - **extra_updates, + } + if not get_resource_attribute( + query, + "attributes.response.meta.more_data_available", + False, + disambiguate=False, # Extremely minor speed-up + ): + # Keep it True, if set to True once. + extra_updates.update( + {"$set": {"response.meta.more_data_available": more_data_available}} ) - else: - meta = { - "data_returned": data_returned, - "data_available": data_available, - "more_data_available": more_data_available, - } - return results, errors, meta + # This ensures an empty list under `response.data.{database_id}` is returned if the case is + # simply that there are no results to return. + if errors: + extra_updates.update({"$addToSet": {"response.errors": {"$each": errors}}}) + await update_query( + query, + f"response.data.{database_id}", + results, + **extra_updates, + ) + + return results diff --git a/optimade_gateway/routers/gateway/__init__.py b/optimade_gateway/routers/gateway/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/optimade_gateway/routers/gateway/info.py b/optimade_gateway/routers/gateway/info.py deleted file mode 100644 index 809d2ba4..00000000 --- a/optimade_gateway/routers/gateway/info.py +++ /dev/null @@ -1,192 +0,0 @@ -"""/gateways/{id}/info/* - -This file describes the router for: - - /gateways/{id}/{version}/info/{entry} - -where `version` and `entry` may be left out. -""" -from fastapi import APIRouter, Request -from optimade import __api_version__ -from optimade.models import ( - BaseInfoAttributes, - BaseInfoResource, - EntryInfoResponse, - InfoResponse, - StructureResource, -) -from optimade.server.routers.utils import get_base_url, meta_values -from optimade.server.schemas import ERROR_RESPONSES - -ROUTER = APIRouter(redirect_slashes=True) - -ENTRY_INFO_SCHEMAS = {"structures": StructureResource.schema} - - -@ROUTER.get( - "/gateways/{gateway_id}/info", - response_model=InfoResponse, - response_model_exclude_defaults=False, - response_model_exclude_none=False, - response_model_exclude_unset=True, - tags=["Info"], - responses=ERROR_RESPONSES, -) -async def get_gateways_info( - request: Request, - gateway_id: str, -) -> InfoResponse: - """`GET /gateways/{gateway_id}/info` - - Return a regular `/info` response for an OPTIMADE implementation, - including extra information from all the gateway's databases. - The general information will be a minimum set from the gateway's databases. - """ - from optimade_gateway.common.config import CONFIG - from optimade_gateway.routers.gateways import GATEWAYS_COLLECTION - from optimade_gateway.routers.utils import get_valid_resource - - gateway = await get_valid_resource(GATEWAYS_COLLECTION, gateway_id) - - return InfoResponse( - data=BaseInfoResource( - id=BaseInfoResource.schema()["properties"]["id"]["const"], - type=BaseInfoResource.schema()["properties"]["type"]["const"], - attributes=BaseInfoAttributes( - api_version=__api_version__, - available_api_versions=[ - { - "url": f"{get_base_url(request.url)}/v{__api_version__.split('.')[0]}/gateways/{gateway_id}/v{__api_version__.split('.')[0]}", - "version": __api_version__, - } - ], - formats=["json"], - entry_types_by_format={"json": list(ENTRY_INFO_SCHEMAS.keys())}, - available_endpoints=sorted( - ["info", "links"] + list(ENTRY_INFO_SCHEMAS.keys()) - ), - is_index=False, - ), - ), - meta=meta_values( - url=request.url, - data_returned=1, - data_available=1, - more_data_available=False, - **{ - f"_{CONFIG.provider.prefix}_gateway": { - "databases": [ - {"id": _.id, "type": _.type} - for _ in gateway.attributes.databases - ], - } - }, - ), - ) - - -@ROUTER.get( - "/gateways/{gateway_id}/info/{entry}", - response_model=EntryInfoResponse, - response_model_exclude_defaults=False, - response_model_exclude_none=False, - response_model_exclude_unset=True, - tags=["Info"], - responses=ERROR_RESPONSES, -) -async def get_gateways_entry_info( - request: Request, gateway_id: str, entry: str -) -> EntryInfoResponse: - """`GET /gateways/{gateway_id}/info/{entry}` - - Get information about the gateway `{gateway_id}`'s entry-listing endpoints. - """ - from optimade.models import EntryInfoResource - from optimade.server.exceptions import NotFound - - from optimade_gateway.routers.gateways import GATEWAYS_COLLECTION - from optimade_gateway.routers.utils import ( - aretrieve_queryable_properties, - validate_resource, - ) - - await validate_resource(GATEWAYS_COLLECTION, gateway_id) - - valid_entry_info_endpoints = ENTRY_INFO_SCHEMAS.keys() - if entry not in valid_entry_info_endpoints: - raise NotFound( - detail=( - f"Entry info not found for {entry}, valid entry info endpoints are: " - f"{', '.join(valid_entry_info_endpoints)}" - ), - ) - - schema = ENTRY_INFO_SCHEMAS[entry]() - queryable_properties = {"id", "type", "attributes"} - properties = await aretrieve_queryable_properties(schema, queryable_properties) - - output_fields_by_format = {"json": list(properties.keys())} - - return EntryInfoResponse( - data=EntryInfoResource( - formats=list(output_fields_by_format.keys()), - description=schema.get("description", "Entry Resources"), - properties=properties, - output_fields_by_format=output_fields_by_format, - ), - meta=meta_values( - url=request.url, - data_returned=1, - data_available=1, - more_data_available=False, - ), - ) - - -@ROUTER.get( - "/gateways/{gateway_id}/{version}/info", - response_model=InfoResponse, - response_model_exclude_defaults=False, - response_model_exclude_none=False, - response_model_exclude_unset=True, - tags=["Info"], - responses=ERROR_RESPONSES, -) -async def get_versioned_gateways_info( - request: Request, - gateway_id: str, - version: str, -) -> InfoResponse: - """`GET /gateways/{gateway_id}/{version}/info` - - Same as `GET /gateways/{gateway_id}/info`. - """ - from optimade_gateway.routers.gateway.utils import validate_version - - await validate_version(version) - return await get_gateways_info(request, gateway_id) - - -@ROUTER.get( - "/gateways/{gateway_id}/{version}/info/{entry}", - response_model=EntryInfoResponse, - response_model_exclude_defaults=False, - response_model_exclude_none=False, - response_model_exclude_unset=True, - tags=["Info"], - responses=ERROR_RESPONSES, -) -async def get_versioned_gateways_entry_info( - request: Request, - gateway_id: str, - version: str, - entry: str, -) -> EntryInfoResponse: - """`GET /gateways/{gateway_id}/{version}/info/{entry}` - - Same as `GET /gateways/{gateway_id}/info/{entry}`. - """ - from optimade_gateway.routers.gateway.utils import validate_version - - await validate_version(version) - return await get_gateways_entry_info(request, gateway_id, entry) diff --git a/optimade_gateway/routers/gateway/links.py b/optimade_gateway/routers/gateway/links.py deleted file mode 100644 index 1abd15d6..00000000 --- a/optimade_gateway/routers/gateway/links.py +++ /dev/null @@ -1,67 +0,0 @@ -"""/gateways/{id}/links - -This file describes the router for: - - /gateways/{id}/{version}/links - -where `version` may be left out. -""" -from fastapi import APIRouter, Depends, Request -from optimade.models import ( - LinksResponse, -) -from optimade.server.query_params import EntryListingQueryParams -from optimade.server.schemas import ERROR_RESPONSES - -ROUTER = APIRouter(redirect_slashes=True) - - -@ROUTER.get( - "/gateways/{gateway_id}/links", - response_model=LinksResponse, - response_model_exclude_defaults=False, - response_model_exclude_none=False, - response_model_exclude_unset=True, - tags=["Links"], - responses=ERROR_RESPONSES, -) -async def get_gateways_links( - request: Request, - gateway_id: str, - params: EntryListingQueryParams = Depends(), -) -> LinksResponse: - """`GET /gateways/{gateway_id}/links` - - Return a regular `/links` response for an OPTIMADE implementation. - """ - from optimade_gateway.routers.gateways import GATEWAYS_COLLECTION - from optimade_gateway.routers.links import get_links - from optimade_gateway.routers.utils import validate_resource - - await validate_resource(GATEWAYS_COLLECTION, gateway_id) - return await get_links(request=request, params=params) - - -@ROUTER.get( - "/gateways/{gateway_id}/{version}/links", - response_model=LinksResponse, - response_model_exclude_defaults=False, - response_model_exclude_none=False, - response_model_exclude_unset=True, - tags=["Links"], - responses=ERROR_RESPONSES, -) -async def get_versioned_gateways_links( - request: Request, - gateway_id: str, - version: str, - params: EntryListingQueryParams = Depends(), -) -> LinksResponse: - """`GET /gateways/{gateway_id}/{version}/links` - - Same as `GET /gateways/{gateway_id}/links`. - """ - from optimade_gateway.routers.gateway.utils import validate_version - - await validate_version(version) - return await get_gateways_links(request, gateway_id, params) diff --git a/optimade_gateway/routers/gateway/queries.py b/optimade_gateway/routers/gateway/queries.py deleted file mode 100644 index 69e90159..00000000 --- a/optimade_gateway/routers/gateway/queries.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Special /queries endpoint for a gateway to perform and list queries - -This file describes the router for: - - /gateways/{id}/{version}/queries/{id} - -where `version` and the last `id` may be left out. -""" -from fastapi import APIRouter, Depends, Request, status -from optimade.server.exceptions import Forbidden -from optimade.server.query_params import EntryListingQueryParams -from optimade.server.schemas import ERROR_RESPONSES - -from optimade_gateway.models import ( - QueryCreate, - QueriesResponse, - QueriesResponseSingle, -) -from optimade_gateway.routers.utils import validate_resource - - -ROUTER = APIRouter(redirect_slashes=True) - - -@ROUTER.get( - "/gateways/{gateway_id}/queries", - response_model=QueriesResponse, - response_model_exclude_defaults=False, - response_model_exclude_none=False, - response_model_exclude_unset=True, - tags=["Gateways", "Queries"], - responses=ERROR_RESPONSES, -) -async def get_gateway_queries( - request: Request, - gateway_id: str, - params: EntryListingQueryParams = Depends(), -) -> QueriesResponse: - """`GET /gateways/{gateway_id}/queries` - - Return overview of all (active) queries for specific gateway. - """ - from optimade_gateway.routers.gateways import GATEWAYS_COLLECTION - from optimade_gateway.routers.queries import get_queries - - await validate_resource(GATEWAYS_COLLECTION, gateway_id) - - params.filter = ( - f'( {params.filter} ) AND ( gateway_id="{gateway_id}" )' - if params.filter - else f'gateway_id="{gateway_id}"' - ) - return await get_queries(request, params) - - -@ROUTER.post( - "/gateways/{gateway_id}/queries", - response_model=QueriesResponseSingle, - response_model_exclude_defaults=False, - response_model_exclude_none=False, - response_model_exclude_unset=True, - tags=["Gateways", "Queries"], - status_code=status.HTTP_202_ACCEPTED, - responses=ERROR_RESPONSES, -) -async def post_gateway_queries( - request: Request, - gateway_id: str, - query: QueryCreate, -) -> QueriesResponseSingle: - """`POST /gateways/{gateway_id}/queries` - - Create or return existing gateway query according to `query`. - """ - from optimade_gateway.routers.gateways import GATEWAYS_COLLECTION - from optimade_gateway.routers.queries import post_queries - - await validate_resource(GATEWAYS_COLLECTION, gateway_id) - - if query.gateway_id and query.gateway_id != gateway_id: - raise Forbidden( - detail=( - f"The gateway ID in the posted data () does not align " - f"with the gateway ID specified in the URL (/{gateway_id}/)." - ), - ) - - return await post_queries(request, query) - - -@ROUTER.get( - "/gateways/{gateway_id}/queries/{query_id}", - response_model=QueriesResponseSingle, - response_model_exclude_defaults=False, - response_model_exclude_none=False, - response_model_exclude_unset=True, - tags=["Gateways", "Queries"], - responses=ERROR_RESPONSES, -) -async def get_gateway_query( - request: Request, gateway_id: str, query_id: str -) -> QueriesResponseSingle: - """`GET /gateways/{gateway_id}/queries/{query_id}` - - Return the response from a gateway query - ([`QueryResource.attributes.response`][optimade_gateway.models.queries.QueryResourceAttributes.response]). - """ - from optimade_gateway.routers.gateways import GATEWAYS_COLLECTION - from optimade_gateway.routers.queries import get_query - - await validate_resource(GATEWAYS_COLLECTION, gateway_id) - - return await get_query(request, query_id) diff --git a/optimade_gateway/routers/gateway/structures.py b/optimade_gateway/routers/gateway/structures.py deleted file mode 100644 index 1e802184..00000000 --- a/optimade_gateway/routers/gateway/structures.py +++ /dev/null @@ -1,319 +0,0 @@ -"""/gateways/{id}/structures/* - -This file describes the router for: - - /gateways/{id}/{version}/structures/{id} - -where `version` and the last `id` may be left out. -""" -import asyncio -from datetime import datetime -import functools -from typing import Union -import urllib.parse -import warnings - -from fastapi import APIRouter, Depends, Request, Response -from optimade.models import ( - ErrorResponse, - Meta, - StructureResponseMany, - StructureResponseOne, - ToplevelLinks, -) -from optimade.server.query_params import EntryListingQueryParams, SingleEntryQueryParams -from optimade.server.routers.utils import meta_values -from optimade.server.schemas import ERROR_RESPONSES - -from optimade_gateway.models import QueryResource -from optimade_gateway.queries import perform_query -from optimade_gateway.routers.gateway.utils import validate_version -from optimade_gateway.routers.gateways import GATEWAYS_COLLECTION -from optimade_gateway.warnings import OptimadeGatewayWarning, SortNotSupported - - -ROUTER = APIRouter(redirect_slashes=True) - - -@ROUTER.get( - "/gateways/{gateway_id}/structures", - response_model=Union[StructureResponseMany, ErrorResponse], - response_model_exclude_defaults=False, - response_model_exclude_none=False, - response_model_exclude_unset=True, - tags=["Structures"], - responses=ERROR_RESPONSES, -) -async def get_structures( - request: Request, - gateway_id: str, - response: Response, - params: EntryListingQueryParams = Depends(), -) -> Union[StructureResponseMany, ErrorResponse]: - """`GET /gateways/{gateway_id}/structures` - - Return a regular `/structures` response for an OPTIMADE implementation, - including responses from all the gateway's databases. - """ - from optimade_gateway.routers.utils import validate_resource - - await validate_resource(GATEWAYS_COLLECTION, gateway_id) - - if getattr(params, "sort", False): - warnings.warn(SortNotSupported()) - params.sort = "" - - gateway_response = await perform_query( - url=request.url, - query=QueryResource( - **{ - "id": "temp", - "type": "queries", - "attributes": { - "last_modified": datetime.utcnow(), - "gateway_id": gateway_id, - "state": "created", - "query_parameters": { - key: value for key, value in params.__dict__.items() if value - }, - "endpoint": "structures", - }, - } - ), - use_query_resource=False, - ) - - if isinstance(gateway_response, ErrorResponse): - for error in gateway_response.errors: - if error.status: - for part in error.status.split(" "): - try: - response.status_code = int(part) - break - except ValueError: - pass - if response.status_code and response.status_code >= 300: - break - else: - response.status_code = 500 - return gateway_response - elif isinstance(gateway_response, StructureResponseMany): - return gateway_response - else: - raise TypeError( - "The response should be either StructureResponseMany or ErrorResponse." - ) - - -@ROUTER.get( - "/gateways/{gateway_id}/structures/{structure_id:path}", - response_model=Union[StructureResponseOne, ErrorResponse], - response_model_exclude_defaults=False, - response_model_exclude_none=False, - response_model_exclude_unset=True, - tags=["Structures"], - responses=ERROR_RESPONSES, -) -async def get_single_structure( - request: Request, - gateway_id: str, - structure_id: str, - response: Response, - params: SingleEntryQueryParams = Depends(), -) -> Union[StructureResponseOne, ErrorResponse]: - """`GET /gateways/{gateway_id}/structures/{structure_id}` - - Return a regular `/structures/{id}` response for an OPTIMADE implementation. - - !!! important - There are two options for the `structure_id`. - - Either one supplies an entry ID similar to what is given in the local databases. - This will result in this endpoint returning the first entry it finds from any database that - matches the given ID. - - Example: `GET /gateway/some_gateway/structures/some_structure`. - - Otherwise, one can supply the database ID (call `GET /databases` to see all available - databases and their IDs), and then the local entry ID, separated by a forward slash. - - Example: `GET /gateways/some_gateway/structures/some_database/some_structure`. - - """ - from optimade_gateway.common.config import CONFIG - from optimade_gateway.models import GatewayResource - from optimade_gateway.queries import db_find - from optimade_gateway.routers.utils import get_valid_resource - - gateway: GatewayResource = await get_valid_resource(GATEWAYS_COLLECTION, gateway_id) - - local_structure_id = None - database_id = None - - for database in gateway.attributes.databases: - if structure_id.startswith(f"{database.id}/"): - # Database found - database_id = database.id - local_structure_id = structure_id[len(f"{database_id}/") :] - break - else: - # Assume the given ID is already a local database ID - find and return the first one - # available. - local_structure_id = structure_id - - errors = [] - parsed_params = urllib.parse.urlencode( - {param: value for param, value in params.__dict__.items() if value} - ) - - if database_id: - (gateway_response, _) = await asyncio.get_running_loop().run_in_executor( - executor=None, # Run in thread with the event loop - func=functools.partial( - db_find, - database=database, - endpoint=f"structures/{local_structure_id}", - response_model=StructureResponseOne, - query_params=parsed_params, - ), - ) - if isinstance(gateway_response, ErrorResponse): - for error in gateway_response.errors: - if isinstance(error.id, str) and error.id.startswith( - "OPTIMADE_GATEWAY" - ): - warnings.warn(error.detail, OptimadeGatewayWarning) - else: - meta = {} - if error.meta: - meta = error.meta.dict() - meta.update( - { - f"_{CONFIG.provider.prefix}_source_gateway": { - "id": gateway.id, - "type": gateway.type, - "links": {"self": gateway.links.self}, - }, - f"_{CONFIG.provider.prefix}_source_database": { - "id": database.id, - "type": database.type, - "links": {"self": database.links.self}, - }, - } - ) - error.meta = Meta(**meta) - errors.append(error) - - meta = meta_values( - url=request.url, - data_returned=gateway_response.meta.data_returned, - data_available=None, # Don't set this, as we'd have to request ALL gateway databases - more_data_available=gateway_response.meta.more_data_available, - ) - del meta.data_available - - if errors: - gateway_response = ErrorResponse(errors=errors, meta=meta) - else: - gateway_response = StructureResponseOne( - links=ToplevelLinks(next=None), data=gateway_response.data, meta=meta - ) - - else: - params.filter = f'id="{local_structure_id}"' - gateway_response = await perform_query( - url=request.url, - query=QueryResource( - **{ - "id": "temp", - "type": "queries", - "attributes": { - "last_modified": datetime.utcnow(), - "gateway_id": gateway_id, - "state": "created", - "query_parameters": { - key: value - for key, value in params.__dict__.items() - if value - }, - "endpoint": "structures", - }, - } - ), - use_query_resource=False, - ) - if isinstance(gateway_response, StructureResponseMany): - gateway_response = gateway_response.dict(exclude_unset=True) - gateway_response["data"] = ( - gateway_response["data"][0] if gateway_response["data"] else None - ) - gateway_response = StructureResponseOne(**gateway_response) - - if isinstance(gateway_response, ErrorResponse): - for error in errors or gateway_response.errors: - if error.status: - for part in error.status.split(" "): - try: - response.status_code = int(part) - break - except ValueError: - pass - if response.status_code and response.status_code >= 300: - break - else: - response.status_code = 500 - - return gateway_response - - -@ROUTER.get( - "/gateways/{gateway_id}/{version}/structures", - response_model=Union[StructureResponseMany, ErrorResponse], - response_model_exclude_defaults=False, - response_model_exclude_none=False, - response_model_exclude_unset=True, - tags=["Structures"], - include_in_schema=False, - responses=ERROR_RESPONSES, -) -async def get_versioned_structures( - request: Request, - gateway_id: str, - version: str, - response: Response, - params: EntryListingQueryParams = Depends(), -) -> Union[StructureResponseMany, ErrorResponse]: - """`GET /gateways/{gateway_id}/{version}/structures` - - Same as `GET /gateways/{gateway_id}/structures`. - """ - await validate_version(version) - return await get_structures(request, gateway_id, response, params) - - -@ROUTER.get( - "/gateways/{gateway_id}/{version}/structures/{structure_id:path}", - response_model=Union[StructureResponseOne, ErrorResponse], - response_model_exclude_defaults=False, - response_model_exclude_none=False, - response_model_exclude_unset=True, - tags=["Structures"], - include_in_schema=False, - responses=ERROR_RESPONSES, -) -async def get_versioned_single_structure( - request: Request, - gateway_id: str, - version: str, - structure_id: str, - response: Response, - params: SingleEntryQueryParams = Depends(), -) -> Union[StructureResponseOne, ErrorResponse]: - """`GET /gateways/{gateway_id}/{version}/structures/{structure_id}` - - Same as `GET /gateways/{gateway_id}/structures/{structure_id}`. - """ - await validate_version(version) - return await get_single_structure( - request, gateway_id, structure_id, response, params - ) diff --git a/optimade_gateway/routers/gateway/utils.py b/optimade_gateway/routers/gateway/utils.py deleted file mode 100644 index 7d2062fd..00000000 --- a/optimade_gateway/routers/gateway/utils.py +++ /dev/null @@ -1,22 +0,0 @@ -from optimade.server.exceptions import NotFound, VersionNotSupported -from optimade.server.routers.utils import BASE_URL_PREFIXES - - -async def validate_version(version: str) -> None: - """Validate version according to `optimade` package. - - Parameters: - version: The OPTIMADE API version. - - """ - valid_versions = [_[1:] for _ in BASE_URL_PREFIXES.values()] - - if version not in valid_versions: - if version.startswith("v"): - raise VersionNotSupported( - detail=f"version {version} is not supported. Supported versions: {valid_versions}" - ) - else: - raise NotFound( - detail=f"version MUST be one of {valid_versions}", - ) diff --git a/optimade_gateway/routers/gateway/versions.py b/optimade_gateway/routers/gateway/versions.py deleted file mode 100644 index cbf162e4..00000000 --- a/optimade_gateway/routers/gateway/versions.py +++ /dev/null @@ -1,32 +0,0 @@ -"""/gateways/{id}/versions - -This file describes the router for: - - /gateways/{id}/versions - -""" -from fastapi import APIRouter, Request -from optimade.server.routers.versions import CsvResponse -from optimade.server.schemas import ERROR_RESPONSES - -ROUTER = APIRouter(redirect_slashes=True) - - -@ROUTER.get( - "/gateways/{gateway_id}/versions", - response_class=CsvResponse, - tags=["Versions"], - responses=ERROR_RESPONSES, -) -async def get_gateway_versions(request: Request, gateway_id: str) -> CsvResponse: - """`GET /gateways/{gateway_id}/versions` - - Return the result of the function used to `GET /versions` from the `optimade` package. - """ - from optimade.server.routers.versions import get_versions - - from optimade_gateway.routers.gateways import GATEWAYS_COLLECTION - from optimade_gateway.routers.utils import validate_resource - - await validate_resource(GATEWAYS_COLLECTION, gateway_id) - return get_versions(request) diff --git a/optimade_gateway/routers/queries.py b/optimade_gateway/routers/queries.py index 3d04e30c..e117905c 100644 --- a/optimade_gateway/routers/queries.py +++ b/optimade_gateway/routers/queries.py @@ -94,9 +94,7 @@ async def post_queries( result, created = await resource_factory(query) if created: - asyncio.create_task( - perform_query(url=request.url, query=result, use_query_resource=True) - ) + asyncio.create_task(perform_query(url=request.url, query=result)) return QueriesResponseSingle( links=ToplevelLinks(next=None), diff --git a/optimade_gateway/routers/search.py b/optimade_gateway/routers/search.py index d9888d27..e0733daf 100644 --- a/optimade_gateway/routers/search.py +++ b/optimade_gateway/routers/search.py @@ -178,9 +178,7 @@ async def post_search(request: Request, search: Search) -> QueriesResponseSingle query, created = await resource_factory(query) if created: - asyncio.create_task( - perform_query(url=request.url, query=query, use_query_resource=True) - ) + asyncio.create_task(perform_query(url=request.url, query=query)) return QueriesResponseSingle( links=ToplevelLinks(next=None), diff --git a/tests/routers/gateway/test_gateway_info.py b/tests/routers/gateway/test_gateway_info.py deleted file mode 100644 index 667e2413..00000000 --- a/tests/routers/gateway/test_gateway_info.py +++ /dev/null @@ -1,291 +0,0 @@ -"""Tests for /gateways/{gateway_id}/info endpoint""" -from typing import Awaitable, Callable - -try: - from typing import Literal -except ImportError: - from typing_extensions import Literal - -from fastapi import FastAPI -import httpx -import pytest - - -pytestmark = pytest.mark.asyncio - - -async def test_get_gateway_info( - client: Callable[ - [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], - Awaitable[httpx.Response], - ], - get_gateway: Callable[[str], Awaitable[dict]], -): - """Test GET /gateways/{gateway_id}/info""" - from optimade.models import InfoResponse - - from optimade_gateway.common.config import CONFIG - - available_endpoints = sorted(["info", "links", "structures"]) - entry_types_by_format = {"json": ["structures"]} - - gateway_id = "twodbs" - - response = await client(f"/gateways/{gateway_id}/info") - - assert response.status_code == 200, f"Request failed: {response.json()}" - response = InfoResponse(**response.json()) - assert response - - assert response.meta.data_returned == 1 - assert response.meta.data_available == 1 - assert not response.meta.more_data_available - assert response.data.attributes.available_endpoints == available_endpoints - assert response.data.attributes.entry_types_by_format == entry_types_by_format - assert ( - not response.data.attributes.is_index - or response.data.attributes.is_index is None - ) - - gateway = await get_gateway(gateway_id) - - assert getattr(response.meta, f"_{CONFIG.provider.prefix}_gateway", {}) == { - "databases": [ - {"id": _.get("id", "NOT FOUND"), "type": _.get("type", "NOT FOUND")} - for _ in gateway.get("databases", []) - ] - } - - -async def test_get_versioned_gateway_info( - client: Callable[ - [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], - Awaitable[httpx.Response], - ], - get_gateway: Callable[[str], Awaitable[dict]], -): - """Test GET /gateways/{gateway_id}/{version}/info""" - from optimade.models import InfoResponse - from optimade.server.routers.utils import BASE_URL_PREFIXES - - from optimade_gateway.common.config import CONFIG - - available_endpoints = sorted(["info", "links", "structures"]) - entry_types_by_format = {"json": ["structures"]} - - gateway_id = "twodbs" - - for version in BASE_URL_PREFIXES.values(): - response = await client(f"/gateways/{gateway_id}{version}/info") - - assert response.status_code == 200, f"Request failed: {response.json()}" - response = InfoResponse(**response.json()) - assert response - - assert response.meta.data_returned == 1 - assert response.meta.data_available == 1 - assert not response.meta.more_data_available - assert response.data.attributes.available_endpoints == available_endpoints - assert response.data.attributes.entry_types_by_format == entry_types_by_format - assert ( - not response.data.attributes.is_index - or response.data.attributes.is_index is None - ) - - gateway = await get_gateway(gateway_id) - - assert getattr(response.meta, f"_{CONFIG.provider.prefix}_gateway", {}) == { - "databases": [ - {"id": _.get("id", "NOT FOUND"), "type": _.get("type", "NOT FOUND")} - for _ in gateway.get("databases", []) - ] - } - - -async def test_bad_versioned_gateway_info( - client: Callable[ - [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], - Awaitable[httpx.Response], - ] -): - """Test GET /gateways/{gateway_id}/{version}/info with wrong version""" - from optimade.models import ErrorResponse, OptimadeError - from optimade.server.exceptions import VersionNotSupported - from optimade.server.routers.utils import BASE_URL_PREFIXES - - gateway_id = "twodbs" - - wrong_versions = [ - ( - # Correct syntax - unsupport version - "v0.1.0", - { - "detail": ( - "version v0.1.0 is not supported. Supported versions: " - f"{[_[1:] for _ in BASE_URL_PREFIXES.values()]}" - ), - "title": "Version Not Supported", - "status": "553", - }, - ), - ( - # Incorrect syntax - supported version - BASE_URL_PREFIXES["patch"][len("/v") :], - { - "detail": f"version MUST be one of {[_[1:] for _ in BASE_URL_PREFIXES.values()]}", - "title": "Not Found", - "status": "404", - }, - ), - ] - - for version, error_data in wrong_versions: - error_resource = OptimadeError(**error_data) - - if version == "v0.1.0": - with pytest.raises( - VersionNotSupported, - match=fr"The parsed versioned base URL '/{version}.*", - ): - response = await client(f"/gateways/{gateway_id}/{version}/info") - return - else: - response = await client(f"/gateways/{gateway_id}/{version}/info") - - assert response.status_code == int( - error_resource.status - ), f"Request succeeded, where it should've failed: {response.json()}" - response = ErrorResponse(**response.json()) - assert response - - assert response.meta.data_returned == 0 - assert response.meta.data_available == 0 - assert not response.meta.more_data_available - - assert len(response.errors) == 1 - assert response.errors == [error_resource] - - -async def test_get_gateway_info_entry( - client: Callable[ - [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], - Awaitable[httpx.Response], - ] -): - """Test GET /gateways/{gateway_id}/info/{entry}""" - from optimade.models import EntryInfoResponse - - from optimade_gateway.routers.gateway.info import ENTRY_INFO_SCHEMAS - - gateway_id = "singledb" - - for entry_endpoint in ENTRY_INFO_SCHEMAS: - response = await client(f"/gateways/{gateway_id}/info/{entry_endpoint}") - - assert response.status_code == 200, f"Request failed: {response.json()}" - response = EntryInfoResponse(**response.json()) - assert response - - assert response.meta.data_returned == 1 - assert response.meta.data_available == 1 - assert not response.meta.more_data_available - - -async def test_get_versioned_gateway_info_entry( - client: Callable[ - [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], - Awaitable[httpx.Response], - ] -): - """Test GET /gateways/{gateway_id}/{version}/info/{entry}""" - from optimade.models import EntryInfoResponse - from optimade.server.routers.utils import BASE_URL_PREFIXES - - from optimade_gateway.routers.gateway.info import ENTRY_INFO_SCHEMAS - - gateway_id = "singledb" - - for version in BASE_URL_PREFIXES.values(): - for entry_endpoint in ENTRY_INFO_SCHEMAS: - response = await client( - f"/gateways/{gateway_id}{version}/info/{entry_endpoint}" - ) - - assert response.status_code == 200, f"Request failed: {response.json()}" - response = EntryInfoResponse(**response.json()) - assert response - - assert response.meta.data_returned == 1 - assert response.meta.data_available == 1 - assert not response.meta.more_data_available - - -async def test_bad_versioned_gateway_info_entry( - client: Callable[ - [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], - Awaitable[httpx.Response], - ] -): - """Test GET /gateways/{gateway_id}/{version}/info/{entry} with wrong version""" - from optimade.models import ErrorResponse, OptimadeError - from optimade.server.exceptions import VersionNotSupported - from optimade.server.routers.utils import BASE_URL_PREFIXES - - from optimade_gateway.routers.gateway.info import ENTRY_INFO_SCHEMAS - - gateway_id = "singledb" - - wrong_versions = [ - ( - # Correct syntax - unsupport version - "v0.1.0", - { - "detail": ( - "version v0.1.0 is not supported. Supported versions: " - f"{[_[1:] for _ in BASE_URL_PREFIXES.values()]}" - ), - "title": "Version Not Supported", - "status": "553", - }, - ), - ( - # Incorrect syntax - supported version - BASE_URL_PREFIXES["patch"][len("/v") :], - { - "detail": f"version MUST be one of {[_[1:] for _ in BASE_URL_PREFIXES.values()]}", - "title": "Not Found", - "status": "404", - }, - ), - ] - - for version, error_data in wrong_versions: - error_resource = OptimadeError(**error_data) - - for entry_endpoint in ENTRY_INFO_SCHEMAS: - if version == "v0.1.0": - with pytest.raises( - VersionNotSupported, - match=fr"The parsed versioned base URL '/{version}.*", - ): - response = await client( - f"/gateways/{gateway_id}/{version}/info/{entry_endpoint}" - ) - return - else: - response = await client( - f"/gateways/{gateway_id}/{version}/info/{entry_endpoint}" - ) - - assert response.status_code == int( - error_resource.status - ), f"Request succeeded, where it should've failed: {response.json()}" - response = ErrorResponse(**response.json()) - assert response - - assert response.meta.data_returned == 0 - assert response.meta.data_available == 0 - assert not response.meta.more_data_available - - assert len(response.errors) == 1 - assert response.errors == [error_resource] diff --git a/tests/routers/gateway/test_gateway_links.py b/tests/routers/gateway/test_gateway_links.py deleted file mode 100644 index 2e9711bd..00000000 --- a/tests/routers/gateway/test_gateway_links.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Tests for /gateways/{gateway_id}/links endpoint""" -from typing import Awaitable, Callable - -try: - from typing import Literal -except ImportError: - from typing_extensions import Literal - -from fastapi import FastAPI -import httpx -import pytest - - -pytestmark = pytest.mark.asyncio - - -async def test_get_gateway_links( - client: Callable[ - [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], - Awaitable[httpx.Response], - ] -): - """Test GET /gateways/{gateway_id}/links""" - from optimade.models import LinksResponse - - gateway_id = "singledb" - - response = await client(f"/gateways/{gateway_id}/links") - - assert response.status_code == 200, f"Request failed: {response.json()}" - response = LinksResponse(**response.json()) - assert response - - assert response.meta.data_returned == 2 - assert response.meta.data_available == 2 - assert not response.meta.more_data_available - - -async def test_get_versioned_gateway_links( - client: Callable[ - [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], - Awaitable[httpx.Response], - ] -): - """Test GET /gateways/{gateway_id}/{version}/links""" - from optimade.models import LinksResponse - from optimade.server.routers.utils import BASE_URL_PREFIXES - - gateway_id = "singledb" - - for version in BASE_URL_PREFIXES.values(): - response = await client(f"/gateways/{gateway_id}{version}/links") - - assert response.status_code == 200, f"Request failed: {response.json()}" - response = LinksResponse(**response.json()) - assert response - - assert response.meta.data_returned == 2 - assert response.meta.data_available == 2 - assert not response.meta.more_data_available - - -async def test_bad_versioned_gateway_links( - client: Callable[ - [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], - Awaitable[httpx.Response], - ] -): - """Test GET /gateways/{gateway_id}/{version}/links with wrong version""" - from optimade.models import ErrorResponse, OptimadeError - from optimade.server.exceptions import VersionNotSupported - from optimade.server.routers.utils import BASE_URL_PREFIXES - - gateway_id = "singledb" - - wrong_versions = [ - ( - # Correct syntax - unsupport version - "v0.1.0", - { - "detail": ( - "version v0.1.0 is not supported. Supported versions: " - f"{[_[1:] for _ in BASE_URL_PREFIXES.values()]}" - ), - "title": "Version Not Supported", - "status": "553", - }, - ), - ( - # Incorrect syntax - supported version - BASE_URL_PREFIXES["patch"][len("/v") :], - { - "detail": f"version MUST be one of {[_[1:] for _ in BASE_URL_PREFIXES.values()]}", - "title": "Not Found", - "status": "404", - }, - ), - ] - - for version, error_data in wrong_versions: - error_resource = OptimadeError(**error_data) - - if version == "v0.1.0": - with pytest.raises( - VersionNotSupported, - match=fr"The parsed versioned base URL '/{version}.*", - ): - response = await client(f"/gateways/{gateway_id}/{version}/links") - return - else: - response = await client(f"/gateways/{gateway_id}/{version}/links") - - assert response.status_code == int( - error_resource.status - ), f"Request succeeded, where it should've failed: {response.json()}" - response = ErrorResponse(**response.json()) - assert response - - assert response.meta.data_returned == 0 - assert response.meta.data_available == 0 - assert not response.meta.more_data_available - - assert len(response.errors) == 1 - assert response.errors == [error_resource] diff --git a/tests/routers/gateway/test_gateway_structures.py b/tests/routers/gateway/test_gateway_structures.py deleted file mode 100644 index adf46334..00000000 --- a/tests/routers/gateway/test_gateway_structures.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Tests for /gateways/{gateway_id}/structures endpoints""" -# pylint: disable=import-error,no-name-in-module -import json -from typing import Awaitable, Callable - -try: - from typing import Literal -except ImportError: - from typing_extensions import Literal - -from fastapi import FastAPI -import httpx -import pytest - - -pytestmark = pytest.mark.asyncio - - -async def test_get_structures( - client: Callable[ - [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], - Awaitable[httpx.Response], - ], - get_gateway: Callable[[str], Awaitable[dict]], - mock_gateway_responses: Callable[[dict], None], -): - """Test GET /gateways/{gateway_id}/structures""" - from optimade.models import StructureResponseMany - - from optimade_gateway.common.config import CONFIG - - gateway_id = "twodbs" - gateway: dict = await get_gateway(gateway_id) - - mock_gateway_responses(gateway) - - response = await client(f"/gateways/{gateway_id}/structures") - - assert response.status_code == 200, f"Request failed: {response.json()}" - response = StructureResponseMany(**response.json()) - assert response - - assert response.meta.more_data_available - - more_data_available = False - data_returned = 0 - data_available = 0 - data = [] - - assert len(response.data) == CONFIG.page_limit * len(gateway["databases"]) - - for database in gateway["databases"]: - url = f"{database['attributes']['base_url']}/structures?page_limit={CONFIG.page_limit}" - db_response = httpx.get(url) - assert ( - db_response.status_code == 200 - ), f"Request to {url} failed: {db_response.json()}" - db_response = db_response.json() - - data_returned += db_response["meta"]["data_returned"] - data_available += db_response["meta"]["data_available"] - if not more_data_available: - more_data_available = db_response["meta"]["more_data_available"] - - for datum in db_response["data"]: - database_id_meta = { - f"_{CONFIG.provider.prefix}_source_database_id": database["id"] - } - if "meta" in datum: - datum["meta"].update(database_id_meta) - else: - datum["meta"] = database_id_meta - data.append(datum) - - assert data_returned == response.meta.data_returned - assert data_available == response.meta.data_available - assert more_data_available == response.meta.more_data_available - - assert data == json.loads(response.json(exclude_unset=True))["data"], ( - f"IDs in test not in response: {set([_['id'] for _ in data]) - set([_['id'] for _ in json.loads(response.json(exclude_unset=True))['data']])}\n\n" - f"IDs in response not in test: {set([_['id'] for _ in json.loads(response.json(exclude_unset=True))['data']]) - set([_['id'] for _ in data])}\n\n" - ) - - -async def test_get_single_structure( - client: Callable[ - [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], - Awaitable[httpx.Response], - ], - get_gateway: Callable[[str], Awaitable[dict]], - mock_gateway_responses: Callable[[dict], None], -): - """TEST GET /gateways/{gateway_id}/structures/{structure_id}""" - from optimade.models import StructureResponseOne - - gateway_id = "single-structure_optimade-sample" - structure_id = "mcloud/optimade-sample_single/1" - gateway = await get_gateway(gateway_id) - - mock_gateway_responses(gateway) - - response = await client(f"/gateways/{gateway_id}/structures/{structure_id}") - - assert response.status_code == 200, f"Request failed: {response.json()}" - response = StructureResponseOne(**response.json()) - assert response - - assert not response.meta.more_data_available - - database = [_ for _ in gateway["databases"] if "_single" in _["id"]][0] - - assert response.data is not None, f"Response:\n{response.json(indent=2)}" - - url = f"{database['attributes']['base_url']}/structures/{structure_id[len(database['id']) + 1:]}" - db_response = httpx.get(url) - assert ( - db_response.status_code == 200 - ), f"Request to {url} failed: {db_response.json()}" - db_response = db_response.json() - - assert db_response["data"] is not None - assert db_response["data"] == json.loads(response.json(exclude_unset=True))["data"] - - assert db_response["meta"]["data_returned"] == response.meta.data_returned - assert response.meta.data_available is None - assert ( - db_response["meta"]["more_data_available"] == response.meta.more_data_available - ) - - -async def test_sort_no_effect( - client: Callable[ - [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], - Awaitable[httpx.Response], - ], - get_gateway: Callable[[str], Awaitable[dict]], - mock_gateway_responses: Callable[[dict], None], -): - """Test GET /gateways/{gateway_id}/structures with the `sort` query parameter - - Currently, the `sort` query parameter should not have an effect when used with this endpoint. - This means if the `sort` parameter is used, the response should not change - it should be - ignored. - """ - from optimade.models import StructureResponseMany, Warnings - - from optimade_gateway.warnings import SortNotSupported - - gateway_id = "twodbs" - gateway: dict = await get_gateway(gateway_id) - - mock_gateway_responses(gateway) - - with pytest.warns(SortNotSupported): - response_asc = await client(f"/gateways/{gateway_id}/structures?sort=id") - with pytest.warns(SortNotSupported): - response_desc = await client(f"/gateways/{gateway_id}/structures?sort=-id") - - assert response_asc.status_code == 200, f"Request failed: {response_asc.json()}" - assert response_desc.status_code == 200, f"Request failed: {response_desc.json()}" - - response_asc = StructureResponseMany(**response_asc.json()) - assert response_asc - response_desc = StructureResponseMany(**response_desc.json()) - assert response_desc - - assert response_asc.data == response_desc.data - - sort_warning = SortNotSupported() - - for response in (response_asc, response_desc): - assert response.meta.warnings, response.json() - assert len(response.meta.warnings) == 1 - assert response.meta.warnings[0] == Warnings( - title=sort_warning.title, - detail=sort_warning.detail, - ) diff --git a/tests/routers/gateway/test_gateway_versions.py b/tests/routers/gateway/test_gateway_versions.py deleted file mode 100644 index 92e52a72..00000000 --- a/tests/routers/gateway/test_gateway_versions.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Tests for /gateways/{gateway_id}/versions endpoint""" -from typing import Awaitable, Callable - -try: - from typing import Literal -except ImportError: - from typing_extensions import Literal - -from fastapi import FastAPI -import httpx -import pytest - - -pytestmark = pytest.mark.asyncio - - -async def test_get_versions( - client: Callable[ - [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], - Awaitable[httpx.Response], - ] -): - """Test GET /versions""" - from optimade import __api_version__ - - response = await client("/versions") - - assert response.status_code == 200, f"Request failed: {response.text}" - - assert response.text == f"version\n{__api_version__.split('.')[0]}" - assert "content-type" in response.headers, response.headers - for media_type_content in ("text/csv", "header=present"): - assert media_type_content in response.headers.get("content-type", "") diff --git a/tests/routers/test_versions.py b/tests/routers/test_versions.py deleted file mode 100644 index 1a70cf17..00000000 --- a/tests/routers/test_versions.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Tests for /versions endpoint - -As this is not implemented in the current package, but rather used directly from the `optimade` -package, we just check it exists and returns the response we would expect. -""" -from typing import Awaitable, Callable - -try: - from typing import Literal -except ImportError: - from typing_extensions import Literal - -from fastapi import FastAPI -import httpx -import pytest - - -pytestmark = pytest.mark.asyncio - - -async def test_get_versions( - client: Callable[ - [str, FastAPI, str, Literal["get", "post", "put", "delete", "patch"]], - Awaitable[httpx.Response], - ] -): - """Test GET /versions""" - from optimade import __api_version__ - - gateway_id = "singledb" - - response = await client(f"/gateways/{gateway_id}/versions") - - assert response.status_code == 200, f"Request failed: {response.text}" - - assert response.text == f"version\n{__api_version__.split('.')[0]}" - assert "content-type" in response.headers, response.headers - for media_type_content in ("text/csv", "header=present"): - assert media_type_content in response.headers.get("content-type", "") From 5fb73e6ade5202ac346aaa89ec9b2b6b339749c9 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Mon, 6 Sep 2021 18:43:41 +0200 Subject: [PATCH 12/13] Update docker CI job --- .github/docker/docker_config.json | 26 -------------------------- .github/workflows/ci.yml | 23 +++++------------------ 2 files changed, 5 insertions(+), 44 deletions(-) delete mode 100644 .github/docker/docker_config.json diff --git a/.github/docker/docker_config.json b/.github/docker/docker_config.json deleted file mode 100644 index 28b7230b..00000000 --- a/.github/docker/docker_config.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "debug": true, - "database_backend": "mongodb", - "mongo_uri": "mongodb://db:27017", - "mongo_database": "docker_optimade_gateway", - "page_limit": 20, - "page_limit_max": 500, - "base_url": "http://gh_actions_host:5000", - "implementation": { - "name": "OPTIMADE Gateway", - "version": "0.1.2", - "source_url": "https://github.com/Materials-Consortia/optimade-gateway", - "maintainer": {"email": "casper.andersen@epfl.ch"} - }, - "provider": { - "name": "OPTIMADE Gateway", - "description": "A gateway server to query multiple OPTIMADE databases.", - "prefix": "gateway", - "homepage": "https://github.com/Materials-Consortia/optimade-gateway" - }, - "provider_fields": {}, - "aliases": {}, - "length_aliases": {}, - "log_level": "debug", - "load_optimade_providers_databases": false -} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7846e448..afc98486 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,29 +102,16 @@ jobs: with: fetch-depth: 2 - - name: Setup and run OPTIMADE Python tools server - run: | - git clone --recurse-submodules https://github.com/Materials-Consortia/optimade-python-tools - docker-compose -f ./optimade-python-tools/docker-compose.yml up optimade & - .github/utils/wait_for_it.sh localhost:3213 -t 120 - - name: Build the Docker app - run: docker-compose build --build-arg CONFIG_FILE=".github/docker/docker_config.json" + run: docker-compose build + env: + OPTIMADE_BASE_URL: "http://gh_actions_host:5000" - - name: Start the Docker app (adding OPT server to 'gateway' network) + - name: Start the Docker app run: | docker-compose up & .github/utils/wait_for_it.sh localhost:5000 -t 120 sleep 5 - docker network connect --alias optimade-sample-server optimade-gateway_gateway optimade-python-tools_optimade_1 - - - name: OPTIMADE Validator for gateway - uses: Materials-Consortia/optimade-validator-action@v2 - with: - port: 5000 - path: /gateways/docker_ci - all versioned paths: True - validate unversioned path: True docs: name: Documentation @@ -141,7 +128,7 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip + python -m pip install -U pip pip install -U setuptools pip install -e .[docs] From 784c5af8ef1c29927056b21bc41598e6a3efcc65 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Tue, 7 Sep 2021 16:58:56 +0200 Subject: [PATCH 13/13] Remove minify plugin Apparently its dependency `jsmin` can result in unstable builds/installations. --- mkdocs.yml | 2 -- requirements_docs.txt | 1 - 2 files changed, 3 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index e5275e64..3a05d68c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -48,8 +48,6 @@ markdown_extensions: plugins: - search: lang: en - - minify: - minify_html: true - mkdocstrings: default_handler: python handlers: diff --git a/requirements_docs.txt b/requirements_docs.txt index 031eb157..60ddc20e 100644 --- a/requirements_docs.txt +++ b/requirements_docs.txt @@ -1,5 +1,4 @@ mkdocs~=1.2 mkdocs-awesome-pages-plugin~=2.5 mkdocs-material~=7.2 -mkdocs-minify-plugin~=0.4.0 mkdocstrings~=0.15.2