diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 94a0972..0e0e0de 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -1,19 +1,18 @@ name: Scheduled validator -on: - push: +on: [push, pull_request] jobs: validate: - name: OPTiMaDe validator + name: OPTIMADE validator runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v1 with: python-version: 3.7 - - name: Install OPTiMaDe + - name: Install OPTIMADE run: | python -m pip install --upgrade pip pip install optimade diff --git a/.gitignore b/.gitignore index e21531b..0de2e5e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ __pycache__/ *.py[cod] *$py.class +*~ +.DS_Store + # C extensions *.so diff --git a/README.md b/README.md index a37d590..81cb5e8 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,12 @@ For convenience, you can also access the most recent list of providers in the fo - [https://providers.optimade.org/providers.json](https://providers.optimade.org/providers.json) -You can contribute a new provider or amend the current information by creating a pull request to [the `providers` repository](https://github.com/Materials-Consortia/providers). +Additionally, there is a providers dashboard that includes the results of validation of listed implementations. +This can be accessed at the following URL: + +- [https://optimade.org/providers-dashboard](https://optimade.org/providers-dashboard) +You can contribute a new provider or amend the current information by creating a pull request to [the `providers` repository](https://github.com/Materials-Consortia/providers). ## Repository organization @@ -27,15 +31,23 @@ The repository is organized this way: - `/src/info//info.json` is the proper response to the info endpoint formatted according to OPTIMADE version `` and any later version that uses a format that is backward compatible with this version. -- `/_redirect` specify http rewrites to map index meta-database URLs `//info` and `//links` to the corresponding files under `src/`, as well as `/providers.json`. - -- `/` directories contain symlinks `info.html` and `links.html` to the corresponding files under `src/` to somewhat support hosting solutions that do not understand the instructions in `_redirect` but which support "pretty URLs". +- `/src/index-metadbs///info.json` and `/src/index-metadbs///links.json` are static Index Meta-Databases that are hosted in this repository for those providers that only have one main sub-database (or very few sub-databases) and do not wish to maintain one on their own. + See more details and instructions in `/src/index-metadbs/README.md`. -- `/providers.json` is a symbolic link continously updated to point at the latest version of the providers.json file under `/src/` to somewhat support hosting solutions that do not understand the instructions in `_redirect`. +- `/_redirects` specifies http rewrites to map index meta-database URLs `//info` and `//links` to the corresponding files under `src/`, as well as `/providers.json`. + It also creates `/index-metadbs///info.json` and `/index-metadbs///links.json` URLs to point to the corresponding files in the `/src/index-metadbs` subfolders. + This is used by the deploy tool Netlify. Presently, only the `v1` version of the formats exist. Hence, to update the list of providers, file a pull-request against the repository to edit `/src/links/v1/providers.json`. +## Requirements to be listed in this providers list + +It is a policy of this providers list ([providers.optimade.org](http://providers.optimade.org)) that links inside `providers.json` must be links to an [OPTIMADE Index Meta-Database](https://github.com/Materials-Consortia/OPTIMADE/blob/develop/optimade.rst#32index-meta-database). + +If you only have one or few databases in your implementation, and you do not want to host an Index Meta-Database yourself, you can host the Index Meta-Database directly in this repository. +You can find instructions [here](./src/index-metadbs). + ## Fork this repository to set up an index meta-database for your own databases - Click on the Fork button in GitHub and follow the instructions. diff --git a/_redirects b/_redirects index e1ecc65..a83c393 100644 --- a/_redirects +++ b/_redirects @@ -1,3 +1,14 @@ +# The exclamation mark at the end of the HTTP code is to make sure +# the redirect is always followed even if there is a file at that page. +# See https://docs.netlify.com/routing/redirects/rewrites-proxies/#shadowing + +# Returning 200 just returns the content without doing a HTTP redirect, i.e. +# without changing the URL, that is what we want to achieve here +# https://docs.netlify.com/routing/redirects/redirect-options/#http-status-codes /providers.json /src/links/v1/providers.json 200! /v1/links /src/links/v1/providers.json 200! /v1/info /src/info/v1/info.json 200! + +# Provide redirects for index meta-databases (removing the JSON at the end) +/index-metadbs/:providerid/:version/links /src/index-metadbs/:providerid/:version/links.json 200! +/index-metadbs/:providerid/:version/info /src/index-metadbs/:providerid/:version/info.json 200! diff --git a/src/index-metadbs/README.md b/src/index-metadbs/README.md new file mode 100644 index 0000000..a2f8880 --- /dev/null +++ b/src/index-metadbs/README.md @@ -0,0 +1,46 @@ +# Static Index Meta-Databases + +## Goal + +This folder hosts static versions of Index Meta-Databases for those providers that only have one main sub-database (or very few sub-databases) and do not wish to maintain an [OPTIMADE Index Meta-Database](https://github.com/Materials-Consortia/OPTIMADE/blob/develop/optimade.rst#32index-meta-database) themselves. + +Note: while providing an Index Meta-Database is not required by the OPTIMADE API specification, it is instead required in order to be listed in this List of Providers ([providers.optimade.org](http://providers.optimade.org)). + +## When should I use a static Index Meta-Database hosted in this repository + +If you add a new provider and do not wish to host an Index Meta-Database yourself, feel free to add an Index Meta-Database here. + +However, if your Index Meta-Database is rapidly changing (e.g., because the list of sub-databases changes often), please host it yourself to avoid multiple commits and pull requests to this repository at every change. +Rare changes are instead of course welcome, e.g., if you have to change your DNS name or the description, or if you need just to add one static new sub-database, like a test database. + +Note that "changes" here refer solely to changes to the *list of sub-databases*; the content of each sub-database can change at any time without the need to modify the Index Meta-Database. + +## Instructions on how to add a new static Index Meta-Database + +1. Go to the folder `src/index-metadbs`, copy the existing template folder (`exmpl`) to a new folder, where the new folder name should be the identifier of your provider (we will use `exmpl2` here and in the following as the name of the new provider). + In particular, this will contain a `v1` subfolder, with two files inside it: `info.json` and `links.json`. + +2. Adapt the content of the `links.json` file. + In particular, provide a new `links` entry resource object for each sub-database that you want to link to. + In the special case in which you have a single "main" sub-database, just change the existing values as follows: + + - change the `id` to your provider identifier: `"exmpl" -> "exmpl2"` + - change the attributes `name`, `description` and `homepage` to contain the correct content. + Please reuse the *same* content as the one you specified in the main `providers.json` file. + - point the `base_url` to the base URL of your OPTIMADE implementation. + Note, this should be the un-versioned base URL (see [the OPTIMADE API specification](https://github.com/Materials-Consortia/OPTiMaDe/blob/develop/optimade.rst#base-url) for more information on this). + +3. Adapt the content of the `info.json` file. + In particular, you should change two fields: + + - change the URL of the `available_api_versions` by replacing `exmpl` with your identifier: `http://providers.optimade.org/index-metadbs/exmpl2/v1/`. + - change the `id` inside `data -> relationships -> default -> data -> id` from `exmpl` to the correct ID from the list of links in the `links.json` file. + As [explained in the OPTIMADE API specification](https://github.com/Materials-Consortia/OPTiMaDe/blob/develop/optimade.rst#base-info-endpoint), this should be the ID of the database that should be considered as the "default" sub-database by clients. + + If you only have one sub-database and you followed the instructions above, you should use here your provider identifier. + If you do not wish to have a default database, set the `relationships` value to an empty dictionary or set the value of `relationships -> default -> data` to `null`. + +4. In the top-level `providers.json` file, point the `base_url` of your provider to `http://providers.optimade.org/index-metadbs/exmpl2/`. + +5. Create a pull request, and check that all automated continuous-integration tests pass. + Also, you can check that the new Index Meta-Database properly works at the expected link using the Netlify preview (just click on the `netlify/optimade-providers/deploy-preview` entry of the GitHub checks that will appear in the GitHub PR Conversation page after a few seconds). diff --git a/src/index-metadbs/cod/v1/info.json b/src/index-metadbs/cod/v1/info.json new file mode 100644 index 0000000..cac8116 --- /dev/null +++ b/src/index-metadbs/cod/v1/info.json @@ -0,0 +1,42 @@ +{ + "meta" : { + "api_version" : "1.0.0", + "query" : { + "representation" : "/info" + }, + "more_data_available" : false, + "schema" : "https://schemas.optimade.org/openapi/v1.0/optimade_index.json" + }, + "data" : { + "attributes" : { + "available_api_versions" : [ + { + "version" : "1.0.0", + "url" : "http://providers.optimade.org/index-metadbs/cod/v1/" + } + ], + "is_index" : true, + "formats" : [ + "json" + ], + "entry_types_by_format" : { + "json" : [] + }, + "available_endpoints" : [ + "info", + "links" + ], + "api_version" : "1.0.0" + }, + "type" : "info", + "relationships" : { + "default" : { + "data" : { + "id" : "cod", + "type" : "links" + } + } + }, + "id" : "/" + } +} diff --git a/src/index-metadbs/cod/v1/links.json b/src/index-metadbs/cod/v1/links.json new file mode 100644 index 0000000..21ab708 --- /dev/null +++ b/src/index-metadbs/cod/v1/links.json @@ -0,0 +1,34 @@ +{ + "meta" : { + "api_version" : "1.0.0", + "query" : { + "representation" : "/links" + }, + "more_data_available" : false, + "schema" : "https://schemas.optimade.org/openapi/v1.0/optimade_index.json" + }, + "data" : [ + { + "id" : "cod", + "type" : "links", + "attributes" : { + "name": "Crystallography Open Database", + "description": "Open-access collection of crystal structures of organic, inorganic, metal-organics compounds and minerals, excluding biopolymers", + "base_url": "https://www.crystallography.net/cod/optimade", + "homepage": "https://www.crystallography.net/cod", + "link_type": "child" + } + }, + { + "id" : "optimade-providers", + "type" : "links", + "attributes" : { + "name": "OPTIMADE Providers Index Meta-Database", + "description": "The list of OPTIMADE providers maintained by Materials-Consortia", + "base_url": "https://providers.optimade.org", + "homepage": "https://www.optimade.org", + "link_type": "providers" + } + } + ] +} diff --git a/src/index-metadbs/exmpl/v1/info.json b/src/index-metadbs/exmpl/v1/info.json new file mode 100644 index 0000000..5562884 --- /dev/null +++ b/src/index-metadbs/exmpl/v1/info.json @@ -0,0 +1,42 @@ +{ + "meta" : { + "api_version" : "1.0.0", + "query" : { + "representation" : "/info" + }, + "more_data_available" : false, + "schema" : "https://schemas.optimade.org/openapi/v1.0/optimade_index.json" + }, + "data" : { + "attributes" : { + "available_api_versions" : [ + { + "version" : "1.0.0", + "url" : "http://providers.optimade.org/index-metadbs/exmpl/v1/" + } + ], + "is_index" : true, + "formats" : [ + "json" + ], + "entry_types_by_format" : { + "json" : [] + }, + "available_endpoints" : [ + "info", + "links" + ], + "api_version" : "1.0.0" + }, + "type" : "info", + "relationships" : { + "default" : { + "data" : { + "id" : "exmpl", + "type" : "links" + } + } + }, + "id" : "/" + } +} diff --git a/src/index-metadbs/exmpl/v1/links.json b/src/index-metadbs/exmpl/v1/links.json new file mode 100644 index 0000000..b7e581a --- /dev/null +++ b/src/index-metadbs/exmpl/v1/links.json @@ -0,0 +1,34 @@ +{ + "meta" : { + "api_version" : "1.0.0", + "query" : { + "representation" : "/links" + }, + "more_data_available" : false, + "schema" : "https://schemas.optimade.org/openapi/v1.0/optimade_index.json" + }, + "data" : [ + { + "id" : "exmpl", + "type" : "links", + "attributes" : { + "name": "Example provider", + "description": "Provider used for examples, not to be assigned to a real database", + "base_url": null, + "homepage": "https://example.com", + "link_type": "child" + } + }, + { + "id" : "optimade-providers", + "type" : "links", + "attributes" : { + "name": "OPTIMADE Providers Index Meta-Database", + "description": "The list of OPTIMADE providers maintained by Materials-Consortia", + "base_url": "https://providers.optimade.org", + "homepage": "https://www.optimade.org", + "link_type": "providers" + } + } + ] +} diff --git a/src/index-metadbs/mp/v1/info.json b/src/index-metadbs/mp/v1/info.json new file mode 100644 index 0000000..3361dae --- /dev/null +++ b/src/index-metadbs/mp/v1/info.json @@ -0,0 +1,43 @@ +{ + "meta" : { + "api_version" : "1.0.0", + "query" : { + "representation" : "/info" + }, + "more_data_available" : false, + "schema" : "https://schemas.optimade.org/openapi/v1.0/optimade_index.json" + }, + "data" : { + "attributes" : { + "available_api_versions" : [ + { + "version" : "1.0.0", + "url" : "http://providers.optimade.org/index-metadbs/mp/v1/" + } + ], + "is_index" : true, + "formats" : [ + "json" + ], + "entry_types_by_format" : { + "json" : [] + }, + "available_endpoints" : [ + "info", + "links" + ], + "api_version" : "1.0.0" + }, + "type" : "info", + "relationships" : { + "default" : { + "data" : { + "id" : "mp", + "type" : "links" + } + } + }, + "id" : "/" + } + } + \ No newline at end of file diff --git a/src/index-metadbs/mp/v1/links.json b/src/index-metadbs/mp/v1/links.json new file mode 100644 index 0000000..abe8cff --- /dev/null +++ b/src/index-metadbs/mp/v1/links.json @@ -0,0 +1,34 @@ +{ + "meta": { + "api_version": "1.0.0", + "query": { + "representation": "/links" + }, + "more_data_available": false, + "schema": "https://schemas.optimade.org/openapi/v1.0/optimade_index.json" + }, + "data": [ + { + "id": "mp", + "type": "links", + "attributes": { + "name": "The Materials Project", + "description": "The Materials Project OPTIMADE endpoint", + "base_url": "https://optimade.materialsproject.org", + "homepage": "https://www.materialsproject.org", + "link_type": "child" + } + }, + { + "id": "optimade-providers", + "type": "links", + "attributes": { + "name": "OPTIMADE Providers Index Meta-Database", + "description": "The list of OPTIMADE providers maintained by Materials-Consortia", + "base_url": "https://providers.optimade.org", + "homepage": "https://www.optimade.org", + "link_type": "providers" + } + } + ] +} diff --git a/src/index-metadbs/tcod/v1/info.json b/src/index-metadbs/tcod/v1/info.json new file mode 100644 index 0000000..76871f8 --- /dev/null +++ b/src/index-metadbs/tcod/v1/info.json @@ -0,0 +1,42 @@ +{ + "meta" : { + "api_version" : "1.0.0", + "query" : { + "representation" : "/info" + }, + "more_data_available" : false, + "schema" : "https://schemas.optimade.org/openapi/v1.0/optimade_index.json" + }, + "data" : { + "attributes" : { + "available_api_versions" : [ + { + "version" : "1.0.0", + "url" : "http://providers.optimade.org/index-metadbs/tcod/v1/" + } + ], + "is_index" : true, + "formats" : [ + "json" + ], + "entry_types_by_format" : { + "json" : [] + }, + "available_endpoints" : [ + "info", + "links" + ], + "api_version" : "1.0.0" + }, + "type" : "info", + "relationships" : { + "default" : { + "data" : { + "id" : "tcod", + "type" : "links" + } + } + }, + "id" : "/" + } +} diff --git a/src/index-metadbs/tcod/v1/links.json b/src/index-metadbs/tcod/v1/links.json new file mode 100644 index 0000000..50606ef --- /dev/null +++ b/src/index-metadbs/tcod/v1/links.json @@ -0,0 +1,34 @@ +{ + "meta" : { + "api_version" : "1.0.0", + "query" : { + "representation" : "/links" + }, + "more_data_available" : false, + "schema" : "https://schemas.optimade.org/openapi/v1.0/optimade_index.json" + }, + "data" : [ + { + "id" : "tcod", + "type" : "links", + "attributes" : { + "name": "Theoretical Crystallography Open Database", + "description": "Open-access collection of theoretically calculated or refined crystal structures of organic, inorganic, metal-organic compounds and minerals, excluding biopolymers", + "base_url": "https://www.crystallography.net/tcod/optimade", + "homepage": "https://www.crystallography.net/tcod", + "link_type": "child" + } + }, + { + "id" : "optimade-providers", + "type" : "links", + "attributes" : { + "name": "OPTIMADE Providers Index Meta-Database", + "description": "The list of OPTIMADE providers maintained by Materials-Consortia", + "base_url": "https://providers.optimade.org", + "homepage": "https://www.optimade.org", + "link_type": "providers" + } + } + ] +} diff --git a/src/info/v1/info.json b/src/info/v1/info.json index 37acd9f..f257c60 100644 --- a/src/info/v1/info.json +++ b/src/info/v1/info.json @@ -1,7 +1,16 @@ { - "data": { + "meta" : { + "api_version" : "1.0.0", + "query" : { + "representation" : "/info" + }, + "more_data_available" : false, + "schema" : "https://schemas.optimade.org/openapi/v1.0/optimade_index.json" + }, + "data": { "type": "info", "id": "/", + "relationships": {}, "attributes": { "api_version": "1.0.0", "available_api_versions": [ diff --git a/src/links/v1/providers.json b/src/links/v1/providers.json index 07c47e3..a6f4112 100644 --- a/src/links/v1/providers.json +++ b/src/links/v1/providers.json @@ -1,163 +1,187 @@ { - "data": [ - { - "type": "provider", + "meta" : { + "api_version" : "1.0.0", + "query" : { + "representation" : "/links" + }, + "more_data_available" : false, + "schema" : "https://schemas.optimade.org/openapi/v1.0/optimade_index.json" + }, + "data": [ + { + "type": "links", "id": "aiida", "attributes": { "name": "AiiDA", "description": "Automated Interactive Infrastructure and Database for Computational Science (AiiDA)", "base_url": null, - "homepage": "http://www.aiida.net" + "homepage": "http://www.aiida.net", + "link_type": "external" } }, { - "type": "provider", + "type": "links", "id": "aflow", "attributes": { "name": "aflow.org", "description": "", "base_url": null, - "homepage": null + "homepage": null, + "link_type": "external" } }, { - "type": "provider", + "type": "links", "id": "cod", "attributes": { "name": "Crystallography Open Database", "description": "Open-access collection of crystal structures of organic, inorganic, metal-organics compounds and minerals, excluding biopolymers", - "base_url": "https://www.crystallography.net/cod/optimade", - "homepage": "https://www.crystallography.net/cod" + "base_url": "http://providers.optimade.org/index-metadbs/cod", + "homepage": "https://www.crystallography.net/cod", + "link_type": "external" } }, { - "type": "provider", + "type": "links", "id": "exmpl", "attributes": { "name": "Example provider", "description": "Provider used for examples, not to be assigned to a real database", - "base_url": "https://example.com/optimade", - "homepage": "https://example.com" + "base_url": "http://providers.optimade.org/index-metadbs/exmpl", + "homepage": "https://example.com", + "link_type": "external" } }, { - "type": "provider", + "type": "links", "id": "httk", "attributes": { "name": "The High-Throughput Toolkit", "description": "Prefix for implementation-specific identifiers used in the httk implementation at http://httk.org/", "base_url": null, - "homepage": null + "homepage": null, + "link_type": "external" } }, { - "type": "provider", + "type": "links", "id": "mcloud", "attributes": { "name": "Materials Cloud", "description": "A platform for Open Science built for seamless sharing of resources in computational materials science", "base_url": "https://www.materialscloud.org/optimade", - "homepage": "https://www.materialscloud.org" + "homepage": "https://www.materialscloud.org", + "link_type": "external" } }, { - "type": "provider", + "type": "links", "id": "mp", "attributes": { - "name": "materialsproject.org", - "description": "", - "base_url": null, - "homepage": null + "name": "The Materials Project", + "description": "An open database of computed materials properties to accelerate materials discovery and design", + "base_url": "http://providers.optimade.org/index-metadbs/mp", + "homepage": "https://www.materialsproject.org", + "link_type": "external" } }, { - "type": "provider", + "type": "links", "id": "mpds", "attributes": { "name": "materials platform for data science", "description": "", "base_url": null, - "homepage": null + "homepage": null, + "link_type": "external" } }, { - "type": "provider", + "type": "links", "id": "nmd", "attributes": { "name": "novel materials discovery (NOMAD)", "description": "A FAIR data sharing platform for materials science data", - "base_url": "https://repository.nomad-coe.eu/v0.8/optimade/index", - "homepage": "https://repository.nomad-coe.eu/v0.8" + "base_url": "https://nomad-lab.eu/prod/rae/optimade/index", + "homepage": "https://nomad-lab.eu", + "link_type": "external" } }, { - "type": "provider", + "type": "links", "id": "odbx", "attributes": { "name": "open database of xtals", "description": "A public database of crystal structures mostly derived from ab initio structure prediction from the group of Dr Andrew Morris at the University of Birmingham https://ajm143.github.io", "base_url": "https://optimade-index.odbx.science", - "homepage": "https://odbx.science" + "homepage": "https://odbx.science", + "link_type": "external" } }, { - "type": "provider", + "type": "links", "id": "omdb", "attributes": { "name": "open materials database", "description": "", "base_url": null, - "homepage": null + "homepage": null, + "link_type": "external" } }, { - "type": "provider", + "type": "links", "id": "optimade", "attributes": { - "name": "OPTiMaDe implementations and libraries", + "name": "OPTIMADE implementations and libraries", "description": "Prefix for implementation-specific identifiers used in API implementations and libraries provided at https://github.com/Materials-Consortia", "base_url": null, - "homepage": "https://www.optimade.org" + "homepage": "https://www.optimade.org", + "link_type": "external" } }, { - "type": "provider", + "type": "links", "id": "oqmd", "attributes": { "name": "open quantum materials database", "description": "", "base_url": null, - "homepage": null + "homepage": null, + "link_type": "external" } }, { - "type": "provider", + "type": "links", "id": "jarvis", "attributes": { "name": "Joint Automated Repository for Various Integrated Simulations (JARVIS)", "description": "JARVIS is a repository designed to automate materials discovery using classical force-field, density functional theory, machine learning calculations and experiments.", "base_url": null, - "homepage": "https://jarvis.nist.gov" + "homepage": "https://jarvis.nist.gov", + "link_type": "external" } }, { - "type": "provider", + "type": "links", "id": "pcod", "attributes": { "name": "Predicted Crystallography Open Database", "description": "", "base_url": null, - "homepage": null + "homepage": null, + "link_type": "external" } }, { - "type": "provider", + "type": "links", "id": "tcod", "attributes": { "name": "Theoretical Crystallography Open Database", "description": "Open-access collection of theoretically calculated or refined crystal structures of organic, inorganic, metal-organic compounds and minerals, excluding biopolymers", - "base_url": "https://www.crystallography.net/tcod/optimade", - "homepage": "https://www.crystallography.net/tcod" + "base_url": "http://providers.optimade.org/index-metadbs/tcod", + "homepage": "https://www.crystallography.net/tcod", + "link_type": "external" } } ] diff --git a/tests/test_providers.py b/tests/test_providers.py index ab3d593..cf795c6 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -2,30 +2,195 @@ import pathlib import json +import ssl import unittest -from optimade.models import InfoResponse, LinksResponse +import urllib.request +import urllib.parse + +from optimade.models import IndexInfoResponse, LinksResponse, Link TOP_DIR = pathlib.Path(__file__).parent.parent -class ProvidersValidator(unittest.TestCase): +# To be disabled if/when we do not wish to allow to point to v0 endpoints +apply_v0_workarounds = True + + +def query_optimade(url): + """Perform a URL request to the given URL endpoint. + + However, for requests to `providers.optimade.org`, uses the local files + rather than going to fetch it on the web. + This is important to allow testing of new endpoints before the PR + is merged in GitHub (and only after things are merged in the main branch, + they will appear on `providers.optimade.org`) + + :param url: a string with the URL to fetch + :return: the raw content. + :raise urllib.error.HTTPError: if the page is not found. Note that this exception + is raised also when emulating a request from the local disk for + `providers.optimade.org` and the file is not found; in this case the code is 404. + """ + # Set to True for now - we might want to decide that this is a problem instead + # and set this to False + ignore_SSL_errors = True + + # Create the context, possibly ignoring SSL errors + ctx = ssl.create_default_context() + if ignore_SSL_errors: + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + parsed_url = urllib.parse.urlparse(url) + if parsed_url.netloc == "providers.optimade.org": + # Strip initial / + path = ( + parsed_url.path[1:] if parsed_url.path.startswith("/") else parsed_url.path + ) + # Remove any pending slashes + path = path[:-1] if path.endswith("/") else path + # Append the .json extension + path += ".json" + # Create the abs path to the file + file_path = TOP_DIR / "src" / path + try: + with open(file_path) as fhandle: + response_content = fhandle.read() + except FileNotFoundError: + raise urllib.error.HTTPError( + url=url, code=404, msg="HTTP Error 404: Not Found", hdrs="", fp="" + ) + else: + with urllib.request.urlopen(url, context=ctx) as url_response: + response_content = url_response.read() + + return response_content + +class ProvidersValidator(unittest.TestCase): def test_info(self): """ Validates the index.html json fudge as a `BaseInfoResource` object. """ - info_dir = TOP_DIR / 'src' / 'info' - versions = [v.parts[-1] for v in info_dir.iterdir() if v.is_dir() and v.parts[-1].startswith('v')] - for v_ind, version in enumerate(versions): + info_dir = TOP_DIR / "src" / "info" + versions = [ + v.parts[-1] + for v in info_dir.iterdir() + if v.is_dir() and v.parts[-1].startswith("v") + ] + for version in versions: path = pathlib.Path(f"/{TOP_DIR}/src/info/{version}/info.json").resolve() - with open(path, 'r') as f: + with open(path, "r") as f: json_rep = json.load(f) - InfoResponse(**json_rep) + IndexInfoResponse(**json_rep) def test_providers(self): """ Validates the providers.json as a list of `Provider`s objects. """ - links_dir = TOP_DIR / 'src' / 'links' - versions = [v.parts[-1] for v in links_dir.iterdir() if v.is_dir() and v.parts[-1].startswith('v')] - for v_ind, version in enumerate(versions): - path = pathlib.Path(f"{TOP_DIR}/src/links/{version}/providers.json").resolve() - with open(path, 'r') as f: + links_dir = TOP_DIR / "src" / "links" + versions = [ + v.parts[-1] + for v in links_dir.iterdir() + if v.is_dir() and v.parts[-1].startswith("v") + ] + for version in versions: + path = pathlib.Path( + f"{TOP_DIR}/src/links/{version}/providers.json" + ).resolve() + with open(path, "r") as f: json_rep = json.load(f) LinksResponse(**json_rep) - + + def test_index_metadb(self): + """ Validates that all (non-null) entries in providers.json point to an index meta-db. """ + # We collect all errors and report all of them at the end, see below + problems = [] + + links_dir = TOP_DIR / "src" / "links" + versions = [ + v.parts[-1] + for v in links_dir.iterdir() + if v.is_dir() and v.parts[-1].startswith("v") + ] + for version in versions: + path = pathlib.Path( + f"{TOP_DIR}/src/links/{version}/providers.json" + ).resolve() + with open(path, "r") as f: + json_rep = json.load(f) + response = LinksResponse(**json_rep) + for entry in response.data: + entry_id = entry.id + if entry.attributes.dict().get("base_url", None) is not None: + # the provider has a non-null base_url + print(f'[INFO] Checking provider "{entry_id}" ({version})') + + # I check the /info endpoint + entry_base_url = ( + entry.attributes.base_url.href + if isinstance(entry.attributes.base_url, Link) + else entry.attributes.base_url + ) + info_endpoint = f"{entry_base_url}/{version}/info" + tested_info_endpoints = [info_endpoint] + try: + try: + response_content = query_optimade(info_endpoint) + except urllib.error.HTTPError as exc: + if ( + apply_v0_workarounds + and version == "v1" + and exc.code == 404 + ): + try: + # Temporary workaround for optimade-python-tools while v1 is released + info_endpoint = f"{entry_base_url}/v0.10/info" + tested_info_endpoints.append(info_endpoint) + response_content = query_optimade(info_endpoint) + except urllib.error.HTTPError as exc: + # Temporary workaround for nomad that uses v0 as a prefix + info_endpoint = f"{entry_base_url}/v0/info" + tested_info_endpoints.append(info_endpoint) + response_content = query_optimade(info_endpoint) + else: + raise + except urllib.error.HTTPError as exc: + fallback_string = ( + "" + if len(tested_info_endpoints) == 1 + else f" (I tried all these URLs: {tested_info_endpoints})" + ) + problems.append( + f'Provider "{entry_id}" {info_endpoint} endpoint is not reachable{fallback_string}. Error: {str(exc)}' + ) + continue + + try: + info_response = IndexInfoResponse( + **json.loads(response_content) + ) + except Exception as exc: + problems.append( + f'Provider "{entry_id}": {info_endpoint} endpoint has problems during validation.\nError message:\n{str(exc)}' + ) + continue + + # If unspecified, it should be assumed as False, according to the OPTIMADE specs + is_index = info_response.data.attributes.dict().get( + "is_index", False + ) + if not is_index: + print(f" > PROBLEM DETECTED with provider '{entry_id}'.") + print(response_content) + problems.append( + f'Provider "{entry_id}" is NOT providing an index meta-database at {info_endpoint}' + ) + continue + + print( + f'[INFO] Provider "{entry_id}" ({version}) validated correctly ({info_endpoint})' + ) + + # I am collecting all problems and printing at the end because in this way we get a full overview + if problems: + err_msg = "PROBLEMS DETECTED!\n\n" + "\n\n".join(problems) + # Prepend with [ERROR] so that it gets colored in the GitHub output + err_msg = "\n".join(f"[ERROR] {line}" for line in err_msg.splitlines()) + print(err_msg) + raise AssertionError(err_msg) diff --git a/v1/info.html b/v1/info.html deleted file mode 120000 index 7a97c76..0000000 --- a/v1/info.html +++ /dev/null @@ -1 +0,0 @@ -../src/info/v1/info.json \ No newline at end of file diff --git a/v1/links.html b/v1/links.html deleted file mode 120000 index 4897ffa..0000000 --- a/v1/links.html +++ /dev/null @@ -1 +0,0 @@ -../src/links/v1/providers.json \ No newline at end of file