From 2a3726c5b29c33970b066528cf290b8c98d68511 Mon Sep 17 00:00:00 2001 From: "Jose M. Pizarro" <112697669+JosePizarro3@users.noreply.github.com> Date: Tue, 5 Mar 2024 10:57:22 +0100 Subject: [PATCH] Plugin distribution as pypi package (#16) * Moved modules to src/nomad_simulations Fix config files with the new path Added Dockerfile, MANIFEST.in, setup.py Added .vscode with settings * Fix pyproject * Added pipeline * Fixed nomad-lab to 1.2.0 --- .github/workflows/actions.yaml | 24 ++++- .github/workflows/publish.yml | 84 ++++++++++++++++ .gitignore | 3 + .vscode/settings.json | 24 +++++ Dockerfile | 15 +++ MANIFEST.in | 1 + README.md | 95 ++++++++++++++++++- nomad.yaml | 12 +++ pyproject.toml | 29 ++++-- setup.py | 4 + simulationdataschema/nomad_plugin.yaml | 4 - .../nomad_simulations}/__init__.py | 0 .../nomad_simulations}/atoms_state.py | 0 .../nomad_simulations}/common.py | 0 .../nomad_simulations}/general.py | 0 .../nomad_simulations}/model_method.py | 0 .../nomad_simulations}/model_system.py | 0 src/nomad_simulations/nomad_plugin.yaml | 4 + .../nomad_simulations}/outputs.py | 0 .../nomad_simulations}/utils/__init__.py | 0 .../nomad_simulations}/utils/utils.py | 0 21 files changed, 283 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/publish.yml create mode 100644 .vscode/settings.json create mode 100644 Dockerfile create mode 100644 MANIFEST.in create mode 100644 nomad.yaml create mode 100644 setup.py delete mode 100644 simulationdataschema/nomad_plugin.yaml rename {simulationdataschema => src/nomad_simulations}/__init__.py (100%) rename {simulationdataschema => src/nomad_simulations}/atoms_state.py (100%) rename {simulationdataschema => src/nomad_simulations}/common.py (100%) rename {simulationdataschema => src/nomad_simulations}/general.py (100%) rename {simulationdataschema => src/nomad_simulations}/model_method.py (100%) rename {simulationdataschema => src/nomad_simulations}/model_system.py (100%) create mode 100644 src/nomad_simulations/nomad_plugin.yaml rename {simulationdataschema => src/nomad_simulations}/outputs.py (100%) rename {simulationdataschema => src/nomad_simulations}/utils/__init__.py (100%) rename {simulationdataschema => src/nomad_simulations}/utils/utils.py (100%) diff --git a/.github/workflows/actions.yaml b/.github/workflows/actions.yaml index a6c1ace3..ecf96a51 100644 --- a/.github/workflows/actions.yaml +++ b/.github/workflows/actions.yaml @@ -16,10 +16,32 @@ jobs: pip install coverage coveralls - name: mypy run: | - python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional simulationdataschema tests + python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional src/nomad_simulations tests - name: Test with pytest run: | python -m coverage run -m pytest -sv tests + - name: Submit to coveralls + continue-on-error: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + coveralls --service=github + build-and-install: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Build the package + run: | + pip install --upgrade pip + pip install build + python -m build --sdist + - name: Install the package + run: | + pip install dist/*.tar.gz --index-url https://gitlab.mpcdf.mpg.de/api/v4/projects/2187/packages/pypi/simple ruff-linting: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..f7fcff6a --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,84 @@ +# Upload python package to pypi server and github release. +# Reference: https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ + +name: Upload Python Package + +on: + release: + types: [published] + +jobs: + # publish-to-pypi: + # name: >- + # Publish distribution to PyPI + # runs-on: ubuntu-latest + # environment: + # name: pypi + # url: https://pypi.org/p/nomad-schema-plugin-example + # permissions: + # id-token: write # IMPORTANT: mandatory for trusted publishing + # + # steps: + # - uses: actions/checkout@v4 + # - name: Set up Python + # uses: actions/setup-python@v4 + # with: + # python-version: "3.9" + # - name: Install pypa/build + # run: >- + # python3 -m + # pip install + # build + # --user + # - name: Build a binary wheel and a source tarball + # run: python3 -m build + # - name: Publish distribution to PyPI + # uses: pypa/gh-action-pypi-publish@release/v1 + + github-release: + name: >- + Sign the Python distribution with Sigstore + and upload them to GitHub Release + runs-on: ubuntu-latest + permissions: + contents: write # IMPORTANT: mandatory for making GitHub Releases + id-token: write # IMPORTANT: mandatory for sigstore + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.9" + - name: Install pypa/build + run: >- + python3 -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Sign the dists with Sigstore + uses: sigstore/gh-action-sigstore-python@v1.2.3 + with: + inputs: >- + ./dist/*.tar.gz + ./dist/*.whl + - name: Create GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + gh release create + '${{ github.ref_name }}' + --repo '${{ github.repository }}' + --notes "" + - name: Upload artifact signatures to GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + # Upload to GitHub Release using the `gh` CLI. + # `dist/` contains the built packages, and the + # sigstore-produced signatures and certificates. + run: >- + gh release upload + '${{ github.ref_name }}' dist/** + --repo '${{ github.repository }}' diff --git a/.gitignore b/.gitignore index 68bc17f9..e818705e 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,6 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# VSCode settings +.vscode/launch.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..fbb1c885 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,24 @@ +{ + "python.defaultInterpreterPath": ".pyenv/bin/python", + "python.terminal.activateEnvInCurrentTerminal": true, + "editor.rulers": [90], + "editor.renderWhitespace": "all", + "editor.tabSize": 4, + "[javascript]": { + "editor.tabSize": 2 + }, + "files.trimTrailingWhitespace": true, + "files.watcherExclude": { + "${workspaceFolder}/.pyenv/**": true + }, + "files.exclude": { + "\"**/*.pyc\": {\"when\": \"$(basename).py\"}": true, + "**/__pycache__": true, + "**/node_modules": true + }, + "python.testing.pytestPath": "pytest", + "python.testing.pytestArgs": ["tests"], + "python.testing.unittestEnabled": false, + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.formatOnSave": true +} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..819e745f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest + +RUN pip install build + +COPY \ + src/nomad_simulations \ + tests \ + README.md \ + LICENSE \ + pyproject.toml \ + . + +RUN python -m build --sdist + +RUN pip install dist/nomad-simulations-*.tar.gz diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..7281e592 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +recursive-include * nomad_plugin.yaml \ No newline at end of file diff --git a/README.md b/README.md index 4c17c20f..0a6b8804 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,96 @@ +![](https://github.com/nomad-coe/nomad-schema-plugin-simulation-data/actions/workflows/actions.yml/badge.svg) +![](https://coveralls.io/repos/github/nomad-coe/nomad-schema-plugin-simulation-data/badge.svg?branch=main) +[![NOMAD](https://img.shields.io/badge/Open%20NOMAD-lightgray?logo=data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDI3LjUuMCwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPgo8c3ZnIHZlcnNpb249IjEuMSIgaWQ9IkxheWVyXzEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIHg9IjBweCIgeT0iMHB4IgoJIHZpZXdCb3g9IjAgMCAxNTAwIDE1MDAiIHN0eWxlPSJlbmFibGUtYmFja2dyb3VuZDpuZXcgMCAwIDE1MDAgMTUwMDsiIHhtbDpzcGFjZT0icHJlc2VydmUiPgo8c3R5bGUgdHlwZT0idGV4dC9jc3MiPgoJLnN0MHtmaWxsOiMxOTJFODY7c3Ryb2tlOiMxOTJFODY7c3Ryb2tlLXdpZHRoOjE0MS4zMjI3O3N0cm9rZS1taXRlcmxpbWl0OjEwO30KCS5zdDF7ZmlsbDojMkE0Q0RGO3N0cm9rZTojMkE0Q0RGO3N0cm9rZS13aWR0aDoxNDEuMzIyNztzdHJva2UtbWl0ZXJsaW1pdDoxMDt9Cjwvc3R5bGU+CjxwYXRoIGNsYXNzPSJzdDAiIGQ9Ik0xMTM2LjQsNjM2LjVjMTUwLjgsMCwyNzMuMS0xMjEuOSwyNzMuMS0yNzIuMlMxMjg3LjIsOTIuMSwxMTM2LjQsOTIuMWMtMTUwLjgsMC0yNzMuMSwxMjEuOS0yNzMuMSwyNzIuMgoJUzk4NS42LDYzNi41LDExMzYuNCw2MzYuNXoiLz4KPHBhdGggY2xhc3M9InN0MSIgZD0iTTEzMjksOTQ2Yy0xMDYuNC0xMDYtMjc4LjgtMTA2LTM4Ni4xLDBjLTk5LjYsOTkuMy0yNTguNywxMDYtMzY1LjEsMTguMWMtNi43LTcuNi0xMy40LTE2LjItMjEuMS0yMy45CgljLTEwNi40LTEwNi0xMDYuNC0yNzgsMC0zODQuOWMxMDYuNC0xMDYsMTA2LjQtMjc4LDAtMzg0LjlzLTI3OC44LTEwNi0zODYuMSwwYy0xMDcuMywxMDYtMTA2LjQsMjc4LDAsMzg0LjkKCWMxMDYuNCwxMDYsMTA2LjQsMjc4LDAsMzg0LjljLTYzLjIsNjMtODkuMSwxNTAtNzYuNywyMzIuMWM3LjcsNTcuMywzMy41LDExMy43LDc3LjYsMTU3LjZjMTA2LjQsMTA2LDI3OC44LDEwNiwzODYuMSwwCgljMTA2LjQtMTA2LDI3OC44LTEwNiwzODYuMSwwYzEwNi40LDEwNiwyNzguOCwxMDYsMzg2LjEsMEMxNDM1LjQsMTIyNCwxNDM1LjQsMTA1MiwxMzI5LDk0NnoiLz4KPC9zdmc+Cg==)](https://nomad-lab.eu/prod/v1/staging/gui/) + # NOMAD's Simulations Schema Plugin +This is a plugin for [NOMAD](https://nomad-lab.eu) which contains the base sections schema definitions for materials science simulations.. + + + + +## Getting started + + +### Install the dependencies + +Clone the project and in the workspace folder, create a virtual environment (note this project uses Python 3.9): +```sh +git clone https://github.com/nomad-coe/nomad-schema-plugin-simulation-data.git +cd nomad-schema-plugin-simulation-data +python3.9 -m venv .pyenv +. .pyenv/bin/activate +``` + +Install the `nomad-lab` package: +```sh +pip install --upgrade pip +pip install '.[dev]' --index-url https://gitlab.mpcdf.mpg.de/api/v4/projects/2187/packages/pypi/simple +``` + +**Note!** +Until we have an official pypi NOMAD release with the plugins functionality. Make +sure to include NOMAD's internal package registry (via `--index-url` in the above command). + + +### Run the tests + +You can the unit testing using the `pytest` package: + +```sh +python -m pytest -sv +``` + +We recommend to install the `coverage` and `coveralls` packages for a more comprehensive output of the testing: +```sh +pip install coverage coveralls +python -m coverage run -m pytest -sv +``` + +## Development + +The plugin is still under development. If you would like to contribute, install the package in editable mode (with the added `-e` flag) with the development dependencies: + +```sh +pip install -e .[dev] --index-url https://gitlab.mpcdf.mpg.de/api/v4/projects/2187/packages/pypi/simple +``` + + +### Setting up plugin on your local installation +Read the [NOMAD plugin documentation](https://nomad-lab.eu/prod/v1/staging/docs/howto/oasis/plugins_install.html) for all details on how to deploy the plugin on your NOMAD instance. + +You need to modify the ```src/nomad_simulations/nomad_plugin.yaml``` to define the plugin adding the following content: +```yaml +plugin_type: schema +name: schemas/nomad_simulations +description: | + This is a collection of NOMAD schemas for simulation data. +``` + +and define the ```nomad.yaml``` configuration file of your NOMAD instance in the root folder with the following content: +```yaml +plugins: + include: 'schema/nomad_simulations' + options: + schema/nomad_simulations: + python_package: nomad_simulations +``` + +You also need to add the package folder to the `PYTHONPATH` of the Python environment of your local NOMAD installation. This can be done by specifying the relative path to this repository. Either run the following command every time you start a new terminal for running the appworker, or add it to your virtual environment in `/.pyenv/bin/activate` file: +```sh +export PYTHONPATH="$PYTHONPATH:" +``` + +If you are working in this repository, you just need to activate the environment to start working using the ```nomad-nomad_simulations``` package. + +### Run linting and auto-formatting + +```sh +ruff check . +``` +```sh +ruff format . --check +``` +Ruff auto-formatting is also a part of the GitHub workflow actions. Make sure that before you make a Pull Request, `ruff format . --check` runs in your local without any errors otherwise the workflow action will fail. -This is a plugin for [NOMAD](https://nomad-lab.eu/nomad-lab/) which contains the base sections schema definitions for materials science simulations. +Alternatively, if you are using VSCode as your IDE, we added the settings configuration file, `.vscode/settings.json`, such that it performs `ruff format` whenever you save progress in a file. - diff --git a/nomad.yaml b/nomad.yaml new file mode 100644 index 00000000..a2becb98 --- /dev/null +++ b/nomad.yaml @@ -0,0 +1,12 @@ +normalize: + normalizers: + include: + - MetainfoNormalizer +plugins: + # We only include our schema here. Without the explicit include, all plugins will be + # loaded. Many build in plugins require more dependencies. Install nomad-lab[parsing] + # to make all default plugins work. + include: 'schema/nomad_simulations' + options: + schema/nomad_simulations: + python_package: nomad_simulations \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index b5936180..931c3ecf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,21 +1,31 @@ [build-system] -requires = ["setuptools"] +requires = [ + "setuptools>=61.0.0", + "setuptools-scm>=8.0", +] build-backend = "setuptools.build_meta" [project] name = 'nomad-schema-plugin-simulation-data' -version = '1.0' -description = 'Simulation data schema plugin for NOMAD.' +dynamic = ["version"] +description = 'A NOMAD plugin for FAIR schemas for simulation data.' readme = "README.md" -authors = [{ name = "The NOMAD Authors" }] +authors = [ + { name = "Jose M. Pizarro", email = "jose.pizarro@physik.hu-berlin.de" }, + { name = "Nathan Daelman", email = "ndaelman@physik.hu-berlin.de" }, + { name = "Joseph F. Rudzinski", email = "joseph.rudzinski@physik.hu-berlin.de" } +] license = { text = "Apache-2.0" } +requires-python = ">=3.9" dependencies = [ "nomad-lab>=1.2.0", 'matid>=2.0.0.dev2' ] [project.urls] -homepage = "https://github.com/nomad-coe/nomad-schema-plugin-simulation-data" +"Homepage" = "https://github.com/nomad-coe/nomad-schema-plugin-simulation-data" +"Bug Tracker" = "https://github.com/nomad-coe/nomad-schema-plugin-simulation-data/issues" +"Documentation" = "https://nomad-coe.github.io/nomad-schema-plugin-simulation-data/" [project.optional-dependencies] dev = [ @@ -23,11 +33,12 @@ dev = [ 'pytest==3.10.0', 'pytest-timeout==1.4.2', 'pytest-cov==2.7.1', - 'ruff==0.1.8' + 'ruff==0.1.8', + "structlog==22.3.0", ] [tool.ruff] -include = ["simulationdataschema/*.py", "tests/*.py"] +include = ["src/nomad_simulations/*.py", "tests/*.py"] lint.select = [ "E", # pycodestyle "W", # pycodestyle @@ -68,7 +79,7 @@ skip-magic-trailing-comma = false line-ending = "auto" [tool.setuptools.packages.find] -include = ["simulationdataschema*"] +where = ["src"] [tool.setuptools.package-data] -simulationdataschema = ["*/nomad_plugin.yaml"] +nomad_simulations = ["*/nomad_plugin.yaml"] diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..6b40b52b --- /dev/null +++ b/setup.py @@ -0,0 +1,4 @@ +from setuptools import setup + +if __name__ == '__main__': + setup() diff --git a/simulationdataschema/nomad_plugin.yaml b/simulationdataschema/nomad_plugin.yaml deleted file mode 100644 index 6ca7a082..00000000 --- a/simulationdataschema/nomad_plugin.yaml +++ /dev/null @@ -1,4 +0,0 @@ -plugin_type: schema -name: simulationdataschema -description: | - This is a collection of schemas for simulation data. diff --git a/simulationdataschema/__init__.py b/src/nomad_simulations/__init__.py similarity index 100% rename from simulationdataschema/__init__.py rename to src/nomad_simulations/__init__.py diff --git a/simulationdataschema/atoms_state.py b/src/nomad_simulations/atoms_state.py similarity index 100% rename from simulationdataschema/atoms_state.py rename to src/nomad_simulations/atoms_state.py diff --git a/simulationdataschema/common.py b/src/nomad_simulations/common.py similarity index 100% rename from simulationdataschema/common.py rename to src/nomad_simulations/common.py diff --git a/simulationdataschema/general.py b/src/nomad_simulations/general.py similarity index 100% rename from simulationdataschema/general.py rename to src/nomad_simulations/general.py diff --git a/simulationdataschema/model_method.py b/src/nomad_simulations/model_method.py similarity index 100% rename from simulationdataschema/model_method.py rename to src/nomad_simulations/model_method.py diff --git a/simulationdataschema/model_system.py b/src/nomad_simulations/model_system.py similarity index 100% rename from simulationdataschema/model_system.py rename to src/nomad_simulations/model_system.py diff --git a/src/nomad_simulations/nomad_plugin.yaml b/src/nomad_simulations/nomad_plugin.yaml new file mode 100644 index 00000000..07cb8346 --- /dev/null +++ b/src/nomad_simulations/nomad_plugin.yaml @@ -0,0 +1,4 @@ +plugin_type: schema +name: schemas/nomad_simulations +description: | + This is a collection of NOMAD schemas for simulation data. diff --git a/simulationdataschema/outputs.py b/src/nomad_simulations/outputs.py similarity index 100% rename from simulationdataschema/outputs.py rename to src/nomad_simulations/outputs.py diff --git a/simulationdataschema/utils/__init__.py b/src/nomad_simulations/utils/__init__.py similarity index 100% rename from simulationdataschema/utils/__init__.py rename to src/nomad_simulations/utils/__init__.py diff --git a/simulationdataschema/utils/utils.py b/src/nomad_simulations/utils/utils.py similarity index 100% rename from simulationdataschema/utils/utils.py rename to src/nomad_simulations/utils/utils.py