Plugin distribution as pypi package (#16)

* Moved modules to src/nomad_simulations

Fix config files with the new path

Added Dockerfile, MANIFEST.in, setup.py

Added .vscode with settings

* Fix pyproject

* Added pipeline

* Fixed nomad-lab to 1.2.0

.github/workflows/actions.yaml

@@ -16,10 +16,32 @@ jobs:
         pip install coverage coveralls
     - name: mypy
       run: |
-        python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional simulationdataschema tests
+        python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional src/nomad_simulations tests
     - name: Test with pytest
       run: |
         python -m coverage run -m pytest -sv tests
+    - name: Submit to coveralls
+      continue-on-error: true
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        coveralls --service=github
+  build-and-install:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+    - name: Build the package
+      run: |
+        pip install --upgrade pip
+        pip install build
+        python -m build --sdist
+    - name: Install the package
+      run: |
+        pip install dist/*.tar.gz --index-url https://gitlab.mpcdf.mpg.de/api/v4/projects/2187/packages/pypi/simple
   ruff-linting:
       runs-on: ubuntu-latest
       steps:

.github/workflows/publish.yml

@@ -0,0 +1,84 @@
+# Upload python package to pypi server and github release.
+# Reference: https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  # publish-to-pypi:
+  #   name: >-
+  #     Publish distribution to PyPI
+  #   runs-on: ubuntu-latest
+  #   environment:
+  #     name: pypi
+  #     url: https://pypi.org/p/nomad-schema-plugin-example
+  #   permissions:
+  #     id-token: write # IMPORTANT: mandatory for trusted publishing
+  #
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - name: Set up Python
+  #       uses: actions/setup-python@v4
+  #       with:
+  #         python-version: "3.9"
+  #     - name: Install pypa/build
+  #       run: >-
+  #         python3 -m
+  #         pip install
+  #         build
+  #         --user
+  #     - name: Build a binary wheel and a source tarball
+  #       run: python3 -m build
+  #     - name: Publish distribution  to PyPI
+  #       uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: >-
+      Sign the Python distribution with Sigstore
+      and upload them to GitHub Release
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write # IMPORTANT: mandatory for making GitHub Releases
+      id-token: write # IMPORTANT: mandatory for sigstore
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - name: Install pypa/build
+        run: >-
+          python3 -m
+          pip install
+          build
+          --user
+      - name: Build a binary wheel and a source tarball
+        run: python3 -m build
+      - name: Sign the dists with Sigstore
+        uses: sigstore/[email protected]
+        with:
+          inputs: >-
+            ./dist/*.tar.gz
+            ./dist/*.whl
+      - name: Create GitHub Release
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >-
+          gh release create
+          '${{ github.ref_name }}'
+          --repo '${{ github.repository }}'
+          --notes ""
+      - name: Upload artifact signatures to GitHub Release
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        # Upload to GitHub Release using the `gh` CLI.
+        # `dist/` contains the built packages, and the
+        # sigstore-produced signatures and certificates.
+        run: >-
+          gh release upload
+          '${{ github.ref_name }}' dist/**
+          --repo '${{ github.repository }}'

.gitignore

@@ -158,3 +158,6 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# VSCode settings
+.vscode/launch.json

.vscode/settings.json

@@ -0,0 +1,24 @@
+{
+  "python.defaultInterpreterPath": ".pyenv/bin/python",
+  "python.terminal.activateEnvInCurrentTerminal": true,
+  "editor.rulers": [90],
+  "editor.renderWhitespace": "all",
+  "editor.tabSize": 4,
+  "[javascript]": {
+    "editor.tabSize": 2
+  },
+  "files.trimTrailingWhitespace": true,
+  "files.watcherExclude": {
+    "${workspaceFolder}/.pyenv/**": true
+  },
+  "files.exclude": {
+    "\"**/*.pyc\": {\"when\": \"$(basename).py\"}": true,
+    "**/__pycache__": true,
+    "**/node_modules": true
+  },
+  "python.testing.pytestPath": "pytest",
+  "python.testing.pytestArgs": ["tests"],
+  "python.testing.unittestEnabled": false,
+  "editor.defaultFormatter": "charliermarsh.ruff",
+  "editor.formatOnSave": true
+}

Dockerfile

@@ -0,0 +1,15 @@
+FROM gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:latest
+
+RUN pip install build
+
+COPY \
+    src/nomad_simulations \
+    tests \
+    README.md \
+    LICENSE \
+    pyproject.toml \
+    .
+
+RUN python -m build --sdist
+
+RUN pip install dist/nomad-simulations-*.tar.gz

MANIFEST.in

@@ -0,0 +1 @@
+recursive-include * nomad_plugin.yaml

README.md

@@ -1,5 +1,96 @@
+![](https://github.com/nomad-coe/nomad-schema-plugin-simulation-data/actions/workflows/actions.yml/badge.svg)
+![](https://coveralls.io/repos/github/nomad-coe/nomad-schema-plugin-simulation-data/badge.svg?branch=main)
+[![NOMAD](https://img.shields.io/badge/Open%20NOMAD-lightgray?logo=data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDI3LjUuMCwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPgo8c3ZnIHZlcnNpb249IjEuMSIgaWQ9IkxheWVyXzEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgeG1sbnM6eGxpbms9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkveGxpbmsiIHg9IjBweCIgeT0iMHB4IgoJIHZpZXdCb3g9IjAgMCAxNTAwIDE1MDAiIHN0eWxlPSJlbmFibGUtYmFja2dyb3VuZDpuZXcgMCAwIDE1MDAgMTUwMDsiIHhtbDpzcGFjZT0icHJlc2VydmUiPgo8c3R5bGUgdHlwZT0idGV4dC9jc3MiPgoJLnN0MHtmaWxsOiMxOTJFODY7c3Ryb2tlOiMxOTJFODY7c3Ryb2tlLXdpZHRoOjE0MS4zMjI3O3N0cm9rZS1taXRlcmxpbWl0OjEwO30KCS5zdDF7ZmlsbDojMkE0Q0RGO3N0cm9rZTojMkE0Q0RGO3N0cm9rZS13aWR0aDoxNDEuMzIyNztzdHJva2UtbWl0ZXJsaW1pdDoxMDt9Cjwvc3R5bGU+CjxwYXRoIGNsYXNzPSJzdDAiIGQ9Ik0xMTM2LjQsNjM2LjVjMTUwLjgsMCwyNzMuMS0xMjEuOSwyNzMuMS0yNzIuMlMxMjg3LjIsOTIuMSwxMTM2LjQsOTIuMWMtMTUwLjgsMC0yNzMuMSwxMjEuOS0yNzMuMSwyNzIuMgoJUzk4NS42LDYzNi41LDExMzYuNCw2MzYuNXoiLz4KPHBhdGggY2xhc3M9InN0MSIgZD0iTTEzMjksOTQ2Yy0xMDYuNC0xMDYtMjc4LjgtMTA2LTM4Ni4xLDBjLTk5LjYsOTkuMy0yNTguNywxMDYtMzY1LjEsMTguMWMtNi43LTcuNi0xMy40LTE2LjItMjEuMS0yMy45CgljLTEwNi40LTEwNi0xMDYuNC0yNzgsMC0zODQuOWMxMDYuNC0xMDYsMTA2LjQtMjc4LDAtMzg0LjlzLTI3OC44LTEwNi0zODYuMSwwYy0xMDcuMywxMDYtMTA2LjQsMjc4LDAsMzg0LjkKCWMxMDYuNCwxMDYsMTA2LjQsMjc4LDAsMzg0LjljLTYzLjIsNjMtODkuMSwxNTAtNzYuNywyMzIuMWM3LjcsNTcuMywzMy41LDExMy43LDc3LjYsMTU3LjZjMTA2LjQsMTA2LDI3OC44LDEwNiwzODYuMSwwCgljMTA2LjQtMTA2LDI3OC44LTEwNiwzODYuMSwwYzEwNi40LDEwNiwyNzguOCwxMDYsMzg2LjEsMEMxNDM1LjQsMTIyNCwxNDM1LjQsMTA1MiwxMzI5LDk0NnoiLz4KPC9zdmc+Cg==)](https://nomad-lab.eu/prod/v1/staging/gui/)
+
 # NOMAD's Simulations Schema Plugin
+This is a plugin for [NOMAD](https://nomad-lab.eu) which contains the base sections schema definitions for materials science simulations..
+
+
+<!-- MOVE THIS TO THE DOCUMENTATION PAGE OF THIS PLUGIN --->
+
+## Getting started
+
+
+### Install the dependencies
+
+Clone the project and in the workspace folder, create a virtual environment (note this project uses Python 3.9):
+```sh
+git clone https://github.com/nomad-coe/nomad-schema-plugin-simulation-data.git
+cd nomad-schema-plugin-simulation-data
+python3.9 -m venv .pyenv
+. .pyenv/bin/activate
+```
+
+Install the `nomad-lab` package:
+```sh
+pip install --upgrade pip
+pip install '.[dev]' --index-url https://gitlab.mpcdf.mpg.de/api/v4/projects/2187/packages/pypi/simple
+```
+
+**Note!**
+Until we have an official pypi NOMAD release with the plugins functionality. Make
+sure to include NOMAD's internal package registry (via `--index-url` in the above command).
+
+
+### Run the tests
+
+You can the unit testing using the `pytest` package:
+
+```sh
+python -m pytest -sv
+```
+
+We recommend to install the `coverage` and `coveralls` packages for a more comprehensive output of the testing:
+```sh
+pip install coverage coveralls
+python -m coverage run -m pytest -sv
+```
+
+## Development
+
+The plugin is still under development. If you would like to contribute, install the package in editable mode (with the added `-e` flag) with the development dependencies:
+
+```sh
+pip install -e .[dev] --index-url https://gitlab.mpcdf.mpg.de/api/v4/projects/2187/packages/pypi/simple
+```
+
+
+### Setting up plugin on your local installation
+Read the [NOMAD plugin documentation](https://nomad-lab.eu/prod/v1/staging/docs/howto/oasis/plugins_install.html) for all details on how to deploy the plugin on your NOMAD instance.
+
+You need to modify the ```src/nomad_simulations/nomad_plugin.yaml``` to define the plugin adding the following content:
+```yaml
+plugin_type: schema
+name: schemas/nomad_simulations
+description: |
+  This is a collection of NOMAD schemas for simulation data.
+```
+
+and define the ```nomad.yaml``` configuration file of your NOMAD instance in the root folder with the following content:
+```yaml
+plugins:
+  include: 'schema/nomad_simulations'
+  options:
+    schema/nomad_simulations:
+      python_package: nomad_simulations
+```
+
+You also need to add the package folder to the `PYTHONPATH` of the Python environment of your local NOMAD installation. This can be done by specifying the relative path to this repository. Either run the following command every time you start a new terminal for running the appworker, or add it to your virtual environment in `<path-to-local-nomad-installation>/.pyenv/bin/activate` file:
+```sh
+export PYTHONPATH="$PYTHONPATH:<path-to-nomad-simulations-cloned-repo>"
+```
+
+If you are working in this repository, you just need to activate the environment to start working using the ```nomad-nomad_simulations``` package.
+
+### Run linting and auto-formatting
+
+```sh
+ruff check .
+```
+```sh
+ruff format . --check
+```
+Ruff auto-formatting is also a part of the GitHub workflow actions. Make sure that before you make a Pull Request, `ruff format . --check` runs in your local without any errors otherwise the workflow action will fail.
 
-This is a plugin for [NOMAD](https://nomad-lab.eu/nomad-lab/) which contains the base sections schema definitions for materials science simulations.
+Alternatively, if you are using VSCode as your IDE, we added the settings configuration file, `.vscode/settings.json`, such that it performs `ruff format` whenever you save progress in a file.
 
-<!--Add Getting started section to install the plugin independently of NOMAD for local and Oasis usage (see e.g. https://github.com/FAIRmat-NFDI/nomad-measurements?tab=readme-ov-file)-->

nomad.yaml

@@ -0,0 +1,12 @@
+normalize:
+  normalizers:
+    include:
+      - MetainfoNormalizer
+plugins:
+  # We only include our schema here. Without the explicit include, all plugins will be
+  # loaded. Many build in plugins require more dependencies. Install nomad-lab[parsing]
+  # to make all default plugins work.
+  include: 'schema/nomad_simulations'
+  options:
+    schema/nomad_simulations:
+      python_package: nomad_simulations

pyproject.toml

@@ -1,33 +1,44 @@
 [build-system]
-requires = ["setuptools"]
+requires = [
+    "setuptools>=61.0.0",
+    "setuptools-scm>=8.0",
+]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = 'nomad-schema-plugin-simulation-data'
-version = '1.0'
-description = 'Simulation data schema plugin for NOMAD.'
+dynamic = ["version"]
+description = 'A NOMAD plugin for FAIR schemas for simulation data.'
 readme = "README.md"
-authors = [{ name = "The NOMAD Authors" }]
+authors = [
+    { name = "Jose M. Pizarro", email = "[email protected]" },
+    { name = "Nathan Daelman", email = "[email protected]" },
+    { name = "Joseph F. Rudzinski", email = "[email protected]" }
+]
 license = { text = "Apache-2.0" }
+requires-python = ">=3.9"
 dependencies = [
     "nomad-lab>=1.2.0",
     'matid>=2.0.0.dev2'
 ]
 
 [project.urls]
-homepage = "https://github.com/nomad-coe/nomad-schema-plugin-simulation-data"
+"Homepage" = "https://github.com/nomad-coe/nomad-schema-plugin-simulation-data"
+"Bug Tracker" = "https://github.com/nomad-coe/nomad-schema-plugin-simulation-data/issues"
+"Documentation" = "https://nomad-coe.github.io/nomad-schema-plugin-simulation-data/"
 
 [project.optional-dependencies]
 dev = [
     'mypy==1.0.1',
     'pytest==3.10.0',
     'pytest-timeout==1.4.2',
     'pytest-cov==2.7.1',
-    'ruff==0.1.8'
+    'ruff==0.1.8',
+    "structlog==22.3.0",
 ]
 
 [tool.ruff]
-include = ["simulationdataschema/*.py", "tests/*.py"]
+include = ["src/nomad_simulations/*.py", "tests/*.py"]
 lint.select = [
     "E", # pycodestyle
     "W", # pycodestyle
@@ -68,7 +79,7 @@ skip-magic-trailing-comma = false
 line-ending = "auto"
 
 [tool.setuptools.packages.find]
-include = ["simulationdataschema*"]
+where = ["src"]
 
 [tool.setuptools.package-data]
-simulationdataschema = ["*/nomad_plugin.yaml"]
+nomad_simulations = ["*/nomad_plugin.yaml"]

setup.py

@@ -0,0 +1,4 @@
+from setuptools import setup
+
+if __name__ == '__main__':
+    setup()

src/nomad_simulations/nomad_plugin.yaml

@@ -0,0 +1,4 @@
+plugin_type: schema
+name: schemas/nomad_simulations
+description: |
+  This is a collection of NOMAD schemas for simulation data.