diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4ef239e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,192 @@ +name: ci + +on: + push: + branches: + - main + - develop + tags: + - "*" + pull_request: + branches: + - main + - develop + pull_request_target: + types: [labeled] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash -l {0} + +jobs: + pre-commit: + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - uses: actions/setup-python@v4 + with: + python-version: 3.x + - uses: pre-commit/action@v3.0.0 + + unit-tests: + name: unit-tests (3.10) + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - name: Install Conda environment with Micromamba + uses: mamba-org/provision-with-micromamba@v14 + with: + environment-file: tests/environment-unit-tests.yml + environment-name: DEVELOP + channels: conda-forge + cache-env: true + extra-specs: | + python=3.10 + - name: Install package + run: | + python -m pip install --no-deps -e . + - name: Run tests + run: | + make unit-tests + + type-check: + needs: [unit-tests] + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - name: Install Conda environment with Micromamba + uses: mamba-org/provision-with-micromamba@v12 + with: + environment-file: environment.yml + environment-name: DEVELOP + channels: conda-forge + cache-env: true + cache-env-key: ubuntu-latest-3.10 + extra-specs: | + python=3.10 + - name: Install package + run: | + python -m pip install --no-deps -e . + - name: Run code quality checks + run: | + echo type-check not used + + documentation: + needs: [unit-tests] + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - name: Install Conda environment with Micromamba + uses: mamba-org/provision-with-micromamba@v12 + with: + environment-file: environment.yml + environment-name: DEVELOP + channels: conda-forge + cache-env: true + cache-env-key: ubuntu-latest-3.10 + extra-specs: | + python=3.10 + - name: Install package + run: | + python -m pip install --no-deps -e . + - name: Build documentation + run: | + make docs-build + + integration-tests: + needs: [unit-tests] + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + + strategy: + matrix: + include: + - python-version: "3.10" + # extra: -minver # This will need to be uncommented and environment-minver.yml updated if we want to publish on conda + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - name: Install Conda environment with Micromamba + uses: mamba-org/provision-with-micromamba@v12 + with: + environment-file: tests/environment-unit-tests${{ matrix.extra }}.yml + environment-name: DEVELOP${{ matrix.extra }} + channels: conda-forge + cache-env: true + cache-env-key: ubuntu-latest-${{ matrix.python-version }}${{ matrix.extra }}. + extra-specs: | + python=${{matrix.python-version }} + - name: Install package + run: | + python -m pip install --no-deps -e . + - name: Run tests + run: | + make unit-tests + + distribution: + needs: [integration-tests, type-check, documentation] + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - name: Build distributions + run: | + $CONDA/bin/python -m pip install build + $CONDA/bin/python -m build + - name: Publish a Python distribution to PyPI + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + + notify: + if: always() && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + needs: + - pre-commit + - unit-tests + - type-check + - documentation + - integration-tests + - distribution + runs-on: ubuntu-latest + steps: + - name: Trigger Teams notification + uses: ecmwf-actions/notify-teams@v1 + with: + incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} + needs_context: ${{ toJSON(needs) }} diff --git a/.github/workflows/label-public-pr.yml b/.github/workflows/label-public-pr.yml new file mode 100644 index 0000000..59b2bfa --- /dev/null +++ b/.github/workflows/label-public-pr.yml @@ -0,0 +1,10 @@ +# Manage labels of pull requests that originate from forks +name: label-public-pr + +on: + pull_request_target: + types: [opened, synchronize] + +jobs: + label: + uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2 diff --git a/.github/workflows/notify-new-issue.yml b/.github/workflows/notify-new-issue.yml new file mode 100644 index 0000000..526bae2 --- /dev/null +++ b/.github/workflows/notify-new-issue.yml @@ -0,0 +1,15 @@ +name: Notify new issue + +on: + issues: + types: + - "opened" + +jobs: + notify: + runs-on: ubuntu-latest + steps: + - name: Notify new issue + uses: ecmwf-actions/notify-teams-issue@v1 + with: + incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} diff --git a/.github/workflows/notify-new-pr.yml b/.github/workflows/notify-new-pr.yml new file mode 100644 index 0000000..50a5305 --- /dev/null +++ b/.github/workflows/notify-new-pr.yml @@ -0,0 +1,15 @@ +name: Notify new PR + +on: + pull_request_target: + types: + - "opened" + +jobs: + notify: + runs-on: ubuntu-latest + steps: + - name: Notify new PR + uses: ecmwf-actions/notify-teams-pr@v1 + with: + incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..36fdbf6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,369 @@ +# setuptools-scm +version.py + +# Sphinx automatic generation of API +docs/_api/ + +# Created by https://www.toptal.com/developers/gitignore/api/python,jupyternotebooks,vim,visualstudiocode,pycharm +# Edit at https://www.toptal.com/developers/gitignore?templates=python,jupyternotebooks,vim,visualstudiocode,pycharm + +### JupyterNotebooks ### +# gitignore template for Jupyter Notebooks +# website: http://jupyter.org/ + +.ipynb_checkpoints +*/.ipynb_checkpoints/* + +# IPython +profile_default/ +ipython_config.py + +# Remove previous ipynb_checkpoints +# git rm -r .ipynb_checkpoints/ + +### PyCharm ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### PyCharm Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +# https://plugins.jetbrains.com/plugin/7973-sonarlint +.idea/**/sonarlint/ + +# SonarQube Plugin +# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin +.idea/**/sonarIssues.xml + +# Markdown Navigator plugin +# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced +.idea/**/markdown-navigator.xml +.idea/**/markdown-navigator-enh.xml +.idea/**/markdown-navigator/ + +# Cache file creation bug +# See https://youtrack.jetbrains.com/issue/JBR-2257 +.idea/$CACHE_FILE$ + +# CodeStream plugin +# https://plugins.jetbrains.com/plugin/12206-codestream +.idea/codestream.xml + +# Azure Toolkit for IntelliJ plugin +# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij +.idea/**/azureSettings.xml + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ +.benchmarks + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ +docs/examples/*.tar +docs/examples/*.zip +docs/examples/_* +docs/examples/earthkit_use_cases/*.grib + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook + +# IPython + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### VisualStudioCode ### +.vscode/ +# .vscode/* +# !.vscode/settings.json +# !.vscode/tasks.json +# !.vscode/launch.json +# !.vscode/extensions.json +# !.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +# Support for Project snippet scope +.vscode/*.code-snippets + +# Ignore code-workspaces +*.code-workspace + +notebooks/data/*/ + +# End of https://www.toptal.com/developers/gitignore/api/python,jupyternotebooks,vim,visualstudiocode,pycharm + +# Ignore cruft update files +*.rej + +# mac +.DS_Store + +# local code +_dev +_util +? +?.* +tempCodeRunnerFile* +dev/ + +# data and matrix files +.grib +*.json +*.npz diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..7874606 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,48 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-json + - id: check-yaml + - id: check-toml + # - id: check-added-large-files + - id: debug-statements + - id: mixed-line-ending +- repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort +- repo: https://github.com/psf/black + rev: 23.9.1 + hooks: + - id: black +- repo: https://github.com/keewis/blackdoc + rev: v0.3.8 + hooks: + - id: blackdoc + additional_dependencies: [black==23.3.0] +- repo: https://github.com/PyCQA/flake8 + rev: 6.1.0 + hooks: + - id: flake8 +- repo: https://github.com/executablebooks/mdformat + rev: 0.7.14 + hooks: + - id: mdformat + exclude: cruft-update-template.md +- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks + rev: v2.4.0 + hooks: + - id: pretty-format-yaml + args: [--autofix, --preserve-quotes] + - id: pretty-format-toml + args: [--autofix] + additional_dependencies: [toml-sort<0.22.0] +- repo: https://github.com/PyCQA/pydocstyle.git + rev: 6.1.1 + hooks: + - id: pydocstyle + additional_dependencies: [toml] + exclude: tests|docs diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..326f882 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,9 @@ +version: 2 +formats: [] +build: + os: "ubuntu-22.04" + tools: + python: "3.9" +python: + install: + - requirements: docs/requirements.txt diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..f327ab8 --- /dev/null +++ b/Makefile @@ -0,0 +1,37 @@ +PROJECT := earthkit-regrid +CONDA := conda +CONDAFLAGS := +COV_REPORT := html + +setup: + pre-commit install + +default: qa unit-tests type-check + +qa: + pre-commit run --all-files + +unit-tests: + python -m pytest -vv --cov=. --cov-report=$(COV_REPORT) + +# type-check: +# python -m mypy . + +conda-env-update: + $(CONDA) env update $(CONDAFLAGS) -f environment.yml + +docker-build: + docker build -t $(PROJECT) . + +docker-run: + docker run --rm -ti -v $(PWD):/srv $(PROJECT) + +template-update: + pre-commit run --all-files cruft -c .pre-commit-config-weekly.yaml + +docs-build: + cd docs && rm -fr _api && make clean && make html + +#integration-tests: +# python -m pytest -vv --cov=. --cov-report=$(COV_REPORT) tests/integration*.py +# python -m pytest -vv --doctest-glob='*.md' diff --git a/README.md b/README.md index 522b88b..fe7f299 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,33 @@ # earthkit-regrid -ECMWF python regridding toolkit. +**DISCLAIMER** -:warning: This project is BETA and will be Experimental for the forseable time. Interfaces and functionality are likely to change, and the project itself may be scrapped. DO NOT use this software in any project/software that is operational. +> This project is BETA and will be Experimental for the foreseeable future. Interfaces and functionality are likely to change, and the project itself may be scrapped. DO NOT use this software in any project/software that is operational. + +ECMWF Python regridding toolkit. + +## Documentation + +The documentation can be found at https://earthkit-regrid.readthedocs.io/. + +## License + +``` +Copyright 2022, European Centre for Medium Range Weather Forecasts. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +In applying this licence, ECMWF does not waive the privileges and immunities +granted to it by virtue of its status as an intergovernmental organisation +nor does it submit to any jurisdiction. +``` diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/.gitkeep b/docs/_static/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/_static/style.css b/docs/_static/style.css new file mode 100644 index 0000000..1a33443 --- /dev/null +++ b/docs/_static/style.css @@ -0,0 +1,48 @@ +.wy-side-nav-search { + background-color: #5d89bb; +} + +/*There is a clash between xarray notebook styles and readthedoc*/ + +.rst-content dl.xr-attrs dt { + all: revert; + font-size: 95%; + white-space: nowrap; +} + +.rst-content dl.xr-attrs dd { + font-size: 95%; +} + +.xr-wrap { + font-size: 85%; +} + +.wy-table-responsive table td, .wy-table-responsive table th { + white-space: inherit; + } + +/* +.wy-table-responsive table td, +.wy-table-responsive table th { + white-space: normal !important; + vertical-align: top !important; +} + +.wy-table-responsive { + margin-bottom: 24px; + max-width: 100%; + overflow: visible; +} */ + +/* Hide notebooks warnings */ +.nboutput .stderr { + display: none; +} + +/* +Set logo size +*/ +.wy-side-nav-search .wy-dropdown > a img.logo, .wy-side-nav-search > a img.logo { + width: 80px; +} diff --git a/docs/_templates/.gitkeep b/docs/_templates/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..d7985f7 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,62 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Import and path setup --------------------------------------------------- + +import datetime +import os +import sys + +sys.path.insert(0, os.path.abspath("./")) +sys.path.insert(0, os.path.abspath("../")) + +# -- Project information ----------------------------------------------------- + +project = "earthkit-regrid" +author = "European Centre for Medium Range Weather Forecasts" + +year = datetime.datetime.now().year +years = "2023-%s" % (year,) +copyright = "%s, European Centre for Medium-Range Weather Forecasts (ECMWF)" % (years,) + +# version = earthkit.meteo.__version__ +# release = earthkit.meteo.__version__ + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx_rtd_theme", + "nbsphinx", +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +# The suffix of source filenames. +source_suffix = ".rst" + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] +html_css_files = ["style.css"] + +# html_logo = "_static/earthkit-regrid.png" diff --git a/docs/development.rst b/docs/development.rst new file mode 100644 index 0000000..93cb46b --- /dev/null +++ b/docs/development.rst @@ -0,0 +1,57 @@ +Development +=========== + +Contributions +------------- + +The code repository is hosted on `Github`_, testing, bug reports and contributions are highly welcomed and appreciated. Feel free to fork it and submit your PRs against the **develop** branch. + +Development setup +----------------------- + +The recommended development environment is based on **conda**. + +First, clone the repository locally. You can use the following command: + +.. code-block:: shell + + git clone --branch develop git@github.com:ecmwf/earthkit-regrid.git + + +Next, enter your git repository and run the following commands: + +.. code-block:: shell + + make conda-env-update + conda activate earthkit-regrid + make setup + pip install -e . + +This will create a new conda environment called "earthkit-regrid" with all the dependencies installed into it. This setup enables the `pre-commit`_ hooks, performing a series of quality control checks on every commit. If any of these checks fails the commit will be rejected. + +Run unit tests +--------------- + +To run the test suite, you can use the following command: + +.. code-block:: shell + + pytest + + +Build documentation +------------------- + +To build the documentation locally, please install the Python dependencies first: + +.. code-block:: shell + + cd docs + pip install -r requirements.txt + make html + +To see the generated HTML documentation open the ``docs/_build/html/index.html`` file in your browser. + + +.. _`Github`: https://github.com/ecmwf/earthkit-regrid +.. _`pre-commit`: https://pre-commit.com/ diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..0ec14d0 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,26 @@ +Welcome to earthkit-regrids's documentation +====================================================== + +.. warning:: + + This project is **BETA** and will be **Experimental** for the foreseeable future. Interfaces and functionality are likely to change, and the project itself may be scrapped. **DO NOT** use this software in any project/software that is operational. + + +**earthkit-regrid** is a Python package for regridding. + +.. toctree:: + :maxdepth: 1 + :caption: Installation + + install + development + licence + + +Indices and tables +================== + +* :ref:`genindex` + +.. * :ref:`modindex` +.. * :ref:`search` diff --git a/docs/install.rst b/docs/install.rst new file mode 100644 index 0000000..1c12172 --- /dev/null +++ b/docs/install.rst @@ -0,0 +1,11 @@ +Installation +============ + +Installing earthkit-regrid +---------------------------- + +Install **earthkit-regrid** with python3 (>= 3.8) and ``pip`` as follows: + +.. code-block:: bash + + python3 -m pip install earthkit-regrid diff --git a/docs/licence.rst b/docs/licence.rst new file mode 100644 index 0000000..2ca0073 --- /dev/null +++ b/docs/licence.rst @@ -0,0 +1,18 @@ +License +======= + +Copyright 2022- European Centre for Medium-Range Weather Forecasts (ECMWF). + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +In applying this licence, ECMWF does not waive the privileges and immunities granted to it by virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction. diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..954237b --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..281da6b --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,6 @@ +# These are the requirements for readthedoc +docutils +Pygments>=2.6.1 +Sphinx +sphinx-rtd-theme +setuptools diff --git a/earthkit/regrid/__init__.py b/earthkit/regrid/__init__.py new file mode 100644 index 0000000..f670301 --- /dev/null +++ b/earthkit/regrid/__init__.py @@ -0,0 +1,102 @@ +import json +import os + +from scipy.sparse import load_npz + +from .mir import mir_cached_matrix_to_file + +try: + # NOTE: the `version.py` file must not be present in the git repository + # as it is generated by setuptools at install time + from .version import __version__ +except ImportError: # pragma: no cover + # Local copy or not installed with setuptools + __version__ = "999" + + +here = os.path.dirname(os.path.abspath(__file__)) +MATRICES = os.path.join(here, "matrices") + + +def match(a, b): + # Just a proof of concept, not the real thing + return a["grid"] == b["grid"] + + +def find_matrix(gridspec_in, gridspec_out): + with open(os.path.join(MATRICES, "index.json")) as f: + index = json.load(f) + for name, entry in index.items(): + if match(gridspec_in, entry["input"]) and match(gridspec_out, entry["output"]): + # The matrix should be hosted elsewhere + z = load_npz(os.path.join(MATRICES, entry["name"] + ".npz")) + return z, entry["output"]["shape"] + + return None, None + + +def interpolate(x, gridspec_in, gridspec_out): + z, shape = find_matrix(gridspec_in, gridspec_out) + + if z is None: + raise ValueError("No matrix found that matches the input and output gridspecs") + + # This should check for 1D (GG) and 2D (LL) matrices + x = x.reshape(-1, 1) + + x = z @ x + + return x.reshape(shape) + + +def regular_ll(entry): + return { + "grid": [entry["increments"][x] for x in ("west_east", "south_north")], + "shape": [entry["nj"], entry["ni"]], + } + + +def reduced_gg(entry): + pl = entry["pl"] + G = "O" if pl[1] - pl[0] == 4 else "N" + N = entry["N"] + + return { + "grid": f"{G}{N}", + "shape": [sum(pl)], + } + + +def make_matrix(path): + with open(path) as f: + entry = json.load(f) + + cache_file = entry.pop("cache_file") + name, _ = os.path.splitext(os.path.basename(cache_file)) + + npz_file = os.path.join(MATRICES, name + ".npz") + + mir_cached_matrix_to_file(cache_file, npz_file) + + index_file = os.path.join(MATRICES, "index.json") + if os.path.exists(index_file): + with open(index_file) as f: + index = json.load(f) + else: + index = {} + + def convert(x): + proc = globals()[x["type"]] + return proc(x) + + index[name] = dict( + name=name, + input=convert(entry["input"]), + output=convert(entry["output"]), + ) + + with open(index_file, "w") as f: + json.dump(index, f, indent=4) + + print("Written", npz_file) + print("Written", index_file) diff --git a/earthkit/regrid/matrices/.gitignore b/earthkit/regrid/matrices/.gitignore new file mode 100644 index 0000000..c2c59e4 --- /dev/null +++ b/earthkit/regrid/matrices/.gitignore @@ -0,0 +1,2 @@ +!*.json +!*.npz diff --git a/earthkit/regrid/matrices/LL-2x2-90:0:-90:358-1a7a9541e3137ad972e7860a4c87827c.npz b/earthkit/regrid/matrices/LL-2x2-90:0:-90:358-1a7a9541e3137ad972e7860a4c87827c.npz new file mode 100644 index 0000000..183f9c5 Binary files /dev/null and b/earthkit/regrid/matrices/LL-2x2-90:0:-90:358-1a7a9541e3137ad972e7860a4c87827c.npz differ diff --git a/earthkit/regrid/matrices/LL-2x2-90:0:-90:358-4de7551e303709fde2dd44435dfe76f1.npz b/earthkit/regrid/matrices/LL-2x2-90:0:-90:358-4de7551e303709fde2dd44435dfe76f1.npz new file mode 100644 index 0000000..2812b05 Binary files /dev/null and b/earthkit/regrid/matrices/LL-2x2-90:0:-90:358-4de7551e303709fde2dd44435dfe76f1.npz differ diff --git a/earthkit/regrid/matrices/index.json b/earthkit/regrid/matrices/index.json new file mode 100644 index 0000000..fde8c57 --- /dev/null +++ b/earthkit/regrid/matrices/index.json @@ -0,0 +1,44 @@ +{ + "LL-2x2-90:0:-90:358-4de7551e303709fde2dd44435dfe76f1": { + "name": "LL-2x2-90:0:-90:358-4de7551e303709fde2dd44435dfe76f1", + "input": { + "grid": [ + 1, + 1 + ], + "shape": [ + 181, + 360 + ] + }, + "output": { + "grid": [ + 2, + 2 + ], + "shape": [ + 91, + 180 + ] + } + }, + "LL-2x2-90:0:-90:358-1a7a9541e3137ad972e7860a4c87827c": { + "name": "LL-2x2-90:0:-90:358-1a7a9541e3137ad972e7860a4c87827c", + "input": { + "grid": "O1280", + "shape": [ + 6599680 + ] + }, + "output": { + "grid": [ + 2, + 2 + ], + "shape": [ + 91, + 180 + ] + } + } +} diff --git a/earthkit/regrid/mir.py b/earthkit/regrid/mir.py new file mode 100644 index 0000000..e4e8dc8 --- /dev/null +++ b/earthkit/regrid/mir.py @@ -0,0 +1,56 @@ +import numpy as np +from scipy.sparse import csr_array, save_npz + +from .stream import Stream + + +def dtype_uint(little_endian, size): + order = "<" if little_endian else ">" + return np.dtype({4: np.uint32}[size]).newbyteorder(order) + + +def dtype_float(little_endian, size): + order = "<" if little_endian else ">" + return np.dtype({4: np.float32, 8: np.float64}[size]).newbyteorder(order) + + +def mir_cached_matrix_to_array(path): + with open(path, "rb") as f: + s = Stream(f) + rows = s.read_unsigned_long() # rows + cols = s.read_unsigned_long() # cols + s.read_unsigned_long() # non-zeros, ignored + + little_endian = s.read_int() != 0 # little_endian + index_item_size = s.read_unsigned_long() # sizeof(index) + scalar_item_size = s.read_unsigned_long() # sizeof(scalar) + s.read_unsigned_long() # sizeof(size), ignored + + outer = s.read_large_blob() # outer + inner = s.read_large_blob() # inner + data = s.read_large_blob() # data + + outer = np.frombuffer( + outer, + dtype=dtype_uint(little_endian, index_item_size), + ) + + inner = np.frombuffer( + inner, + dtype=dtype_uint(little_endian, index_item_size), + ) + + data = np.frombuffer( + data, + dtype=dtype_float(little_endian, scalar_item_size), + ) + + return csr_array((data, inner, outer), shape=(rows, cols)) + + +def mir_cached_matrix_to_file(path, target): + if not target.endswith(".npz"): + raise ValueError("target must end with .npz") + + z = mir_cached_matrix_to_array(path) + save_npz(target, z) diff --git a/earthkit/regrid/stream.py b/earthkit/regrid/stream.py new file mode 100644 index 0000000..897a71e --- /dev/null +++ b/earthkit/regrid/stream.py @@ -0,0 +1,224 @@ +from struct import pack, unpack + +TAG_ZERO = 0 +TAG_START_OBJ = 1 +TAG_END_OBJ = 2 +TAG_CHAR = 3 +TAG_UNSIGNED_CHAR = 4 +TAG_INT = 5 +TAG_UNSIGNED_INT = 6 +TAG_SHORT = 7 +TAG_UNSIGNED_SHORT = 8 +TAG_LONG = 9 +TAG_UNSIGNED_LONG = 10 +TAG_LONG_LONG = 11 +TAG_UNSIGNED_LONG_LONG = 12 +TAG_FLOAT = 13 +TAG_DOUBLE = 14 +TAG_STRING = 15 +TAG_BLOB = 16 +TAG_EXCEPTION = 17 +TAG_START_REC = 18 +TAG_END_REC = 19 +TAG_EOF = 20 +TAG_LARGE_BLOB = 21 # For blobs >= 2Gb +LAST_TAG = 22 + +TAG_NAME = ( + "0", + "start of object", + "end of object", + "char", + "unsigned char", + "int", + "unsigned int", + "short", + "unsigned short", + "long", + "unsigned long", + "long long", + "unsigned long long", + "float", + "double", + "string", + "blob", + "exception", + "start of record", + "end of record", +) + + +class Stream: + def __init__(self, stream): + self.stream = stream + + def _read(self, n): + return self.stream.read(n) + + def read_tag(self, expected_tag): + tag = self._read(1) + tag = ord(tag) + + while tag == TAG_END_OBJ: + tag = self._read(1) + if tag: + tag = ord(tag) + + # TODO There is some suspicious code on the perl side for the following. + # TODO Maybe worth having a look at it sometime. + if tag != expected_tag: + if tag < len(TAG_NAME): + raise RuntimeError( + "Unexpected tag: %s, wanted %s" + % (TAG_NAME[tag], TAG_NAME[expected_tag]) + ) + else: + raise RuntimeError("Invalid tag with id: '%s'" % tag) + + return tag + + def write_tag(self, tag): + self._write(pack("b", tag), 1) + + def next_object(self): + while 1: + tag = self.stream.recv(1) + if tag == "": + return 0 + + tag = unpack("b", tag)[0] + + if tag == TAG_START_OBJ: + return 1 + else: + if tag < len(TAG_NAME): + raise RuntimeError( + "Unexpected tag: '%s', wanted '%s'" + % (TAG_NAME[tag], TAG_NAME[TAG_START_OBJ]) + ) + else: + raise RuntimeError("Invalid tag with id: '%d'" % tag) + + def write_char(self, c): + self.write_tag(TAG_CHAR) + self._write(c, 1) + + def write_unsigned_char(self, c): + self.write_tag(TAG_UNSIGNED_CHAR) + self._write(c, 1) + + def write_int(self, n): + self.write_tag(TAG_INT) + self._write(pack("!L", n), 4) + + def write_unsigned_int(self, n): + self.write_tag(TAG_UNSIGNED_INT) + self._write(pack("!L", n), 4) + + def write_long(self, n): + self.write_tag(TAG_LONG) + self._write(pack("!L", n), 4) + + def write_double(self, n): + self.write_tag(TAG_DOUBLE) + raise RuntimeError("write_doble") + + def write_unsigned_long(self, n): + self.write_tag(TAG_UNSIGNED_LONG) + self._write(pack("!L", n), 4) + + def write_long_long(self, n): + self.write_tag(TAG_LONG_LONG) + + # TODO Error handling when the number is too long. + # TODO See the perl. + + self._write(pack("!L", n), 4) + + def write_unsigned_long_long(self, n): + self.write_tag(TAG_UNSIGNED_LONG_LONG) + + # TODO Error handling when input value too big. + + self._write(pack("!L", 0), 4) + self._write(pack("!L", n), 4) + + def write_string(self, s): + self.write_tag(TAG_STRING) + n = len(s) + self._write(pack("!L", n), 4) + self._write(s, n) + + def read_char(self): + self.read_tag(TAG_CHAR) + return self._read(1) + + def read_unsigned_char(self): + self.read_tag(TAG_UNSIGNED_CHAR) + return self._read(1) + + def read_int(self): + self.read_tag(TAG_INT) + return unpack("!L", self._read(4))[0] + + def read_unsigned_int(self): + self.read_tag(TAG_UNSIGNED_INT) + return unpack("!L", self._read(4))[0] + + def read_long(self): + self.read_tag(TAG_LONG) + return unpack("!L", self._read(4))[0] + + def read_double(self): + self.read_tag(TAG_DOUBLE) + raise RuntimeError("read_double") + + def read_unsigned_long(self): + self.read_tag(TAG_UNSIGNED_LONG) + return unpack("!L", self._read(4))[0] + + def read_long_long(self): + self.read_tag(TAG_LONG_LONG) + raise RuntimeError("read_long_long") + + def _read_long_long(self): + return unpack("!Q", self._read(8))[0] + # TODO: use 'Q' instead of 'L' for 64-bit ints + x = unpack("!L", self._read(4))[0] + y = unpack("!L", self._read(4))[0] + + return (x << 32) | y + + def read_unsigned_long_long(self): + self.read_tag(TAG_UNSIGNED_LONG_LONG) + return unpack("!Q", self._read(8))[0] + + def read_string(self): + self.read_tag(TAG_STRING) + n = unpack("!L", self._read(4))[0] + return self._read(n) + + def read_large_blob(self): + self.read_tag(TAG_LARGE_BLOB) + len = unpack("!Q", self._read(8))[0] + return self._read(len) + + def start_object(self): + self.write_tag(TAG_START_OBJ) + + def end_object(self): + self.write_tag(TAG_END_OBJ) + + def read_object(self): + if not self.next_object(): + return None + klass = self.read_string() + return klass + + def write_object(self, name, ref): + self.start_object() + + self.write_string(name) + self.write_string(ref) + + self.end_object() diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..527bf9a --- /dev/null +++ b/environment.yml @@ -0,0 +1,22 @@ +name: earthkit-regrid +channels: +- conda-forge +- nodefaults +dependencies: +- pip +- scipy +- make +- mypy +- myst-parser +- pre-commit +- pydata-sphinx-theme +- pytest +- pytest-cov +- sphinx +- sphinx-autoapi +- sphinx_rtd_theme +- sphinxcontrib-apidoc +- nbformat +- nbconvert +- nbsphinx +- ipykernel diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f24cbe7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"] + +[tool.coverage.run] +branch = true + +[tool.isort] +profile = "black" + +[tool.pydocstyle] +convention = "numpy" +add_ignore = ["D1", "D200", "D205", "D400", "D401"] + +[tool.setuptools_scm] +write_to = "earthkit/regrid/version.py" +write_to_template = ''' +# Do not change! Do not track in version control! +__version__ = "{version}" +''' diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..d276e0f --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +addopts=-s --verbose +testpaths = tests diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..552c835 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,39 @@ +[metadata] +name = earthkit-regrid +license = Apache License 2.0 +description = ECMWF python regridding toolkit + Development Status :: 2 - Pre-Alpha + Intended Audience :: Science/Research + License :: OSI Approved :: Apache Software License + Operating System :: OS Independent + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Topic :: Scientific/Engineering +long_description_content_type=text/markdown +long_description = file: README.md +test_suite = tests + +[options] +packages = find_namespace: +install_requires = + scipy + +[options.packages.find] +include = earthkit.* + +[options.extras_require] +test = + pytest + pytest-cov + +[flake8] +max-line-length = 110 +extend-ignore = E203, W503 + +[mypy] +strict = False +ignore_missing_imports = True diff --git a/tests/environment-unit-tests.yml b/tests/environment-unit-tests.yml new file mode 100644 index 0000000..527bf9a --- /dev/null +++ b/tests/environment-unit-tests.yml @@ -0,0 +1,22 @@ +name: earthkit-regrid +channels: +- conda-forge +- nodefaults +dependencies: +- pip +- scipy +- make +- mypy +- myst-parser +- pre-commit +- pydata-sphinx-theme +- pytest +- pytest-cov +- sphinx +- sphinx-autoapi +- sphinx_rtd_theme +- sphinxcontrib-apidoc +- nbformat +- nbconvert +- nbsphinx +- ipykernel diff --git a/tests/test_interpolate.py b/tests/test_interpolate.py new file mode 100644 index 0000000..fa18f33 --- /dev/null +++ b/tests/test_interpolate.py @@ -0,0 +1,48 @@ +# (C) Copyright 2023 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +import os + +import numpy as np +import pytest + +from earthkit.regrid import interpolate + +PATH = os.path.dirname(__file__) + + +def file_in_testdir(filename): + return os.path.join(PATH, filename) + + +def test_regular_ll_1x1() -> None: + a = np.ones(181 * 360) + r = interpolate(a, {"grid": [1, 1]}, {"grid": [2, 2]}) + + assert r.shape == (91, 180) + assert np.isclose(r[0, 0], 1.0) + + +def test_o1280() -> None: + a = np.ones(6599680) + r = interpolate(a, {"grid": "O1280"}, {"grid": [2, 2]}) + + assert r.shape == (91, 180) + assert np.isclose(r[0, 0], 1.0) + + +def test_unsupported_input_grid() -> None: + a = np.ones(91 * 180) + with pytest.raises(ValueError): + _ = interpolate(a, {"grid": [2, 2]}, {"grid": [1, 1]}) + + +def test_unsupported_output_grid() -> None: + a = np.ones(181 * 360) + with pytest.raises(ValueError): + _ = interpolate(a, {"grid": [1, 1]}, {"grid": [5, 5]}) diff --git a/tests/test_version.py b/tests/test_version.py new file mode 100644 index 0000000..b73e2d2 --- /dev/null +++ b/tests/test_version.py @@ -0,0 +1,13 @@ +# (C) Copyright 2023 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +import earthkit.regrid + + +def test_version() -> None: + assert earthkit.regrid.__version__ != "999" diff --git a/tools/make-matrix.sh b/tools/make-matrix.sh new file mode 100644 index 0000000..ec1a298 --- /dev/null +++ b/tools/make-matrix.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -eux + +input=$(echo $1 | tr 'onf/' 'ONFx' ) +output=$(echo $2 | tr 'onf/' 'ONFx' ) +output="$input-$output" + +if [[ ! -f $input.grib ]]; then +mars<