Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/main' into hamamtsu-signal
Browse files Browse the repository at this point in the history
  • Loading branch information
jlaehne committed Oct 21, 2024
2 parents 2939ff7 + 34b55db commit e0a2636
Show file tree
Hide file tree
Showing 155 changed files with 4,010 additions and 1,427 deletions.
11 changes: 0 additions & 11 deletions .github/workflows/black.yml

This file was deleted.

8 changes: 7 additions & 1 deletion .github/workflows/docs.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
name: Documentation

on: [push, pull_request]
on:
pull_request:
push:
branches-ignore:
- 'dependabot/**'
- 'pre-commit-ci-update-config'
workflow_dispatch:

jobs:
Build:
Expand Down
10 changes: 9 additions & 1 deletion .github/workflows/package_and_test.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
name: Package & Test

on: [push, pull_request]
on:
pull_request:
push:
branches-ignore:
- 'dependabot/**'
- 'pre-commit-ci-update-config'
workflow_dispatch:

jobs:
package_and_test:
Expand All @@ -12,3 +18,5 @@ jobs:
# "github.event.pull_request.head.repo.full_name" is for "pull request" event while github.repository is for "push" event
# "github.event.pull_request.head.ref" is for "pull request" event while "github.ref_name" is for "push" event
POOCH_BASE_URL: https://github.com/${{ github.event.pull_request.head.repo.full_name || github.repository }}/raw/${{ github.event.pull_request.head.ref || github.ref_name }}/rsciio/tests/data/
# "-s" is used to show of output when downloading the test files
PYTEST_ARGS: "-n 2"
106 changes: 63 additions & 43 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,32 +8,37 @@ on:
push:
# Sequence of patterns matched against refs/tags
tags:
- 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
- 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
workflow_dispatch:

jobs:
build_wheels:
name: Build wheels on ${{ matrix.os }} ${{ matrix.CIBW_ARCHS }}
runs-on: ${{ matrix.os }}-latest
runs-on: ${{ matrix.os }}
env:
CIBW_ENVIRONMENT: POOCH_BASE_URL=https://github.com/${{ github.repository }}/raw/${{ github.ref_name }}/rsciio/tests/data/
CIBW_TEST_COMMAND: "pytest --pyargs rsciio"
CIBW_TEST_EXTRAS: "tests"
# Skip testing arm64 builds with python 3.8
CIBW_TEST_SKIP: "cp38-macosx_arm64"
# No need to build wheels for pypy because the pure python wheels can be used
# PyPy documentation recommends no to build the C extension
CIBW_SKIP: "{pp*,*-musllinux*,*win32,*-manylinux_i686}"
# CPython 3.13 not supported yet because of pint
CIBW_SKIP: "{pp*,cp313*,*-musllinux*,*win32,*-manylinux_i686}"
strategy:
fail-fast: false
matrix:
include:
- os: "ubuntu"
- os: "ubuntu-latest"
CIBW_ARCHS: "x86_64"
- os: "ubuntu"
- os: "ubuntu-latest"
CIBW_ARCHS: "aarch64"
- os: "windows"
- os: "windows-latest"
CIBW_ARCHS: "AMD64"
- os: "macos"
CIBW_ARCHS: "x86_64 universal2 arm64"
- os: "macos-13"
CIBW_ARCHS: "x86_64"
- os: "macos-14"
CIBW_ARCHS: "arm64"

steps:
- name: Set up QEMU
Expand All @@ -45,7 +50,7 @@ jobs:
- uses: actions/checkout@v4

- name: Build wheels for CPython
uses: pypa/cibuildwheel@v2.16.2
uses: pypa/cibuildwheel@v2.21.0
env:
CIBW_ARCHS: ${{ matrix.CIBW_ARCHS }}

Expand All @@ -55,63 +60,78 @@ jobs:
- uses: actions/upload-artifact@v4
with:
name: wheels
name: artifacts-${{ matrix.os }}-${{ matrix.CIBW_ARCHS }}
path: ./wheelhouse/*.whl
if-no-files-found: error

make_sdist:
name: Make SDist
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v4

- name: Build SDist
run: pipx run build --sdist
- name: Build SDist
run: pipx run build --sdist

- uses: actions/upload-artifact@v4
with:
path: dist/*.tar.gz
- name: List SDist
run: |
ls ./dist
- uses: actions/upload-artifact@v4
with:
name: artifacts-${{ matrix.os }}-sdist
path: dist/*.tar.gz

pure_python_wheel:
# Build pure python without C extention to be used by pyodide
name: Make pure python wheel
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v4

- name: Build pure python wheel
run: DISABLE_C_EXTENTIONS=1 pipx run build --wheel

- name: List SDist
run: |
ls ./dist
- name: Build pure python wheel
run: DISABLE_C_EXTENTIONS=1 pipx run build --wheel
- uses: actions/upload-artifact@v4
with:
name: artifacts-${{ matrix.os }}-pure_python
path: dist/*.whl

- uses: actions/upload-artifact@v4
with:
path: dist/*.whl
# Merge all disttribution files into the same directory
merge_artifacts:
runs-on: ubuntu-latest
needs: [ build_wheels, make_sdist, pure_python_wheel ]
steps:
- name: Merge Artifacts
uses: actions/upload-artifact/merge@v4
with:
name: artifacts
pattern: artifacts-*

upload_to_pypi:
needs: [build_wheels, make_sdist]
needs: merge_artifacts
runs-on: ubuntu-latest
permissions:
# IMPORTANT: this permission is mandatory for trusted publishing
id-token: write
steps:
- name: Download dist
uses: actions/download-artifact@v4
with:
name: artifact
path: dist

- name: Download wheels
uses: actions/download-artifact@v4
with:
name: wheels
path: dist

- name: Display structure of downloaded files
run: ls -R
working-directory: dist

- uses: pypa/gh-action-pypi-publish@release/v1
if: ${{ startsWith(github.ref, 'refs/tags/') && github.repository_owner == 'hyperspy' }}
# See https://docs.pypi.org/trusted-publishers/using-a-publisher/
- name: Download wheels
uses: actions/download-artifact@v4
with:
name: artifacts
path: dist

- name: Display structure of downloaded files
run: ls -R
working-directory: dist

- uses: pypa/gh-action-pypi-publish@release/v1
if: ${{ startsWith(github.ref, 'refs/tags/') && github.repository_owner == 'hyperspy' }}
# See https://docs.pypi.org/trusted-publishers/using-a-publisher/

create_release:
# TODO: once we are happy with the workflow
Expand All @@ -126,4 +146,4 @@ jobs:
uses: actions/checkout@v4
- name: Create Release
id: create_release
uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844
uses: softprops/action-gh-release@c062e08bd532815e2082a85e87e3ef29c3e6d191
88 changes: 71 additions & 17 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
@@ -1,41 +1,61 @@
name: Tests

on: [push, pull_request, workflow_dispatch]
on:
pull_request:
push:
branches-ignore:
- 'dependabot/**'
- 'pre-commit-ci-update-config'
workflow_dispatch:

jobs:
run_test_site:
name: ${{ matrix.os }}-py${{ matrix.PYTHON_VERSION }}${{ matrix.LABEL }}
runs-on: ${{ matrix.os }}-latest
name: ${{ matrix.os }}-${{ matrix.os_version }}-py${{ matrix.PYTHON_VERSION }}${{ matrix.LABEL }}
runs-on: ${{ matrix.os }}-${{ matrix.os_version }}
timeout-minutes: 30
env:
MPLBACKEND: agg
strategy:
fail-fast: false
matrix:
os: [ubuntu, windows, macos]
os_version: [latest]
PYTHON_VERSION: ['3.9', '3.10']
LABEL: ['']
include:
# test oldest supported version of main dependencies on python 3.8
- os: ubuntu
os_version: latest
PYTHON_VERSION: '3.8'
# Set pillow and scikit-image version to be compatible with imageio and scipy
# matplotlib needs 3.5 to support markers in hyperspy 2.0 (requires `collection.set_offset_transform`)
DEPENDENCIES: matplotlib==3.5 numpy==1.20.0 imagecodecs==2020.1.31 tifffile==2020.2.16 dask[array]==2021.3.1 numba==0.52 imageio==2.16 pillow==8.3.2 scikit-image==0.18.0
DEPENDENCIES: matplotlib==3.5 numpy==1.20.0 tifffile==2022.7.28 dask[array]==2021.5.1 distributed==2021.5.1 numba==0.52 imageio==2.16 pillow==8.3.2 scikit-image==0.18.0 python-box==6.0.0
LABEL: '-oldest'
# test minimum requirement
- os: ubuntu
os_version: latest
PYTHON_VERSION: '3.9'
LABEL: '-minimum'
- os: ubuntu
os_version: latest
PYTHON_VERSION: '3.12'
LABEL: '-minimum-without-hyperspy'
- os: ubuntu
os_version: latest
PYTHON_VERSION: '3.11'
LABEL: '-hyperspy-dev'
- os: ubuntu
os_version: latest
PYTHON_VERSION: '3.9'
LABEL: '-without-hyperspy'
- os: ubuntu
os_version: latest
PYTHON_VERSION: '3.8'
- os: ubuntu
os_version: latest
PYTHON_VERSION: '3.11'
- os: macos
os_version: '13'
PYTHON_VERSION: '3.11'

steps:
Expand All @@ -59,7 +79,19 @@ jobs:
name: Install Python
with:
python-version: ${{ matrix.PYTHON_VERSION }}


- name: Get the number of CPUs
id: cpus
run: |
import os, platform
num_cpus = os.cpu_count()
print(f"Number of CPU: {num_cpus}")
print(f"Architecture: {platform.machine()}")
output_file = os.environ["GITHUB_OUTPUT"]
with open(output_file, "a", encoding="utf-8") as output_stream:
output_stream.write(f"count={num_cpus}\n")
shell: python

- name: Set Environment Variable
shell: bash
# Set PIP_SELECTOR environment variable according to matrix.LABEL
Expand All @@ -76,35 +108,57 @@ jobs:
python --version
pip --version
- name: Install oldest supported version
if: contains(matrix.LABEL, 'oldest')
run: |
pip install ${{ matrix.DEPENDENCIES }}
- name: Install (HyperSpy dev)
- name: Install hyperspy and exspy
if: ${{ ! contains(matrix.LABEL, 'without-hyperspy') }}
# Need to install hyperspy dev until hyperspy 2.0 is released
run: |
pip install git+https://github.com/hyperspy/hyperspy.git@RELEASE_next_major
pip install hyperspy exspy
- name: Install (exspy)
if: ${{ ! contains(matrix.LABEL, '-minimum') && ! contains(matrix.LABEL, 'without-hyperspy') }}
- name: Install hyperspy and exspy (dev)
if: ${{ contains(matrix.LABEL, 'hyperspy-dev') }}
run: |
pip install git+https://github.com/hyperspy/hyperspy.git
pip install git+https://github.com/hyperspy/exspy.git
- name: Install pint and python-mrcz dev
# for numpy 2.0 support for python >= 3.9
# https://github.com/em-MRCZ/python-mrcz/pull/15
# https://github.com/hgrecco/pint/issues/1974
if: ${{ ! contains(matrix.LABEL, 'oldest') && matrix.PYTHON_VERSION != '3.8' }}
run: |
pip install git+https://github.com/ericpre/[email protected]_and_deprecation_fixes
pip install git+https://github.com/hgrecco/pint
- name: Install
shell: bash
run: |
pip install --upgrade -e .'${{ env.PIP_SELECTOR }}'
- name: Uninstall pyUSID
# remove when pyUSID supports numpy 2
if: ${{ ! contains(matrix.LABEL, 'oldest') && matrix.PYTHON_VERSION != '3.8' }}
run: |
pip uninstall -y pyUSID
- name: Install oldest supported version
if: contains(matrix.LABEL, 'oldest')
run: |
pip install ${{ matrix.DEPENDENCIES }}
- name: Install numpy 2.0
if: ${{ ! contains(matrix.LABEL, 'oldest') && matrix.PYTHON_VERSION != '3.8' }}
run: |
pip install numpy==2
- name: Pip list
run: |
pip list
- name: Run test suite
run: |
pytest --pyargs rsciio --reruns 3 -n 2 --cov=. --cov-report=xml
pytest --pyargs rsciio --reruns 3 -n ${{ steps.cpus.outputs.count }} --cov=. --cov-report=xml
- name: Upload coverage to Codecov
if: ${{ always() }}
uses: codecov/codecov-action@v3
uses: codecov/codecov-action@v4
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
12 changes: 8 additions & 4 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
repos:
- repo: https://github.com/psf/black
# Version can be updated by running "pre-commit autoupdate"
rev: 23.12.0
- repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff version.
rev: v0.6.8
hooks:
- id: black
# Run the linter.
- id: ruff
args: [ --fix ]
# Run the formatter.
- id: ruff-format
- repo: local
hooks:
- id: registry
Expand Down
Loading

0 comments on commit e0a2636

Please sign in to comment.