Merge branch 'dev' into docs/ext_autosummary

.git_archival.txt

@@ -0,0 +1,4 @@
+node: $Format:%H$
+node-date: $Format:%cI$
+describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$
+ref-names: $Format:%D$

.gitattributes

@@ -1 +1 @@
-src/hdmf/_version.py export-subst
+.git_archival.txt export-subst

.github/CODE_OF_CONDUCT.md

@@ -44,5 +44,3 @@ This Code of Conduct is adapted from the [Contributor Covenant][homepage], versi
 
 [homepage]: http://contributor-covenant.org
 [version]: http://contributor-covenant.org/version/1/4/
-
-

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -24,7 +24,7 @@ body:
         Please copy and paste the code you were trying to run that caused the error.
 
         Feel free to include as little or as much as you think is relevant. This section will be automatically formatted into code, so no need for backticks.
-      render: shell
+      render: python
     validations:
       required: true
   - type: textarea
@@ -35,16 +35,14 @@ body:
         Please copy and paste the full traceback produced by the error.
 
         This section will be automatically formatted into code, so no need for backticks.
-      render: shell
-    validations:
-      required: true
+      render: pytb
   - type: dropdown
     id: os
     attributes:
       label: Operating System
       options:
-        - Windows
         - macOS
+        - Windows
         - Linux
     validations:
       required: true
@@ -62,9 +60,11 @@ body:
     attributes:
       label: Python Version
       options:
-        - 3.7
-        - 3.8
-        - 3.9
+        - "3.12"
+        - "3.11"
+        - "3.10"
+        - "3.9"
+        - "3.8"
     validations:
       required: true
   - type: textarea
@@ -78,15 +78,11 @@ body:
 
         You can attach images or log files by clicking this area to highlight it and then dragging files in.
         If GitHub upload is not working, you can also copy and paste the output into this section.
-  - type: checkboxes
-    id: terms
+  - type: markdown
     attributes:
-      label: Code of Conduct
-      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/hdmf-dev/hdmf/blob/dev/.github/CODE_OF_CONDUCT.md)
-      options:
-        - label: I agree to follow this project's [Code of Conduct](https://github.com/hdmf-dev/hdmf/blob/dev/.github/CODE_OF_CONDUCT.md)
-          required: true
-        - label: Have you checked the [Contributing](https://github.com/hdmf-dev/hdmf/blob/dev/docs/CONTRIBUTING.rst) document?
-          required: true
-        - label: Have you ensured this bug was not already [reported](https://github.com/hdmf-dev/hdmf/issues)?
-          required: true
+      value: |
+        By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/hdmf-dev/hdmf/blob/dev/.github/CODE_OF_CONDUCT.md).
+
+        Before submitting this issue, please review the [Contributing Guide](https://github.com/hdmf-dev/hdmf/blob/dev/docs/CONTRIBUTING.rst).
+
+        Please also ensure that this issue has not already been [reported](https://github.com/hdmf-dev/hdmf/issues). Thank you!

.github/ISSUE_TEMPLATE/documentation.yml

@@ -32,15 +32,11 @@ body:
         - No.
     validations:
       required: true
-  - type: checkboxes
-    id: terms
+  - type: markdown
     attributes:
-      label: Code of Conduct
-      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/hdmf-dev/hdmf/blob/dev/.github/CODE_OF_CONDUCT.md)
-      options:
-        - label: I agree to follow this project's [Code of Conduct](https://github.com/hdmf-dev/hdmf/blob/dev/.github/CODE_OF_CONDUCT.md)
-          required: true
-        - label: Have you checked the [Contributing](https://github.com/hdmf-dev/hdmf/blob/dev/docs/CONTRIBUTING.rst) document?
-          required: true
-        - label: Have you ensured this change was not already [requested](https://github.com/hdmf-dev/hdmf/issues)?
-          required: true
+      value: |
+        By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/hdmf-dev/hdmf/blob/dev/.github/CODE_OF_CONDUCT.md).
+
+        Before submitting this issue, please review the [Contributing Guide](https://github.com/hdmf-dev/hdmf/blob/dev/docs/CONTRIBUTING.rst).
+
+        Please also ensure that this issue has not already been [reported](https://github.com/hdmf-dev/hdmf/issues). Thank you!

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -19,13 +19,10 @@ body:
         What are you trying to achieve with **HDMF**?
 
         Is this a more convenient way to do something that is already possible, or is a workaround currently unfeasible?
+
+        If the change is related to a problem, please provide a clear and concise description of the problem.
     validations:
       required: true
-  - type: textarea
-    id: problem
-    attributes:
-      label: Is your feature request related to a problem?
-      description: A clear and concise description of what the problem is.
   - type: textarea
     id: solution
     attributes:
@@ -50,15 +47,11 @@ body:
         - No.
     validations:
       required: true
-  - type: checkboxes
-    id: terms
+  - type: markdown
     attributes:
-      label: Code of Conduct
-      description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/hdmf-dev/hdmf/blob/dev/.github/CODE_OF_CONDUCT.md)
-      options:
-        - label: I agree to follow this project's [Code of Conduct](https://github.com/hdmf-dev/hdmf/blob/dev/.github/CODE_OF_CONDUCT.md)
-          required: true
-        - label: Have you checked the [Contributing](https://github.com/hdmf-dev/hdmf/blob/dev/docs/CONTRIBUTING.rst) document?
-          required: true
-        - label: Have you ensured this change was not already [requested](https://github.com/hdmf-dev/hdmf/issues)?
-          required: true
+      value: |
+        By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/hdmf-dev/hdmf/blob/dev/.github/CODE_OF_CONDUCT.md).
+
+        Before submitting this issue, please review the [Contributing Guide](https://github.com/hdmf-dev/hdmf/blob/dev/docs/CONTRIBUTING.rst).
+
+        Please also ensure that this issue has not already been [reported](https://github.com/hdmf-dev/hdmf/issues). Thank you!

.github/PULL_REQUEST_TEMPLATE/release.md

@@ -1,28 +1,38 @@
 Prepare for release of HDMF [version]
 
 ### Before merging:
+- [ ] Make sure all PRs to be included in this release have been merged to `dev`.
 - [ ] Major and minor releases: Update package versions in `requirements.txt`, `requirements-dev.txt`,
-  `requirements-doc.txt`, `requirements-min.txt`, `setup.py` as needed
-  See https://requires.io/github/hdmf-dev/hdmf/requirements/?branch=dev
+  `requirements-doc.txt`, `requirements-opt.txt`, and `environment-ros3.yml` to the latest versions,
+  and update dependency ranges in `pyproject.toml` and minimums in `requirements-min.txt` as needed.
+  Run `pip install pur && pur -r requirements-dev.txt -r requirements.txt -r requirements-opt.txt`
+  and manually update `environment-ros3.yml`.
 - [ ] Check legal file dates and information in `Legal.txt`, `license.txt`, `README.rst`, `docs/source/conf.py`,
   and any other locations as needed
-- [ ] Update `setup.py` as needed
+- [ ] Update `pyproject.toml` as needed
 - [ ] Update `README.rst` as needed
-- [ ] Update `src/hdmf/common/hdmf-common-schema` submodule as needed. Check the version number and commit SHA manually
+- [ ] Update `src/hdmf/common/hdmf-common-schema` submodule as needed. Check the version number and commit SHA
+  manually. Make sure we are using the latest release and not the latest commit on the `main` branch.
 - [ ] Update changelog (set release date) in `CHANGELOG.md` and any other docs as needed
 - [ ] Run tests locally including gallery tests, and inspect all warnings and outputs
-  (`python test.py -v -u -e > out.txt 2>&1`)
+  (`pytest && python test_gallery.py`). Try to remove all warnings.
 - [ ] Run PyNWB tests locally including gallery and validation tests, and inspect all warnings and outputs
-  (`python test.py -v > out.txt 2>&1`)
-- [ ] Test docs locally (`make clean`, `make html`)
-- [ ] Push changes to this PR and make sure all PRs to be included in this release have been merged
-- [ ] Check that the readthedocs build for this PR succeeds (build latest to pull the new branch, then activate and
-  build docs for new branch): https://readthedocs.org/projects/hdmf/builds/
+  (`cd pynwb; git checkout dev; git pull; python test.py -v > out.txt 2>&1`)
+- [ ] Run HDMF-Zarr tests locally including gallery and validation tests, and inspect all warnings and outputs
+  (`cd hdmf-zarr; git checkout dev; git pull; pytest && python test_gallery.py`)
+- [ ] Test docs locally and inspect all warnings and outputs `cd docs; make clean && make html`
+- [ ] After pushing this branch to GitHub, manually trigger the "Run all tests" GitHub Actions workflow on this
+  branch by going to https://github.com/hdmf-dev/hdmf/actions/workflows/run_all_tests.yml, selecting
+  "Run workflow" on the right, selecting this branch, and clicking "Run workflow". Make sure all tests pass.
+- [ ] Check that the readthedocs build for this PR succeeds (see the PR check)
 
 ### After merging:
 1. Create release by following steps in `docs/source/make_a_release.rst` or use alias `git pypi-release [tag]` if set up
 2. After the CI bot creates the new release (wait ~10 min), update the release notes on the
    [GitHub releases page](https://github.com/hdmf-dev/hdmf/releases) with the changelog
-3. Check that the readthedocs "latest" and "stable" builds run and succeed
-4. Update [conda-forge/hdmf-feedstock](https://github.com/conda-forge/hdmf-feedstock) with the latest version number
-   and SHA256 retrieved from PyPI > HDMF > Download Files > View hashes for the `.tar.gz` file. Re-render as needed
+3. Check that the readthedocs "stable" build runs and succeeds
+4. Either monitor [conda-forge/hdmf-feedstock](https://github.com/conda-forge/hdmf-feedstock) for the
+   regro-cf-autotick-bot bot to create a PR updating the version of HDMF to the latest PyPI release, usually within
+   24 hours of release, or manually create a PR updating `recipe/meta.yaml` with the latest version number
+   and SHA256 retrieved from PyPI > HDMF > Download Files > View hashes for the `.tar.gz` file. Re-render and update
+   dependencies as needed.

.github/dependabot.yml

@@ -0,0 +1,18 @@
+version: 2
+updates:
+  # disable checking python requirements files because there are too
+  # many updates and dependabot will not ignore requirements-min.txt
+  # until https://github.com/dependabot/dependabot-core/issues/2883 is resolved
+  # workaround is to continue updating these files manually
+
+  # - package-ecosystem: "pip"
+  #   directory: "/"
+  #   schedule:
+  #     # Check for updates to requirements files and pyproject.toml every week
+  #     interval: "weekly"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      # Check for updates to GitHub Actions every week
+      interval: "weekly"

.github/pull_request_template.md

@@ -9,9 +9,7 @@ Show how to reproduce the new behavior (can be a bug fix or a new feature)
 
 ## Checklist
 
-- [ ] Did you update CHANGELOG.md with your changes?
-- [ ] Have you checked our [Contributing](https://github.com/hdmf-dev/hdmf/blob/dev/docs/CONTRIBUTING.rst) document?
-- [ ] Have you ensured the PR clearly describes the problem and the solution?
-- [ ] Is your contribution compliant with our coding style? This can be checked running `flake8` from the source directory.
-- [ ] Have you checked to ensure that there aren't other open [Pull Requests](https://github.com/hdmf-dev/hdmf/pulls) for the same change?
-- [ ] Have you included the relevant issue number using "Fix #XXX" notation where XXX is the issue number? By including "Fix #XXX" you allow GitHub to close issue #XXX when the PR is merged.
+- [ ] Did you update `CHANGELOG.md` with your changes?
+- [ ] Does the PR clearly describe the problem and the solution?
+- [ ] Have you reviewed our [Contributing Guide](https://github.com/hdmf-dev/hdmf/blob/dev/docs/CONTRIBUTING.rst)?
+- [ ] Does the PR use "Fix #XXX" notation to tell GitHub to close the relevant issue numbered XXX when the PR is merged?

.github/workflows/check_sphinx_links.yml

@@ -0,0 +1,33 @@
+name: Check Sphinx links
+on:
+  pull_request:
+  schedule:
+    - cron: '0 5 * * *'  # once per day at midnight ET
+  workflow_dispatch:
+
+jobs:
+  check-sphinx-links:
+    runs-on: ubuntu-latest
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+    steps:
+      - name: Checkout repo with submodules
+        uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+          fetch-depth: 0  # tags are required to determine the version
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'  # TODO update to 3.12 when optional reqs (e.g., oaklib) support 3.12
+
+      - name: Install Sphinx dependencies and package
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements-doc.txt -r requirements-opt.txt
+          python -m pip install .
+
+      - name: Check Sphinx internal and external links
+        run: sphinx-build -W -b linkcheck ./docs/source ./test_build

.github/workflows/codespell.yml

@@ -0,0 +1,14 @@
+name: Codespell
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  codespell:
+    name: Check for spelling errors
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Codespell
+        uses: codespell-project/actions-codespell@v2

.github/workflows/deploy_release.yml

@@ -1,24 +1,24 @@
 name: Deploy release
 on:
   push:
-    tags:  # run only on new tags that follow semver
-      - '/^[0-9]+(\.[0-9]+)?(\.[0-9]+)?$/'
+    tags:  # run only on new tags that follow semver MAJOR.MINOR.PATCH
+      - '[0-9]+.[0-9]+.[0-9]+'
 
 jobs:
   deploy-release:
     name: Deploy release from tag
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo with submodules
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: 'recursive'
           fetch-depth: 0  # tags are required for versioneer to determine the version
 
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.12'
 
       - name: Install build dependencies
         run: |
@@ -28,28 +28,31 @@ jobs:
 
       - name: Run tox tests
         run: |
-          tox -e py310-upgraded
+          tox -e py312-upgraded
 
       - name: Build wheel and source distribution
         run: |
-          tox -e build-py310-upgraded
+          tox -e build
           ls -1 dist
 
       - name: Test installation from a wheel
         run: |
-          tox -e wheelinstall --recreate --installpkg dist/*-none-any.whl
+          tox -e wheelinstall --installpkg dist/*-none-any.whl
 
-      - name: Upload wheel and source distributions to PyPI  # TODO remove the --repository flag after testing
+      - name: Test installation from a source distribution
+        run: |
+          tox -e wheelinstall --installpkg dist/*.tar.gz
+
+      - name: Upload wheel and source distributions to PyPI
         run: |
           python -m pip install twine
           ls -1 dist
-          twine upload --repository testpypi -u ${{ secrets.BOT_PYPI_USER }} -p ${{ secrets.BOT_PYPI_PASSWORD }} --skip-existing dist/*
+          # twine upload --repository-url https://test.pypi.org/legacy/ -u ${{ secrets.BOT_PYPI_USER }} -p ${{ secrets.BOT_TEST_PYPI_PASSWORD }} --skip-existing dist/*
+          twine upload -u ${{ secrets.BOT_PYPI_USER }} -p ${{ secrets.BOT_PYPI_PASSWORD }} --skip-existing dist/*
 
       - name: Publish wheel and source distributions as a GitHub release
         run: |
-          # use click<8 until https://github.com/j0057/github-release/issues/62 is resolved
-          python -m pip install "click<8" githubrelease
-          githubrelease release hdmf-dev/hdmf \
+          python -m pip install "githubrelease>=1.5.9"
+          githubrelease --github-token ${{ secrets.BOT_GITHUB_TOKEN }} release hdmf-dev/hdmf \
             create ${{ github.ref_name }} --name ${{ github.ref_name }} \
-            --github-token ${{ secrets.BOT_GITHUB_TOKEN }} \
             --publish dist/*