Merge pull request #939 from hed-standard/develop

Merge in preparation for new release --- not quite in final form but need to test web tools.

.codeclimate.yml

@@ -60,6 +60,7 @@ exclude_patterns:
 - "**/spec/"
 - "**/test/"
 - "**/tests/"
+- "**/spec_tests/"
 - "**/docs/"
 - "Tests/"
 - "**/vendor/"

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+per-file-ignores =
+    # Ignore unused import errors in init files
+    __init__.py: F401

.github/workflows/ci.yaml

@@ -16,10 +16,10 @@ jobs:
         run: |
           if [[ "${{ github.event_name }}" == 'push' && "${{ github.ref }}" == 'refs/heads/master' ]]; then
             # Push to master branch
-            echo 'matrix=["3.7", "3.9", "3.10", "3.11"]' >> $GITHUB_OUTPUT
+            echo 'matrix=["3.8", "3.9", "3.10", "3.11"]' >> $GITHUB_OUTPUT
           elif [[ "${{ github.event_name }}" == 'pull_request' && "${{ github.event.pull_request.base.ref }}" == 'master' ]]; then
             # PR to master branch
-            echo 'matrix=["3.7", "3.9", "3.10", "3.11"]' >> $GITHUB_OUTPUT
+            echo 'matrix=["3.8", "3.9", "3.10", "3.11"]' >> $GITHUB_OUTPUT
           else
             echo 'matrix=["3.9"]' >> $GITHUB_OUTPUT
           fi
@@ -39,11 +39,11 @@ jobs:
           submodules: true
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: ${{ env.pythonLocation }}
           key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('docs/requirements.txt') }}
@@ -64,15 +64,20 @@ jobs:
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
 
       - name: Test with unittest
+        env:
+          HED_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          HED_GITHUB_TOKEN=${{ secrets.HED_GITHUB_TOKEN }} coverage run -m unittest
+          coverage run -m unittest
 
       - name: Run spec_test coverage
-        run: HED_GITHUB_TOKEN=${{ secrets.HED_GITHUB_TOKEN }} coverage run --append -m unittest spec_tests/test_errors.py
+        env:
+          HED_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: coverage run --append -m unittest spec_tests/test_errors.py
+        continue-on-error: true
 
       - name: Archive code coverage results
         if: ${{matrix.python-version == '3.9'}}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: code-coverage-report
           path: .coverage
@@ -102,11 +107,11 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python 3.9
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.9
 
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: ${{ env.pythonLocation }}
           key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('docs/requirements.txt') }}
@@ -120,15 +125,15 @@ jobs:
           pip install -r docs/requirements.txt
 
       - name: Download a single artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: code-coverage-report
 
       - name: publish-coverages
         with:
           coverageCommand: coverage xml
           debug: true
-        uses: paambaati/codeclimate-action@v5.0.0
+        uses: paambaati/codeclimate-action@v6.0.0
         env:
           CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
 

.github/workflows/spec_tests.yaml

@@ -22,7 +22,7 @@ jobs:
           submodules: true
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -31,13 +31,30 @@ jobs:
           python -m pip install --upgrade --upgrade-strategy eager pip
           pip install -r requirements.txt
 
-      - name: Test with unittest
+      - name: Spec Error Tests
+        id: spec_error_tests
+        continue-on-error: true
         run: |
-           python -m unittest spec_tests/test_errors.py > test_results.txt
+           echo "::group::Spec Test Results"
+           python -m unittest spec_tests/test_errors.py
+           echo "::endgroup::"
 
-      - name: Upload spec test results
-        uses: actions/upload-artifact@v3
-        with:
-          name: spec-test-results
-          path: test_results.txt
+      - name: Bids Validation Test
+        id: bids_validation_test
+        continue-on-error: true
+        run: |
+           echo "::group::Bids Results"
+           python -m unittest spec_tests/validate_bids.py
+           echo "::endgroup::"
+
+      - name: Hed Cache Test
+        env:
+          HED_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        id: hed_cache_test
+        continue-on-error: true
+        run: |
+           python -m unittest spec_tests/test_hed_cache.py
 
+      - name: Fail if Tests Failed
+        if: steps.spec_error_tests.outcome == 'failure' || steps.bids_validation_test.outcome == 'failure' || steps.hed_cache_test.outcome == 'failure'
+        run: exit 1

.github/workflows/test_installer.yaml

@@ -0,0 +1,44 @@
+on:
+  push:
+    branches: ["develop"]
+  pull_request:
+    branches: ["develop"]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.11"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Create work directory
+      run: |
+        mkdir workdir
+        echo "WORKDIR=$(pwd)/workdir" >> $GITHUB_OUTPUT
+
+    - name: Create and activate virtual environment
+      run: |
+        cd $WORKDIR
+        python -m venv .venv
+        source .venv/bin/activate
+
+    - name: Install package
+      run: |
+        cd $WORKDIR
+        source .venv/bin/activate
+        python -m pip install --upgrade pip
+        pip install $GITHUB_WORKSPACE
+
+    - name: Run post-installation test
+      run: |
+        cd $WORKDIR
+        source .venv/bin/activate
+        python -c "from hed.models.hed_string import HedString; print('Import test passed.')"

.gitmodules

@@ -2,3 +2,8 @@
 	path = spec_tests/hed-specification
 	url = https://github.com/hed-standard/hed-specification/
 	branch = develop
+
+[submodule "spec_tests/hed-examples"]
+	path = spec_tests/hed-examples
+	url = https://github.com/hed-standard/hed-examples/
+	branch = develop

docs/requirements.txt

@@ -1,5 +1,6 @@
 defusedxml>=0.7.1
 inflect>=6.0.5
+jsonschema>=4.17.3
 numpy>=1.21.6
 openpyxl>=3.1.0
 pandas>=1.3.5
@@ -8,4 +9,4 @@ semantic_version>=2.10.0
 myst-parser>=1.0.0
 Sphinx>=5.2.2
 sphinx_rtd_theme>=1.0.0
-wordcloud==1.9.2
+wordcloud==1.9.3

docs/source/_static/customize_icon.css

@@ -0,0 +1,13 @@
+.icon-home:before {
+  content: '';
+  /* Other styles to adjust size, positioning, etc. */
+  display: inline-block;
+  width: 24px;
+  height: 24px;
+  background: url('images/croppedWideLogo.png') no-repeat center center;
+  background-size: contain;
+}
+
+.icon-home {
+  font-family: inherit;
+}

docs/source/_templates/custom-class-template.rst

@@ -8,34 +8,24 @@
 .. rubric:: {{ _('Methods') }}
 
 .. autosummary::
-{% for item in methods %}
-   {{ module }}.{{ objname }}.{{ item }}
+{%- for item in methods %}
+   {{ objname }}.{{ item }}
 {%- endfor %}
 
 .. rubric:: {{ _('Attributes') }}
 
 .. autosummary::
-{% for item in attributes %}
-   {{ module }}.{{ objname }}.{{ item }}
+{%- for item in attributes %}
+   {{ objname }}.{{ item }}
 {%- endfor %}
 
-.. toctree::
-   :hidden:
-
-{% for item in methods %}
-   {{ fullname }}#method-{{ item }}
-{%- endfor %}
-{% for item in attributes %}
-   {{ fullname }}#attribute-{{ item }}
-{%- endfor %}
-
-{% for item in methods %}
+{%- for item in methods %}
 .. _method-{{ item }}:
 
 .. automethod:: {{ module }}.{{ objname }}.{{ item }}
 {%- endfor %}
 
-{% for item in attributes %}
+{%- for item in attributes %}
 .. _attribute-{{ item }}:
 
 .. autoattribute:: {{ module }}.{{ objname }}.{{ item }}

docs/source/_templates/custom-module-template.rst

@@ -38,7 +38,7 @@
    .. rubric:: {{ _('Classes') }}
 
    .. autosummary::
-      :toctree:
+      :toctree: _generated_classes
       :template: custom-class-template.rst
    {% for item in classes %}
       {{ item }}

docs/source/conf.py

@@ -39,7 +39,7 @@
     "myst_parser",
     "sphinx.ext.autodoc",
     "sphinx.ext.autosummary",
-    "sphinx.ext.autosectionlabel",
+    # "sphinx.ext.autosectionlabel",
     "sphinx.ext.intersphinx",
     "sphinx.ext.coverage",
     "sphinx.ext.mathjax",
@@ -97,3 +97,7 @@
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
+
+html_css_files = [
+    'customize_icon.css',
+]

docs/source/introduction.rst

@@ -42,10 +42,6 @@ Finding help
 
     The `HED online tools <https://hedtools.org>`_ provide an easy-to-use interface that requires no programming. 
 
-:Mailing lists and forums:
-
-    * Don't hesitate to ask questions about the python hedtools on `NeuroStars
-      <https://neurostars.org/tags/hedtools>`_.
 
 :Issues and problems:
     * If you notice a bug in the python hedtools code or encounter other problems using the tools, please `open an issue`_ in the

hed/__init__.py

@@ -8,15 +8,33 @@
 from hed.models.tabular_input import TabularInput
 from hed.models.sidecar import Sidecar
 from hed.models.definition_dict import DefinitionDict
-
+from hed.models.query_handler import QueryHandler
+from hed.models.query_service import get_query_handlers, search_strings
 
 from hed.schema.hed_schema import HedSchema
 from hed.schema.hed_schema_group import HedSchemaGroup
 from hed.schema.hed_schema_io import load_schema, load_schema_version
 
+from hed.tools.bids.bids_dataset import BidsDataset
+from hed.tools.analysis.event_manager import EventManager
+from hed.tools.analysis.file_dictionary import FileDictionary
+from hed.tools.analysis.hed_tag_manager import HedTagManager
+from hed.tools.analysis.hed_type_defs import HedTypeDefs
+from hed.tools.analysis.hed_type_factors import HedTypeFactors
+from hed.tools.analysis.hed_type import HedType
+from hed.tools.analysis.hed_type_manager import HedTypeManager
+from hed.tools.analysis.hed_type_counts import HedTypeCount
+from hed.tools.analysis.key_map import KeyMap
+from hed.tools.analysis.tabular_summary import TabularSummary
+from hed.tools.analysis.temporal_event import TemporalEvent
+from hed.tools.analysis.hed_tag_manager import HedTagManager
+from hed.tools.analysis.annotation_util import (check_df_columns, extract_tags, generate_sidecar_entry, 
+    get_bids_dataset, hed_to_df, df_to_hed, merge_hed_dict, str_to_tabular, strs_to_sidecar, to_strlist)
 
-# from hed import errors, models, schema, tools, validator
-
+from hed.tools.util.hed_logger import HedLogger
+from hed.tools.util.data_util import get_new_dataframe, get_value_dict, replace_values, reorder_columns
+from hed.tools.util.io_util import check_filename, clean_filename, extract_suffix_path, get_file_list, make_path
+from hed.tools.util.io_util import get_dir_dictionary, get_file_list, get_path_components, parse_bids_filename
 
 from . import _version
 __version__ = _version.get_versions()['version']