From 390eb24ed9f5a6b357ec5c7968c782d010508d4f Mon Sep 17 00:00:00 2001 From: Marcel Zwiers Date: Tue, 14 Jan 2025 12:49:54 +0100 Subject: [PATCH] Ditch ruamel-yaml (keeps messing up due to the comment anchors) in favor of pyyaml --- bidscoin/bcoin.py | 62 ++++-------------------------- bidscoin/bids.py | 13 ++++--- bidscoin/bidseditor.py | 6 +-- docs/_static/dictionary-custom.txt | 3 +- pyproject.toml | 2 +- tests/test_bidsmapper.py | 7 ++-- tests/test_heuristics.py | 6 +-- 7 files changed, 27 insertions(+), 72 deletions(-) diff --git a/bidscoin/bcoin.py b/bidscoin/bcoin.py index e7f8ad3f..1dd40c57 100755 --- a/bidscoin/bcoin.py +++ b/bidscoin/bcoin.py @@ -20,7 +20,6 @@ from importlib.util import spec_from_file_location, module_from_spec from pathlib import Path from typing import Union -from ruamel.yaml import YAML from tqdm import tqdm from tqdm.contrib.logging import logging_redirect_tqdm from importlib.util import find_spec @@ -28,9 +27,6 @@ sys.path.append(str(Path(__file__).parents[1])) from bidscoin import templatefolder, pluginfolder, bidsmap_template, tutorialurl, trackusage, tracking, configdir, configfile, config, DEBUG -yaml = YAML() -yaml.representer.ignore_aliases = lambda *data: True # Expand aliases (https://stackoverflow.com/questions/58091449/disabling-alias-for-yaml-file-in-python) - LOGGER = logging.getLogger(__name__) @@ -266,7 +262,7 @@ def list_plugins(show: bool=False) -> tuple[list[Path], list[Path]]: def install_plugins(filenames: list[str]=()) -> None: """ - Installs template bidsmaps and plugins and adds the plugin Options and data format section to the default template bidsmap + Installs template bidsmaps and plugins :param filenames: Fullpath filenames of the and template bidsmaps plugins that need to be installed :return: Nothing @@ -276,13 +272,15 @@ def install_plugins(filenames: list[str]=()) -> None: files = [Path(file) for file in filenames if file.endswith('.yaml') or file.endswith('.py')] - # Load the default template bidsmap - with open(bidsmap_template, 'r') as stream: - template = yaml.load(stream) - # Install the template bidsmaps and plugins in their targetfolder for file in files: + # Check if we can import the plugin + module = import_plugin(file.resolve()) + if not module: + LOGGER.error(f"Plugin failure, please re-install a valid version of '{file.name}'") + continue + # Copy the file to their target folder targetfolder = templatefolder if file.suffix == '.yaml' else pluginfolder LOGGER.info(f"Installing: '{file}'") @@ -295,30 +293,12 @@ def install_plugins(filenames: list[str]=()) -> None: LOGGER.success(f"The '{file.name}' template bidsmap was successfully installed") continue - # Check if we can import the plugin - module = import_plugin(file) - if not module: - LOGGER.error(f"Plugin failure, please re-install a valid version of '{file.name}'") - continue - - # Add the Options and data format section of the plugin to the default template bidsmap - if hasattr(module, 'OPTIONS') or hasattr(module, 'BIDSMAP'): - if hasattr(module, 'OPTIONS'): - LOGGER.info(f"Adding default {file.name} bidsmap options to the {bidsmap_template.stem} template") - template['Options']['plugins'][file.stem] = module.OPTIONS - if hasattr(module, 'BIDSMAP'): - for key, value in module.BIDSMAP.items(): - LOGGER.info(f"Adding default {key} bidsmappings to the {bidsmap_template.stem} template") - template[key] = value - with open(bidsmap_template, 'w') as stream: - yaml.dump(template, stream) - LOGGER.success(f"The '{file.name}' plugin was successfully installed") def uninstall_plugins(filenames: list[str]=(), wipe: bool=False) -> None: """ - Uninstalls template bidsmaps and plugins and removes the plugin Options and data format section from the default template bidsmap + Uninstalls template bidsmaps and plugins :param filenames: Fullpath filenames of the and template bidsmaps plugins that need to be uninstalled :param wipe: Removes the plugin bidsmapping section if True @@ -329,19 +309,9 @@ def uninstall_plugins(filenames: list[str]=(), wipe: bool=False) -> None: files = [Path(file) for file in filenames if file.endswith('.yaml') or file.endswith('.py')] - # Load the default template bidsmap - with open(bidsmap_template, 'r') as stream: - template = yaml.load(stream) - # Uninstall the plugins for file in files: - # First check if we can import the plugin - if file.suffix == '.py': - module = import_plugin(pluginfolder/file.name) - else: - module = None - # Remove the file from the target folder LOGGER.info(f"Uninstalling: '{file}'") sourcefolder = templatefolder if file.suffix == '.yaml' else pluginfolder @@ -354,22 +324,6 @@ def uninstall_plugins(filenames: list[str]=(), wipe: bool=False) -> None: LOGGER.success(f"The '{file.name}' template bidsmap was successfully uninstalled") continue - # Remove the Options and data format section from the default template bidsmap - if not module: - LOGGER.warning(f"Cannot remove any {file.stem} bidsmap options from the {bidsmap_template.stem} template") - continue - if removed := hasattr(module, 'OPTIONS'): - LOGGER.info(f"Removing default {file.stem} bidsmap options from the {bidsmap_template.stem} template") - template['Options']['plugins'].pop(file.stem, None) - if wipe and hasattr(module, 'BIDSMAP'): - removed = True - for key, value in module.BIDSMAP.items(): - LOGGER.info(f"Removing default {key} bidsmappings from the {bidsmap_template.stem} template") - template.pop(key, None) - if removed: - with open(bidsmap_template, 'w') as stream: - yaml.dump(template, stream) - LOGGER.success(f"The '{file.stem}' plugin was successfully uninstalled") diff --git a/bidscoin/bids.py b/bidscoin/bids.py index 7a270ede..51509825 100644 --- a/bidscoin/bids.py +++ b/bidscoin/bids.py @@ -13,6 +13,7 @@ import pandas as pd import ast import datetime +import yaml import jsonschema import bidsschematools.schema as bst import dateutil.parser @@ -25,9 +26,6 @@ sys.path.append(str(Path(__file__).parents[1])) from bidscoin import bcoin, schemafolder, templatefolder, is_hidden, __version__ from bidscoin.plugins import EventsParser -from ruamel.yaml import YAML -yaml = YAML() -yaml.representer.ignore_aliases = lambda *data: True # Expand aliases (https://stackoverflow.com/questions/58091449/disabling-alias-for-yaml-file-in-python) # Define custom data types (replace with proper classes or TypeAlias of Python >= 3.10) Plugin = NewType('Plugin', dict[str, Any]) @@ -53,6 +51,11 @@ """The possible extensions of BIDS data files""" +class NoAliasDumper(yaml.SafeDumper): + def ignore_aliases(self, data): + return True + + class DataSource: """Reads properties, attributes and BIDS-related features to sourcefiles of a supported dataformat (e.g. DICOM or PAR)""" @@ -920,7 +923,7 @@ def __init__(self, yamlfile: Path, folder: Path=templatefolder, plugins: Iterabl if any(checks): LOGGER.info(f"Reading: {yamlfile}") with yamlfile.open('r') as stream: - bidsmap_data = yaml.load(stream) + bidsmap_data = yaml.safe_load(stream) self._data = bidsmap_data """The raw YAML data""" @@ -1082,7 +1085,7 @@ def save(self, filename: Path=None): filename.parent.mkdir(parents=True, exist_ok=True) LOGGER.info(f"Saving bidsmap in: {filename}") with filename.open('w') as stream: - yaml.dump(self._data, stream) + yaml.dump(self._data, stream, NoAliasDumper, sort_keys=False) def validate(self, level: int=1) -> bool: """ diff --git a/bidscoin/bidseditor.py b/bidscoin/bidseditor.py index d107cf83..d38a8f5e 100755 --- a/bidscoin/bidseditor.py +++ b/bidscoin/bidseditor.py @@ -933,11 +933,11 @@ def save_options(self): if yamlfile: LOGGER.info(f"Saving bidsmap options in: {yamlfile}") with open(yamlfile, 'r') as stream: - bidsmap = bids.yaml.load(stream) + bidsmap = bids.yaml.safe_load(stream) bidsmap.options = self.output_bidsmap.options bidsmap.plugins = self.output_bidsmap.plugins with open(yamlfile, 'w') as stream: - bids.yaml.dump(bidsmap, stream) + bids.yaml.safe_dump(bidsmap, stream, sort_keys=False) def sample_doubleclicked(self, item): """When source file is double-clicked in the samples_table, show the inspect- or edit-window""" @@ -1346,7 +1346,7 @@ def import_menu(self, pos): if Path(metafile).suffix == '.json': metadata = json.load(meta_fid) elif Path(metafile).suffix in ('.yaml', '.yml'): - metadata = bids.yaml.load(meta_fid) + metadata = bids.yaml.safe_load(meta_fid) else: dialect = csv.Sniffer().sniff(meta_fid.read()) meta_fid.seek(0) diff --git a/docs/_static/dictionary-custom.txt b/docs/_static/dictionary-custom.txt index 28ada1e0..28d7b275 100644 --- a/docs/_static/dictionary-custom.txt +++ b/docs/_static/dictionary-custom.txt @@ -96,6 +96,7 @@ RDP README Radboud Refactored +RunItem SBREF SDAT SPM @@ -280,9 +281,9 @@ rawmapper rebranded refactoring regex +ruamel runindex runitem -RunItem runtime sbatch screenshot diff --git a/pyproject.toml b/pyproject.toml index abf753c0..a08776f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ dependencies = ['pandas', 'numpy', 'pydicom >= 2', 'PyQt6', - 'ruamel-yaml > 0.17.2', + 'pyyaml', 'tomli >= 1.1.0 ; python_version < "3.11"', 'coloredlogs', 'tqdm >= 4.60.0', diff --git a/tests/test_bidsmapper.py b/tests/test_bidsmapper.py index f0304832..a790e44a 100644 --- a/tests/test_bidsmapper.py +++ b/tests/test_bidsmapper.py @@ -1,19 +1,18 @@ import pytest import re -import ruamel.yaml from bidscoin import bcoin, bidsmapper, bidsmap_template bcoin.setup_logging() -@pytest.mark.parametrize('subprefix', ['Doe']) -@pytest.mark.parametrize('sesprefix', ['0']) +@pytest.mark.parametrize('subprefix', ['Doe', 'Doe^', '*']) +@pytest.mark.parametrize('sesprefix', ['0', '*']) @pytest.mark.parametrize('store', [False, True]) def test_bidsmapper(raw_dicomdir, bids_dicomdir, bidsmap_dicomdir, subprefix, sesprefix, store): resubprefix = '' if subprefix=='*' else re.escape(subprefix).replace(r'\-','-') resesprefix = '' if sesprefix=='*' else re.escape(sesprefix).replace(r'\-','-') bidsmap = bidsmapper.bidsmapper(raw_dicomdir, bids_dicomdir, bidsmap_dicomdir, bidsmap_template, [], subprefix, sesprefix, unzip='', store=store, automated=True, force=True) - assert isinstance(bidsmap.dataformats[0]._data, ruamel.yaml.CommentedMap) + assert isinstance(bidsmap.dataformats[0]._data, dict) assert bidsmap.options['subprefix'] == subprefix assert bidsmap.options['sesprefix'] == sesprefix assert bidsmap.dataformat('DICOM').subject == f"<>" diff --git a/tests/test_heuristics.py b/tests/test_heuristics.py index 0f6608d8..239d8273 100644 --- a/tests/test_heuristics.py +++ b/tests/test_heuristics.py @@ -1,9 +1,7 @@ import jsonschema import json +import yaml from bidscoin import bcoin, bidscoinroot -from ruamel.yaml import YAML -yaml = YAML() -yaml.representer.ignore_aliases = lambda *data: True # Expand aliases (https://stackoverflow.com/questions/58091449/disabling-alias-for-yaml-file-in-python) bcoin.setup_logging() @@ -15,5 +13,5 @@ def test_jsonschema_validate_bidsmaps(): schema = json.load(stream) for template in (bidscoinroot/'heuristics').glob('*.yaml'): with template.open('r') as stream: - bidsmap = yaml.load(stream) + bidsmap = yaml.safe_load(stream) jsonschema.validate(bidsmap, schema)