diff --git a/docs/README.md b/docs/README.md index 8c4fd662..f9f6d193 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,4 +11,5 @@ Here are the available topics : * :microscope: [testing](/docs/testing.md) details the testing features of the project, i.e.: how is the code tested ? * :package: [ci-cd](/docs/ci-cd.md) contains the information on how continuous integration and delivery (knowned as CI/CD) is set up. * :writing_hand: [pipeline](/docs/pipelines.md) tells you all you need to know in order to write pipelines +* :compass: [core](/docs/core.md) a list of helpful functions when writing pipelines * :vertical_traffic_light: [status](/docs/status.md) contains the information on how to get the work progress status for a pipeline. diff --git a/docs/core.md b/docs/core.md new file mode 100644 index 00000000..2ea8e536 --- /dev/null +++ b/docs/core.md @@ -0,0 +1,117 @@ +# Core functions you can use to write pipelines + +Here are a few functions that could be useful for creating a pipeline with Nipype. These functions are meant to stay as unitary as possible. + +These are intended to be inserted in a nipype.Workflow inside a [nipype.Function](https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.utility.wrappers.html#function) interface, or for some of them (see associated docstring) as part of a [nipype.Workflow.connect](https://nipype.readthedocs.io/en/latest/api/generated/nipype.pipeline.engine.workflows.html#nipype.pipeline.engine.workflows.Workflow.connect) method. + +In the following example, we use the `list_intersection` function of `narps_open.core.common`, in both of the mentioned cases. + +```python +from nipype import Node, Function, Workflow +from narps_open.core.common import list_intersection + +# First case : a Function Node +intersection_node = Node(Function( + function = list_intersection, + input_names = ['list_1', 'list_2'], + output_names = ['output'] + ), name = 'intersection_node') +intersection_node.inputs.list_1 = ['001', '002', '003', '004'] +intersection_node.inputs.list_2 = ['002', '004', '005'] +print(intersection_node.run().outputs.output) # ['002', '004'] + +# Second case : inside a connect node +# We assume that there is a node_0 returning ['001', '002', '003', '004'] as `output` value +test_workflow = Workflow( + base_dir = '/path/to/base/dir', + name = 'test_workflow' + ) +test_workflow.connect([ + # node_1 will receive the evaluation of : + # list_intersection(['001', '002', '003', '004'], ['002', '004', '005']) + # as in_value + (node_0, node_1, [(('output', list_intersection, ['002', '004', '005']), 'in_value')]) + ]) +test_workflow.run() +``` + +> [!TIP] +> Use a [nipype.MapNode](https://nipype.readthedocs.io/en/latest/api/generated/nipype.pipeline.engine.nodes.html#nipype.pipeline.engine.nodes.MapNode) to run these functions on lists instead of unitary contents. E.g.: the `remove_file` function of `narps_open.core.common` only removes one file at a time, but feel free to pass a list of files using a `nipype.MapNode`. + +```python +from nipype import MapNode, Function +from narps_open.core.common import remove_file + +# Create the MapNode so that the `remove_file` function handles lists of files +remove_files_node = MapNode(Function( + function = remove_file, + input_names = ['_', 'file_name'], + output_names = [] + ), name = 'remove_files_node', iterfield = ['file_name']) + +# ... A couple of lines later, in the Worlflow definition +test_workflow = Workflow(base_dir = '/home/bclenet/dev/tests/nipype_merge/', name = 'test_workflow') +test_workflow.connect([ + # ... + # Here we assume the select_node's output `out_files` is a list of files + (select_node, remove_files_node, [('out_files', 'file_name')]) + # ... + ]) +``` + +## narps_open.core.common + +This module contains a set of functions that nearly every pipeline could use. + +* `remove_file` remove a file when it is not needed anymore (to save disk space) + +```python +from narps_open.core.common import remove_file + +# Remove the /path/to/the/image.nii.gz file +remove_file('/path/to/the/image.nii.gz') +``` + +* `elements_in_string` : return the first input parameter if it contains one element of second parameter (None otherwise). + +```python +from narps_open.core.common import elements_in_string + +# Here we test if the file 'sub-001_file.nii.gz' belongs to a group of subjects. +elements_in_string('sub-001_file.nii.gz', ['005', '006', '007']) # Returns None +elements_in_string('sub-001_file.nii.gz', ['001', '002', '003']) # Returns 'sub-001_file.nii.gz' +``` + +> [!TIP] +> This can be generalised to a group of files, using a `nipype.MapNode`! + +* `clean_list` : remove elements of the first input parameter (list) if it is equal to the second parameter. + +```python +from narps_open.core.common import clean_list + +# Here we remove subject 002 from a group of subjects. +clean_list(['002', '005', '006', '007'], '002') +``` + +* `list_intersection` : return the intersection of two lists. + +```python +from narps_open.core.common import list_intersection + +# Here we keep only subjects that are in the equalRange group and selected for the analysis. +equal_range_group = ['002', '004', '006', '008'] +selected_for_analysis = ['002', '006', '010'] +list_intersection(equal_range_group, selected_for_analysis) # Returns ['002', '006'] +``` + +## narps_open.core.image + +This module contains a set of functions dedicated to computations on images. + + * `get_voxel_dimensions` : returns the voxel dimensions of an image + +```python +# Get dimensions of voxels along x, y, and z in mm (returns e.g.: [1.0, 1.0, 1.0]). +get_voxel_dimensions('/path/to/the/image.nii.gz') +``` diff --git a/narps_open/core/__init__.py b/narps_open/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/narps_open/core/common.py b/narps_open/core/common.py new file mode 100644 index 00000000..e40d4e9a --- /dev/null +++ b/narps_open/core/common.py @@ -0,0 +1,65 @@ +#!/usr/bin/python +# coding: utf-8 + +""" Common functions to write pipelines """ + +def remove_file(_, file_name: str) -> None: + """ + Fully remove files generated by a Node, once they aren't needed anymore. + This function is meant to be used in a Nipype Function Node. + + Parameters: + - _: input only used for triggering the Node + - file_name: str, a single absolute filename of the file to remove + """ + # This import must stay inside the function, as required by Nipype + from os import remove + + try: + remove(file_name) + except OSError as error: + print(error) + +def elements_in_string(input_str: str, elements: list) -> str: #| None: + """ + Return input_str if it contains one element of the elements list. + Return None otherwise. + This function is meant to be used in a Nipype Function Node. + + Parameters: + - input_str: str + - elements: list of str, elements to be searched in input_str + """ + if any(e in input_str for e in elements): + return input_str + return None + +def clean_list(input_list: list, element = None) -> list: + """ + Remove elements of input_list that are equal to element and return the resultant list. + This function is meant to be used in a Nipype Function Node. It can be used inside a + nipype.Workflow.connect call as well. + + Parameters: + - input_list: list + - element: any + + Returns: + - input_list with elements equal to element removed + """ + return [f for f in input_list if f != element] + +def list_intersection(list_1: list, list_2: list) -> list: + """ + Returns the intersection of two lists. + This function is meant to be used in a Nipype Function Node. It can be used inside a + nipype.Workflow.connect call as well. + + Parameters: + - list_1: list + - list_2: list + + Returns: + - list, the intersection of list_1 and list_2 + """ + return [e for e in list_1 if e in list_2] diff --git a/narps_open/core/image.py b/narps_open/core/image.py new file mode 100644 index 00000000..35323e45 --- /dev/null +++ b/narps_open/core/image.py @@ -0,0 +1,25 @@ +#!/usr/bin/python +# coding: utf-8 + +""" Image functions to write pipelines """ + +def get_voxel_dimensions(image: str) -> list: + """ + Return the voxel dimensions of a image in millimeters. + + Arguments: + image: str, string that represent an absolute path to a Nifti image. + + Returns: + list, size of the voxels in the image in millimeters. + """ + # This import must stay inside the function, as required by Nipype + from nibabel import load + + voxel_dimensions = load(image).header.get_zooms() + + return [ + float(voxel_dimensions[0]), + float(voxel_dimensions[1]), + float(voxel_dimensions[2]) + ] diff --git a/narps_open/data/participants.py b/narps_open/data/participants.py index a9cc65a5..835e834f 100644 --- a/narps_open/data/participants.py +++ b/narps_open/data/participants.py @@ -49,3 +49,11 @@ def get_participants(team_id: str) -> list: def get_participants_subset(nb_participants: int = 108) -> list: """ Return a list of participants of length nb_participants """ return get_all_participants()[0:nb_participants] + +def get_group(group_name: str) -> list: + """ Return a list containing all the participants inside the group_name group + + Warning : the subject ids are return as written in the participants file (i.e.: 'sub-*') + """ + participants = get_participants_information() + return participants.loc[participants['group'] == group_name]['participant_id'].values.tolist() diff --git a/tests/core/__init__.py b/tests/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/core/test_common.py b/tests/core/test_common.py new file mode 100644 index 00000000..3e00fd1b --- /dev/null +++ b/tests/core/test_common.py @@ -0,0 +1,319 @@ +#!/usr/bin/python +# coding: utf-8 + +""" Tests of the 'narps_open.core.common' module. + +Launch this test with PyTest + +Usage: +====== + pytest -q test_common.py + pytest -q test_common.py -k +""" +from os import mkdir +from os.path import join, exists, abspath +from shutil import rmtree +from pathlib import Path + +from pytest import mark, fixture +from nipype import Node, Function, Workflow + +from narps_open.utils.configuration import Configuration +import narps_open.core.common as co + +TEMPORARY_DIR = join(Configuration()['directories']['test_runs'], 'test_common') + +@fixture +def remove_test_dir(): + """ A fixture to remove temporary directory created by tests """ + + rmtree(TEMPORARY_DIR, ignore_errors = True) + mkdir(TEMPORARY_DIR) + yield # test runs here + rmtree(TEMPORARY_DIR, ignore_errors = True) + +class TestCoreCommon: + """ A class that contains all the unit tests for the common module.""" + + @staticmethod + @mark.unit_test + def test_remove_file(remove_test_dir): + """ Test the remove_file function """ + + # Create a single file + test_file_path = abspath(join(TEMPORARY_DIR, 'file1.txt')) + Path(test_file_path).touch() + + # Check file exist + assert exists(test_file_path) + + # Create a Nipype Node using remove_files + test_remove_file_node = Node(Function( + function = co.remove_file, + input_names = ['_', 'file_name'], + output_names = [] + ), name = 'test_remove_file_node') + test_remove_file_node.inputs._ = '' + test_remove_file_node.inputs.file_name = test_file_path + test_remove_file_node.run() + + # Check file is removed + assert not exists(test_file_path) + + @staticmethod + @mark.unit_test + def test_node_elements_in_string(): + """ Test the elements_in_string function as a nipype.Node """ + + # Inputs + string = 'test_string' + elements_false = ['z', 'u', 'warning'] + elements_true = ['z', 'u', 'warning', '_'] + + # Create a Nipype Node using elements_in_string + test_node = Node(Function( + function = co.elements_in_string, + input_names = ['input_str', 'elements'], + output_names = ['output'] + ), name = 'test_node') + test_node.inputs.input_str = string + test_node.inputs.elements = elements_true + out = test_node.run().outputs.output + + # Check return value + assert out == string + + # Change input and check return value + test_node = Node(Function( + function = co.elements_in_string, + input_names = ['input_str', 'elements'], + output_names = ['output'] + ), name = 'test_node') + test_node.inputs.input_str = string + test_node.inputs.elements = elements_false + out = test_node.run().outputs.output + assert out is None + + @staticmethod + @mark.unit_test + def test_connect_elements_in_string(remove_test_dir): + """ Test the elements_in_string function as evaluated in a connect """ + + # Inputs + string = 'test_string' + elements_false = ['z', 'u', 'warning'] + elements_true = ['z', 'u', 'warning', '_'] + function = lambda in_value: in_value + + # Create Nodes + node_1 = Node(Function( + function = function, + input_names = ['in_value'], + output_names = ['out_value'] + ), name = 'node_1') + node_1.inputs.in_value = string + node_true = Node(Function( + function = function, + input_names = ['in_value'], + output_names = ['out_value'] + ), name = 'node_true') + node_false = Node(Function( + function = function, + input_names = ['in_value'], + output_names = ['out_value'] + ), name = 'node_false') + + # Create Workflow + test_workflow = Workflow( + base_dir = TEMPORARY_DIR, + name = 'test_workflow' + ) + test_workflow.connect([ + # elements_in_string is evaluated as part of the connection + (node_1, node_true, [( + ('out_value', co.elements_in_string, elements_true), 'in_value')]), + (node_1, node_false, [( + ('out_value', co.elements_in_string, elements_false), 'in_value')]) + ]) + + test_workflow.run() + + test_file_t = join(TEMPORARY_DIR, 'test_workflow', 'node_true', '_report', 'report.rst') + with open(test_file_t, 'r', encoding = 'utf-8') as file: + assert '* out_value : test_string' in file.read() + + test_file_f = join(TEMPORARY_DIR, 'test_workflow', 'node_false', '_report', 'report.rst') + with open(test_file_f, 'r', encoding = 'utf-8') as file: + assert '* out_value : None' in file.read() + + @staticmethod + @mark.unit_test + def test_node_clean_list(): + """ Test the clean_list function as a nipype.Node """ + + # Inputs + input_list = ['z', '_', 'u', 'warning', '_', None] + element_to_remove_1 = '_' + output_list_1 = ['z', 'u', 'warning', None] + element_to_remove_2 = None + output_list_2 = ['z', '_', 'u', 'warning', '_'] + + # Create a Nipype Node using clean_list + test_node = Node(Function( + function = co.clean_list, + input_names = ['input_list', 'element'], + output_names = ['output'] + ), name = 'test_node') + test_node.inputs.input_list = input_list + test_node.inputs.element = element_to_remove_1 + + # Check return value + assert test_node.run().outputs.output == output_list_1 + + # Change input and check return value + test_node = Node(Function( + function = co.clean_list, + input_names = ['input_list', 'element'], + output_names = ['output'] + ), name = 'test_node') + test_node.inputs.input_list = input_list + test_node.inputs.element = element_to_remove_2 + + assert test_node.run().outputs.output == output_list_2 + + @staticmethod + @mark.unit_test + def test_connect_clean_list(remove_test_dir): + """ Test the clean_list function as evaluated in a connect """ + + # Inputs + input_list = ['z', '_', 'u', 'warning', '_', None] + element_to_remove_1 = '_' + output_list_1 = ['z', 'u', 'warning', None] + element_to_remove_2 = None + output_list_2 = ['z', '_', 'u', 'warning', '_'] + function = lambda in_value: in_value + + # Create Nodes + node_0 = Node(Function( + function = function, + input_names = ['in_value'], + output_names = ['out_value'] + ), name = 'node_0') + node_0.inputs.in_value = input_list + node_1 = Node(Function( + function = function, + input_names = ['in_value'], + output_names = ['out_value'] + ), name = 'node_1') + node_2 = Node(Function( + function = function, + input_names = ['in_value'], + output_names = ['out_value'] + ), name = 'node_2') + + # Create Workflow + test_workflow = Workflow( + base_dir = TEMPORARY_DIR, + name = 'test_workflow' + ) + test_workflow.connect([ + # elements_in_string is evaluated as part of the connection + (node_0, node_1, [(('out_value', co.clean_list, element_to_remove_1), 'in_value')]), + (node_0, node_2, [(('out_value', co.clean_list, element_to_remove_2), 'in_value')]) + ]) + test_workflow.run() + + test_file_1 = join(TEMPORARY_DIR, 'test_workflow', 'node_1', '_report', 'report.rst') + with open(test_file_1, 'r', encoding = 'utf-8') as file: + assert f'* out_value : {output_list_1}' in file.read() + + test_file_2 = join(TEMPORARY_DIR, 'test_workflow', 'node_2', '_report', 'report.rst') + with open(test_file_2, 'r', encoding = 'utf-8') as file: + assert f'* out_value : {output_list_2}' in file.read() + + @staticmethod + @mark.unit_test + def test_node_list_intersection(): + """ Test the list_intersection function as a nipype.Node """ + + # Inputs / outputs + input_list_1 = ['001', '002', '003', '004'] + input_list_2 = ['002', '004'] + input_list_3 = ['001', '003', '005'] + output_list_1 = ['002', '004'] + output_list_2 = ['001', '003'] + + # Create a Nipype Node using list_intersection + test_node = Node(Function( + function = co.list_intersection, + input_names = ['list_1', 'list_2'], + output_names = ['output'] + ), name = 'test_node') + test_node.inputs.list_1 = input_list_1 + test_node.inputs.list_2 = input_list_2 + + # Check return value + assert test_node.run().outputs.output == output_list_1 + + # Change input and check return value + test_node = Node(Function( + function = co.list_intersection, + input_names = ['list_1', 'list_2'], + output_names = ['output'] + ), name = 'test_node') + test_node.inputs.list_1 = input_list_1 + test_node.inputs.list_2 = input_list_3 + + assert test_node.run().outputs.output == output_list_2 + + @staticmethod + @mark.unit_test + def test_connect_list_intersection(remove_test_dir): + """ Test the list_intersection function as evaluated in a connect """ + + # Inputs / outputs + input_list_1 = ['001', '002', '003', '004'] + input_list_2 = ['002', '004'] + input_list_3 = ['001', '003', '005'] + output_list_1 = ['002', '004'] + output_list_2 = ['001', '003'] + function = lambda in_value: in_value + + # Create Nodes + node_0 = Node(Function( + function = function, + input_names = ['in_value'], + output_names = ['out_value'] + ), name = 'node_0') + node_0.inputs.in_value = input_list_1 + node_1 = Node(Function( + function = function, + input_names = ['in_value'], + output_names = ['out_value'] + ), name = 'node_1') + node_2 = Node(Function( + function = function, + input_names = ['in_value'], + output_names = ['out_value'] + ), name = 'node_2') + + # Create Workflow + test_workflow = Workflow( + base_dir = TEMPORARY_DIR, + name = 'test_workflow' + ) + test_workflow.connect([ + # elements_in_string is evaluated as part of the connection + (node_0, node_1, [(('out_value', co.list_intersection, input_list_2), 'in_value')]), + (node_0, node_2, [(('out_value', co.list_intersection, input_list_3), 'in_value')]) + ]) + test_workflow.run() + + test_file_1 = join(TEMPORARY_DIR, 'test_workflow', 'node_1', '_report', 'report.rst') + with open(test_file_1, 'r', encoding = 'utf-8') as file: + assert f'* out_value : {output_list_1}' in file.read() + + test_file_2 = join(TEMPORARY_DIR, 'test_workflow', 'node_2', '_report', 'report.rst') + with open(test_file_2, 'r', encoding = 'utf-8') as file: + assert f'* out_value : {output_list_2}' in file.read() diff --git a/tests/core/test_image.py b/tests/core/test_image.py new file mode 100644 index 00000000..d3b83ac5 --- /dev/null +++ b/tests/core/test_image.py @@ -0,0 +1,48 @@ +#!/usr/bin/python +# coding: utf-8 + +""" Tests of the 'narps_open.core.image' module. + +Launch this test with PyTest + +Usage: +====== + pytest -q test_image.py + pytest -q test_image.py -k +""" + +from os.path import abspath, join +from numpy import isclose + +from pytest import mark +from nipype import Node, Function + +from narps_open.utils.configuration import Configuration +import narps_open.core.image as im + +class TestCoreImage: + """ A class that contains all the unit tests for the image module.""" + + @staticmethod + @mark.unit_test + def test_get_voxel_dimensions(): + """ Test the get_voxel_dimensions function """ + + # Path to the test image + test_file_path = abspath(join( + Configuration()['directories']['test_data'], + 'core', + 'image', + 'test_image.nii.gz')) + + # Create a Nipype Node using get_voxel_dimensions + test_get_voxel_dimensions_node = Node(Function( + function = im.get_voxel_dimensions, + input_names = ['image'], + output_names = ['voxel_dimensions'] + ), name = 'test_get_voxel_dimensions_node') + test_get_voxel_dimensions_node.inputs.image = test_file_path + outputs = test_get_voxel_dimensions_node.run().outputs + + # Check voxel sizes + assert isclose(outputs.voxel_dimensions, [8.0, 8.0, 9.6]).all() diff --git a/tests/data/test_participants.py b/tests/data/test_participants.py index d30cd23e..f36f0a05 100644 --- a/tests/data/test_participants.py +++ b/tests/data/test_participants.py @@ -10,28 +10,46 @@ pytest -q test_participants.py pytest -q test_participants.py -k """ +from os.path import join -from pytest import mark +from pytest import mark, fixture import narps_open.data.participants as part +from narps_open.utils.configuration import Configuration + +@fixture +def mock_participants_data(mocker): + """ A fixture to provide mocked data from the test_data directory """ + + mocker.patch( + 'narps_open.data.participants.Configuration', + return_value = { + 'directories': { + 'dataset': join( + Configuration()['directories']['test_data'], + 'data', 'participants') + } + } + ) class TestParticipants: """ A class that contains all the unit tests for the participants module.""" @staticmethod @mark.unit_test - def test_get_participants_information(): + def test_get_participants_information(mock_participants_data): """ Test the get_participants_information function """ - """p_info = part.get_participants_information() - assert len(p_info) == 108 - assert p_info.at[5, 'participant_id'] == 'sub-006' - assert p_info.at[5, 'group'] == 'equalRange' - assert p_info.at[5, 'gender'] == 'M' - assert p_info.at[5, 'age'] == 30 - assert p_info.at[12, 'participant_id'] == 'sub-015' - assert p_info.at[12, 'group'] == 'equalIndifference' - assert p_info.at[12, 'gender'] == 'F' - assert p_info.at[12, 'age'] == 26""" + + p_info = part.get_participants_information() + assert len(p_info) == 4 + assert p_info.at[1, 'participant_id'] == 'sub-002' + assert p_info.at[1, 'group'] == 'equalRange' + assert p_info.at[1, 'gender'] == 'M' + assert p_info.at[1, 'age'] == 25 + assert p_info.at[2, 'participant_id'] == 'sub-003' + assert p_info.at[2, 'group'] == 'equalIndifference' + assert p_info.at[2, 'gender'] == 'F' + assert p_info.at[2, 'age'] == 27 @staticmethod @mark.unit_test @@ -87,3 +105,12 @@ def test_get_participants_subset(): assert len(participants_list) == 80 assert participants_list[0] == '020' assert participants_list[-1] == '003' + + @staticmethod + @mark.unit_test + def test_get_group(mock_participants_data): + """ Test the get_group function """ + + assert part.get_group('') == [] + assert part.get_group('equalRange') == ['sub-002', 'sub-004'] + assert part.get_group('equalIndifference') == ['sub-001', 'sub-003'] diff --git a/tests/test_data/core/image/test_image.nii.gz b/tests/test_data/core/image/test_image.nii.gz new file mode 100644 index 00000000..06fb9aa3 Binary files /dev/null and b/tests/test_data/core/image/test_image.nii.gz differ diff --git a/tests/test_data/data/participants/participants.tsv b/tests/test_data/data/participants/participants.tsv new file mode 100644 index 00000000..312dbcde --- /dev/null +++ b/tests/test_data/data/participants/participants.tsv @@ -0,0 +1,5 @@ +participant_id group gender age +sub-001 equalIndifference M 24 +sub-002 equalRange M 25 +sub-003 equalIndifference F 27 +sub-004 equalRange M 25 \ No newline at end of file